From b2bc3a4081b6eff947acdb13e32e2d64ceb1823c Mon Sep 17 00:00:00 2001
From: Sofer Athlan-Guyot <sathlang@redhat.com>
Date: Thu, 02 Feb 2017 12:04:15 +0100
Subject: [PATCH] Ensure working compute node during the whole migration.

Fix subtile error introduced during the attempt to get the vm migration
after controller upgrade working.  Namely the server.cnf in the my.cnf.d
directory was updated with default values, which include listening to
127.0.0.1.  This overwrite the galera.cnf configuration and break the
link between the haproxy and the database, making the database
unavailable for services.  This happens only at convergence step where
the database is eventually restarted.

Beside fixing this bug, this is a rewrite at having the possibility for
the operator to manipulate instance during those states:
 - controllers upgraded/compute not upgraded (vm migration)
 - controllers upgraded/part of compute upgraded (vm migration)
 - controllers upgraded/all computes upgraded/no convergence

Change-Id: I22a9f1cde99c010d00c653861cc7cf34ab4cc06e
Closes-Bug: #1661202
---

diff --git a/extraconfig/tasks/liberty_to_mitaka_nova_api_upgrade.pp b/extraconfig/tasks/liberty_to_mitaka_nova_api_upgrade.pp
index fd090b8..94fc9ff 100644
--- a/extraconfig/tasks/liberty_to_mitaka_nova_api_upgrade.pp
+++ b/extraconfig/tasks/liberty_to_mitaka_nova_api_upgrade.pp
@@ -15,23 +15,58 @@
 
 # This puppet manifest is to be used only during a Liberty->Mitaka upgrade
 # It creates the nova api database during the controller upgrade instead of
-# during the converge step.
-
-class { '::mysql::server':
-  service_manage => false,
-}
-
+# during the converge step.  Executed on the controllers only.
 include ::tripleo::packages
 $pacemaker_master   = hiera('bootstrap_nodeid')
 
+# Will prevent any reboot of any service.
+Service <| |> {
+  hasrestart => true,
+  restart    => '/bin/true',
+  start      => '/bin/true',
+  stop       => '/bin/true',
+  enable     => '/bin/true',
+  provider   => 'base',
+}
+
+# Restart of the nova-* services will be done in another step.
+nova_config {
+  'api_database/connection': value => hiera('nova::api_database_connection')
+}
+
 if downcase($pacemaker_master) == $::hostname {
-    class {'::nova::db::mysql_api':
-        user          => hiera('nova::db::mysql_api::dbname'),
-        password      => hiera('nova::db::mysql_api::password'),
-        dbname        => hiera('nova::db::mysql_api::dbname'),
-        host          => hiera('nova_api_vip'),
-        charset       => 'utf8',
-        collate       => 'utf8_general_ci',
-        allowed_hosts => undef,
-      }
+  # You have to open up the class here to avoid any change to the
+  # /etc/my.cnf.d/server.cnf.  Ultimately, "include ::mysql::server"
+  # is the one that modify it.
+  $dbname = 'nova_api'
+  mysql_database { $dbname:
+    ensure  => present,
+    charset => 'utf8',
+    collate => 'utf8_general_ci',
+    notify  => Exec['nova-db-sync-api'],
+  }
+
+  $allowed_hosts_list = unique(concat(any2array(hiera('nova::db::mysql::allowed_hosts')), [hiera('nova_api_vip')]))
+  $real_allowed_hosts = prefix($allowed_hosts_list, "${dbname}_")
+
+  openstacklib::db::mysql::host_access { $real_allowed_hosts:
+    user          => hiera('nova::db::mysql_api::dbname'),
+    password_hash => mysql_password(hiera('nova::db::mysql_api::password')),
+    database      => $dbname,
+    privileges    => 'ALL',
+    notify        => Exec['nova-db-sync-api'],
+  }
+
+  exec { 'nova-db-sync-api':
+    command     => '/usr/bin/nova-manage api_db sync',
+    refreshonly => true,
+    logoutput   => on_failure,
+    notify      => Exec['nova-db-online-data-migration']
+  }
+
+  exec { 'nova-db-online-data-migration':
+    command     => '/usr/bin/nova-manage db online_data_migration',
+    refreshonly => true,
+    logoutput   => on_failure,
+  }
 }
diff --git a/extraconfig/tasks/liberty_to_mitaka_nova_api_upgrade_conf.pp b/extraconfig/tasks/liberty_to_mitaka_nova_api_upgrade_conf.pp
new file mode 100644
index 0000000..94bc8e5
--- /dev/null
+++ b/extraconfig/tasks/liberty_to_mitaka_nova_api_upgrade_conf.pp
@@ -0,0 +1,32 @@
+# Copyright 2017 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This puppet manifest is to be used only during a Liberty->Mitaka
+# upgrade It ensures proper configuration of nova.conf on controller
+# and compute nodes.
+nova_config {
+  'neutron/auth_plugin':         value => 'v3password';
+  'neutron/username':            value => hiera('nova::network::neutron::neutron_username');
+  'neutron/password':            value => hiera('nova::network::neutron::neutron_password');
+  'neutron/auth_url':            value => hiera('nova::network::neutron::neutron_auth_url');
+  'neutron/project_name':        value => hiera('nova::network::neutron::neutron_project_name');
+  'neutron/user_domain_name':    value => 'Default';
+  'neutron/project_domain_name': value => 'Default';
+  'DEFAULT/use_neutron':         value => 'True';
+  'neutron/auth_strategy':       ensure => absent;
+  'neutron/admin_tenant_name':   ensure => absent;
+  'neutron/admin_username':      ensure => absent;
+  'neutron/admin_auth_url':      ensure => absent;
+}
diff --git a/extraconfig/tasks/major_upgrade_compute.sh b/extraconfig/tasks/major_upgrade_compute.sh
index 24bf092..ade6367 100644
--- a/extraconfig/tasks/major_upgrade_compute.sh
+++ b/extraconfig/tasks/major_upgrade_compute.sh
@@ -33,10 +33,18 @@
 yum -y install python-zaqarclient  # needed for os-collect-config
 yum -y update
 
-# Update nova neutron auth parameters, to be able to launch and migrate the
-# workloads before running the converge step
-puppet apply /root/liberty_to_mitaka_neutron_auth_param.pp
-systemctl restart openstack-nova-compute
+# Problem creating vif if not restarted.
+if systemctl is-enabled openvswitch; then
+   systemctl restart openvswitch
+fi
+
+# Look like it is required after the installation of the new openvswitch.
+if systemctl is-enabled neutron-openvswitch-agent; then
+    if systemctl is-failed neutron-openvswitch-agent; then
+        systemctl restart neutron-openvswitch-agent
+    fi
+fi
+
 ENDOFCAT
 
 # ensure the permissions are OK
diff --git a/extraconfig/tasks/major_upgrade_pacemaker.yaml b/extraconfig/tasks/major_upgrade_pacemaker.yaml
index cdf9473..f12ed76 100644
--- a/extraconfig/tasks/major_upgrade_pacemaker.yaml
+++ b/extraconfig/tasks/major_upgrade_pacemaker.yaml
@@ -100,7 +100,10 @@
     properties:
       group: puppet
       config:
-        get_file: liberty_to_mitaka_nova_api_upgrade.pp
+        list_join:
+          - ''
+          - - get_file: liberty_to_mitaka_nova_api_upgrade.pp
+            - get_file: liberty_to_mitaka_nova_api_upgrade_conf.pp
 
   NovaApiDatabaseUpgradeDeployment:
     type: OS::Heat::SoftwareDeploymentGroup
@@ -110,26 +113,54 @@
       servers: {get_param: controller_servers}
       config: {get_resource: NovaApiDatabaseUpgradeConfig}
 
-  NovaApiDatabasePostConfig:
+  NovaApiDatabasePostUgradeRestartConfig:
     type: OS::Heat::SoftwareConfig
     properties:
       group: script
       config: |
         #!/bin/bash
-        set -eux
-        crudini --set /etc/nova/nova.conf api_database connection "$(hiera nova::api_database_connection)"
-        crudini --set /etc/nova/nova.conf neutron auth_plugin v3password
-        crudini --set /etc/nova/nova.conf neutron auth_url "$(hiera nova::api::identity_uri)"
-        if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]; then
-            nova-manage api_db sync
-            nova-manage db online_data_migrations
-        fi
-        systemctl restart openstack-nova-api
 
-  NovaApiDatabasePostUpgradeDeployment:
+        set -eux
+        # As per http://docs.openstack.org/developer/nova/upgrade.html
+        # restart conductor first and nova-api list.
+
+        # This is the best way to restart the whole nova stack.  It
+        # 100% of time leads to no transiant errors in pcs resources.
+        # conductor is the leaf of the pacemaker dependency graph and
+        # must be restarted first.  consoleauth is the root of this
+        # branch and will restart all nova service in this order:
+        # openstack-nova-consoleauth openstack-nova-novncproxy openstack-nova-api openstack-nova-scheduler openstack-nova-conductor
+        if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]; then
+            for res in openstack-nova-conductor openstack-nova-consoleauth ; do
+                pcs resource restart ${res}-clone --wait=600
+            done
+        fi
+
+  NovaApiDatabaseUpgradePostUpgradeRestartDeployment:
     type: OS::Heat::SoftwareDeploymentGroup
     depends_on: NovaApiDatabaseUpgradeDeployment
     properties:
-      name: NovaApiDatabasePostUpgradeDeployment
+      name: NovaApiDatabaseUpgradePostUpgradeRestartDeployment
       servers: {get_param: controller_servers}
-      config: {get_resource: NovaApiDatabasePostConfig}
+      config: {get_resource: NovaApiDatabasePostUgradeRestartConfig}
+
+  NovaApiComputePostConfig:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: puppet
+      config:
+        list_join:
+        - ''
+        - - get_file: liberty_to_mitaka_nova_api_upgrade_conf.pp
+          - |
+            exec { '/bin/systemctl restart openstack-nova-compute': refreshonly => true }
+            Nova_config<| |> ~> Exec['/bin/systemctl restart openstack-nova-compute']
+
+
+  NovaApiComputePostUpgradeDeployment:
+    type: OS::Heat::SoftwareDeploymentGroup
+    depends_on: NovaApiDatabaseUpgradePostUpgradeRestartDeployment
+    properties:
+      name: NovaApiComputePostUpgradeDeployment
+      servers: {get_param: compute_servers}
+      config: {get_resource: NovaApiComputePostConfig}
diff --git a/extraconfig/tasks/major_upgrade_pacemaker_init.yaml b/extraconfig/tasks/major_upgrade_pacemaker_init.yaml
index 3f1f7ea..623549a 100644
--- a/extraconfig/tasks/major_upgrade_pacemaker_init.yaml
+++ b/extraconfig/tasks/major_upgrade_pacemaker_init.yaml
@@ -97,13 +97,6 @@
                 UPGRADE_LEVEL_NOVA_COMPUTE: {get_param: UpgradeLevelNovaCompute}
           - get_file: major_upgrade_compute.sh
 
-  NovaNeutronAuthParamUpgradeConfig:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: script
-      config:
-        get_file: liberty_to_mitaka_neutron_auth_param.sh
-
   ComputeDeliverUpgradeScriptDeployment:
     type: OS::Heat::SoftwareDeploymentGroup
     properties:
@@ -111,13 +104,6 @@
       config: {get_resource: ComputeDeliverUpgradeScriptConfig}
       input_values: {get_param: input_values}
 
-  NovaNeutronAuthParamUpgradeDeployment:
-    type: OS::Heat::SoftwareDeploymentGroup
-    properties:
-      servers:  {get_param: compute_servers}
-      config: {get_resource: NovaNeutronAuthParamUpgradeConfig}
-      input_values: {get_param: input_values}
-
   ObjectStorageDeliverUpgradeScriptConfig:
     type: OS::Heat::SoftwareConfig
     properties: