Fix 8.0 update procedure

There is a bug around lvm, systemd and xfs. So we need
to change update procedure.
Now it reboots master node after update.

Related-Bug: #1642232

Change-Id: Ie59e339d6876264fe463a47061089281acbcdc78

diff --git a/jenkins/build_cluster/build_cluster.py b/jenkins/build_cluster/build_cluster.py
index 08d9e8919cf68e269ce66285aa53e2e78481dc9f..030bdbcc9d70c3c774ba58279b5693b118d01f6d 100755 (executable)
--- a/jenkins/build_cluster/build_cluster.py
+++ b/jenkins/build_cluster/build_cluster.py
@@ -18,7 +18,8 @@ import scancodes
 UPDATE_HELPER = "update_helper.sh"
 REPO_HELPER = "repo_helper.sh"
 SSH_PARAMS = ["-o", "UserKnownHostsFile=/dev/null",
-              "-o", "StrictHostKeyChecking=no"]
+              "-o", "StrictHostKeyChecking=no",
+              "-o", "LogLevel=quiet"] # That supress most of warnings and info messages
 DNS_SERVER = "172.18.16.10" #Moscow DNS
 
 cfg = dict()
@@ -512,10 +513,21 @@ def add_cent_repo(node, repolist):
 
 
 def do_update(node):
+    if not node.put_file(UPDATE_HELPER):
+        print ("ERROR: Unable to copy update script to admin node")
+        return False
+
+    if not node.execute(["/tmp/"+UPDATE_HELPER,"first"]):
+        print ("ERROR: First phase of upgrade faileduuu")
+        return False
+        
+    node.execute(["/tmp/"+UPDATE_HELPER,"reboot"])
+    time.sleep(60 * 5)
+
     if node.put_file(UPDATE_HELPER):
-        return node.execute(["/tmp/"+UPDATE_HELPER])
+        return node.execute(["/tmp/"+UPDATE_HELPER,"second"])
     else:
-        print ("ERROR: Unable to copy update script to admin node")
+        print ("ERROR: Unable to copy update script to admin node") 
         return False
 
 

diff --git a/jenkins/build_cluster/update_helper.sh b/jenkins/build_cluster/update_helper.sh
index 3eb2e72d6c5896a7b7ccb00591ba2141f0758bf9..8619f8570c9e732c94eac3bff07ebba3f8a88e7b 100755 (executable)
--- a/jenkins/build_cluster/update_helper.sh
+++ b/jenkins/build_cluster/update_helper.sh
@@ -40,6 +40,19 @@ upgrade_with_docker () {
        RUN_WITH_LOGGER "dockerctl start all"  || return 1
 }
 
+upgrade_first_phase_8x (){
+       RUN_WITH_LOGGER "dockerctl destroy all" || return 1
+       RUN_WITH_LOGGER "docker rmi -f $(docker images -q -a)"
+       RUN_WITH_LOGGER "systemctl stop docker.service" || return 1
+       RUN_WITH_LOGGER "yum update -y " || return 1
+}
+
+upgrade_second_phase_8x (){
+       #RUN_WITH_LOGGER "systemctl start docker.service" || return 1
+       RUN_WITH_LOGGER "docker load -i /var/www/nailgun/docker/images/fuel-images.tar" || return 1
+       RUN_WITH_LOGGER "dockerctl start all" || return 1
+}
+
 upgrade_to_9x () {
     RUN_WITH_LOGGER yum clean all || return 1
     RUN_WITH_LOGGER yum install python-cudet -y || return 1
@@ -63,8 +76,70 @@ wait_for () {
        done;
 }
 
-version=$(get_fuel_version) || crap "error: can't get fuel version"
+first_phase () {
+       case $version in 
+               # Check that we have update repo in yum.repos.d
+               7.0.0|8.0)
+                       wait_for '[ -f /etc/yum.repos.d/mos[78].0-updates.repo ]' 30 60 || crap "error: timeout waiting of update repo"
+                       ;;&
+
+               9.0)
+                       wait_for '[ -f /etc/yum.repos.d/mos-updates.repo ]' 30 60 || crap "error: timeout waiting of update repo"
+                       ;;&
+
+               # Check that we have no bootstrap builder right now
+               7.0.0)
+                       wait_for '[ $(ps -ef | grep fuel-bootstrap | grep -v grep | wc -l) -eq 0 ]' 30 60 || crap "error: timeout of bootstrap waiting" 
+                       ;;&
+
+               # Upgrade
+               7.0.0)
+                       upgrade_with_docker  || crap "error: upgrade failed for version $version. See log $upgrade_log on master node."
+                       ;;&
+
+               8.0)
+                       upgrade_first_phase_8x  || crap "error: upgrade failed for version $version. See log $upgrade_log on master node."
+                       ;;&
+
+               9.0)
+                       upgrade_to_9x  || crap "error: upgrade failed for version $version. See log $upgrade_log on master node."
+                       ;;&
+       esac
+}
 
+reboot_phase () {
+       case $version in
+               8.0)
+                       echo "Going to reboot node..."
+                       shutdown -r +1 
+                       exit 0
+                       ;;
+               *)
+                       echo "No action on reboot phase for $version"
+               ;;
+       esac
+}
+
+second_phase () {
+       case $version in 
+               # Upgrade
+               8.0)
+                       upgrade_second_phase_8x  || crap "error: upgrade failed for version $version. See log $upgrade_log on master node."
+                       ;;&
+
+               # Run bootstap builder
+               # NOTE: No need to do this step for MOS9 because it's implemented in update procedure
+               7.0.0)
+                       RUN_WITH_LOGGER "fuel-bootstrap-image" || crap "error: fuel-bootstrap failed. See log $upgrade_log on master node."
+                       ;;
+
+               8.0)
+                       RUN_WITH_LOGGER "fuel-bootstrap build --activate" || crap "error: fuel-bootstrap failed. See log $upgrade_log on master node."
+                       ;;
+       esac
+}
+
+version=$(get_fuel_version) || crap "error: can't get fuel version"
 
 case $version in
        7.0.0|8.0|9.0)
@@ -75,37 +150,15 @@ case $version in
                ;;
 esac
 
-case $version in 
-       # Check that we have update repo in yum.repos.d
-       7.0.0|8.0)
-               wait_for '[ -f /etc/yum.repos.d/mos[78].0-updates.repo ]' 30 60 || crap "error: timeout waiting of update repo"
-               ;;&
-
-       9.0)
-               wait_for '[ -f /etc/yum.repos.d/mos-updates.repo ]' 30 60 || crap "error: timeout waiting of update repo"
-               ;;&
-
-       # Check that we have no bootstrap builder right now
-       7.0.0)
-               wait_for '[ $(ps -ef | grep fuel-bootstrap | grep -v grep | wc -l) -eq 0 ]' 30 60 || crap "error: timeout of bootstrap waiting" 
-               ;;&
-
-       # Upgrade
-       7.0.0|8.0)
-               upgrade_with_docker  || crap "error: upgrade failed for version $version. See log $upgrade_log on master node."
-               ;;&
-
-       9.0)
-               upgrade_to_9x  || crap "error: upgrade failed for version $version. See log $upgrade_log on master node."
-               ;;&
-
-       # Run bootstap builder
-    # NOTE: No need to do this step for MOS9 because it's implemented in update procedure
-       7.0.0)
-               RUN_WITH_LOGGER "fuel-bootstrap-image" || crap "error: fuel-bootstrap failed. See log $upgrade_log on master node."
-               ;;
+# For some of the upgrade procedures we have to split upgrade on two phase with reboot between.
+phase=${1:-"second"}
 
-       8.0)
-               RUN_WITH_LOGGER "fuel-bootstrap build --activate" || crap "error: fuel-bootstrap failed. See log $upgrade_log on master node."
+case $phase in
+       first|second|reboot)
+               action="${phase}_phase"
+               $action
+               ;;
+       *)
+               crap "error: Unable to undestand phase name: $phase"
                ;;
 esac