From: Anton Chevychalov Date: Wed, 21 Dec 2016 13:35:44 +0000 (+0300) Subject: Fix 8.0 update procedure X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F82%2F29482%2F6;p=tools%2Fsustaining.git Fix 8.0 update procedure There is a bug around lvm, systemd and xfs. So we need to change update procedure. Now it reboots master node after update. Related-Bug: #1642232 Change-Id: Ie59e339d6876264fe463a47061089281acbcdc78 --- diff --git a/jenkins/build_cluster/build_cluster.py b/jenkins/build_cluster/build_cluster.py index 08d9e89..030bdbc 100755 --- a/jenkins/build_cluster/build_cluster.py +++ b/jenkins/build_cluster/build_cluster.py @@ -18,7 +18,8 @@ import scancodes UPDATE_HELPER = "update_helper.sh" REPO_HELPER = "repo_helper.sh" SSH_PARAMS = ["-o", "UserKnownHostsFile=/dev/null", - "-o", "StrictHostKeyChecking=no"] + "-o", "StrictHostKeyChecking=no", + "-o", "LogLevel=quiet"] # That supress most of warnings and info messages DNS_SERVER = "172.18.16.10" #Moscow DNS cfg = dict() @@ -512,10 +513,21 @@ def add_cent_repo(node, repolist): def do_update(node): + if not node.put_file(UPDATE_HELPER): + print ("ERROR: Unable to copy update script to admin node") + return False + + if not node.execute(["/tmp/"+UPDATE_HELPER,"first"]): + print ("ERROR: First phase of upgrade faileduuu") + return False + + node.execute(["/tmp/"+UPDATE_HELPER,"reboot"]) + time.sleep(60 * 5) + if node.put_file(UPDATE_HELPER): - return node.execute(["/tmp/"+UPDATE_HELPER]) + return node.execute(["/tmp/"+UPDATE_HELPER,"second"]) else: - print ("ERROR: Unable to copy update script to admin node") + print ("ERROR: Unable to copy update script to admin node") return False diff --git a/jenkins/build_cluster/update_helper.sh b/jenkins/build_cluster/update_helper.sh index 3eb2e72..8619f85 100755 --- a/jenkins/build_cluster/update_helper.sh +++ b/jenkins/build_cluster/update_helper.sh @@ -40,6 +40,19 @@ upgrade_with_docker () { RUN_WITH_LOGGER "dockerctl start all" || return 1 } +upgrade_first_phase_8x (){ + RUN_WITH_LOGGER "dockerctl destroy all" || return 1 + RUN_WITH_LOGGER "docker rmi -f $(docker images -q -a)" + RUN_WITH_LOGGER "systemctl stop docker.service" || return 1 + RUN_WITH_LOGGER "yum update -y " || return 1 +} + +upgrade_second_phase_8x (){ + #RUN_WITH_LOGGER "systemctl start docker.service" || return 1 + RUN_WITH_LOGGER "docker load -i /var/www/nailgun/docker/images/fuel-images.tar" || return 1 + RUN_WITH_LOGGER "dockerctl start all" || return 1 +} + upgrade_to_9x () { RUN_WITH_LOGGER yum clean all || return 1 RUN_WITH_LOGGER yum install python-cudet -y || return 1 @@ -63,8 +76,70 @@ wait_for () { done; } -version=$(get_fuel_version) || crap "error: can't get fuel version" +first_phase () { + case $version in + # Check that we have update repo in yum.repos.d + 7.0.0|8.0) + wait_for '[ -f /etc/yum.repos.d/mos[78].0-updates.repo ]' 30 60 || crap "error: timeout waiting of update repo" + ;;& + + 9.0) + wait_for '[ -f /etc/yum.repos.d/mos-updates.repo ]' 30 60 || crap "error: timeout waiting of update repo" + ;;& + + # Check that we have no bootstrap builder right now + 7.0.0) + wait_for '[ $(ps -ef | grep fuel-bootstrap | grep -v grep | wc -l) -eq 0 ]' 30 60 || crap "error: timeout of bootstrap waiting" + ;;& + + # Upgrade + 7.0.0) + upgrade_with_docker || crap "error: upgrade failed for version $version. See log $upgrade_log on master node." + ;;& + + 8.0) + upgrade_first_phase_8x || crap "error: upgrade failed for version $version. See log $upgrade_log on master node." + ;;& + + 9.0) + upgrade_to_9x || crap "error: upgrade failed for version $version. See log $upgrade_log on master node." + ;;& + esac +} +reboot_phase () { + case $version in + 8.0) + echo "Going to reboot node..." + shutdown -r +1 + exit 0 + ;; + *) + echo "No action on reboot phase for $version" + ;; + esac +} + +second_phase () { + case $version in + # Upgrade + 8.0) + upgrade_second_phase_8x || crap "error: upgrade failed for version $version. See log $upgrade_log on master node." + ;;& + + # Run bootstap builder + # NOTE: No need to do this step for MOS9 because it's implemented in update procedure + 7.0.0) + RUN_WITH_LOGGER "fuel-bootstrap-image" || crap "error: fuel-bootstrap failed. See log $upgrade_log on master node." + ;; + + 8.0) + RUN_WITH_LOGGER "fuel-bootstrap build --activate" || crap "error: fuel-bootstrap failed. See log $upgrade_log on master node." + ;; + esac +} + +version=$(get_fuel_version) || crap "error: can't get fuel version" case $version in 7.0.0|8.0|9.0) @@ -75,37 +150,15 @@ case $version in ;; esac -case $version in - # Check that we have update repo in yum.repos.d - 7.0.0|8.0) - wait_for '[ -f /etc/yum.repos.d/mos[78].0-updates.repo ]' 30 60 || crap "error: timeout waiting of update repo" - ;;& - - 9.0) - wait_for '[ -f /etc/yum.repos.d/mos-updates.repo ]' 30 60 || crap "error: timeout waiting of update repo" - ;;& - - # Check that we have no bootstrap builder right now - 7.0.0) - wait_for '[ $(ps -ef | grep fuel-bootstrap | grep -v grep | wc -l) -eq 0 ]' 30 60 || crap "error: timeout of bootstrap waiting" - ;;& - - # Upgrade - 7.0.0|8.0) - upgrade_with_docker || crap "error: upgrade failed for version $version. See log $upgrade_log on master node." - ;;& - - 9.0) - upgrade_to_9x || crap "error: upgrade failed for version $version. See log $upgrade_log on master node." - ;;& - - # Run bootstap builder - # NOTE: No need to do this step for MOS9 because it's implemented in update procedure - 7.0.0) - RUN_WITH_LOGGER "fuel-bootstrap-image" || crap "error: fuel-bootstrap failed. See log $upgrade_log on master node." - ;; +# For some of the upgrade procedures we have to split upgrade on two phase with reboot between. +phase=${1:-"second"} - 8.0) - RUN_WITH_LOGGER "fuel-bootstrap build --activate" || crap "error: fuel-bootstrap failed. See log $upgrade_log on master node." +case $phase in + first|second|reboot) + action="${phase}_phase" + $action + ;; + *) + crap "error: Unable to undestand phase name: $phase" ;; esac