[Jenkins] Fix racing with puppet and bootstrap

There is a racing between update script and fuel-bootstrap
image builder.

Change-Id: I2941257bf889ee80fed09bb31e496a1d4a1f3e0b

diff --git a/jenkins/build_cluster/build_cluster.py b/jenkins/build_cluster/build_cluster.py
index 50e1ae7c0c5609b50175e5dc86632243d5094f60..d8305e63dfde8e90e7ae301998650a91477817c1 100755 (executable)
--- a/jenkins/build_cluster/build_cluster.py
+++ b/jenkins/build_cluster/build_cluster.py
@@ -703,7 +703,6 @@ def main():
 
     inject_ifconfig_ssh()
 
-    start_slaves()
 
     wait_for_api_is_ready()
 
@@ -715,6 +714,8 @@ def main():
         else:
             print("ERROR: unable to update fuel")
 
+    start_slaves()
+
     configure_nailgun()
 
     wait_for_cluster_is_ready()

diff --git a/jenkins/build_cluster/update_helper.sh b/jenkins/build_cluster/update_helper.sh
index bb29a9b314144b3cf1e5e208f7ba7285754dca44..a3af65c8a3edac3228e0da6b6051271a5ed8bc51 100755 (executable)
--- a/jenkins/build_cluster/update_helper.sh
+++ b/jenkins/build_cluster/update_helper.sh
@@ -45,14 +45,50 @@ crap () {
        exit ${2:-1}
 }
 
+wait_for () {
+    local cnt=0
+    while true; do
+               eval $1 && return 0
+               ((cnt++))
+               echo -en "$cnt... "
+               [[ $cnt -eq $2 ]] && return 1
+               sleep $3
+       done;
+}
 
 version=$(get_fuel_version) || crap "error: can't get fuel version"
 
+
+case $version in
+       7.0.0|8.0)
+               echo "Gues version is $version"
+               ;;
+       *)
+               crap "Version $version is not supported by update script"
+               ;;
+esac
+
 case $version in 
+       # Check that we have update repo in yum.repos.d
+       7.0.0|8.0)
+               wait_for '[ -f /etc/yum.repos.d/mos[78].0-updates.repo ]' 30 60 || crap "error: timeout waiting of update repo"
+               ;;&
+
+       # Check that we have no bootstrap builder right now
+       7.0.0)
+               wait_for '[ $(ps -ef | grep fuel-bootstrap | grep -v grep | wc -l) -eq 0 ]' 30 60 || crap "error: timeout of bootstrap waiting" 
+               ;;&
+
+       # Upgrade
        7.0.0|8.0)
                upgrade_with_docker  || crap "error: upgrade failed for version $version. See log $upgrade_log on master node."
+               ;;&
+
+       # Run bootstap builder
+       7.0.0)
+               RUN_WITH_LOGGER "fuel-bootstrap-image" || crap "error: fuel-bootstrap failed. See log $upgrade_log on master node."
                ;;
-       *) 
-               crap "error: upgrade procedure for version $version not implemented"
+       8.0)
+               RUN_WITH_LOGGER "fuel-bootstrap build --activate" || crap "error: fuel-bootstrap failed. See log $upgrade_log on master node."
                ;;
 esac