From: Denis V. Meltsaykin Date: Thu, 5 Nov 2015 14:34:25 +0000 (+0300) Subject: Refactoring mos_apply_mu.py X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=d012fe1fa6310441726a0160cb4a554daadf9719;p=tools%2Fsustaining.git Refactoring mos_apply_mu.py Total refactoring of script: * Changed structure to classful * Got rid of custom packages installation * Support for 5.1.1, 6.0, 6.1, 7.0 added * Added logging to file * Better wording in help message * Changed way how multiple repos are handled in apt-get * 5.1.1 now uses dist-upgrade as it's necessary for the installation of nova and qemu. Change-Id: I07d8ee3eb8e2e467800c19b4deb2de009aab367f --- diff --git a/scripts/mos_apply_mu.py b/scripts/mos_apply_mu.py old mode 100644 new mode 100755 index 7cdf209..b1d8158 --- a/scripts/mos_apply_mu.py +++ b/scripts/mos_apply_mu.py @@ -14,484 +14,397 @@ # under the License. import json +import logging import os import subprocess import sys import urllib2 import yaml - -username = 'admin' -password = 'admin' -tenant = 'admin' -keystone = 'http://127.0.0.1:5000/v2.0' -nailgun = 'http://127.0.0.1:8000/' -logfile = 'mos_apply_mu.log' -mos_version = None -mos_repos_to_install = set(["updates"]) -new_fuel = ['6.1', '7.0', '8.0'] - -env_id = 0 -all_envs = False -try_offline = False -really = False -check = False -no_rsync = False -use_latest = False - -master_ip = None -path = "custom.pkg" -install_custom = False -pkgs = None -dest = "/var/www/nailgun/updates/custom/" - -mos_repo_template = { - "ubuntu": { - "rsync": - "rsync://mirror.fuel-infra.org/" - "mirror/mos/ubuntu/dists/mos{MOS_VERSION}-{REPO_NAME}/", - "local_path": - "/var/www/nailgun/mos-ubuntu/dists/mos{MOS_VERSION}-{REPO_NAME}/", - "repo_file": - "/etc/apt/sources.list.d/mos-{REPO_NAME}.list", - "repo_text": - "deb http://{MASTER_IP}:8080/mos-ubuntu " - "mos{MOS_VERSION}-{REPO_NAME} main restricted", - "prio": "1150" - }, - "centos": { - "rsync": - "rsync://mirror.fuel-infra.org/mirror/" - "mos/centos-6/mos{MOS_VERSION}/{REPO_NAME}/", - "local_path": - "/var/www/nailgun/mos-centos/mos{MOS_VERSION}/{REPO_NAME}", - "repo_text": - "[mos-{REPO_NAME}]\n" - "name=mos-{REPO_NAME}\n" - "baseurl=http://{MASTER_IP}:8080/mos-centos/mos{MOS_VERSION}/" - "{REPO_NAME}/\ngpgcheck=0\n", - "repo_file": - "/etc/yum.repos.d/mos-{REPO_NAME}.repo", - "prio": "100" - } -} - -latest_mos_repo_template = { - "ubuntu": { - "rsync": "rsync://mirror.fuel-infra.org/mirror/mos/snapshots/" - "ubuntu-latest/dists/mos{MOS_VERSION}-{REPO_NAME}/", - "local_path": "/var/www/nailgun/mos-ubuntu/dists/" - "latest-mos{MOS_VERSION}-{REPO_NAME}/", - "repo_file": "/etc/apt/sources.list.d/latest-mos-{REPO_NAME}.list", - "repo_text": "deb http://{MASTER_IP}:8080/mos-ubuntu latest" - "-mos{MOS_VERSION}-{REPO_NAME} main restricted", - "prio": "1150" - }, - "centos": { - "rsync": - "rsync://mirror.fuel-infra.org/mirror/mos/snapshots/" - "centos-6-latest/mos{MOS_VERSION}/{REPO_NAME}/", - "local_path": - "/var/www/nailgun/mos-centos/mos{MOS_VERSION}/latest-{REPO_NAME}", - "repo_text": - "[latest-mos-{REPO_NAME}]\n" - "name=latest-mos-{REPO_NAME}\n" - "baseurl=http://{MASTER_IP}:8080/mos-centos/" - "mos{MOS_VERSION}/latest-{REPO_NAME}/\ngpgcheck=0\n", - "repo_file": - "/etc/yum.repos.d/latest-mos-{REPO_NAME}.repo", - "prio": "100" +LOG = logging.getLogger("updater") +LOG.setLevel(logging.DEBUG) + +ch = logging.StreamHandler(sys.stdout) +ch.setFormatter(logging.Formatter("%(message)s")) +ch.setLevel(logging.INFO) +LOG.addHandler(ch) + +fh = logging.FileHandler("mos_apply_mu.log") +fh.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")) +LOG.addHandler(fh) + + +class Config(object): + def __init__(self): + """ + :rtype: Config + """ + self.cfg = { + "username": "admin", + "password": "admin", + "tenant": "admin", + "keystone": "http://127.0.0.1:5000/v2.0", + "nailgun": "http://127.0.0.1:8000/", + "mos_version": None, + "mos_repos_to_install": set(["updates"]), + "env_id": 0, + "all_envs": False, + "try_offline": False, + "really": False, + "check": False, + "no_rsync": False, + "use_latest": False, + "ubuntu_pool": None, + "apache_user": { + 'ubuntu': 'horizon', + 'centos': 'apache' + }, + "apache_restart": { + 'ubuntu': 'service apache2 restart', + 'centos': 'service httpd restart' + }, + "master_ip": None, + "repo_install_text": 'echo "{REPO_TEXT}" > {REPO_FILE};\n', + "repo_use_text": { + 'ubuntu': 'apt-get -o Dir::Etc::sourceparts="/root/mos_update_repo/"' + ' -o APT::Get::List-Cleanup="0" update\n' + 'apt-get -o Dir::Etc::sourceparts="/root/mos_update_repo/"' + ' -o APT::Get::List-Cleanup="0"' + ' -o Dpkg::Options::="--force-confdef"' + ' -o Dpkg::Options::="--force-confold" -y upgrade\n', + 'centos': 'yum --disablerepo="*" {REPOS_ACTIVATE} update' + ' --skip-broken -y --nogpgcheck\n' + }, + "repo_activate": { + "ubuntu": ' -o Dir::Etc::sourcelist="{REPO_FILE}" ', + "centos": ' --enablerepo="mos-{REPO_NAME}" ' + } } -} - - -def arg_parse(): - global env_id - global all_envs - global try_offline - global really - global username - global password - global tenant - global path - global install_custom - global repo_install - global master_ip - global check - global mos_version - global keystone - global nailgun - global use_latest - global no_rsync - - usage = """ -Tool to apply maintenance updates and custom packages into nodes. -Usage: - python mos_apply_mu.py --env-id|--all-envs --check|--update + for cmd in sys.argv[1:]: + if '--env-id' in cmd: + self.cfg['env_id'] = int(cmd.split('=')[1]) + if '--user' in cmd: + self.cfg['username'] = cmd.split('=')[1] + if '--pass' in cmd: + self.cfg['password'] = cmd.split('=')[1] + if '--tenant' in cmd: + self.cfg['tenant'] = cmd.split('=')[1] + if '--all-envs' in cmd: + self.cfg['all_envs'] = True + if '--offline' in cmd: + self.cfg['try_offline'] = True + if '--check' in cmd: + self.cfg['check'] = True + if '--update' in cmd: + self.cfg['really'] = True + if '--fuel-ip' in cmd: + fuel_ip = cmd.split('=')[1] + self.cfg['keystone'] = 'http://' + fuel_ip + ':5000/v2.0' + self.cfg['nailgun'] = 'http://' + fuel_ip + ':8000/' + if '--master-ip' in cmd: + self.cfg['master_ip'] = cmd.split('=')[1] + if '--mos-version' in cmd: + self.cfg['mos_version'] = cmd.split('=')[1] + if '--mos-proposed' in cmd: + self.cfg['mos_repos_to_install'].add("proposed") + if '--mos-security' in cmd: + self.cfg['mos_repos_to_install'].add("security") + if '--use-latest' in cmd: + self.cfg['use_latest'] = True + if '--no-rsync' in cmd: + self.cfg['no_rsync'] = True + if '--version' in cmd: + self.errexit(msg="VER_ID: 05112015", code=19) + + # validate all the data to find out incompatibles + if (self.cfg['env_id'] > 0) and (self.cfg['all_envs']): + self.errexit( + msg="ERROR: You should only select either --env-id or --all-envs.", + code=5 + ) + if (self.cfg['env_id'] == 0) and (not self.cfg['all_envs']): + self.errexit( + msg="ERROR: At least one option (env-id or all-envs) must be set.", + code=6 + ) + if self.cfg['really'] and self.cfg['check']: + self.errexit( + msg="ERROR: You should use either --check or --update. Not both.", + code=7 + ) + if not self.cfg['really'] and not self.cfg['check']: + self.errexit( + msg="ERROR: Either --check or --update must be set.", + code=8 + ) + if self.cfg['master_ip'] is None: + try: + self.cfg['master_ip'] = self.guess_master_ip() + LOG.info("\tMASTER_IP: {0}".format(self.cfg['master_ip'])) + except: + self.errexit( + msg="ERROR: --master-ip is required! Set with --master-ip=X.X.X.X", + code=110 + ) + if self.cfg['mos_version'] is None: + try: + self.cfg['mos_version'] = self.guess_mos_version() + LOG.info("\tMOS_VERSION: {0}".format(self.cfg['mos_version'])) + except: + self.errexit( + msg="ERROR: MOS version is not determined, please set it with --mos-version", + code=111 + ) -Required arguments: + def guess_mos_version(self): + try: + with open('/etc/fuel/version.yaml', 'r') as fp: + return yaml.load(fp)['VERSION']['release'] + except: + raise + + def guess_master_ip(self): + try: + with open('/etc/fuel/astute.yaml', 'r') as fp: + return yaml.load(fp)['ADMIN_NETWORK']['ipaddress'] + except: + raise - --env-id ID of operational environment which needs to be updated - OR - --all-envs Update all operational environments + def errexit(self, msg=None, code=1): + usage = """ +Tool to install maintenance updates to the nodes. +Usage: + python mos_apply_mu.py {--env-id=N | --all-envs} {--check | --update} + [user=X] [--pass=Y] [--tenant=Z] [--offline] + [--master-ip=A.B.C.D] [--mos-version=E.F] - --update Make real update (without this nothing will be updated) - OR - --check Check status of selected nodes +Required arguments: + --env-id=N ID of operational environment which needs to be updated + --all-envs Update all operational environments + --update Make real update (without this nothing will be updated) + --check Check status of selected nodes Optional arguments: - - --offline Add to update nodes which are currently offline in fuel - (can cause significant timeouts) - --user Username used in Fuel-Keystone authentication - default: admin - --pass Password suitable to username - default: admin - --tenant Suitable tenant - default: admin - --file Name of the config file in json-format with list of - custom packages to download and install - --mos-version Version of MOS ('old', '6.1', '7.0', etc.) - default: taken from /etc/fuel/version.yaml - --master-ip IP-Address of Fuel-master node - default: taken from /etc/fuel/astute.yaml + --offline Add to update nodes which are currently offline in fuel + (may cause significant timeouts) + --user Username used in Fuel-Keystone authentication, default: admin + --pass Password suitable to username, default: admin + --tenant Suitable tenant, default: admin + --mos-version Version of MOS ('5.1.1', '6.1', '7.0', etc.) + default: taken from /etc/fuel/version.yaml + --master-ip IP-Address of Fuel-master node + default: taken from /etc/fuel/astute.yaml Fuel 6.1 and higher options: + By default only 'mos-updates' repository will + be installed into an environment. - By default only 'mos-updates' repository will - be installed into an environment. - - --mos-proposed Enable proposed updates repository - --mos-security Enable security updates repository - --no-rsync Don't download chosen repositories + --mos-proposed Enable proposed updates repository + --mos-security Enable security updates repository + --no-rsync Don't download chosen repositories Examples: - python mos_apply_mu.py --env-id=11 --user=op --pass=V3ryS3Cur3 \ - --master-ip="10.100.15.2" - - Inspects Fuel configuration with op's credentials and shows which nodes - will be updated in environment (cluster) #11 - python mos_apply_mu.py --env-id=11 --user=op --pass=V3ryS3Cur3 \ --update --master-ip="10.100.15.2" - Makes real update on nodes in environment #11 + Runs update process on nodes in the environment #11 python mos_apply_mu.py --env-id=11 --user=op --pass=V3ryS3Cur3 --check - Checks current state of update process on nodes in environment #11 - States may be the following (separate for each node): - STARTED - script has been started and it's still working - PACKAGES_INSTALLING - script is installing custom packages - REPO_UPD=OK;CUSTOM=0 of 2 INSTALLED - execution is over, - maintenance update has been successfuly installed - but installation of custom packages was unsuccessful + Checks the state of update process on the nodes in the environment #11 + States may be the following (separate for each node): + * STARTED - script has been started and it's still working + * REPO_UPD=OK - execution is over, maintenance update has been + successfully installed. To get detailed log examine /var/log/mos_apply_mu.log on remote nodes. Questions: dmeltsaykin@mirantis.com - Mirantis, 2015 """ - - for cmd in sys.argv[1:]: - if '--env-id' in cmd: - env_id = int(cmd.split('=')[1]) - if '--user' in cmd: - username = cmd.split('=')[1] - if '--pass' in cmd: - password = cmd.split('=')[1] - if '--tenant' in cmd: - tenant = cmd.split('=')[1] - if '--all-envs' in cmd: - all_envs = True - if '--offline' in cmd: - try_offline = True - if '--check' in cmd: - check = True - if '--update' in cmd: - really = True - if '--fuel-ip' in cmd: - fuel_ip = cmd.split('=')[1] - keystone = 'http://' + fuel_ip + ':5000/v2.0' - nailgun = 'http://' + fuel_ip + ':8000/' - if '--file' in cmd: - path = cmd.split('=')[1] - if '--with-custom' in cmd: - install_custom = True - if '--master-ip' in cmd: - master_ip = cmd.split('=')[1] - if '--mos-version' in cmd: - mos_version = cmd.split('=')[1] - if '--mos-proposed' in cmd: - mos_repos_to_install.add("proposed") - if '--mos-security' in cmd: - mos_repos_to_install.add("security") - if '--use-latest' in cmd: - use_latest = True - if '--no-rsync' in cmd: - no_rsync = True - if '--version' in cmd: - print ("VER_ID: 22092015") - sys.exit(19) - - if (env_id > 0) and (all_envs): - print (usage) - print ("ERROR: You should only select either --env-id or --all-envs.") - sys.exit(5) - if (env_id == 0) and (not all_envs): - print (usage) - print ("ERROR: At least one option (env-id or all-envs) must be set.") - sys.exit(6) - if really and check: - print (usage) - print ("ERROR: You should use either --check or --update. Not both.") - sys.exit(7) - if not really and not check: - print (usage) - print ("ERROR: Either --check or --update must be set.") - sys.exit(8) - if master_ip is None: - try: - master_ip = guess_master_ip() - print ("\tMASTER_IP: {0}".format(master_ip)) - except: - print (usage) - print ( - "ERROR: --master-ip is required! Set with --master-ip=X.X.X.X" - ) - sys.exit(110) - if mos_version is None: - try: - mos_version = guess_mos_version() - print ("\tMOS_VERSION: {0}".format(mos_version)) - except: - print ("MOS Version cannot be detected, setting to 'old'") - mos_version = 'old' - - -def guess_mos_version(): - try: - with open('/etc/fuel/version.yaml', 'r') as fp: - ver = yaml.load(fp)['VERSION']['release'] - if ver not in new_fuel: - return 'old' - return ver - except: - raise - - -def guess_master_ip(): - try: - with open('/etc/fuel/astute.yaml', 'r') as fp: - return yaml.load(fp)['ADMIN_NETWORK']['ipaddress'] - except: - raise - - -def get_downloads_list(): - global pkgs - try: - file = open(path, 'r') - pkgs = json.load(file) - file.close() - except: - return None - return True - - -def packages_download(): - # check if dst dir exists if not create it (and others) - try: - os.makedirs(dest) - except os.error as err: - if err.args[0] != 17: - print ("Error during creating directory {0}: {1}".format( - dest, err.args[1])) - return (None) - - retval = 0 - for pkg in pkgs.values(): - for t in pkg: - cmd = "wget -c -P{0} \"{1}\"".format(dest, t) - print ("Running: {0}".format(cmd)) - retval += os.system(cmd) - - if retval != 0: - print ("Some downloads are failed!") - return (retval) - - -def get_nodes(): - req = urllib2.Request(nailgun + '/api/v1/nodes/') - req.add_header('X-Auth-Token', token) - nodes = json.load(urllib2.urlopen(req)) - return nodes - - -def get_operational_envs(nodes, env_list): - for node in nodes: - if (node['status'] == "ready"): - if try_offline: - env_list.add(node['cluster']) - elif node['online']: - env_list.add(node['cluster']) - - -def check_status(ip): - """Checks state of node by reading last line of status-file""" - cmd = ["ssh", ip, "tail -n1 /var/log/mos_apply_mu.status"] - proc = subprocess.Popen(cmd, stdin=None, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - proc.wait() - if proc.returncode == 0: - state = proc.communicate()[0] - print("Node {0} state: {1}".format(ip, state)) - return True - else: - print("Node {0}: no updates information found.".format(ip)) - return False - - -def do_node_update(nodes, env_list): - to_update = set() - for env in env_list: - for node in nodes: - if node['cluster'] == env: - if try_offline: - to_update.add((node['ip'], node['os_platform'])) - elif node['online']: - to_update.add((node['ip'], node['os_platform'])) - - print ("Selected nodes: " + ", ".join([x[0] for x in to_update])) - packages_to_install = get_downloads_list() - if pkgs is not None: - print ("Custom packages will be installed") - else: - print("No custom packages will be installed") - - if really: - if packages_to_install is not None: - packages_download() + print(usage) + if msg: + LOG.info(msg) + sys.exit(code) + + def getcfg(self): + return dict(self.cfg) + + +class BasicUpdater(object): + def __init__(self, cfg): + self.cfg = cfg + + def run(self): + """ + main entry point for each subclass, + returns nothing + :return: None + """ + # download repos if allowed + if not self.cfg['no_rsync'] and self.cfg['really']: + self.rsync() + # get full list of nodes and envs from nailgun + self.cfg['nodes_list'] = self.get_node_list() + # decide either to update or to check + if self.cfg['check']: + self.check() else: - print ("Unable to get packages list from file {0}".format(path)) - - for ip, os_version in to_update: - print ("{0}'s log: /var/log/remote/" - "{0}/mos_apply_mu.log".format(ip)) - send_shell_script(ip, os_version) - if check: - check_status(ip) - - -def get_md5_from_file(file): - """ Gets md5 checksum from file by calling external tool.""" - run = subprocess.Popen(["md5sum", file], stdin=None, - stdout=subprocess.PIPE, stderr=None) - run.wait() - - if run.returncode == 0: - md5 = run.communicate()[0].split(' ')[0] - else: - md5 = None - - return md5 - - -def send_shell_script(ip, os_version): - """ This function generates a shell-script and sends it to the node. - Then the script will be run in nohup.""" - - repo_install = { - 'ubuntu': "(grep -q \"updates\" /etc/apt/sources.list " - "|| echo -e \"\\ndeb http://{0}:8080/updates/ubuntu " - "precise main\" >> /etc/apt/sources.list)\n" - "apt-get update\n" - "apt-get -o Dpkg::Options::=\"--force-confdef\" -o " - "Dpkg::Options::=\"--force-confold\" -y " - "upgrade\n".format(master_ip), - - 'centos': "yum-config-manager --add-repo=http://{0}:8080/updates/" - "centos/os/x86_64/\nyum update --skip-broken -y " - "--nogpgcheck\n".format(master_ip) - } - - pkg_mgr_string = { - 'ubuntu': 'apt-get {repos} -o Dir::Etc::sourceparts="-"' - ' -o APT::Get::List-Cleanup="0" update\n' - 'apt-get {repos} -o Dir::Etc::sourceparts="-"' - ' -o APT::Get::List-Cleanup="0"' - ' -o Dpkg::Options::="--force-confdef"' - ' -o Dpkg::Options::="--force-confold" -y upgrade\n', - 'centos': 'yum --disablerepo="*" {repos} update' - ' --skip-broken -y --nogpgcheck\n' - } - - if mos_version != 'old': - repo_string = "" - tmpl = '' - for repo in mos_repos_to_install: - if os_version == "ubuntu": - repo_string += ' -o Dir::Etc::sourcelist="{repo_file}"'.format( - repo_file=mos_repo_template[os_version]['repo_file'].format( - REPO_NAME=repo)) - else: - repo_string += ' --enablerepo="mos-{repo_name}"'.format( - repo_name=repo) - - tmpl += 'echo "{repo_text}" > {repo_file};\n'.format( - repo_text=mos_repo_template[os_version]['repo_text'].format( - MASTER_IP=master_ip, - MOS_VERSION=mos_version, - REPO_NAME=repo - ), - repo_file=mos_repo_template[os_version]['repo_file'].format( - REPO_NAME=repo + self.update() + + def get_node_list(self): + """ + Function to get list of nodes to update or to check + if the update process was successful + + :return: set + """ + def get_operational_envs(nodes): + envs = set() + for node in nodes: + if self.cfg['try_offline'] and node['status'] == 'ready': + envs.add(node['cluster']) + elif node['online'] and node['status'] == 'ready': + envs.add(node['cluster']) + return envs + + def keystone_get_token(): + try: + jrequest = """ + {{ + "auth": {{ + "tenantName": "{tenant}", + "passwordCredentials": {{ + "username": "{username}", + "password": "{password}" + }} + }} + }} + """.format( + tenant=self.cfg['tenant'], + username=self.cfg['username'], + password=self.cfg['password'] ) - ) - if use_latest: - for repo in mos_repos_to_install: - if os_version == "ubuntu": - repo_string += ' -o Dir::Etc::sourcelist="{repo_file}"'.format( - repo_file=latest_mos_repo_template[os_version]['repo_file'].format( - REPO_NAME=repo)) - else: - repo_string += ' --enablerepo="latest-mos-{repo_name}"'.format( - repo_name=repo) - - tmpl += 'echo "{repo_text}" > {repo_file};\n'.format( - repo_text=latest_mos_repo_template[os_version]['repo_text'].format( - MASTER_IP=master_ip, - MOS_VERSION=mos_version, - REPO_NAME=repo - ), - repo_file=latest_mos_repo_template[os_version]['repo_file'].format( - REPO_NAME=repo - ) + req = urllib2.Request(self.cfg['keystone'] + '/tokens/') + req.add_header('Content-Type', 'application/json') + req.add_data(jrequest) + out = json.load(urllib2.urlopen(req)) + return out['access']['token']['id'] + except: + self.errexit( + msg="Cannot obtain token from keystone!", + code=95 ) - tmpl += pkg_mgr_string[os_version].format(repos=repo_string) - - repo_install = { - os_version: tmpl - } + req = urllib2.Request(self.cfg['nailgun'] + '/api/v1/nodes/') + req.add_header('X-Auth-Token', keystone_get_token()) + nodes = json.load(urllib2.urlopen(req)) + # we fetched all the nodes from nailgun, here we must filter them out + if self.cfg['env_id'] != 0: + self.cfg['envs_list'] = set([self.cfg['env_id']]) + else: + self.cfg['envs_list'] = get_operational_envs(nodes) + return nodes + + def errexit(self, msg=None, code=1): + LOG.info("Error occurred!") + if msg: + LOG.info(msg) + sys.exit(code) + + def _run_helper(self, cmd, send_text=None, nostderr=True): + """ + run command + :param cmd: set[] + :param nostderr: if True stderr redirected to /dev/null + :return: tuple + """ + + devnull = open(os.devnull, 'w') + if nostderr: + STDERR = devnull + else: + STDERR = subprocess.PIPE + STDOUT = subprocess.PIPE + if send_text: + STDIN = subprocess.PIPE + else: + STDIN = None + LOG.debug("Started _run_helper() with: {0}".format(" ".join(cmd))) + try: + proc = subprocess.Popen( + cmd, + stdin=STDIN, + stderr=STDERR, + stdout=STDOUT + ) + result = "" + if send_text: + proc.communicate(input=send_text) + proc.wait() + while not send_text: + out = proc.stdout.readline() + if out == '' and proc.poll() is not None: + break + result += out + devnull.close() + LOG.debug(result) + return proc.returncode, result + + except Exception as e: + LOG.debug("Exception in _run_helper()", exc_info=True) + return 254, "Exception in _run_helper(): {0}".format(e.message) + + def _affected_nodes(self): + """ + Returns a set(ip, os) of the selected nodes + :return: set + """ + _nodes = set() + for env in self.cfg['envs_list']: + for node in self.cfg['nodes_list']: + if node['cluster'] == env: + if self.cfg['try_offline']: + _nodes.add((node['ip'], node['os_platform'])) + elif node['online']: + _nodes.add((node['ip'], node['os_platform'])) + return _nodes + + def check(self): + """ + Checks what is the current state of nodes + :return: + """ + for ip, os in self._affected_nodes(): + cmd = ["ssh", ip, "tail -n1 /var/log/mos_apply_mu.status"] + (state, msg) = self._run_helper(cmd=cmd) + if state == 0: + LOG.info("Node {0} state: {1}".format(ip, msg)) + else: + LOG.info("Node {0}: no updates information found.".format(ip)) + + def update(self): + """ + does the real update of the selected nodes + :return: None + """ + update_script = """#!/bin/bash +exec > >(logger -tmos_apply_mu) 2>&1 +set -x - pkg_install_tool = { - 'ubuntu': '/usr/bin/dpkg -iE', - 'centos': '/bin/rpm -Uvh' - } +STATUS="/var/log/mos_apply_mu.status" +echo "STARTED" > $STATUS - apache_user = { - 'ubuntu': 'horizon', - 'centos': 'apache' - } +mkdir /root/mos_update_repo || rm /root/mos_update_repo/* - apache_restart = { - 'ubuntu': 'service apache2 restart', - 'centos': 'service httpd restart' - } +%%repo_install%% - murano_fix = """ +retval=$? +if [ $retval != 0 ]; then + REPO_STATE="FAIL" +else + REPO_STATE="OK" +fi if [ -a /usr/bin/modify-horizon-config.sh ] then /usr/bin/modify-horizon-config.sh uninstall @@ -499,231 +412,304 @@ then export MURANO_SSL_ENABLED=False export USE_KEYSTONE_ENDPOINT=True export USE_SQLITE_BACKEND=False - export APACHE_USER="{apache_user}" - export APACHE_GROUP="{apache_user}" + export APACHE_USER="%%apache_user%%" + export APACHE_GROUP="%%apache_user%%" /usr/bin/modify-horizon-config.sh install /usr/share/openstack-dashboard/manage.py collectstatic --noinput - {apache_restart} + %%apache_restart%% else echo "Murano is not installed. Fix skipped." fi - """.format( - apache_user=apache_user[os_version], - apache_restart=apache_restart[os_version] - ) - - package_template = """ -FILE="{0}" -FILEMD="{1}" -TOTAL_COUNT=`expr $TOTAL_COUNT + 1` - -install_package $FILE $FILEMD -retval=$? -if [ $retval -eq 0 ]; then - SUCCESS_COUNT=`expr $SUCCESS_COUNT + 1` -fi -""" - package_text = "" - try: - for package in pkgs[os_version]: - name = package.split("/")[-1] - package_text += package_template.format(name, get_md5_from_file( - dest+name)) - except: - pass - - head_of_script = """#!/bin/bash -exec > >(logger -tmos_apply_mu) 2>&1 - -set -x -TMPDIR="/tmp" -TOTAL_COUNT=0 -SUCCESS_COUNT=0 -WGET=/usr/bin/wget -MD5="/usr/bin/md5sum" -INSTALL="%%install%%" -URL="http://%%master_ip%%:8080/updates/custom" -STATUS="/var/log/mos_apply_mu.status" -echo "STARTED" > $STATUS - -%%repo_install%% -retval=$? - -if [ $retval != 0 ]; then - REPO_STATE="FAIL" -else - REPO_STATE="OK" -fi -echo "PACKAGES_INSTALLING" >> $STATUS -install_package() -{ - OBJ=$1 - MD=$2 - $WGET -c -P $TMPDIR $URL/$OBJ - retval=$? - if [ $retval != 0 ]; then - echo "$OBJ FAILED TO DOWNLOAD" - return 1 - fi - (echo "$MD $TMPDIR/$OBJ" | $MD5 -c - --quiet) - retval=$? - if [ $retval != 0 ]; then - echo "$OBJ CHECKSUM FAILED!" - return 2 - fi - $INSTALL $TMPDIR/$OBJ - retval=$? - if [ $retval != 0 ]; then - echo "$OBJ FAILED TO INSTALL" - return 3 - fi - echo "$OBJ SUCCESS" - return 0 -} - -%%packages_install%% - -%%murano_fix%% - -echo "REPO_UPD=$REPO_STATE;CUSTOM=$SUCCESS_COUNT of $TOTAL_COUNT INSTALLED" >> $STATUS -""" - total = head_of_script\ - .replace("%%install%%", pkg_install_tool[os_version])\ - .replace("%%master_ip%%", master_ip)\ - .replace("%%packages_install%%", package_text)\ - .replace("%%repo_install%%", repo_install[os_version])\ - .replace("%%murano_fix%%", murano_fix) - - if really: - cmd = [ - "ssh", - ip, - "cat - > /root/mos_update.sh ; chmod +x /root/mos_update.sh; " - "(nohup /root/mos_update.sh > /dev/null 2>&1) &" - ] - with open(os.devnull, 'w') as DNULL: - proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=DNULL) - proc.communicate(input=total) - proc.wait() - if proc.returncode == 0: - print ("Script is running on {0}".format(ip)) - return True - print ("Error during sending script to {0}".format(ip)) - return False - else: - return True - - -def keystone_get_token(): - try: - jrequest = """ - {{ - "auth": {{ - "tenantName": "{tenant}", - "passwordCredentials": {{ - "username": "{username}", - "password": "{password}" - }} - }} - }} - """.format( - tenant=tenant, - username=username, - password=password - ) - req = urllib2.Request(keystone + '/tokens/') - req.add_header('Content-Type', 'application/json') - req.add_data(jrequest) - out = json.load(urllib2.urlopen(req)) - return out['access']['token']['id'] - except: - return None - - -def rsync_repos(): - if not really or no_rsync: - print ("Download of repository is skipped") - return None - # work around the mos-ubuntu repos - # that have no individual pools - try: - os.makedirs("/var/www/nailgun/mos-ubuntu/pool") - except: - pass - cmdline = "rsync -vap --chmod=Dugo+x "\ - "rsync://mirror.fuel-infra.org/mirror/mos/ubuntu/pool/ "\ - "/var/www/nailgun/mos-ubuntu/pool/;" - if use_latest: - cmdline += "rsync -vap --chmod=Dugo+x "\ - "rsync://mirror.fuel-infra.org/mirror/mos/snapshots/ubuntu-latest/pool/ "\ - "/var/www/nailgun/mos-ubuntu/pool/;" - for distro in ['ubuntu', 'centos']: - for repo in mos_repos_to_install: - try: - os.makedirs(mos_repo_template[distro]['local_path'].format( - MOS_VERSION=mos_version, - REPO_NAME=repo - ) - ) - if use_latest: - os.makedirs(latest_mos_repo_template[distro]['local_path'].format( - MOS_VERSION=mos_version, - REPO_NAME=repo - ) +echo "UPDATE=$REPO_STATE" >> $STATUS + """ + + def _get_repo_install(os_version): + """ + generates the text of repository installation + :return: string + """ + _result = "" + _repos_activate ="" + # making header per each repo + for repo in self.cfg['mos_repos_to_install']: + REPO_TEXT = self.cfg['repo_template'][os_version]['repo_text'].format( + MOS_VERSION=self.cfg['mos_version'], + MASTER_IP=self.cfg['master_ip'], + REPO_NAME=repo ) - except: - pass - cmdline += 'rsync -vap --chmod=Dugo+x {url} {folder};'.format( - url=mos_repo_template[distro]['rsync'].format( - MOS_VERSION=mos_version, + REPO_FILE = self.cfg['repo_template'][os_version]['repo_file'].format( REPO_NAME=repo - ), - folder=mos_repo_template[distro]['local_path'].format( - MOS_VERSION=mos_version, + ) + _result += self.cfg['repo_install_text'].format( + REPO_TEXT=REPO_TEXT, + REPO_FILE=REPO_FILE + ) + _repos_activate += self.cfg['repo_activate'][os_version].format( + REPO_FILE=REPO_FILE, REPO_NAME=repo ) + # this part adds yum update/apt-get upgrade + _result += self.cfg['repo_use_text'][os_version].format( + REPOS_ACTIVATE=_repos_activate ) - if use_latest: - cmdline += 'rsync -vap --chmod=Dugo+x {url} {folder};'.format( - url=latest_mos_repo_template[distro]['rsync'].format( - MOS_VERSION=mos_version, + return _result + + for ip, distro in self._affected_nodes(): + script_text = update_script.replace( + "%%repo_install%%", + _get_repo_install(distro) + ).replace( + "%%apache_user%%", + self.cfg['apache_user'][distro] + ).replace( + "%%apache_restart%%", + self.cfg['apache_restart'][distro] + ) + cmd = [ + "ssh", ip, + "cat - > /root/mos_update.sh;chmod +x /root/mos_update.sh;" + "(nohup /root/mos_update.sh > /dev/null 2>&1) &" + ] + + state, out = self._run_helper(cmd=cmd, send_text=script_text) + if state != 0: + LOG.info("Node {0}: failure! [{1}:{2}]" + "Examine log at /var/log/remote/{0}/" + "mos_apply_mu.log".format(ip, state, out)) + else: + LOG.info("Node {0}: started update. " + "Log at /var/log/remote/{0}/" + "mos_apply_mu.log".format(ip)) + + def rsync(self): + """ + downloads repositories to the fuel node + :return: + """ + def _dir_check_or_create(dir): + """ + check whether dir exists, if not create it + with all the subdirs + :param dir: + :return: None + """ + try: + os.makedirs(dir) + except: + pass + + for distro in ['ubuntu', 'centos']: + for repo in self.cfg['mos_repos_to_install']: + dest_dir = self.cfg['repo_template'][distro]["local_path"].format( + MOS_VERSION=self.cfg['mos_version'], REPO_NAME=repo - ), - folder=latest_mos_repo_template[distro]['local_path'].format( - MOS_VERSION=mos_version, + ) + rsync_url = self.cfg['repo_template'][distro]["rsync"].format( + MOS_VERSION=self.cfg['mos_version'], REPO_NAME=repo ) - ) - print (cmdline) - retval = os.system(cmdline) - if retval != 0: - print ("Error during rsync!") - sys.exit(21) + _dir_check_or_create(dest_dir) + cmd = ["rsync", "-vap", "--chmod=Dugo+x", rsync_url, dest_dir] + LOG.info("RSYNC: Downloading {0} repository for {1}".format( + repo, distro)) + retval, text = self._run_helper(cmd=cmd) + if retval != 0: + self.errexit("RSYNC: Error downloading repository! [{0}:{1}]".format( + retval, text)) + + if self.cfg['ubuntu_pool']: + LOG.info("RSYNC: Additional pool is downloading.") + _dir_check_or_create(self.cfg['ubuntu_pool']['local_path']) + cmd = ["rsync", "-vap", "--chmod=Dugo+x", + self.cfg['ubuntu_pool']['rsync'], self.cfg['ubuntu_pool']['local_path']] + retval, text = self._run_helper(cmd=cmd) + if retval != 0: + self.errexit("RSYNC: Error downloading pool! [{0}:{1}]".format( + retval, text)) + + def rsync_pool_if_needed(self): + pass -if __name__ == "__main__": +class Updater511(BasicUpdater): + def __init__(self, cfg): + super(self.__class__, self).__init__(cfg) + # we have only updates repository for 5.1.1 + # regardless what was selected by the user + self.cfg['mos_repos_to_install'] = set(['updates']) + self.cfg['repo_template'] = { + "ubuntu": { + "rsync": + "rsync://mirror.fuel-infra.org/" + "mirror/fwm/{MOS_VERSION}/{REPO_NAME}/ubuntu/", + "local_path": + "/var/www/nailgun/{REPO_NAME}/ubuntu/", + "repo_file": + "/root/mos_update_repo/mos-{REPO_NAME}.list", + "repo_text": + "deb http://{MASTER_IP}:8080/updates/ubuntu " + "precise main restricted", + "prio": "1150" + }, + "centos": { + "rsync": + "rsync://mirror.fuel-infra.org/" + "mirror/fwm/{MOS_VERSION}/{REPO_NAME}/centos/", + "local_path": + "/var/www/nailgun/{REPO_NAME}/centos/", + "repo_text": + "[mos-{REPO_NAME}]\n" + "name=mos-{REPO_NAME}\n" + "baseurl=http://{MASTER_IP}:8080/updates/centos/" + "os/x86_64/\ngpgcheck=0\n", + "repo_file": + "/etc/yum.repos.d/mos-{REPO_NAME}.repo", + "prio": "100" + } + } + self.cfg["repo_use_text"] = { + 'ubuntu': 'apt-get -o Dir::Etc::sourceparts="/root/mos_update_repo/"' + ' -o APT::Get::List-Cleanup="0" update\n' + 'apt-get -o Dir::Etc::sourceparts="/root/mos_update_repo/"' + ' -o APT::Get::List-Cleanup="0"' + ' -o Dpkg::Options::="--force-confdef"' + ' -o Dpkg::Options::="--force-confold" -y dist-upgrade\n', + 'centos': 'yum --disablerepo="*" {REPOS_ACTIVATE} update' + ' --skip-broken -y --nogpgcheck\n' + } - arg_parse() - token = keystone_get_token() +class Updater60(BasicUpdater): + def __init__(self, cfg): + super(self.__class__, self).__init__(cfg) + # we have only updates repository for 6.0 + # regardless what was selected by the user + self.cfg['mos_repos_to_install'] = set(['updates']) + self.cfg['repo_template'] = { + "ubuntu": { + "rsync": + "rsync://mirror.fuel-infra.org/" + "mirror/fwm/{MOS_VERSION}/{REPO_NAME}/ubuntu/", + "local_path": + "/var/www/nailgun/{REPO_NAME}/ubuntu/", + "repo_file": + "/root/mos_update_repo/mos-{REPO_NAME}.list", + "repo_text": + "deb http://{MASTER_IP}:8080/updates/ubuntu " + "precise main restricted", + "prio": "1150" + }, + "centos": { + "rsync": + "rsync://mirror.fuel-infra.org/" + "mirror/fwm/{MOS_VERSION}/{REPO_NAME}/centos/", + "local_path": + "/var/www/nailgun/{REPO_NAME}/centos/", + "repo_text": + "[mos-{REPO_NAME}]\n" + "name=mos-{REPO_NAME}\n" + "baseurl=http://{MASTER_IP}:8080/updates/centos/" + "os/x86_64/\ngpgcheck=0\n", + "repo_file": + "/etc/yum.repos.d/mos-{REPO_NAME}.repo", + "prio": "100" + } + } - env_list = set() - nodes = get_nodes() - if (env_id > 0): - env_list.add(env_id) - else: - get_operational_envs(nodes, env_list) - if mos_version == 'old': - print ("Environments to update: " + ",".join( - [str(x) for x in env_list])) - do_node_update(nodes, env_list) +class Updater61(BasicUpdater): + def __init__(self, cfg): + super(self.__class__, self).__init__(cfg) + self.cfg['repo_template'] = { + "ubuntu": { + "rsync": + "rsync://mirror.fuel-infra.org/" + "mirror/mos/ubuntu/dists/mos{MOS_VERSION}-{REPO_NAME}/", + "local_path": + "/var/www/nailgun/mos-ubuntu/dists/mos{MOS_VERSION}-{REPO_NAME}/", + "repo_file": + "/root/mos_update_repo/mos-{REPO_NAME}.list", + "repo_text": + "deb http://{MASTER_IP}:8080/mos-ubuntu " + "mos{MOS_VERSION}-{REPO_NAME} main restricted", + "prio": "1150" + }, + "centos": { + "rsync": + "rsync://mirror.fuel-infra.org/mirror/" + "mos/centos-6/mos{MOS_VERSION}/{REPO_NAME}/", + "local_path": + "/var/www/nailgun/mos-centos/mos{MOS_VERSION}/{REPO_NAME}", + "repo_text": + "[mos-{REPO_NAME}]\n" + "name=mos-{REPO_NAME}\n" + "baseurl=http://{MASTER_IP}:8080/mos-centos/mos{MOS_VERSION}/" + "{REPO_NAME}/\ngpgcheck=0\n", + "repo_file": + "/etc/yum.repos.d/mos-{REPO_NAME}.repo", + "prio": "100" + } + } + self.cfg['ubuntu_pool'] = { + 'rsync': "rsync://mirror.fuel-infra.org" + "/mirror/mos/ubuntu/pool/", + 'local_path': "/var/www/nailgun/mos-ubuntu/pool/" + } + + +class Updater70(BasicUpdater): + def __init__(self, cfg): + super(self.__class__, self).__init__(cfg) + self.cfg['repo_template'] = { + "ubuntu": { + "rsync": + "rsync://mirror.fuel-infra.org/" + "mirror/mos-repos/ubuntu/{MOS_VERSION}/dists/mos{MOS_VERSION}-{REPO_NAME}/", + "local_path": + "/var/www/nailgun/mos-ubuntu/dists/mos{MOS_VERSION}-{REPO_NAME}/", + "repo_file": + "/root/mos_update_repo/mos-{REPO_NAME}.list", + "repo_text": + "deb http://{MASTER_IP}:8080/mos-ubuntu " + "mos{MOS_VERSION}-{REPO_NAME} main restricted", + "prio": "1150" + }, + "centos": { + "rsync": + "rsync://mirror.fuel-infra.org/mirror/" + "mos-repos/centos/mos{MOS_VERSION}-centos6-fuel/{REPO_NAME}/", + "local_path": + "/var/www/nailgun/mos-centos/mos{MOS_VERSION}/{REPO_NAME}", + "repo_text": + "[mos-{REPO_NAME}]\n" + "name=mos-{REPO_NAME}\n" + "baseurl=http://{MASTER_IP}:8080/mos-centos/mos{MOS_VERSION}/" + "{REPO_NAME}/\ngpgcheck=0\n", + "repo_file": + "/etc/yum.repos.d/mos-{REPO_NAME}.repo", + "prio": "100" + } + } + self.cfg['ubuntu_pool'] = { + 'rsync': "rsync://mirror.fuel-infra.org" + "/mirror/mos-repos/ubuntu/7.0/pool/", + 'local_path': "/var/www/nailgun/mos-ubuntu/pool/" + } + + +if __name__ == "__main__": + v = { + "5.1.1": Updater511, + "6.0": Updater60, + "6.1": Updater61, + "7.0": Updater70 + } + config = Config() + ver = v.get(config.cfg['mos_version'], None) + if ver: + inst = ver(config.getcfg()) + inst.run() else: - rsync_repos() - print ("Selected repositories: {0}".format( - ",".join(mos_repos_to_install))) - do_node_update(nodes, env_list) - sys.exit(0) + LOG.info("This script is not designed to update Fuel {0}".format( + config.cfg['mos_version']))