From 9979cd8e1ab2ce652fcb68911123bfe77c2b4e9a Mon Sep 17 00:00:00 2001 From: vrovachev Date: Mon, 3 Aug 2015 18:55:44 +0400 Subject: [PATCH] Add check for affected packages for erratum file Added method for search difference between affected packages in erratum file and compiled packages from patches. Closes-bug: #1480986 Change-Id: I575c6526f795a65e3199e0a6ec1886b36d2e57c3 --- scripts/erratumvalidation.py | 249 ++++++++++++++++++++++++++++++++++- 1 file changed, 244 insertions(+), 5 deletions(-) diff --git a/scripts/erratumvalidation.py b/scripts/erratumvalidation.py index 7616ad7..fcfdc92 100644 --- a/scripts/erratumvalidation.py +++ b/scripts/erratumvalidation.py @@ -12,15 +12,31 @@ # License for the specific language governing permissions and limitations # under the License. -from jsonschema import validate -import yaml +import os +import re import sys import urllib2 +import logging + +import yaml +from jsonschema import validate + +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) +ch = logging.StreamHandler() +ch.setLevel(logging.INFO) +formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') +ch.setFormatter(formatter) +logger.addHandler(ch) SCHEMA = "http://json-schema.org/draft-04/schema" CVE_URL = "https://cve.mitre.org/cgi-bin/cvename.cgi?name={bug}" +MIRROR_URLS = ['http://osci-obs.vm.mirantis.net:82/', + 'http://obs-1.mirantis.com:82/'] + CONFIG_SCHEMA = { "type": "object", "$schema": SCHEMA, @@ -280,6 +296,11 @@ CONFIG_SCHEMA_V2 = {} def validate_schema(erratum_file): + """ + Check that erratum schema is correctly + :param erratum_file: erratum.yaml file + :return: raise or return 0 + """ with open(erratum_file) as f: content = f.read() @@ -290,32 +311,250 @@ def validate_schema(erratum_file): validate_schema = CONFIG_SCHEMA_V2 # validate schema + logger.info("[ Check erratum schema ]") validate(dict_content, validate_schema) + logger.info("[ Done ]\n") # Check that OS in affected-pkgs has in patch-scenario and # in verify-scenario if verify-scenario specified + + logger.info("[ Check that OS in affected-pkgs has in " + "patch-scenario ]") os_error_msg = ("{system} OS specified in affected-pkgs but not " "specified in {vol}-scenario steps") for distro in dict_content['affected-pkgs']: for target in dict_content['targets']: if not target['patch-scenario'].get(distro): raise Exception(os_error_msg.format(system=distro, - vol="patch")) + vol="patch")) if target.get('verify-scenario'): if not target['verify-scenario'].get(distro): raise Exception(os_error_msg.format(system=distro, - vol="verify")) + vol="verify")) + logger.info("[ Done ]\n") # Check CVE bug if cve bug number specified cve_error_msg = ("CVE bub with number: {number} specified in erratum" " file but this bug not found." " URL for search: {search_url}") if dict_content.get("cve"): + logger.info("[ Check CVE bug ]") url_for_cve = CVE_URL.format(bug=dict_content["cve"]) resp = urllib2.urlopen(url_for_cve) if resp.read().find("ERROR:") != -1: raise Exception(cve_error_msg.format( number=dict_content['cve'], search_url=url_for_cve)) + logger.info("[ Done ]\n") + return dict_content + + +def _get_packages_from_url(mirror, package_url, message=None): + + rpm_regexp = '(?<=href=")([a-z0-9-.]+).rpm' + deb_regexp = '(?<=href=")([a-z0-9-~_+%.]+).deb' + + if package_url.startswith('centos'): + distro = 'centos' + postfix = 'centos/noarch/' + package_regexp = rpm_regexp + elif package_url.startswith(('ubuntu', 'trusty')): + distro = 'ubuntu' + postfix = 'ubuntu/all/' + package_regexp = deb_regexp + mirror_repo_url = "{glob_url}{pkg_url}{postfix}".format( + glob_url=mirror, pkg_url=package_url, postfix=postfix) + packages = urllib2.urlopen( + mirror_repo_url).read().split('\n') + + if message: + logger.info("-"*40) + logger.info(message) + logger.info("-"*40) + logger.info(mirror_repo_url) + logger.info("-"*40) + + packages_list = [] + for package in packages: + package_regexp_result = re.search(package_regexp, + package) + if package_regexp_result: + package_name = package_regexp_result.group() + + if message: + logger.info(package_name) + + packages_list.append((distro, package_name)) + return packages_list + + +def check_affected_packages(erratum_dict): + """ + Check that all affected packages has in mirrors + :param erratum_dict: dictionary with erratum variables + :return: raise or return 0 + """ + logger.info("[ Check affected packages ]") + branch = os.environ.get('GERRIT_BRANCH').split('/')[-1] + bug = os.environ.get('GERRIT_TOPIC').split('/')[-1] + logger.info("GERRIT_BRANCH: {branch}".format(branch=branch)) + logger.info("GERRIT_TOPIC: {topic}".format(topic=bug)) + + compiled_packages = {'centos': [], + 'ubuntu': []} + stable_packages = {'centos': {}, + 'ubuntu': {}} + get_pkg_part = { + 'centos': { + 'name': (lambda pkg: "-".join(pkg.split('-')[:-2])), + 'version': (lambda pkg, regex=( + lambda pkg: re.search('(?<=mira)[1-9]+(?<=.)', pkg)): + int(regex(pkg).group()) if regex(pkg) else 0), + 'full_name': (lambda pkg: pkg.replace(".noarch.rpm", "")) + }, + 'ubuntu': { + 'name': (lambda pkg: pkg.split('_')[0]), + 'version': (lambda pkg, regex=( + lambda pkg: re.search('(?<=mos)[1-9]+(?<=.)', pkg)): + int(regex(pkg).group()) if regex(pkg) else 0), + 'full_name': (lambda pkg: pkg.replace("_all.deb", "")) + } + } + affected_packages = { + "centos": [x.split("=")[0] for x + in erratum_dict['affected-pkgs']['centos']], + "ubuntu": [x.split("=")[0] for x + in erratum_dict['affected-pkgs']['ubuntu']] + } + affected_pkgs_with_version = { + "centos": [x.replace("=", "-") for x + in erratum_dict['affected-pkgs']['centos']], + "ubuntu": [x.replace("=", "-") for x + in erratum_dict['affected-pkgs']['ubuntu']], + } + patch_numbers_in_branch = [] + bug_regexp = ('(?<=href=")(centos|ubuntu|trusty)' + '-fuel-{branch}-([a-z-]+)LP{bug}/') + stable_regexp = ('(?<=href=")(centos|ubuntu|trusty)' + '-fuel-{branch}-stable(-updates/|/)') + + for mirror in MIRROR_URLS: + all_mirror_packages = urllib2.urlopen(mirror).read().split('\n') + + # Find all packages which built in patches for bug on specified mirror + for package in all_mirror_packages: + bug_regexp_result = re.search( + bug_regexp.format( + branch=branch, bug=bug), package) + stable_regexp_result = re.search( + stable_regexp.format(branch=branch), package) + + if bug_regexp_result: + package_url = bug_regexp_result.group() + + packages = _get_packages_from_url( + mirror, package_url, "found mirror packages repo") + + for distro, package in packages: + compiled_packages[distro].append( + get_pkg_part[distro]['name'](package)) + + if stable_regexp_result: + stable_url = stable_regexp_result.group() + packages = _get_packages_from_url(mirror, stable_url) + for distro, package in packages: + package_name = get_pkg_part[distro]['name'](package) + package_ver = get_pkg_part[distro]['version'](package) + full_name = get_pkg_part[distro]['full_name'](package) + if package_name in affected_packages[distro]: + if package_name in stable_packages[distro]: + if (stable_packages[ + distro][package_name]['version'] + < package_ver): + stable_packages[distro][package_name] = { + 'full': full_name, + 'version': package_ver + } + else: + stable_packages[distro][package_name] = { + 'full': full_name, + 'version': package_ver + } + + # Check versions for affected packages + + compiled_packages['ubuntu'].sort() + compiled_packages['centos'].sort() + affected_packages['centos'].sort() + affected_packages['ubuntu'].sort() + + logger.info("-"*40) + + if set(compiled_packages['ubuntu']).symmetric_difference( + set(affected_packages['ubuntu'])): + compiled = set(compiled_packages['ubuntu']).difference( + affected_packages['ubuntu']) + affected = set(affected_packages['ubuntu']).difference( + compiled_packages['ubuntu']) + if compiled: + logger.warn("Compiled Ubuntu packages but not in erratum " + "file:\n{}".format("\n".join(list(compiled)))) + if affected: + raise BaseException("Ubuntu packages in erratum file but not " + "compiled:\n{}".format( + "\n".join(list(affected)))) + + if set(compiled_packages['centos']).symmetric_difference( + affected_packages['centos']): + compiled = set(compiled_packages['centos']).difference( + affected_packages['centos']) + affected = set(affected_packages['centos']).difference( + compiled_packages['centos']) + if compiled: + logger.warn("Compiled CentOS packages but not in erratum " + "file:\n{}".format("\n".join(list(compiled)))) + if affected: + raise BaseException("Ubuntu packages in erratum file but not " + "compiled:\n{}".format( + "\n".join(list(affected)))) + + logger.info("[ Done ]\n") + logger.info("[ Check that in affected_pkgs specified " + "last package versions ]") + + found_affected_centos = [value['full'] for key, value + in stable_packages['centos'].iteritems()] + found_affected_ubuntu = [value['full'].replace('%2b', '+') + for key, value + in stable_packages['ubuntu'].iteritems()] + + found_affected_centos.sort() + found_affected_ubuntu.sort() + affected_pkgs_with_version['centos'].sort() + affected_pkgs_with_version['ubuntu'].sort() + + centos_difference = set(found_affected_centos).symmetric_difference( + set(affected_pkgs_with_version['centos'])) + ubuntu_difference = set(found_affected_ubuntu).symmetric_difference( + set(affected_pkgs_with_version['ubuntu'])) + + err_msg = ('Found difference in {} affected packages in erratum file and ' + 'in mirrors.\n ' + 'Packages in erratum file:\n{}\n Found Packages:\n{}\n') + if centos_difference: + logger.warning(err_msg.format("CentOS", + "\n".join( + affected_pkgs_with_version[ + 'centos']), + "\n".join(found_affected_centos))) + if ubuntu_difference: + logger.warning(err_msg.format("Ubuntu", + "\n".join( + affected_pkgs_with_version[ + 'ubuntu']), + "\n".join(found_affected_ubuntu))) + logger.info("[ Done ]\n") + if __name__ == "__main__": - validate_schema(sys.argv[1]) + erratum_dict = validate_schema(sys.argv[1]) + check_affected_packages(erratum_dict) -- 2.45.2