From 67f103d5ef8cd4a392cad594bc892aa2c7e4389d Mon Sep 17 00:00:00 2001 From: Alex Ermolov Date: Mon, 21 Mar 2016 16:55:58 +0300 Subject: [PATCH] jenkins: added merge_upstream job and scripts Change-Id: I2b73b351d7a5132050e0a6a578971c7386f0f32f --- jenkins/codesync/code-sync.yaml | 151 ++++++++++ jenkins/codesync/codesync.py | 472 ++++++++++++++++++++++++++++++++ jenkins/deploy/install.yml | 10 + 3 files changed, 633 insertions(+) create mode 100644 jenkins/codesync/code-sync.yaml create mode 100755 jenkins/codesync/codesync.py diff --git a/jenkins/codesync/code-sync.yaml b/jenkins/codesync/code-sync.yaml new file mode 100644 index 0000000..f007c31 --- /dev/null +++ b/jenkins/codesync/code-sync.yaml @@ -0,0 +1,151 @@ +- options: + gerrit-base-uri: ssh://jenkins@review.fuel-infra.org:29418 + gerrit-topic: sync/stable/liberty + downstream-branch: openstack-ci/fuel-8.0/liberty + upstream-branch: stable/liberty + + project: + - nova: + repo: openstack/nova + mail-to: mos-nova@mirantis.com + - python-novaclient: + repo: openstack/python-novaclient + mail-to: mos-nova@mirantis.com + + - neutron: + repo: openstack/neutron + mail-to: mos-neutron@mirantis.com + - python-neutronclient: + repo: openstack/python-neutronclient + mail-to: mos-neutron@mirantis.com + + - keystone: + repo: openstack/keystone + mail-to: mos-keystone@mirantis.com + - python-keystoneclient: + repo: openstack/python-keystoneclient + mail-to: mos-keystone@mirantis.com + - keystonemiddleware: + repo: openstack/keystonemiddleware + mail-to: mos-keystone@mirantis.com + + - glance: + repo: openstack/glance + mail-to: mos-glance@mirantis.com + - python-glanceclient: + repo: openstack/python-glanceclient + mail-to: mos-glance@mirantis.com + - glance_store: + repo: openstack/glance_store + mail-to: mos-glance@mirantis.com + + - cinder: + repo: openstack/cinder + mail-to: mos-cinder@mirantis.com + - python-cinderclient: + repo: openstack/python-cinderclient + mail-to: mos-cinder@mirantis.com + + - heat: + repo: openstack/heat + mail-to: mos-heat@mirantis.com + - python-heatclient: + repo: openstack/python-heatclient + mail-to: mos-heat@mirantis.com + + - horizon: + repo: openstack/horizon + mail-to: mos-horizon@mirantis.com + + - sahara: + repo: openstack/sahara + mail-to: mos-sahara@mirantis.com + - python-saharaclient: + repo: openstack/python-saharaclient + mail-to: mos-sahara@mirantis.com + + - murano: + repo: openstack/murano + mail-to: mos-murano@mirantis.com + - python-muranoclient: + repo: openstack/python-muranoclient + mail-to: mos-murano@mirantis.com + + - ceilometer: + repo: openstack/ceilometer + mail-to: mos-ceilometer@mirantis.com + - python-ceilometerclient: + repo: openstack/python-ceilometerclient + mail-to: mos-ceilometer@mirantis.com + + - swift: + repo: openstack/swift + mail-to: akhivin@mirantis.com # :) + - python-swiftclient: + repo: openstack/python-swiftclient + mail-to: akhivin@mirantis.com # :) + + - ironic: + repo: openstack/ironic + mail-to: mos-ironic@mirantis.com + - python-ironicclient: + repo: openstack/python-ironicclient + mail-to: mos-ironic@mirantis.com + + - oslo.cache: + repo: openstack/oslo.cache + mail-to: mos-oslo@mirantis.com + - oslo.concurrency: + repo: openstack/oslo.concurrency + mail-to: mos-oslo@mirantis.com + - oslo.config: + repo: openstack/oslo.config + mail-to: mos-oslo@mirantis.com + - oslo.context: + repo: openstack/oslo.context + mail-to: mos-oslo@mirantis.com + - oslo.db: + repo: openstack/oslo.db + mail-to: mos-oslo@mirantis.com + - oslo.i18n: + repo: openstack/oslo.i18n + mail-to: mos-oslo@mirantis.com + - oslo.log: + repo: openstack/oslo.log + mail-to: mos-oslo@mirantis.com + - oslo.messaging: + repo: openstack/oslo.messaging + mail-to: mos-oslo@mirantis.com + - oslo.middleware: + repo: openstack/oslo.middleware + mail-to: mos-oslo@mirantis.com + - oslo.policy: + repo: openstack/oslo.policy + mail-to: mos-oslo@mirantis.com + - oslo.reports: + repo: openstack/oslo.reports + mail-to: mos-oslo@mirantis.com + - oslo.rootwrap: + repo: openstack/oslo.rootwrap + mail-to: mos-oslo@mirantis.com + - oslo.serialization: + repo: openstack/oslo.serialization + mail-to: mos-oslo@mirantis.com + - oslo.service: + repo: openstack/oslo.service + mail-to: mos-oslo@mirantis.com + - oslo.utils: + repo: openstack/oslo.utils + mail-to: mos-oslo@mirantis.com + - oslo.versionedobjects: + repo: openstack/oslo.versionedobjects + mail-to: mos-oslo@mirantis.com + - oslo.vmware: + repo: openstack/oslo.vmware + mail-to: mos-oslo@mirantis.com + - oslosphinx: + repo: openstack/oslosphinx + mail-to: mos-oslo@mirantis.com + - oslotest: + repo: openstack/oslotest + mail-to: mos-oslo@mirantis.com diff --git a/jenkins/codesync/codesync.py b/jenkins/codesync/codesync.py new file mode 100755 index 0000000..d45ccef --- /dev/null +++ b/jenkins/codesync/codesync.py @@ -0,0 +1,472 @@ +#!/usr/bin/env python +# coding: utf-8 + +import argparse +import collections +import hashlib +import logging +import os +import re +import requests +import subprocess +import sys +import urlparse +import yaml + + +LOG = logging.getLogger('codesync') + +config_data = {} +commit_stats = { + "total_commits": 0, + "total_regexp_errors": 0 +} + + +class FailedToMerge(Exception): + '''Raised when automatic merge fails due to conflicts.''' + + +def _clone_or_fetch(gerrit_uri): + LOG.info('Cloning %s...', gerrit_uri) + + repo = os.path.basename(urlparse.urlsplit(gerrit_uri).path) + + retcode = subprocess.call( + ['git', 'clone', '-q', gerrit_uri], + stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + if retcode: + if not os.path.exists(repo): + LOG.error('Failed to clone repo: %s', gerrit_uri) + raise RuntimeError('Failed to clone repo: %s' % gerrit_uri) + else: + LOG.info('Repo already exists, fetching the latest state...') + + subprocess.check_call( + ['git', 'reset', '--hard', 'HEAD'], + cwd=repo, + stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + subprocess.check_call( + ['git', 'remote', 'update'], + cwd=repo, + stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + + path = os.path.join(os.getcwd(), repo) + LOG.info('Updated repo at: %s', path) + return path + + +def _get_commit_id(repo, ref='HEAD'): + return subprocess.check_output( + ['git', 'show', ref], + cwd=repo + ).splitlines()[0].split()[1] + + +def _get_merge_commit_message(repo, downstream_branch, upstream_branch): + downstream = _get_commit_id(repo, downstream_branch) + upstream = _get_commit_id(repo, upstream_branch) + + LOG.info('Downstream commit id: %s', downstream) + LOG.info('Upstream commit id: %s', upstream) + + commits_range = '%s..%s' % (downstream_branch, upstream_branch) + commits = subprocess.check_output( + ['git', 'log', '--no-merges', '--pretty=format:%h %s', commits_range], + cwd=repo + ) + + hashsum = hashlib.sha1() + hashsum.update(downstream) + changeid = 'I' + hashsum.hexdigest() + + template = ('Merge the tip of %(upstream)s into %(downstream)s' + '\n\n%(commits)s' + '\n\nChange-Id: %(changeid)s') + + return template % {'upstream': upstream_branch, + 'downstream': downstream_branch, + 'changeid': changeid, + 'commits': commits} + + +def _merge_tip(repo, downstream_branch, upstream_branch): + LOG.info('Trying to merge the tip of %s into %s...', + upstream_branch, downstream_branch) + + if not downstream_branch.startswith('origin/'): + downstream_branch = 'origin/' + downstream_branch + if not upstream_branch.startswith('origin/'): + upstream_branch = 'origin/' + upstream_branch + + # print merge information for visibility purposes + commits_range = '%s..%s' % (downstream_branch, upstream_branch) + graph = subprocess.check_output( + ['git', 'log', '--graph', '--pretty=format:%h %s', commits_range], + cwd=repo + ) + LOG.info('Commits graph to be merged:\n\n%s', graph) + + subprocess.check_call( + ['git', 'checkout', downstream_branch], + cwd=repo, + stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + try: + m = _get_merge_commit_message(repo, downstream_branch, upstream_branch) + LOG.info('Commit message:\n\n%s\n\n', m) + + subprocess.check_call( + ['git', 'merge', '--no-ff', '-m', m, upstream_branch], + cwd=repo, + stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + except subprocess.CalledProcessError: + raise FailedToMerge + else: + commit = _get_commit_id(repo) + LOG.info('Merge commit id: %s', commit) + return commit + + +def _upload_for_review(repo, commit, branch, topic=None): + LOG.info('Uploading commit %s to %s for review...', commit, branch) + + pusharg = '%s:refs/for/%s' % (commit, branch) + if topic: + pusharg += '%topic=' + str(topic) + + process = subprocess.Popen( + ['git', 'push', 'origin', pusharg], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + stdout, stderr = process.communicate() + + if process.returncode: + if 'no changes made' in stdout or 'no changes made' in stderr: + LOG.info('No changes since the last sync. Skip.') + else: + LOG.error('Failed to push the commit %s to %s', commit, branch) + raise RuntimeError( + 'Failed to push the commit %s to %s' % (commit, branch) + ) + + +def _cleanup(repo): + LOG.info('Running cleanups (hard reset + checkout of master + gc)...') + + subprocess.check_call( + ['git', 'reset', '--hard', 'HEAD'], + cwd=repo, + stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + subprocess.check_call( + ['git', 'checkout', 'master'], + cwd=repo, + stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + subprocess.check_call( + ['git', 'gc'], + cwd=repo, + stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + + LOG.info('Cleanups done.') + + +def sync_project(gerrit_uri, downstream_branch, upstream_branch, topic=None, + dry_run=False): + '''Merge the tip of the tracked upstream branch and upload it for review. + + Tries to clone (fetch, if path already exists) the git repo and do a + non-fastforward merge of the tip of the tracked upstream branch into + downstream one, and then upload the resulting merge commit for review. + + If automatic merge fails due to conflicts, FailedToMerge exception is + raised. + + :param gerrit_uri: gerrit git repo uri + :param downstream_branch: name of the downstream branch + :param upstream_branch: name of the corresponding upstream branch + :param topic: a Gerrit topic to be used + :param dry_run: don't actually upload commits to Gerrit, just try to merge + the branch locally + + :returns merge commit id + + ''' + + repo = _clone_or_fetch(gerrit_uri) + try: + commit = _merge_tip(repo, downstream_branch, upstream_branch) + + if not dry_run: + _upload_for_review(repo, commit, downstream_branch, topic=topic) + else: + LOG.info('Dry run, do not attempt to upload the merge commit') + + return commit + finally: + _cleanup(repo) + + +def merge_bug_fixes(gerrit_uri, downstream_branch, upstream_branch, topic=None, + dry_run=False): + LOG.info("Trying to cherry-pick only High/Critical bug fixes of %s into " + "%s...", upstream_branch, downstream_branch) + + repo = _clone_or_fetch(gerrit_uri) + + if not downstream_branch.startswith("origin/"): + downstream_branch = "origin/" + downstream_branch + if not upstream_branch.startswith('origin/'): + upstream_branch = "origin/" + upstream_branch + + # print difference information + commits_range = '%s..%s' % (downstream_branch, upstream_branch) + graph = subprocess.check_output( + ["git", "log", "--no-merges", "--graph", + "--pretty=format:'%Cred%h%Creset -%C(yellow)%d%Creset %s " + "%Cgreen(%cr)%Creset'", "--abbrev-commit", "--date=relative", + commits_range], + cwd=repo + ) + + # possible output: + # * a0ffd8c - Validate translations (7 days ago) + # * 1f594f9 - Imported Translations from Zanata (3 days ago) + # * 9ed4489 - Imported Translations from Zanata (5 days ago) + # * 8ffca40 - Imported Translations from Zanata (9 days ago) + if graph: + LOG.info("Commits, that may contain needed bug fixes:\n\n%s\n", graph) + + commits_list = subprocess.check_output( + ["git", "log", "--no-merges", + "--pretty=format:'%Cred%h%Creset -%C(yellow)%d%Creset %s " + "%Cgreen(%cr)%Creset'", "--abbrev-commit", "--date=relative", + commits_range], + cwd=repo + ) + + commit_lines = commits_list.split("\n") + + bugs = collections.OrderedDict() + bug_suffixes = ["closes-bug:", "partial-bug:", "fixes-bug", + "partially-fixes-bug", "closes bug:", "partial bug:", + "fixes bug", "partially fixes bug"] + commits_count = 0 + regexp_error_count = 0 + for ind, commit_line in enumerate(commit_lines): + try: + commit_line = re.sub('\x1b[^m]*m', '', + commit_line).replace("'", "") + commit_lines[ind] = commit_line + if not commit_line: + continue + # commit_id = re.search('\* (.*) - *', commit_line).group(1) + commit_id = commit_line.split('-')[0].strip() + commit_msg = subprocess.check_output( + ["git", "log", "--format=%B", "-n", "1", commit_id], + cwd=repo + ) + + for bug_suffix in bug_suffixes: + if bug_suffix in commit_msg.lower(): + # line may looks like: + # Closes-Bug: #1536214 + # We're setting the following values in bugs dict: + # bugs["a0ffd8c"] = "1536214" + bugs[commit_id] = re.search('%s #?(.+?)\\n' % bug_suffix, + commit_msg.lower()).group(1) + break + commits_count += 1 + except AttributeError as e: + LOG.info("Encountered commit_line '{0}', skipping...".format( + commit_line)) + regexp_error_count += 1 + continue + except subprocess.CalledProcessError as e: + LOG.info("Git returned error: '{0}', skipping...".format( + e)) + LOG.info("Commit line was: '{0}'".format( + commit_line)) + regexp_error_count += 1 + continue + commit_stats["total_commits"] += commits_count + LOG.info("Processed commits: {0}".format(commits_count)) + if regexp_error_count: + commit_stats["total_regexp_errors"] += regexp_error_count + LOG.info("Regexp failures encountered: {0}".format( + regexp_error_count)) + if len(bugs.keys()): + LOG.info("Commits, that are bug fixes:\n%s\n", bugs.keys()) + + important_bugs = collections.OrderedDict() + + for commit_id, bug in bugs.iteritems(): + resp = requests.get("https://api.launchpad.net/devel/bugs/%s/bug_tasks" + % bug).json() + for entry in resp["entries"]: + if "liberty" in entry["bug_target_name"] and \ + entry["importance"] in ["High", "Critical"]: + important_bugs[commit_id] = bug + + if len(important_bugs.keys()): + LOG.info("Commits, that are important bug fixes:\n%s\n", + important_bugs.keys()) + + if important_bugs: + subprocess.check_call( + ['git', 'checkout', downstream_branch], + cwd=repo, + stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + try: + items = important_bugs.items() + items.reverse() + for bug in collections.OrderedDict(items): + subprocess.check_call( + ['git', 'cherry-pick', '-x', bug], + cwd=repo, + stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + except subprocess.CalledProcessError: + raise FailedToMerge + else: + commit = _get_commit_id(repo) + LOG.info('Current commit id: %s', commit) + if not dry_run: + _upload_for_review(repo, commit, downstream_branch, + topic=topic) + else: + LOG.info('Dry run, do not attempt to upload the merge commit') + return commit + + +def read_config(config_file): + global config_data + with open(config_file, 'r') as f: + config_data = yaml.load(f) + + +def process_repos(action, downstream_branch, upstream_branch, + topic, dry_run=True): + repos_list = config_data[0]['options']['project'] + gerrit_base_uri = config_data[0]['options']['gerrit-base-uri'] + upstream_branch = upstream_branch or \ + config_data[0]['options']['upstream-branch'] + downstream_branch = downstream_branch or \ + config_data[0]['options']['downstream-branch'] + topic = topic or config_data[0]['options']['gerrit-topic'] + print "Using gerrit URI: {0}".format(gerrit_base_uri) + + if action == 'merge_tip': + func = sync_project + elif action == 'merge_bug_fixes': + func = merge_bug_fixes + + for repo in repos_list: + LOG.info("========================================================") + LOG.info("processing project: {0}".format(repo.keys()[0])) + repo_name = repo[repo.keys()[0]]['repo'] + LOG.info("repo name: {0}".format(repo_name)) + try: + commit = func(gerrit_uri=gerrit_base_uri + "/" + repo_name, + downstream_branch=downstream_branch, + upstream_branch=upstream_branch, + topic=topic, + dry_run=dry_run) + if commit: + print(commit) + except FailedToMerge: + LOG.info("Automatic merge failed, trying next repo from batch.") + continue + + if func == merge_bug_fixes: + LOG.info("======================TOTAL==================================") + LOG.info("Commits found: {0}".format(commit_stats["total_commits"])) + LOG.info("Regexp errors encountered: {0}".format( + commit_stats["total_regexp_errors"])) + + +def main(): + logging.basicConfig( + format='%(asctime)s - %(levelname)s - %(message)s' + ) + LOG.setLevel(logging.INFO) + + parser = argparse.ArgumentParser( + description=('Merge the tip of the upstream tracking branch and ' + 'upload it for review. Merge commit id is printed ' + 'to stdout on success. If automatic merge fails ' + 'the process ends with a special exit code - 1. ' + 'All other exit codes (except 0 and 1) are runtime ' + 'errors.') + ) + + parser.add_argument( + 'config', type=str, + help="Name of file, containing the List of very basic parameter " + "defaults and list of projects to process.", + metavar="config" + ) + + parser.add_argument( + '--action', + help="What action is expected to happen. By default script will try" + "to upload on review the merge commit of upstream branch to " + "downstream branch. Also it's possible to merge only resolutions " + "of High and Critical bugs from the upstream.", + default='merge_tip', + choices=['merge_tip', 'merge_bug_fixes'], + ) + + parser.add_argument( + '--downstream-branch', + help=('downstream branch to upload merge commit to ' + '(defaults to $SYNC_DOWNSTREAM_BRANCH)'), + default=os.getenv('SYNC_DOWNSTREAM_BRANCH') + ) + parser.add_argument( + '--upstream-branch', + help=('upstream branch to sync the state from ' + '(defaults to $SYNC_UPSTREAM_BRANCH)'), + default=os.getenv('SYNC_UPSTREAM_BRANCH') + ) + parser.add_argument( + '--topic', + help='a Gerrit topic to be used', + default=os.getenv('SYNC_GERRIT_TOPIC') + ) + parser.add_argument( + '--dry-run', + help="do not upload a merge commit on review - just try local merge", + action='store_true' + ) + + try: + args = parser.parse_args() + if not args.action: + parser.print_usage() + raise ValueError('Required arguments not passed') + read_config(args.config) + dry_run = bool(os.getenv('SYNC_DRY_RUN') == 'true') or args.dry_run + process_repos(args.action, + args.downstream_branch, + args.upstream_branch, + args.topic, + dry_run=dry_run) + except Exception: + # unhandled runtime errors + LOG.exception('Runtime error: ') + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/jenkins/deploy/install.yml b/jenkins/deploy/install.yml index ac1ad8e..71b8d9e 100644 --- a/jenkins/deploy/install.yml +++ b/jenkins/deploy/install.yml @@ -6,6 +6,7 @@ rally_job_name: rally tempest_job_name: tempest patching_job_name: patch_environment + codesync_job_name: merge_upstream ta_subpath: testing_automation/ansible tasks: @@ -62,3 +63,12 @@ - common_vars.yml - patch_environment.yml - cleanup_knownhosts.yml + + - name: codesync | mkdir + file: path={{ jobs_dir }}/{{ codesync_job_name }} state=directory + + - name: codesync | upload script + config + copy: src=../codesync/{{ item.file }} dest={{ jobs_dir }}/{{ codesync_job_name }} mode={{ item.mode }} + with_items: + - { file: "codesync.py", mode: "755"} + - { file: "code-sync.yaml", mode: "644"} -- 2.45.2