From dc2c533f608a603958a016e8bba7905d70c2be88 Mon Sep 17 00:00:00 2001 From: Vladimir Khlyunev Date: Fri, 8 Feb 2019 10:44:07 +0400 Subject: [PATCH] Maintenance CI JJB initial commit The commit introduces new directory for JJB configs for brand new Maintenance-CI cloud-based Jenkins instance. First set of jobs includes following prototypes of jobs: - JJB update mechanism - Stack create job - Stack delete job It also includes very common 1-instance heat stack but it covers current needs. Also tox configuration file for JJB checks is added. Change-Id: I5f65ea7331590d51dae94d1c415e34a83a5343d1 --- .gitignore | 3 +- maintenance-ci/common/data/base-heat.yml | 73 +++++++++++++++++++ maintenance-ci/common/jobs/delete_stack.yaml | 37 ++++++++++ maintenance-ci/common/jobs/deploy_stack.yaml | 47 ++++++++++++ .../common/jobs/update-jenkins-jobs.yaml | 62 ++++++++++++++++ .../common/macro/openstack_creds.yaml | 11 +++ .../common/scripts/delete_heat_stack.sh | 24 ++++++ .../common/scripts/deploy_heat_stack.sh | 36 +++++++++ .../common/scripts/update-jenkins-cleanup.sh | 5 ++ .../common/scripts/update-jenkins-jobs.sh | 29 ++++++++ .../common/scripts/update-jenkins-views.sh | 62 ++++++++++++++++ maintenance-ci/conf/jenkins_job.ini.example | 14 ++++ maintenance-ci/conf/requirements-jobs.txt | 2 + maintenance-ci/conf/requirements-views.txt | 3 + maintenance-ci/tox.ini | 18 +++++ 15 files changed, 425 insertions(+), 1 deletion(-) create mode 100644 maintenance-ci/common/data/base-heat.yml create mode 100644 maintenance-ci/common/jobs/delete_stack.yaml create mode 100644 maintenance-ci/common/jobs/deploy_stack.yaml create mode 100644 maintenance-ci/common/jobs/update-jenkins-jobs.yaml create mode 100644 maintenance-ci/common/macro/openstack_creds.yaml create mode 100755 maintenance-ci/common/scripts/delete_heat_stack.sh create mode 100755 maintenance-ci/common/scripts/deploy_heat_stack.sh create mode 100644 maintenance-ci/common/scripts/update-jenkins-cleanup.sh create mode 100644 maintenance-ci/common/scripts/update-jenkins-jobs.sh create mode 100644 maintenance-ci/common/scripts/update-jenkins-views.sh create mode 100644 maintenance-ci/conf/jenkins_job.ini.example create mode 100644 maintenance-ci/conf/requirements-jobs.txt create mode 100644 maintenance-ci/conf/requirements-views.txt create mode 100644 maintenance-ci/tox.ini diff --git a/.gitignore b/.gitignore index d111a46..778733e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /jenkins/build_cluster/*pyc /jenkins/build_cluster/.*sw? - +/maintenance-ci/output +/maintenance-ci/.tox diff --git a/maintenance-ci/common/data/base-heat.yml b/maintenance-ci/common/data/base-heat.yml new file mode 100644 index 0000000..380a185 --- /dev/null +++ b/maintenance-ci/common/data/base-heat.yml @@ -0,0 +1,73 @@ + +heat_template_version: 2017-09-01 + +parameters: + key_name: + type: string + description: Name of keypair to assign to servers + default: maintenance-ci-public-key + image: + type: string + description: Name of image to use for servers + default: xenial-server-cloudimg-amd64-qcow + flavor: + type: string + description: Flavor to use for servers + private_net: + type: string + description: > + ID or name of public network for which floating IP addresses will be allocated + default: maintenance-ci-network + private_subnet: + type: string + default: maintenance-ci-subnet + description: Id of the private sub network for the compute server + public_net: + type: string + default: public + description: Id of the public network for the compute server + jenkins_labels: + type: string + default: '' + description: List of labels for jenkins swarm agent separated by whitespace + +resources: + + public_port: + type: OS::Neutron::Port + properties: + network_id: { get_param: private_net } + fixed_ips: + - subnet_id: { get_param: private_subnet } + security_groups: + - dc56d304-e75e-437a-8ef5-656f9bfcd2a0 + + floating_ip: + type: OS::Neutron::FloatingIP + properties: + floating_network_id: { get_param: public_net } + port_id: { get_resource: public_port } + + jenkins-slave: + type: OS::Nova::Server + properties: + image: { get_param: image } + flavor: { get_param: flavor } + key_name: { get_param: key_name } + networks: + - port: { get_resource: public_port } + user_data: + str_replace: + template: | + #!/bin/bash + mkdir -p /etc/jenkins-agent/ + echo "$FLAVOR $LABELS" > /etc/jenkins-agent/labels + echo "127.0.0.1 $(hostname)" >> /etc/hosts + params: + $FLAVOR: {get_param: flavor} + $LABELS: {get_param: jenkins_labels} + +outputs: + floating_ip: + description: Floating IP + value: { get_attr: [ floating_ip, floating_ip_address ] } diff --git a/maintenance-ci/common/jobs/delete_stack.yaml b/maintenance-ci/common/jobs/delete_stack.yaml new file mode 100644 index 0000000..c05e776 --- /dev/null +++ b/maintenance-ci/common/jobs/delete_stack.yaml @@ -0,0 +1,37 @@ +- job: + name: 'delete-heat-stack' + description: | + Delete given heat stack on internal cloud from maintenance-team tenant + + concurrent: false + node: 'jenkins-master' + + parameters: + - string: + name: STACK_NAME + default: "" + description: | + Full name of pending for deletion stack. + If not given - STACK_PREFIX will be proceed + - string: + name: STACK_PREFIX + description: | + All stacks named with prefix on name's start will be deleted. + Used only if STACK_NAME is not defined + default: "" + - string: + name: OPENSTACK_CLIENTS_VENV + default: /home/jenkins/venv-openstack-clients + description: Path to venv with openstack clients inside + + + wrappers: + - timestamps + - openstack-creds + - timeout: + fail: true + timeout: 30 + + builders: + - shell: + !include-raw: common/scripts/delete_heat_stack.sh diff --git a/maintenance-ci/common/jobs/deploy_stack.yaml b/maintenance-ci/common/jobs/deploy_stack.yaml new file mode 100644 index 0000000..5ddcaaa --- /dev/null +++ b/maintenance-ci/common/jobs/deploy_stack.yaml @@ -0,0 +1,47 @@ +- job: + name: 'deploy-heat-stack' + description: | + Create given heat stack on internal cloud in maintenance-team tenant + + concurrent: false + node: 'jenkins-master' + + parameters: + - string: + name: HEAT_STACK_YAML + description: "Stack yaml file" + default: "maintenance-ci/common/data/base_heat.yml" + - string: + name: STACK_PREFIX + description: "Stack prefix" + default: "swarm-slave" + - string: + name: FLAVOR_NAME + default: "dev.share" + - string: + name: IMAGE_NAME + default: xenial-server-cloudimg-amd64-qcow + - string: + name: JENKINS_LABELS + default: '' + - string: + name: OPENSTACK_CLIENTS_VENV + default: /home/jenkins/venv-openstack-clients + description: Path to venv with openstack clients inside + + scm: + - git: + url: 'https://review.fuel-infra.org/tools/sustaining/' + branches: + - origin/master + + wrappers: + - timestamps + - openstack-creds + - timeout: + fail: true + timeout: 30 + + builders: + - shell: + !include-raw: common/scripts/deploy_heat_stack.sh diff --git a/maintenance-ci/common/jobs/update-jenkins-jobs.yaml b/maintenance-ci/common/jobs/update-jenkins-jobs.yaml new file mode 100644 index 0000000..8ee9980 --- /dev/null +++ b/maintenance-ci/common/jobs/update-jenkins-jobs.yaml @@ -0,0 +1,62 @@ +- job: + name: 'update-jenkins-jobs' + description: | +

Update jenkins jobs configuration

+

Requires python-tox package and user credentials stored as JJB_USER and JJB_PASS using Jenkins Global Passwords feature

+ + + concurrent: false + node: 'jenkins-master' + + parameters: + - string: + name: JOBS_LIST + description: 'Space separated list of jobs to update. Will update all jobs if empty' + + scm: + - git: + remotes: + - gerrit: + url: 'https://review.fuel-infra.org/tools/sustaining/' + branches: + - master + choosing-strategy: default + +# triggers: +# - timed: 'H * * * *' +# - gerrit: +# trigger-on: +# - change-merged-event +# projects: +# - project-compare-type: PLAIN +# project-pattern: 'fuel-infra/jenkins-jobs' +# branches: +# - branch-compare-type: PLAIN +# branch-pattern: 'master' +# file-paths: +# - compare-type: ANT +# pattern: 'common/**' +# - compare-type: ANT +# pattern: 'servers/{ci-name}/**' + + wrappers: + - timestamps + - credentials-binding: + - username-password-separated: + credential-id: maintenance-ci-robot-password + username: JJB_USER + password: JJB_PASS + - timeout: + fail: true + timeout: 30 + + builders: + - shell: + !include-raw-escape: common/scripts/update-jenkins-jobs.sh + + publishers: + - post-tasks: + - matches: + - log-text: '.' + escalate-status: true + script: !include-raw-escape: common/scripts/update-jenkins-cleanup.sh \ No newline at end of file diff --git a/maintenance-ci/common/macro/openstack_creds.yaml b/maintenance-ci/common/macro/openstack_creds.yaml new file mode 100644 index 0000000..e4c69df --- /dev/null +++ b/maintenance-ci/common/macro/openstack_creds.yaml @@ -0,0 +1,11 @@ +- wrapper: + name: openstack-creds + wrappers: + - credentials-binding: + - file: + credential-id: maintenance-ci-openrc-password-free + variable: OPENRC_FILE + - username-password-separated: + credential-id: maintenance-ci-robot-password + username: OS_USERNAME + password: OS_PASSWORD \ No newline at end of file diff --git a/maintenance-ci/common/scripts/delete_heat_stack.sh b/maintenance-ci/common/scripts/delete_heat_stack.sh new file mode 100755 index 0000000..c60b384 --- /dev/null +++ b/maintenance-ci/common/scripts/delete_heat_stack.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -ex + +if [ ! -d "${OPENSTACK_CLIENTS_VENV}" ] ; then + virtualenv "${OPENSTACK_CLIENTS_VENV}" + source "${OPENSTACK_CLIENTS_VENV}"/bin/activate + pip install openstackclient + deactivate +fi + +source "${OPENSTACK_CLIENTS_VENV}"/bin/activate + source "${OPENRC_FILE?}" + + if [ -z "${STACK_NAME}" ] ; then + openstack stack delete -y "${STACK_NAME}" + elif [ -z "${STACK_PREFIX}" ] ; then + for stack in $(openstack stack list -c 'Stack Name' -f value) ; do + if [[ ${stack} == ${STACK_PREFIX}* ]] ; then + openstack stack delete -y "${stack}" + fi + done + fi +deactivate diff --git a/maintenance-ci/common/scripts/deploy_heat_stack.sh b/maintenance-ci/common/scripts/deploy_heat_stack.sh new file mode 100755 index 0000000..a648294 --- /dev/null +++ b/maintenance-ci/common/scripts/deploy_heat_stack.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +set -ex + +if [ ! -d "${OPENSTACK_CLIENTS_VENV}" ] ; then + virtualenv "${OPENSTACK_CLIENTS_VENV}" + source "${OPENSTACK_CLIENTS_VENV}"/bin/activate + pip install openstackclient + deactivate +fi + + +parameter_string='' +if [ -z ${IMAGE_NAME} ] ; then + parameter_string="${parameter_string} --parameter 'image=${IMAGE_NAME}'" +fi +if [ -z ${FLAVOR_NAME} ] ; then + parameter_string="${parameter_string} --parameter 'flavor=${FLAVOR_NAME}'" +fi +if [ -z ${JENKINS_LABELS} ] ; then + parameter_string="${parameter_string} --parameter 'jenkins_labels=${JENKINS_LABELS}'" +fi + +# MAC OS compatibility, for manual usage +if [ $(which md5sum) ] ; then + md5_cmd="md5sum" +else + md5_cmd="md5" +fi + +STACK_NAME="${STACK_PREFIX:-swarm_slave}_$(date +%s | "${md5_cmd}" | head -c 4)" + +source "${OPENSTACK_CLIENTS_VENV}"/bin/activate + source "${OPENRC_FILE?}" + openstack stack create -t "${HEAT_STACK_YAML}" "${STACK_NAME}" +deactivate \ No newline at end of file diff --git a/maintenance-ci/common/scripts/update-jenkins-cleanup.sh b/maintenance-ci/common/scripts/update-jenkins-cleanup.sh new file mode 100644 index 0000000..0173c3f --- /dev/null +++ b/maintenance-ci/common/scripts/update-jenkins-cleanup.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -ex + +rm "${WORKSPACE}/../tmp/${JOB_NAME}/jenkins_jobs.ini" diff --git a/maintenance-ci/common/scripts/update-jenkins-jobs.sh b/maintenance-ci/common/scripts/update-jenkins-jobs.sh new file mode 100644 index 0000000..b166d04 --- /dev/null +++ b/maintenance-ci/common/scripts/update-jenkins-jobs.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -ex + +tox --help || apt-get install -y python-tox +tox -e jobs + +source ".tox/jobs/bin/activate" + +CONFIG_PATH="${WORKSPACE}/../tmp/${JOB_NAME}" +# JOBS_LIST - Jenkins string parameter. Space separated list of job names to update. +# Will update all jobs if empty. Example: verify-mos-docs docker-rebuild-fuel-ci +JOBS_LIST=(${JOBS_LIST}) + +umask 0077 +mkdir -p "${CONFIG_PATH}" +cat > "${CONFIG_PATH}/jenkins_jobs.ini" << EOF +[jenkins] +user=${JJB_USER} +password=${JJB_PASS} +url=${JENKINS_URL} +query_plugins_info=False +[job_builder] +ignore_cache=True +recursive=True +[__future__] +param_order_from_yaml=true +EOF + +jenkins-jobs --conf "${CONFIG_PATH}/jenkins_jobs.ini" update "common:mos:mcp" "${JOBS_LIST[@]}" diff --git a/maintenance-ci/common/scripts/update-jenkins-views.sh b/maintenance-ci/common/scripts/update-jenkins-views.sh new file mode 100644 index 0000000..b2a436d --- /dev/null +++ b/maintenance-ci/common/scripts/update-jenkins-views.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# +# :mod:`update-jenkins-views` -- Updating jenkins views +# ===================================================== +# +# .. module:: sample-bash-script +# :platform: Unix, Windows +# :synopsis: Update views on Jenkins envs +# .. vesionadded:: MOS-10.0 +# .. vesionchanged:: MOS-10.0 +# .. author:: Artur Mihura +# +# +# This script is used to update views on jenkins environments using +# jenkins-view-builder and YAML files with view descriptions +# +# .. envvar:: +# :var CI_NAME: Id of Jenkins build under which this +# script is running, defaults to ``0`` +# :type CI_NAME: string +# :var VIEWS_LIST: Space separated list of view YAMLS to update. +# Will update all views if empty +# :type VIEWS_LIST: string +# :var WORKSPACE: Location where build is started, defaults to ``.`` +# :type WORKSPACE: path +# +# +# .. requirements:: +# +# * ``tox`` in ``/usr/bin/tox` +# +# +# .. seealso:: https://bugs.launchpad.net/fuel/+bug/1496987 +# .. warnings:: never use on production for test purposes + +set -ex + +tox --help || apt install -y python-tox +tox -e views + +source ".tox/ci-views/bin/activate" + +CONFIG_PATH="${WORKSPACE}/../tmp/${JOB_NAME}" + +umask 0077 +mkdir -p "${CONFIG_PATH}" +cat > "${CONFIG_PATH}/jenkins_jobs.ini" << EOF +[jenkins] +user=${JJB_USER} +password=${JJB_PASS} +url=${JENKINS_URL} +EOF + +cd "views/${CI_NAME}" + +if [ -n "$VIEWS_LIST" ] ; then + VIEWS_LIST=(${VIEWS_LIST}) +else + VIEWS_LIST=(*) +fi + +jenkins-view-builder update --conf "${CONFIG_PATH}/jenkins_jobs.ini" "${VIEWS_LIST[@]}" diff --git a/maintenance-ci/conf/jenkins_job.ini.example b/maintenance-ci/conf/jenkins_job.ini.example new file mode 100644 index 0000000..9955a9c --- /dev/null +++ b/maintenance-ci/conf/jenkins_job.ini.example @@ -0,0 +1,14 @@ +[jenkins] +user=my_username +password=my_secret_password +url=https://my.jenkins.com/ +query_plugins_info=False + +[job_builder] +ignore_cache=True +keep_descriptions=False +recursive=True +include_path=.:scripts + +[__future__] +param_order_from_yaml=true diff --git a/maintenance-ci/conf/requirements-jobs.txt b/maintenance-ci/conf/requirements-jobs.txt new file mode 100644 index 0000000..10948fe --- /dev/null +++ b/maintenance-ci/conf/requirements-jobs.txt @@ -0,0 +1,2 @@ +jenkins-job-builder +cmd2 diff --git a/maintenance-ci/conf/requirements-views.txt b/maintenance-ci/conf/requirements-views.txt new file mode 100644 index 0000000..448245f --- /dev/null +++ b/maintenance-ci/conf/requirements-views.txt @@ -0,0 +1,3 @@ +jenkins-view-builder +cmd2 +cliff diff --git a/maintenance-ci/tox.ini b/maintenance-ci/tox.ini new file mode 100644 index 0000000..35dd3c4 --- /dev/null +++ b/maintenance-ci/tox.ini @@ -0,0 +1,18 @@ +[tox] +minversion = 1.6 +skipsdist = True +envlist = jobs,views + +[testenv] +basepython = python2 +usedevelop = False + +[testenv:jobs] +deps = -r{toxinidir}/conf/requirements-jobs.txt +commands = + jenkins-jobs --conf conf/jenkins_job.ini.example test -x common/data common:mos:mcp -o {toxinidir}/output/jobs --config-xml + +[testenv:views] +deps = -r{toxinidir}/conf/requirements-views.txt +commands = + /bin/bash -c 'jenkins-view-builder test views/* -o {toxinidir}/output/views/patching-ci' -- 2.45.2