From: Vladimir Khlyunev Date: Mon, 18 Feb 2019 07:19:02 +0000 (+0400) Subject: Initial MOS templates commit X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F36%2F40536%2F30;p=tools%2Fsustaining.git Initial MOS templates commit Soul of patching-ci now lives here... Change-Id: If45d64c506540a4f4edaee9c1c416b93061cc8ba --- diff --git a/maintenance-ci/common/data/mos-slave.yml b/maintenance-ci/common/data/mos-slave.yml new file mode 100644 index 0000000..5896e46 --- /dev/null +++ b/maintenance-ci/common/data/mos-slave.yml @@ -0,0 +1,90 @@ + +heat_template_version: 2017-09-01 + +parameters: + key_name: + type: string + description: Name of keypair to assign to servers + default: maintenance-ci-public-key + image: + type: string + description: Name of image to use for servers + default: xenial-server-cloudimg-amd64-qcow + flavor: + type: string + description: Flavor to use for servers + private_net: + type: string + description: > + ID or name of public network for which floating IP addresses will be allocated + default: maintenance-ci-network + private_subnet: + type: string + default: maintenance-ci-subnet + description: Id of the private sub network for the compute server + public_net: + type: string + default: public + description: Id of the public network for the compute server + jenkins_labels: + type: string + default: '' + description: List of labels for jenkins swarm agent separated by whitespace + +resources: + + wait_condition: + type: OS::Heat::WaitCondition + properties: + handle: {get_resource: wait_handle} + count: 1 + timeout: 300 + + wait_handle: + type: OS::Heat::WaitConditionHandle + + public_port: + type: OS::Neutron::Port + properties: + network_id: { get_param: private_net } + fixed_ips: + - subnet_id: { get_param: private_subnet } + security_groups: + - dc56d304-e75e-437a-8ef5-656f9bfcd2a0 + + floating_ip: + type: OS::Neutron::FloatingIP + properties: + floating_network_id: { get_param: public_net } + port_id: { get_resource: public_port } + + node: + type: OS::Nova::Server + properties: + image: { get_param: image } + availability_zone: nova + flavor: { get_param: flavor } + key_name: { get_param: key_name } + networks: + - port: { get_resource: public_port } + user_data: + str_replace: + template: | + #!/bin/bash + mkdir -p /etc/jenkins-agent/ + echo "FLAVOR LABELS" > /etc/jenkins-agent/labels + echo "127.0.0.1 $(hostname)" >> /etc/hosts + echo "FLOATING" > /etc/jenkins-agent/description + service jenkins-swarm-agent restart + wc_notify --data-binary '{"status": "SUCCESS"}' + params: + FLAVOR: {get_param: flavor} + LABELS: {get_param: jenkins_labels} + wc_notify: { get_attr: [wait_handle, curl_cli] } + FLOATING: { get_attr: [ floating_ip, floating_ip_address ] } + + +outputs: + floating_ip: + description: Floating IP + value: { get_attr: [ floating_ip, floating_ip_address ] } diff --git a/maintenance-ci/common/jobs/build-swarm-image.yaml b/maintenance-ci/common/jobs/build-swarm-image.yaml index 7334d85..5a0e8b3 100644 --- a/maintenance-ci/common/jobs/build-swarm-image.yaml +++ b/maintenance-ci/common/jobs/build-swarm-image.yaml @@ -1,7 +1,6 @@ - job: name: 'build-swarm-image' - description: | - Create given heat stack on internal cloud in maintenance-team tenant + description: "Create given heat stack on internal cloud in maintenance-team tenant" concurrent: false node: 'image-builder' @@ -27,19 +26,19 @@ description: 'Number of gerrit review' scm: - - git: - url: 'https://review.fuel-infra.org/tools/sustaining/' - branches: - - origin/master + - git: + url: 'https://review.fuel-infra.org/tools/sustaining/' + branches: + - origin/master wrappers: - timestamps - openstack-creds - credentials-binding: - username-password-separated: - credential-id: maintenance-ci-robot-password - username: JJB_USER - password: JJB_PASS + credential-id: maintenance-ci-robot-password + username: JJB_USER + password: JJB_PASS - timeout: fail: true diff --git a/maintenance-ci/common/jobs/delete_stack.yaml b/maintenance-ci/common/jobs/delete-stack.yaml similarity index 60% rename from maintenance-ci/common/jobs/delete_stack.yaml rename to maintenance-ci/common/jobs/delete-stack.yaml index 9b612d1..2a1e088 100644 --- a/maintenance-ci/common/jobs/delete_stack.yaml +++ b/maintenance-ci/common/jobs/delete-stack.yaml @@ -1,7 +1,7 @@ - job: name: 'delete-heat-stack' description: | - Delete given heat stack on internal cloud from maintenance-team tenant + Delete given heat stack on internal cloud from maintenance-team tenant concurrent: false node: 'jenkins-master' @@ -11,13 +11,13 @@ name: STACK_NAME default: "" description: | - Full name of pending for deletion stack. - If not given - STACK_PREFIX will be proceed + Full name of pending for deletion stack. + If not given - STACK_PREFIX will be proceed - string: name: STACK_PREFIX description: | - All stacks named with prefix on name's start will be deleted. - Used only if STACK_NAME is not defined + All stacks named with prefix on name's start will be deleted. + Used only if STACK_NAME is not defined default: "" diff --git a/maintenance-ci/common/jobs/deploy-jenkins-swarm.yaml b/maintenance-ci/common/jobs/deploy-jenkins-swarm.yaml index f3fcb64..41d66f3 100644 --- a/maintenance-ci/common/jobs/deploy-jenkins-swarm.yaml +++ b/maintenance-ci/common/jobs/deploy-jenkins-swarm.yaml @@ -1,60 +1,106 @@ - job: name: 'deploy-jenkins-swarm' description: | - Create given jenkins slaves on internal cloud in maintenance-team tenant + Create given jenkins slaves on internal cloud in maintenance-team tenant concurrent: false node: 'jenkins-master' parameters: - - string: - name: HEAT_STACK_YAML - description: "Stack yaml file" - default: "maintenance-ci/common/data/jenkins-slave.yml" - - string: - name: STACK_PREFIX - description: "Stack prefix" - default: "swarm-slave" - - string: - name: MOS_BASE_NODE_COUNT - default: "0" - - string: - name: MOS_HUGE_NODE_COUNT - default: "0" - - string: - name: MOS_TEMPEST_NODE_COUNT - default: "0" - - - string: - name: IMAGE_NAME - default: xenial-server-cloudimg-amd64-qcow - - string: - name: JENKINS_LABELS - default: '' - - - string: - name: CUSTOM_FLAVOR_NAME - default: "0" - - string: - name: CUSTOM_FLAVOR_JENKINS_LABELS - default: "" - - string: - name: CUSTOM_FLAVOR_NODE_COUNT - default: "0" + - mos-jenkins-swarm-params scm: - - git: - url: 'https://review.fuel-infra.org/tools/sustaining/' - branches: - - origin/master + - git: + url: 'https://review.fuel-infra.org/tools/sustaining/' + branches: + - origin/master wrappers: - - timestamps - - openstack-creds - - timeout: - fail: true - timeout: 30 + - timestamps + - openstack-creds + - timeout: + fail: true + timeout: 30 builders: - - shell: - !include-raw: common/scripts/deploy_jenkins_swarm.sh + - conditional-step: + condition-kind: num-comp + lhs: $MOS_BASE_NODE_COUNT + rhs: 0 + comparator: greater-than + steps: + - trigger-builds: + - project: "deploy-heat-stack" + block: true + parameter-factories: + - factory: counterbuild + from: 1 + to: $MOS_BASE_NODE_COUNT + step: 1 + predefined-parameters: | + HEAT_STACK_YAML=${HEAT_STACK_YAML} + STACK_PREFIX=mos-slave + FLAVOR_NAME=mos.fuel.base + IMAGE_NAME=${IMAGE_NAME} + GERRIT_REVIEW=${GERRIT_REVIEW} + - conditional-step: + condition-kind: num-comp + lhs: $MOS_HUGE_NODE_COUNT + rhs: 0 + comparator: greater-than + steps: + - trigger-builds: + - project: "deploy-heat-stack" + block: true + parameter-factories: + - factory: counterbuild + from: 1 + to: $MOS_HUGE_NODE_COUNT + step: 1 + predefined-parameters: | + HEAT_STACK_YAML=${HEAT_STACK_YAML} + STACK_PREFIX=mos-slave + FLAVOR_NAME=mos.fuel.huge + IMAGE_NAME=${IMAGE_NAME} + GERRIT_REVIEW=${GERRIT_REVIEW} + - conditional-step: + condition-kind: num-comp + lhs: $MOS_TEMPEST_NODE_COUNT + rhs: 0 + comparator: greater-than + steps: + - trigger-builds: + - project: "deploy-heat-stack" + block: true + parameter-factories: + - factory: counterbuild + from: 1 + to: $MOS_TEMPEST_NODE_COUNT + step: 1 + predefined-parameters: | + HEAT_STACK_YAML=${HEAT_STACK_YAML} + STACK_PREFIX=mos-slave + FLAVOR_NAME=mos.tempest + IMAGE_NAME=${IMAGE_NAME} + GERRIT_REVIEW=${GERRIT_REVIEW} + - conditional-step: + condition-kind: num-comp + lhs: $CUSTOM_FLAVOR_NODE_COUNT + rhs: 0 + comparator: greater-than + steps: + - trigger-builds: + - project: "deploy-heat-stack" + block: true + parameter-factories: + - factory: counterbuild + from: 1 + to: $CUSTOM_FLAVOR_NODE_COUNT + step: 1 + predefined-parameters: | + HEAT_STACK_YAML=${CUSTOM_HEAT_STACK_YAML} + STACK_PREFIX=mos-slave + FLAVOR_NAME=${CUSTOM_FLAVOR_NAME} + JENKINS_LABELS=${CUSTOM_FLAVOR_JENKINS_LABELS} + IMAGE_NAME=${IMAGE_NAME} + GERRIT_REVIEW=${GERRIT_REVIEW} diff --git a/maintenance-ci/common/jobs/deploy-stack.yaml b/maintenance-ci/common/jobs/deploy-stack.yaml index c09a58b..882269c 100644 --- a/maintenance-ci/common/jobs/deploy-stack.yaml +++ b/maintenance-ci/common/jobs/deploy-stack.yaml @@ -20,16 +20,19 @@ default: "dev.share" - string: name: IMAGE_NAME - default: xenial-server-cloudimg-amd64-qcow + default: maintenance-ci-base-slave - string: name: JENKINS_LABELS default: '' + - string: + name: GERRIT_REVIEW + description: 'Number of gerrit review' scm: - git: url: 'https://review.fuel-infra.org/tools/sustaining/' branches: - - origin/master + - origin/master wrappers: - timestamps diff --git a/maintenance-ci/common/jobs/multijob-build-image.yaml b/maintenance-ci/common/jobs/multijob-build-image.yaml index 2e54ccc..05afaa6 100644 --- a/maintenance-ci/common/jobs/multijob-build-image.yaml +++ b/maintenance-ci/common/jobs/multijob-build-image.yaml @@ -1,7 +1,7 @@ - job: name: 'multijob-build-image' description: | - Build image for future use + Build image for future use concurrent: false node: 'jenkins-master' @@ -20,8 +20,8 @@ - choice: name: UPDATE_SCRIPT choices: - - "maintenance-ci/common/scripts/build_base_swarm_slave_image.sh" - - "maintenance-ci/common/scripts/build_mos_swarm_slave_image.sh" + - "maintenance-ci/common/scripts/build_base_swarm_slave_image.sh" + - "maintenance-ci/common/scripts/build_mos_swarm_slave_image.sh" - string: name: CUSTOM_UPDATE_SCRIPT description: "Overwrite UPDATE_SCRIPT if needed" @@ -32,28 +32,28 @@ builders: - trigger-builds: - - project: "deploy-heat-stack" - predefined-parameters: | - HEAT_STACK_YAML=maintenance-ci/common/data/jenkins-slave.yml - STACK_PREFIX=image-builder - FLAVOR_NAME=dev.cmn - JENKINS_LABELS=image-builder - IMAGE_NAME=${DEPLOY_IMAGE_NAME} - GERRIT_REVIEW=${GERRIT_REVIEW} - block: true + - project: "deploy-heat-stack" + predefined-parameters: | + HEAT_STACK_YAML=maintenance-ci/common/data/jenkins-slave.yml + STACK_PREFIX=image-builder + FLAVOR_NAME=dev.cmn + JENKINS_LABELS=image-builder + IMAGE_NAME=${DEPLOY_IMAGE_NAME} + GERRIT_REVIEW=${GERRIT_REVIEW} + block: true - trigger-builds: - - project: "build-swarm-image" - block: true - predefined-parameters: | - IMAGE_NAME=${BUILD_IMAGE_NAME} - GERRIT_REVIEW=${GERRIT_REVIEW} - TEST_AFTER_BUILD=${TEST_AFTER_BUILD} - UPDATE_SCRIPT=${UPDATE_SCRIPT} - CUSTOM_UPDATE_SCRIPT=${CUSTOM_UPDATE_SCRIPT} + - project: "build-swarm-image" + block: true + predefined-parameters: | + IMAGE_NAME=${BUILD_IMAGE_NAME} + GERRIT_REVIEW=${GERRIT_REVIEW} + TEST_AFTER_BUILD=${TEST_AFTER_BUILD} + UPDATE_SCRIPT=${UPDATE_SCRIPT} + CUSTOM_UPDATE_SCRIPT=${CUSTOM_UPDATE_SCRIPT} - trigger-builds: - - project: "delete-heat-stack" - predefined-parameters: | - STACK_PREFIX=image-builder - block: true + - project: "delete-heat-stack" + predefined-parameters: | + STACK_PREFIX=image-builder + block: true diff --git a/maintenance-ci/common/jobs/update-jenkins-jobs.yaml b/maintenance-ci/common/jobs/update-jenkins-jobs.yaml new file mode 100644 index 0000000..b3589df --- /dev/null +++ b/maintenance-ci/common/jobs/update-jenkins-jobs.yaml @@ -0,0 +1,62 @@ +- job: + name: 'update-jenkins-jobs' + description: | +

Update jenkins jobs configuration

+

Requires python-tox package and user credentials stored as JJB_USER and JJB_PASS using Jenkins Global Passwords feature

+ + + concurrent: false + node: 'jenkins-master' + + build-discarder: + numToKeep: '5' + + parameters: + - string: + name: GERRIT_REVIEW + description: 'Number of gerrit review' + + scm: + - git: + url: 'https://review.fuel-infra.org/tools/sustaining/' + branches: + - origin/master + + # triggers: + # - timed: 'H * * * *' + # - gerrit: + # trigger-on: + # - change-merged-event + # projects: + # - project-compare-type: PLAIN + # project-pattern: 'fuel-infra/jenkins-jobs' + # branches: + # - branch-compare-type: PLAIN + # branch-pattern: 'master' + # file-paths: + # - compare-type: ANT + # pattern: 'common/**' + # - compare-type: ANT + # pattern: 'servers/{ci-name}/**' + + wrappers: + - timestamps + - credentials-binding: + - username-password-separated: + credential-id: maintenance-ci-robot-password + username: JJB_USER + password: JJB_PASS + - timeout: + fail: true + timeout: 30 + + builders: + - shell: + !include-raw: common/scripts/update_jenkins_jobs.sh + + publishers: + - post-tasks: + - matches: + - log-text: '.' + escalate-status: true + script: !include-raw: common/scripts/update_jenkins_cleanup.sh \ No newline at end of file diff --git a/maintenance-ci/common/jobs/update_jenkins_jobs.yaml b/maintenance-ci/common/jobs/update_jenkins_jobs.yaml deleted file mode 100644 index 7bdd713..0000000 --- a/maintenance-ci/common/jobs/update_jenkins_jobs.yaml +++ /dev/null @@ -1,62 +0,0 @@ -- job: - name: 'update-jenkins-jobs' - description: | -

Update jenkins jobs configuration

-

Requires python-tox package and user credentials stored as JJB_USER and JJB_PASS using Jenkins Global Passwords feature

- - - concurrent: false - node: 'jenkins-master' - - parameters: - - string: - name: GERRIT_REVIEW - description: 'Number of gerrit review' - - scm: - - git: - remotes: - - gerrit: - url: 'https://review.fuel-infra.org/tools/sustaining/' - branches: - - master - choosing-strategy: default - -# triggers: -# - timed: 'H * * * *' -# - gerrit: -# trigger-on: -# - change-merged-event -# projects: -# - project-compare-type: PLAIN -# project-pattern: 'fuel-infra/jenkins-jobs' -# branches: -# - branch-compare-type: PLAIN -# branch-pattern: 'master' -# file-paths: -# - compare-type: ANT -# pattern: 'common/**' -# - compare-type: ANT -# pattern: 'servers/{ci-name}/**' - - wrappers: - - timestamps - - credentials-binding: - - username-password-separated: - credential-id: maintenance-ci-robot-password - username: JJB_USER - password: JJB_PASS - - timeout: - fail: true - timeout: 30 - - builders: - - shell: - !include-raw: common/scripts/update_jenkins_jobs.sh - - publishers: - - post-tasks: - - matches: - - log-text: '.' - escalate-status: true - script: !include-raw: common/scripts/update_jenkins_cleanup.sh \ No newline at end of file diff --git a/maintenance-ci/common/macroses/inject-text-parameter.yaml b/maintenance-ci/common/macroses/inject-text-parameter.yaml new file mode 100644 index 0000000..df0eec7 --- /dev/null +++ b/maintenance-ci/common/macroses/inject-text-parameter.yaml @@ -0,0 +1,10 @@ +- builder: + name: inject-text-parameter + builders: + - shell: | + #!/bin/bash + set -e + echo {properties-content} | tr " " "\n" > "inject.text.sh" + + - inject: + properties-file: 'inject.text.sh' \ No newline at end of file diff --git a/maintenance-ci/common/macroses/mos-jenkins-swarm-params.yaml b/maintenance-ci/common/macroses/mos-jenkins-swarm-params.yaml new file mode 100644 index 0000000..16eed54 --- /dev/null +++ b/maintenance-ci/common/macroses/mos-jenkins-swarm-params.yaml @@ -0,0 +1,41 @@ +- parameter: + name: mos-jenkins-swarm-params + + parameters: + - string: + name: HEAT_STACK_YAML + description: "Stack yaml file" + default: "maintenance-ci/common/data/jenkins-slave.yml" + - string: + name: STACK_PREFIX + description: "Stack prefix" + default: "swarm-slave" + - string: + name: MOS_BASE_NODE_COUNT + default: "0" + - string: + name: MOS_HUGE_NODE_COUNT + default: "0" + - string: + name: MOS_TEMPEST_NODE_COUNT + default: "0" + + - string: + name: IMAGE_NAME + default: maintenance-ci-mos-image + - string: + name: JENKINS_LABELS + default: '' + + - string: + name: CUSTOM_FLAVOR_NAME + default: "" + - string: + name: CUSTOM_HEAT_STACK_YAML + default: "" + - string: + name: CUSTOM_FLAVOR_JENKINS_LABELS + default: "" + - string: + name: CUSTOM_FLAVOR_NODE_COUNT + default: "0" \ No newline at end of file diff --git a/maintenance-ci/common/macroses/openstack_creds.yaml b/maintenance-ci/common/macroses/openstack_creds.yaml index e4c69df..2991fa6 100644 --- a/maintenance-ci/common/macroses/openstack_creds.yaml +++ b/maintenance-ci/common/macroses/openstack_creds.yaml @@ -1,11 +1,11 @@ - wrapper: name: openstack-creds wrappers: - - credentials-binding: - - file: - credential-id: maintenance-ci-openrc-password-free - variable: OPENRC_FILE - - username-password-separated: - credential-id: maintenance-ci-robot-password - username: OS_USERNAME - password: OS_PASSWORD \ No newline at end of file + - credentials-binding: + - file: + credential-id: maintenance-ci-openrc-password-free + variable: OPENRC_FILE + - username-password-separated: + credential-id: maintenance-ci-robot-password + username: OS_USERNAME + password: OS_PASSWORD \ No newline at end of file diff --git a/maintenance-ci/common/scripts/deploy_heat_stack.sh b/maintenance-ci/common/scripts/deploy_heat_stack.sh index 0f48202..bfec3ac 100755 --- a/maintenance-ci/common/scripts/deploy_heat_stack.sh +++ b/maintenance-ci/common/scripts/deploy_heat_stack.sh @@ -2,6 +2,11 @@ set -ex +if [ ! -z ${GERRIT_REVIEW} ] ; then +refspec=$(curl "https://review.fuel-infra.org/changes/?q=${GERRIT_REVIEW}&o=CURRENT_REVISION" | grep '"ref":' | awk -F ": " '{print $2}' | tr -d '"') +git fetch https://review.fuel-infra.org/tools/sustaining "${refspec}" && git cherry-pick -n FETCH_HEAD +fi + OPENSTACK_CLIENTS_VENV="${OPENSTACK_CLIENTS_VENV:-/home/jenkins/venv-openstack-clients}" if [[ ! -d "${OPENSTACK_CLIENTS_VENV}" ]] ; then @@ -13,14 +18,15 @@ fi parameter_string="" if [[ ! -z "${FLAVOR_NAME}" ]] ; then - parameter_string="$parameter_string --parameter \"flavor=${FLAVOR_NAME}\"" + parameter_string="flavor=${FLAVOR_NAME};$parameter_string" fi if [[ ! -z "${IMAGE_NAME}" ]] ; then - parameter_string="$parameter_string --parameter \"image=${IMAGE_NAME}\"" + parameter_string="image=${IMAGE_NAME};$parameter_string" fi if [[ ! -z "${JENKINS_LABELS}" ]] ; then - parameter_string="$parameter_string --parameter \"jenkins_labels=${JENKINS_LABELS}\"" + parameter_string="jenkins_labels=${JENKINS_LABEL};$parameter_string" fi +parameter_string=$(sed 's/;$//' <<< $parameter_string) # MAC OS compatibility, for manual usage if command -v md5sum ; then @@ -35,7 +41,7 @@ set +x source "${OPENSTACK_CLIENTS_VENV}"/bin/activate set -x source "${OPENRC_FILE?}" - echo "openstack stack create --wait -t ${HEAT_STACK_YAML} ${parameter_string} ${STACK_NAME}" | bash - + openstack stack create --wait -t ${HEAT_STACK_YAML} --parameter "${parameter_string}" ${STACK_NAME} openstack stack output show "${STACK_NAME}" --all cat >stack_data.sh <> snapshots.params + echo "$1=\"$2\"" >> snapshots.sh +} + + + +# Create and store ID of this snapshot file, which will be used as PK for snapshot set + +# id for testrail to report test results +__snapshot_id_default="snapshot #${BUILD_NUMBER?}" +__snapshot_id="${CUSTOM_VERSION:-$__snapshot_id_default}" +store "CUSTOM_VERSION" "${__snapshot_id}" +# unix timestamp of snapshot +__snapshot_timestamp="$(date +%s)" +store "SNAPSHOT_TIMESTAMP" "${__snapshot_timestamp}" + + +# Create and store MAGNET_LINK + +store "MAGNET_LINK" "${MAGNET_LINK?}" + + + +# Store FUEL_QA_COMMIT + +store "FUEL_QA_COMMIT" "$(git -C . rev-parse HEAD)" + + +# Store snapshot for copy of Ubuntu deb repo + +# http://mirror.seed-cz1.fuel-infra.org/pkgs/snapshots/ubuntu-2016-07-13-172538 +# ^^^^^^^^^^^^^^^^^^^^^^^^ +__ubuntu_latest_repo_snaphot_id=$(curl -sSf "${MIRROR_HOST}/pkgs/snapshots/ubuntu-latest.target.txt" | sed '1p;d') +store "UBUNTU_MIRROR_ID" "${__ubuntu_latest_repo_snaphot_id}" + + + +# Store snapshot for copy of Centos rpm repo + +# http://mirror.fuel-infra.org/pkgs/snapshots/centos-7.2.1511-2016-05-31-083834/ +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +__centos_latest_repo_snaphot_id=$(curl -sSf "${MIRROR_HOST}/pkgs/snapshots/centos-7.3.1611-latest.target.txt" | sed '1p;d') +store "CENTOS_MIRROR_ID" "${__centos_latest_repo_snaphot_id}" + + + +# Store snapshot for MOS deb repo + +# 9.0-2016-06-23-164100 +# ^^^^^^^^^^^^^^^^^^^^^ +__mos_latest_deb_mirror_id="$(\ + curl "http://${MIRROR_HOST}/mos-repos/ubuntu/snapshots/9.0-latest.target.txt" \ + | head -1)" +store "MOS_UBUNTU_MIRROR_ID" "${__mos_latest_deb_mirror_id}" + + + +# Store snapshots for full set of MOS rpm repos + +# -2016-07-14-082020 +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +for _dn in "os" \ + "proposed" \ + "updates" \ + "holdback" \ + "hotfix" \ + "security" ; do + __dt_snapshot="$(\ + curl "http://${MIRROR_HOST}/mos-repos/centos/mos9.0-centos7/snapshots/${_dn}-latest.target.txt" \ + | head -1)" + store "MOS_CENTOS_$(to_uppercase "${_dn}")_MIRROR_ID" "${__dt_snapshot}" +done + diff --git a/maintenance-ci/mos/scripts/freeze_deps_for_systest_plugin.sh b/maintenance-ci/mos/scripts/freeze_deps_for_systest_plugin.sh new file mode 100644 index 0000000..5217807 --- /dev/null +++ b/maintenance-ci/mos/scripts/freeze_deps_for_systest_plugin.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +set -ex +# TBD: This script will be extended with plugin-related data + +# Variables: +# whole os plugin url example: +# PLUGIN_REPO_SUB_URL="extras/murano-plugin-repos/" +# PLUGIN_VERSION="release/1.0.0" or "" +# PLUGIN_MOS_VERSION="mos9.0" +# PLUGIN_PKG_DIST="centos7" +# http://mirror.fuel-infra.org/extras/murano-plugin-repos/centos/mos9.0-centos7/os +# +# For guess rpm filename +# PLUGIN_RPM_MASK="detach-murano" + +# fixme: mirror.fuel-infra.org could point to brokem mirror +MIRROR_HOST="${MIRROR_HOST:-mirror.seed-cz1.fuel-infra.org}" + +function to_uppercase() { + echo "$1" | awk '{print toupper($0)}' +} + +function store() { + echo "$1=$2" >> snapshots.params + echo "$1=\"$2\"" >> snapshots.sh +} + + +function guess_rpm_filename(){ + + # Dirty hack, for determine rpm filename from repo, by mask + # Should be rewrited to yumdownloared based soluthion + local t_file;t_file=$(mktemp) + local file_mask;file_mask=$1 + local snapshot_id;snapshot_id=$2 + + wget "${plugin_repo_long_url}/snapshots/${snapshot_id}/x86_64/Packages/" -O "${t_file}" + store "PLUGIN_RPM_FILENAME_FROM_REPO" "$(grep "${file_mask}" "${t_file}" | sed -n 's/.*href="\([^"]*\).*/\1/p' | sort -ru | head -n 1)" + rm -vf "${t_file}" +} + +store PLUGIN_REPO_SUB_URL "${PLUGIN_REPO_SUB_URL?}" +store PLUGIN_VERSION "${PLUGIN_VERSION?}" +store PLUGIN_MOS_VERSION "${PLUGIN_MOS_VERSION?}" +store PLUGIN_PKG_DIST "${PLUGIN_PKG_DIST?}" +store PLUGIN_RPM_MASK "${PLUGIN_RPM_MASK?}" + +# todo: cd into project and get sha of commit +store PLUGIN_TEST_REPO "${PLUGIN_TEST_REPO?}" +store PLUGIN_TEST_COMMIT "${PLUGIN_TEST_COMMIT?}" # not frozen. + +# Store snapshot for copy of rpm repo +# http://mirror.fuel-infra.org/extras/murano-plugin-repos/centos/mos9.0-centos7/os.target.txt >> +# http://mirror.fuel-infra.org/extras/murano-plugin-repos/centos/mos9.0-centos7/snapshots/os-2016-09-21-210927/ +# ^^^^^^^^^^^^^^^^^^^^ +plugin_repo_long_url="http://${MIRROR_HOST}/${PLUGIN_REPO_SUB_URL}/${PLUGIN_VERSION}/centos/${PLUGIN_MOS_VERSION}-${PLUGIN_PKG_DIST}/" + +__plugin_latest_repo_snaphot_url="$(\ + curl "${plugin_repo_long_url}/os.target.txt" \ + | head -1)" +plugin_latest_repo_snaphot_id="${__plugin_latest_repo_snaphot_url##*/}" +store "PLUGIN_OS_REPO_ID" "${plugin_latest_repo_snaphot_id}" + +guess_rpm_filename "${PLUGIN_RPM_MASK}" "${plugin_latest_repo_snaphot_id}" diff --git a/maintenance-ci/mos/scripts/fuel_qa_launch_test.sh b/maintenance-ci/mos/scripts/fuel_qa_launch_test.sh new file mode 100644 index 0000000..9935926 --- /dev/null +++ b/maintenance-ci/mos/scripts/fuel_qa_launch_test.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -ex + +mkdir -p logs +rm -rf logs/* + +# for cloud-based testing we will use "one OS instance - one devops lab" +# in this case we want to use the same ENV_NAME without additions like BUILD_ID +export ENV_NAME="9.x_virtual_lab" +echo "ENV_NAME=${ENV_NAME}" > ${WORKSPACE}/${DOS_ENV_NAME_PROPS_FILE:=.dos_environment_name} +export ISO_PATH=$(seedclient-wrapper -d -m "${MAGNET_LINK?}" -v --force-set-symlink -o "${WORKSPACE?}") +export MAKE_SNAPSHOT=${MAKE_SNAPSHOT:-false} + +echo "Description string: ${TEST_GROUP?} on ${CUSTOM_VERSION?}" + +set +x +echo "======================================================" +echo "=====================ENV VARS=========================" +env +echo "======================================================" +echo "======================================================" +set -x +set +x +echo "======================================================" +echo "=====================MOS VARS=========================" +echo "UPDATE_FUEL_MIRROR: ${UPDATE_FUEL_MIRROR}" +echo "EXTRA_DEB_REPOS: ${EXTRA_DEB_REPOS}" +echo "======================================================" +echo "======================================================" +set -x + +source "${VENV_PATH}/bin/activate" + pip install -U -r fuelweb_test/requirements.txt > logs/pip_install_output.log + export PYTHONPATH="${WORKSPACE}" + if [[ $DELETE_ENV_BEFORE_RUN = true ]] ; then dos.py erase $ENV_NAME || true; fi + python run_system_test.py run -q --nologcapture --with-xunit "${TEST_GROUP}" +deactivate + +# VMs will be shut down by post-build step diff --git a/maintenance-ci/mos/scripts/guess-mirror.sh b/maintenance-ci/mos/scripts/guess-mirror.sh new file mode 100644 index 0000000..00b31f1 --- /dev/null +++ b/maintenance-ci/mos/scripts/guess-mirror.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +set -o xtrace + +# fixme: this section is for disabling our mirror snapshots for xenial builds, need rework +if [ "${GUESS_MIRROR:-true}" == false ] ; then + + echo > mirror.setenvfile + echo > mirror.jenkins-injectfile + exit 0 +fi + +###################### Guess mirror host ############### + +# Top guess ever +MIRROR_HOST="mirror.seed-us1.fuel-infra.org" + +###################### Get MIRROR_UBUNTU ############### + +# If UBUNTU_MIRROR_ARTIFACT is set get UBUNTU_MIRROR_ID from artifact +if [[ -n "${UBUNTU_MIRROR_ARTIFACT}" ]]; then + export $(curl -sSf "${UBUNTU_MIRROR_ARTIFACT}") +fi + +UBUNTU_MIRROR_ID=${UBUNTU_MIRROR_ID:-latest} +UBUNTU_DIST=${UBUNTU_DIST:-trusty} + +# By default disable Ubuntu proposed repository +ENABLE_UBUNTU_PROPOSED=${ENABLE_UBUNTU_PROPOSED:-false} + +if [ -z "${MIRROR_UBUNTU}" ]; then + if [ "${UBUNTU_MIRROR_ID}" = "latest" ]; then + UBUNTU_MIRROR_ID=$(curl -sSf "http://${MIRROR_HOST}/pkgs/snapshots/ubuntu-${UBUNTU_MIRROR_ID}.target.txt" | sed '1p;d') + fi + UBUNTU_MIRROR_URL="http://${MIRROR_HOST}/pkgs/snapshots/${UBUNTU_MIRROR_ID}/" + + MIRROR_UBUNTU="deb ${UBUNTU_MIRROR_URL} ${UBUNTU_DIST} main universe multiverse|deb ${UBUNTU_MIRROR_URL} ${UBUNTU_DIST}-updates main universe multiverse|deb ${UBUNTU_MIRROR_URL} ${UBUNTU_DIST}-security main universe multiverse" + + # Add proposed repository only when required + if [[ "${ENABLE_UBUNTU_PROPOSED}" = true ]]; then + MIRROR_UBUNTU="${MIRROR_UBUNTU}|deb ${UBUNTU_MIRROR_URL} ${UBUNTU_DIST}-proposed main universe multiverse" + fi +fi + +# Save parameters to file in format required by source in bash +cat > mirror.setenvfile < mirror.jenkins-injectfile <> snapshots.params + echo "$1=\"$2\"" >> snapshots.sh + echo "export $1=\"$2\"" >> snapshots.export.sh +} + + +store "CUSTOM_VERSION" "snapshot #${BUILD_NUMBER?}" +store "MIRROR_HOST" "${MIRROR_HOST?}" +store "SNAPSHOT_TIMESTAMP" "$(date +%s)" +store "MAGNET_LINK" "${MAGNET_LINK?}" + +# snapshot for copy of ubuntu deb repo +if [[ -n "${SRC_UBUNTU_MIRROR}" ]]; then + __ubuntu_latest_repo_snaphot_id=$(curl -sSf "${MIRROR_HOST}/${SRC_UBUNTU_MIRROR}" | sed '1p;d') + store "UBUNTU_MIRROR_ID" "${__ubuntu_latest_repo_snaphot_id}" +fi + +# snapshot for copy of centos rpm repo +if [[ -n "${SRC_CENTOS_MIRROR}" ]]; then + __centos_latest_repo_snaphot_id=$(curl -sSf "${MIRROR_HOST}/${SRC_CENTOS_MIRROR}" | sed '1p;d') + store "CENTOS_MIRROR_ID" "${__centos_latest_repo_snaphot_id}" +fi + +# snapshot for mos ubuntu deb repo +__mos_latest_deb_mirror_id="$(curl "${MIRROR_HOST}/${SRC_MOS_UBUNTU_MIRROR}" | head -1)" +store "MOS_UBUNTU_MIRROR_ID" "${__mos_latest_deb_mirror_id}" + +# snapshots for all mos centos rpm repos +for _dn in "os" "proposed" "updates" "holdback" "hotfix" "security"; do + echo "Using auto-generated ${_dn} repo" + __dt_snapshot="$(curl "${MIRROR_HOST}/${SRC_MOS_CENTOS_REPOS_PREFIX}/${_dn}-latest.target.txt" | head -1)" + store "MOS_CENTOS_$(to_uppercase "${_dn}")_MIRROR_ID" "${__dt_snapshot}" +done + +if [[ -n "${SRC_CENTOS_SECURITY_PROPOSED}" ]]; then + store "CENTOS_SECURITY_PROPOSED" "${SRC_CENTOS_SECURITY_PROPOSED}" +fi + diff --git a/maintenance-ci/mos/scripts/systest_repos_parsing.sh b/maintenance-ci/mos/scripts/systest_repos_parsing.sh new file mode 100644 index 0000000..e953299 --- /dev/null +++ b/maintenance-ci/mos/scripts/systest_repos_parsing.sh @@ -0,0 +1,138 @@ +#!/bin/bash + +set -ex + +function join() { + local __sep="${1}" + local __head="${2}" + local __tail="${3}" + + if [[ -n "${__head}" ]]; then + echo "${__head}${__sep}${__tail}" + else + echo "${__tail}" + fi +} + +function to_uppercase() { + echo "$1" | awk '{print toupper($0)}' +} + +__space=' ' +__pipe='|' + +# Fetch params from snapshot job +if [[ ! "${SNAPSHOT_PARAMS_ID}" ]]; then + SNAPSHOT_PARAMS_ID="lastSuccessfulBuild" +fi + +curl -s "http://maintenance-ci.scc.mirantis.net/job/9.x.snapshot.params/${SNAPSHOT_PARAMS_ID}/artifact/snapshots.sh" > snapshots.sh + +while read line ; do + var_name=$(echo "${line}" | awk -F '=' '{print $1}') + var_overwrite="$(join "_" "${var_name}" "$(to_uppercase "overwrite")")" + if [[ ! -z ${!var_overwrite} ]] + then + declare ${var_name}="${!var_overwrite}" + else + eval "${line}" + fi +done REPOS PARSING RESULTS <============>" +echo "MAGNET_LINK: ${MAGNET_LINK}" +echo "UPDATE_MASTER: ${UPDATE_MASTER}" +echo "UPDATE_FUEL_MIRROR: ${UPDATE_FUEL_MIRROR}" +echo "MIRROR_UBUNTU: ${MIRROR_UBUNTU}" +echo "EXTRA_RPM_REPOS: ${EXTRA_RPM_REPOS}" +echo "EXTRA_DEB_REPOS: ${EXTRA_DEB_REPOS}" +echo "MIRROR_HOST: ${MIRROR_HOST}" +echo "<=================================================>" + +cat > systest_repos.jenkins-injectfile < logs/pip_install_output.log + export PYTHONPATH="$(pwd):$PYTHONPATH" + +# Report tests results from swarm (Ubuntu) + export USE_UBUNTU=true + export USE_CENTOS=false +# shellcheck disable=SC2086 + python ${TESTRAIL_REPORTER_PATH} -v -s +deactivate \ No newline at end of file diff --git a/maintenance-ci/mos/scripts/testrail-upload-cases-9.x.sh b/maintenance-ci/mos/scripts/testrail-upload-cases-9.x.sh new file mode 100644 index 0000000..d5a2577 --- /dev/null +++ b/maintenance-ci/mos/scripts/testrail-upload-cases-9.x.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# +# TestRail + +set -ex +mkdir -p logs + +export TESTRAIL_PROJECT="${TESTRAIL_PROJECT:-"Mirantis OpenStack"}" +export TESTRAIL_URL="https://mirantis.testrail.com" + +# Prepare venv +source "${VENV_PATH}/bin/activate" + pip install -U -r fuelweb_test/requirements.txt > logs/pip_install_output.log + export PYTHONPATH="$(pwd):$PYTHONPATH" + python fuelweb_test/testrail/upload_cases_description.py -v -j "${TESTS_RUNNER}" + # stub for debugging part of migration + python fuelweb_test/testrail/upload_cases_description.py -v -j "9.x.swarm.micro.runner" +deactivate \ No newline at end of file diff --git a/maintenance-ci/mos/templates/snapshot.yaml b/maintenance-ci/mos/templates/snapshot.yaml new file mode 100644 index 0000000..be90593 --- /dev/null +++ b/maintenance-ci/mos/templates/snapshot.yaml @@ -0,0 +1,47 @@ +- job-template: + name: '{version}.snapshot.params' + + timer: '' + magnet-link: '' + mirror-host: 'mirror.seed-us1.fuel-infra.org' + src-ubuntu-mirror: 'pkgs/snapshots/ubuntu-latest.target.txt' + src-centos-mirror: '' + src-mos-ubuntu-mirror: '' + src-mos-centos-repos-prefix: '' + src-centos-security-proposed: '' + + node: 'jenkins-master' + + triggers: + - timed: '{timer}' + + builders: + - inject: + properties-content: | + MAGNET_LINK={magnet-link} + MIRROR_HOST={mirror-host} + SRC_UBUNTU_MIRROR={src-ubuntu-mirror} + SRC_CENTOS_MIRROR={src-centos-mirror} + SRC_MOS_UBUNTU_MIRROR={src-mos-ubuntu-mirror} + SRC_MOS_CENTOS_REPOS_PREFIX={src-mos-centos-repos-prefix} + SRC_CENTOS_SECURITY_PROPOSED={src-centos-security-proposed} + - shell: + !include-raw-escape: + '../scripts/snapshot.params.sh' + - inject-properties-file: + properties-file: 'snapshots.params' + - write-description: + description-string: $CUSTOM_VERSION + + publishers: + - description-setter: + regexp: "'Description string: (.*)'" + - archive: + artifacts: 'snapshots.params' + allow-empty: false + - archive: + artifacts: 'snapshots.sh' + allow-empty: false + - archive: + artifacts: 'snapshots.export.sh' + allow-empty: false diff --git a/maintenance-ci/mos/templates/swarm-test.yaml b/maintenance-ci/mos/templates/swarm-test.yaml new file mode 100644 index 0000000..e277971 --- /dev/null +++ b/maintenance-ci/mos/templates/swarm-test.yaml @@ -0,0 +1,92 @@ +- job-template: + name: '9.x.{testgroup}' + id: mos/templates/swarm-test + + swarm-test/node: 'mos.fuel.base' + swarm-test/weight: '7' + + swarm-test/venvpath: '/home/jenkins/qa-venv-9.x' + + swarm-test/custom-overrides-plain-properties: '' + swarm-test/custom-overrides-properties-file: '' + swarm-test/timeout: '180' + swarm-test/make-snapshot: 'false' + + swarm-test/build-discarder/artifactDaysToKeep: -1 + swarm-test/build-discarder/artifactNumToKeep: -1 + swarm-test/build-discarder/daysToKeep: 60 + swarm-test/build-discarder/numToKeep: 20 + + description: 'Run Swarm Test {testgroup}' + node: '{swarm-test/node}' + concurrent: true + + build-discarder: + artifactDaysToKeep: '{swarm-test/build-discarder/artifactDaysToKeep}' + artifactNumToKeep: '{swarm-test/build-discarder/artifactNumToKeep}' + daysToKeep: '{swarm-test/build-discarder/daysToKeep}' + numToKeep: '{swarm-test/build-discarder/numToKeep}' + + parameters: + - 9x-systest-params + - text: + name: EXTRA_OPTS + default: "" + + properties: + - heavy-job: + weight: '{swarm-test/job-weight}' + + scm: + - git: + url: 'https://review.fuel-infra.org/openstack/{swarm-test/reponame}.git' + branches: + - '{swarm-test/repobranch}' + basedir: '' + + wrappers: + - timeout: + fail: false + timeout: '{swarm-test/timeout}' + write-description: false + - ansicolor: + colormap: xterm + + builders: + - inject: # Defaults + properties-content: | + OPENSTACK_RELEASE=ubuntu + TEST_GROUP={testgroup} + VENV_PATH={swarm-test/venvpath} + MAKE_SNAPSHOT={swarm-test/make-snapshot} + SLAVE_NODE_MEMORY=4096 + SLAVE_NODE_CPU=2 + ADMIN_NODE_MEMORY=4096 + ADMIN_NODE_CPU=2 + + - inject: # Overrides + properties-content: '{swarm-test/custom-overrides-plain-properties}' + - inject-text-parameter: + properties-content: $MOS_MIRROR_OVERWRITES + - 9x-run-sys-test + + publishers: + + - archive: + allow-empty: true + artifacts: '**/nosetests.xml,logs/*,*.txt' + latest-only: false + - junit: + keep-long-stdio: false + results: '**/nosetests.xml' + + - post-destroy-vms + + - description-setter: + regexp: "'Description string: (.*)'" + regexp-for-failed: "'Description string: (.*)'" + + - trigger-parameterized-builds: + - project: '9.x.swarm.test-reports' + condition: ALWAYS + trigger-with-no-params: true diff --git a/maintenance-ci/mos/templates/testrail-report-9.x.yaml b/maintenance-ci/mos/templates/testrail-report-9.x.yaml new file mode 100644 index 0000000..1e5845e --- /dev/null +++ b/maintenance-ci/mos/templates/testrail-report-9.x.yaml @@ -0,0 +1,48 @@ +- job-template: + id: mos/templates/test-report-9.x + name: '{version}.swarm.test-reports' + + build-discarder: + daysToKeep: '{testrail_reporter/logrotate/daysToKeep}' + numToKeep: '{testrail_reporter/logrotate/numToKeep}' + + + node: '{testrail_reporter/node}' + + triggers: + - timed: '' + + scm: + - git: + url: 'https://review.fuel-infra.org/openstack/fuel-qa.git' + branches: + - '{testrail_reporter/fuel-qa-branch}' + basedir: '' + + wrappers: + - credentials-binding: + - username-password-separated: + credential-id: testrail_credentials + username: TESTRAIL_USER + password: TESTRAIL_PASSWORD + + builders: + - inject: + properties-content: | + VENV_PATH={testrail_reporter/venv_path} + TESTS_RUNNER={testrail_reporter/test_runner_job} + - inject: + properties-content: | + TESTRAIL_TEST_SUITE={testrail_reporter/testrail_suite} + TESTRAIL_MILESTONE={testrail_reporter/testrail_milestone} + LAUNCHPAD_MILESTONE={testrail_reporter/launchpad_milestone} + - shell: + !include-raw-escape: ../scripts/testrail-upload-cases-9.x.sh + - shell: + !include-raw-escape: ../scripts/testrail-report-9.x.sh + + + publishers: + - description-setter: + regexp: '^.*Report URL:\s*(.*)' + description: 'TestRail Report URL' diff --git a/maintenance-ci/views/.keep b/maintenance-ci/views/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/maintenance-ci/views/9.x.tempest.yaml b/maintenance-ci/views/9.x.tempest.yaml new file mode 100644 index 0000000..7112fdd --- /dev/null +++ b/maintenance-ci/views/9.x.tempest.yaml @@ -0,0 +1,18 @@ +- view: + type: list + name: '9.x.tempest' + regex: '9.x.tempest.*' +# jobs: +# - 9.x.tempest +# - maintenance-tempest-reports-Testrail +# - maintenance-tempest-runner + columns: + - status + - weather + - job + - last_success + - last_failure + - last_duration + - build_button + recurse: False + diff --git a/maintenance-ci/views/9.x.yaml b/maintenance-ci/views/9.x.yaml new file mode 100644 index 0000000..3e42384 --- /dev/null +++ b/maintenance-ci/views/9.x.yaml @@ -0,0 +1,23 @@ +- view: + type: list + name: '9.x' + regex: '9.x.*' +# jobs: +# - 9.x.snapshot.params +# - 9.x.swarm.runner +# - 9.x.swarm.micro.runner +# - 9.x.acceptance.runner +# - 9.x.acceptance.test-reports +# - 9.x.backup-restore.test-reports +# - 9.x.baremetal.test-reports +# - 9.0.proposed-to-updates + columns: + - status + - weather + - job + - last_success + - last_failure + - last_duration + - build_button + recurse: False +