From: Vladimir Khlyunev Date: Thu, 7 Apr 2022 06:11:16 +0000 (+0400) Subject: Initial rework MOS9 tempest X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F89%2F41889%2F15;p=tools%2Fsustaining.git Initial rework MOS9 tempest Change-Id: I006d7b5148d9eb0ff543192a9b63fa59ad2d354e --- diff --git a/maintenance-ci/mos/jobs/maintenance-tempest-runner.yaml b/maintenance-ci/mos/jobs/maintenance-tempest-runner.yaml index 238538d..de29fb5 100644 --- a/maintenance-ci/mos/jobs/maintenance-tempest-runner.yaml +++ b/maintenance-ci/mos/jobs/maintenance-tempest-runner.yaml @@ -11,15 +11,9 @@ weight: 1 scm: - git: - url: "https://review.fuel-infra.org/fuel-infra/dockerfiles" - branches: - - "origin/master" - skip-tag: true - - git: - url: "https://github.com/Mirantis/mos-ci-deployment-scripts.git" + url: 'https://review.fuel-infra.org/tools/sustaining/' branches: - - 'origin/master' - basedir: 'mos-ci-deployment-scripts' + - origin/master parameters: - node: name: NODE_LABELS @@ -31,19 +25,6 @@ - string: name: VENV_PATH default: /home/jenkins/qa-venv-9.x - - choice: - name: TEMPEST_RUNNER - choices: - - 'mos-tempest-runner' - - 'rally' - - 'rally_without_docker' - description: | - mos-tempest-runner for 7.0 - rally for 8.0 - rally_without_docker for 9.x - - string: - name: MILESTONE - default: '7.0' - string: name: REPORT_PREFIX description: directory, where verification.xml will been copyed diff --git a/maintenance-ci/mos/rally-docker/Dockerfile b/maintenance-ci/mos/rally-docker/Dockerfile new file mode 100644 index 0000000..4f2a187 --- /dev/null +++ b/maintenance-ci/mos/rally-docker/Dockerfile @@ -0,0 +1,44 @@ +FROM ubuntu:16.04 + +LABEL rebuild="build1" + +ENV DEBIAN_FRONTEND=noninteractive \ + DEBCONF_NONINTERACTIVE_SEEN=true \ + LANG=C.UTF-8 \ + LANGUAGE=$LANG +SHELL ["/bin/bash", "-xec"] + +# Base apt config +RUN echo 'Acquire::Languages "none";' > /etc/apt/apt.conf.d/docker-no-languages \ + && echo 'Acquire::GzipIndexes "true"; Acquire::CompressionTypes::Order:: "gz";' > /etc/apt/apt.conf.d/docker-gzip-indexes \ + && echo 'APT::Get::Install-Recommends "false"; APT::Get::Install-Suggests "false";' > /etc/apt/apt.conf.d/docker-recommends \ + && apt-get update \ + && apt-get -yy upgrade \ + && apt-get install -y vim sshuttle sshpass git curl gettext-base python-dev \ + python-virtualenv iputils-ping libffi-dev libpq-dev \ + libssl-dev libxml2-dev libxslt1-dev wget \ + && apt-get -y install python-pip && pip install -U "pip<21" && pip install bindep + +WORKDIR /root +RUN git clone https://github.com/openstack/rally.git + +WORKDIR /root/rally +RUN git checkout 01a1b5fc6e242775d126a0f357ea91c38c783404 && \ + sed -i "s|https://bootstrap.pypa.io/get-pip.py|https://bootstrap.pypa.io/pip/2.7/get-pip.py|g" install_rally.sh && \ + ./install_rally.sh --branch 0.7.0 -d rally-venv/ -y + +# EOL-ed python workarounds +RUN egrep -lR "^import queue$" rally-venv/ | xargs -tI{} sed -i 's/import queue/import Queue as queue/' {} && \ + sed -i '/if self.version:/a \ check_output(["git", "checkout", "--", "."],\n cwd=self.path())' /root/rally/rally-venv/local/lib/python2.7/site-packages/rally/verification/tempest/tempest.py && \ + sed -i '/multiple Tempest instances/a \ check_output([self.venv_wrapper, "pip", "install", "voluptuous==0.8.9"],\n cwd=self.path())' /root/rally/rally-venv/local/lib/python2.7/site-packages/rally/verification/tempest/tempest.py + +WORKDIR /root + +RUN wget https://raw.githubusercontent.com/Mirantis/mos-ci-deployment-scripts/master/jenkins-job-builder/shell_scripts/skip_ceph.list \ + https://raw.githubusercontent.com/Mirantis/mos-ci-deployment-scripts/master/jenkins-job-builder/shell_scripts/ceph \ + https://raw.githubusercontent.com/Mirantis/mos-ci-deployment-scripts/master/jenkins-job-builder/shell_scripts/skip_lvm.list \ + https://raw.githubusercontent.com/Mirantis/mos-ci-deployment-scripts/master/jenkins-job-builder/shell_scripts/lvm + +COPY run_rally_tempest.sh /root/run_rally_tempest.sh + +CMD /bin/bash run_rally_tempest.sh \ No newline at end of file diff --git a/maintenance-ci/mos/rally-docker/run_rally_tempest.sh b/maintenance-ci/mos/rally-docker/run_rally_tempest.sh new file mode 100644 index 0000000..01b6f74 --- /dev/null +++ b/maintenance-ci/mos/rally-docker/run_rally_tempest.sh @@ -0,0 +1,112 @@ +#!/bin/bash +set -ex +#source /root/helpers.sh + +SSH_OPTIONS=(-o "ConnectTimeout=20" -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null") +ssh_to_fuel_master() { + # $1 - command to execute + sshpass -p "${VM_PASSWORD}" ssh "${SSH_OPTIONS[@]}" "${VM_USERNAME}@${FUEL_ADMIN_IP}" "$1" +} +ssh_to_controller() { + # $1 - command to execute + ssh "${SSH_OPTIONS[@]}" "${VM_USERNAME}@${CTL_ADMIN_IP}" "$1" +} +scp_to_controller() { + # $1 - file, can be 'flagged' with --recursive + # $2 - target path + SCP_ARGS=() + case $1 in + -r|--recursive) + SCP_ARGS+=("-r") + shift + ;; + esac + targetpath=$2 + scp "${SSH_OPTIONS[@]}" "${SCP_ARGS[@]}" "$1" "${VM_USERNAME}@${CTL_ADMIN_IP}:${targetpath:-\"/tmp/\"}" +} +scp_from_controller() { + # $1 - remote file, can be 'flagged' with --recursive + # $2 - local path + SCP_ARGS=() + case $1 in + -r|--recursive) + SCP_ARGS+=("-r") + shift + ;; + esac + scp "${SSH_OPTIONS[@]}" "${SCP_ARGS[@]}" "${VM_USERNAME}@${CTL_ADMIN_IP}:$1" "$2" +} + +mkdir -p /root/.ssh +cp /root/shared/id_rsa* /root/.ssh +chmod 600 /root/.ssh/* + +OPENRC_FILE=/root/openrc + +source /root/shared/ssh_connection_data.sh + +echo "Fuel admin IP: ${FUEL_ADMIN_IP}" +echo "CTL node admin IP: ${CTL_ADMIN_IP}" +echo "VM_USERNAME= ${VM_USERNAME}" +echo "VM_PASSWORD= ${VM_PASSWORD}" +echo "Management Network: ${MGMT_NET}" + +ssh_to_fuel_master "ssh ${CTL_ADMIN_IP} echo \"$(cat /root/shared/id_rsa.pub) >> /root/.ssh/authorized_keys\"" +pkill sshuttle || true +sshuttle -D -e "ssh ${SSH_OPTIONS[*]}" -r "${VM_USERNAME}@${CTL_ADMIN_IP}" ${MGMT_NET} + +scp_from_controller /root/openrc ${OPENRC_FILE} +if ssh_to_controller "test -f /var/lib/astute/haproxy/public_haproxy.pem" ; then + scp_from_controller /var/lib/astute/haproxy/public_haproxy.pem /root/public_haproxy.pem + echo "export OS_CACERT='/root/public_haproxy.pem'" >> ${OPENRC_FILE} +fi +sed -i "s/:5000\/'/:5000\/v3\/'/" ${OPENRC_FILE} + +echo "export OS_PROJECT_DOMAIN_NAME='Default'" >> ${OPENRC_FILE} +echo "export OS_USER_DOMAIN_NAME='Default'" >> ${OPENRC_FILE} +echo "export OS_IDENTITY_API_VERSION='3'" >> ${OPENRC_FILE} + +NOVA_FLTR=$(ssh_to_controller "sed -n '/scheduler_default_filters=/p' /etc/nova/nova.conf | cut -f2 -d=") +check_ceph=$(ssh_to_controller "cat /etc/cinder/cinder.conf |grep '\[RBD-backend\]' | wc -l") + +if [ ${check_ceph} == '1' ]; then + storage_protocol="ceph" + sed -i 's|#swift_operator_role = Member|swift_operator_role = swiftoperator|g' /root/rally/rally-venv/etc/rally/rally.conf + echo 'scheduler_available_filters = '$NOVA_FLTR >> ceph +else + storage_protocol="lvm" + sed -i 's|#swift_operator_role = Member|swift_operator_role = SwiftOperator|g' /root/rally/rally-venv/etc/rally/rally.conf + echo 'scheduler_available_filters = '$NOVA_FLTR >> lvm +fi + +source /root/rally/rally-venv/bin/activate +# shellcheck disable=SC1090 +source ${OPENRC_FILE} +ironic node-create -d fake + +rally-manage db recreate +rally deployment create --fromenv --name=tempest +rally verify install --version 14.0.0 +rally verify genconfig --add-options $storage_protocol +rally verify showconfig + +if [ $storage_protocol == 'ceph' ]; then + rally verify start --skip-list skip_ceph.list --concurrency 2 | tee /root/rally/log.log +else + rally verify start --skip-list skip_lvm.list --concurrency 2 | tee /root/rally/log.log +fi + +set +e +rally verify results --json --output-file output.json +rally verify showconfig > /root/rally/tempest.conf +cp "$(find / -name tempest.log)" /root/rally/tempest.log +git clone https://github.com/EduardFazliev/rally_json2junit +python rally_json2junit/rally_json2junit/results_parser.py output.json + +mkdir -p /root/shared/results +chmod 777 /root/shared/results +cp /root/rally/log.log /root/shared/results/ +cp /root/rally/tempest.log /root/shared/results/ +cp "$(find / -name verification.xml)" /root/rally/results +cp "$(find / -name output.json)" /root/shared/results/ +chmod -R +w /root/shared \ No newline at end of file diff --git a/maintenance-ci/mos/scripts/maintenance-tempest-runner.sh b/maintenance-ci/mos/scripts/maintenance-tempest-runner.sh index 7f2be88..08e1f00 100644 --- a/maintenance-ci/mos/scripts/maintenance-tempest-runner.sh +++ b/maintenance-ci/mos/scripts/maintenance-tempest-runner.sh @@ -6,93 +6,14 @@ set -xe # REPORT_PREFIX=path to report directory # ENV_NAME=devops name # SNAPSHOT_NAME=tempest group name -# MILESTONE=7.0 # VENV_PATH=path to venv with devops -# TEMPEST_RUNNER=tempest runner type # clean previous results to prevent double-reporting of same run -if [ -f "${REPORT_PREFIX}/verification.xml" ]; then - mv -f "${REPORT_PREFIX}/verification.xml" "${REPORT_PREFIX}/verification.xml.unreported" -fi -rm -rf log.log verification.xml tmepest.log tempest.conf - -source "${VENV_PATH}/bin/activate" - -MILESTONE_MAJOR=$(echo "${MILESTONE}" | cut -c 1) - -# retry 3 times, because dos.py is not stable sometimes (ntp problem) -for i in $(seq 3); do - if [ "${MILESTONE_MAJOR}" -ge "7" ]; then - dos.py revert-resume "${ENV_NAME}" "${SNAPSHOT_NAME}" && break - else - dos.py revert-resume "${ENV_NAME}" --snapshot-name "${SNAPSHOT_NAME}" && break - fi - echo "Revert-resume attempt $i is failed. Retrying..." - dos.py destroy "${ENV_NAME}" - sleep 15 -done - -VM_USERNAME="root" -VM_PASSWORD="r00tme" -VM_IP=$(dos.py list --ips|grep "${ENV_NAME}"|awk '{print $2}') - -deactivate - SSH_OPTIONS=(-o "ConnectTimeout=20" -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null") ssh_to_fuel_master() { # $1 - command to execute - sshpass -p "${VM_PASSWORD}" ssh "${SSH_OPTIONS[@]}" "${VM_USERNAME}@${VM_IP}" "$1" -} - -scp_to_fuel_master() { - # $1 - file, can be 'flagged' with --recursive - # $2 - target path - SCP_ARGS=() - case $1 in - -r|--recursive) - SCP_ARGS+="-r" - shift - ;; - esac - targetpath=$2 - sshpass -p "${VM_PASSWORD}" scp "${SSH_OPTIONS[@]}" "${SCP_ARGS[@]}" "$1" "${VM_USERNAME}@${VM_IP}:${targetpath:-\"/tmp/\"}" -} - -scp_from_fuel_master() { - # $1 - remote file, can be 'flagged' with --recursive - # $2 - local path - SCP_ARGS=() - case $1 in - -r|--recursive) - SCP_ARGS+="-r" - shift - ;; - esac - sshpass -p "${VM_PASSWORD}" scp "${SSH_OPTIONS[@]}" "${SCP_ARGS[@]}" "${VM_USERNAME}@${VM_IP}:$1" "$2" -} - -check_return_code_after_command_execution() { - if [ "$1" -ne 0 ]; then - if [ -n "$2" ]; then - echo "$2" - fi - exit 1 - fi -} - -enable_public_ip() { - source "${VENV_PATH}/bin/activate" - public_mac=$(virsh dumpxml "${ENV_NAME}_admin" | grep -B 1 "${ENV_NAME}_public" | awk -F"'" '{print $2}' | head -1) - public_ip=$(dos.py net-list "${ENV_NAME}" | awk '/public/{print $2}' | grep -E -o "[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}") - public_net=$(dos.py net-list "${ENV_NAME}" | awk -F/ '/public/{print $2}') - deactivate - - ssh_to_fuel_master < ${WORK_FLDR}/log.log -EOF - - scp_from_fuel_master -r /home/developer/mos-tempest-runner/tempest-reports/* . - scp_from_fuel_master "${WORK_FLDR}/log.log" ./ - - mv tempest-report.xml verification.xml - set -e -elif [[ "${TEMPEST_RUNNER}" == "rally" ]]; then - # Workaround for run on master node. install dependencies for tempest commit b39bbce80c69a57c708ed1b672319f111c79bdd5 - sed -i 's|rally verify install --source /var/lib/tempest --no-tempest-venv|rally verify install --source /var/lib/tempest --system-wide --version b39bbce80c69a57c708ed1b672319f111c79bdd5|g' rally-tempest/latest/setup_tempest.sh - - sed -i 's|FROM rallyforge/rally:latest|FROM rallyforge/rally:0.5.0|g' rally-tempest/latest/Dockerfile - sed -i 's|RUN git clone https://git.openstack.org/openstack/tempest|RUN git clone https://git.openstack.org/openstack/tempest; cd tempest; git checkout b39bbce80c69a57c708ed1b672319f111c79bdd5|g' rally-tempest/latest/Dockerfile - sed -i 's|pip install tempest/|pip install -U -r requirements.txt|g' rally-tempest/latest/Dockerfile +LOCAL_STORAGE="$(pwd)"/shared +rm -rf "${LOCAL_STORAGE}" +mkdir -p "${LOCAL_STORAGE}" - docker build -t rally-tempest rally-tempest/latest - docker save -o ./dimage rally-tempest - - scp_to_fuel_master dimage "${WORK_FLDR}/rally" - ssh_to_fuel_master "ln -sf ${WORK_FLDR}/rally /root/rally" - ssh_to_fuel_master "cd ${WORK_FLDR} && wget https://raw.githubusercontent.com/fuel-infra/jenkins-jobs/master/servers/patching-ci/builders/maintenance-tempest-rally-runner.sh" - ssh_to_fuel_master "chmod +x ${WORK_FLDR}/maintenance-tempest-rally-runner.sh" - - echo "Run tempest tests" - set +e - ssh_to_fuel_master "/bin/bash -xe ${WORK_FLDR}/maintenance-tempest-rally-runner.sh > ${WORK_FLDR}/log.log" - - scp_from_fuel_master /var/lib/rally-tempest-container-home-dir/verification.xml ./ - scp_from_fuel_master "${WORK_FLDR}/log.log" ./ - set -e -elif [[ "${TEMPEST_RUNNER}" == "rally_without_docker" ]]; then - scp_to_fuel_master mos-ci-deployment-scripts/jenkins-job-builder/shell_scripts/run_tempest_without_docker.sh "${WORK_FLDR}/tempest.sh" - - CONTROLLER_ID=$(ssh_to_fuel_master "fuel node | grep -m1 controller | awk '{print \$1}'") - ssh_to_fuel_master "scp ${WORK_FLDR}/tempest.sh node-${CONTROLLER_ID}:/root/tempest.sh" - - # Workaround for 'There are problems and -y was used without --force-yes problem' - ssh_to_fuel_master "ssh node-${CONTROLLER_ID} 'echo \"APT::Get::AllowUnauthenticated 1;\" >> /etc/apt/apt.conf.d/02allow-unathenticated'" +IMAGE_ID=$(docker build -q maintenance-ci/mos/rally-docker) +for i in VM_USERNAME VM_PASSWORD FUEL_ADMIN_IP MGMT_NET CTL_ADMIN_IP; do + echo "export ${i}=${!i}" >> "${LOCAL_STORAGE}"/ssh_connection_data.sh +done +cp "${HOME}"/.ssh/id_rsa* "${LOCAL_STORAGE}"/ +docker run --cap-add=NET_ADMIN --network host --rm -v "${LOCAL_STORAGE}":/root/shared "${IMAGE_ID}" - echo "Run tempest tests" - set +e - ssh_to_fuel_master "ssh node-${CONTROLLER_ID} 'bash -xe /root/tempest.sh'" - - # collect logs - ssh_to_fuel_master "scp node-${CONTROLLER_ID}:/root/rally/verification.xml ${WORK_FLDR}/verification.xml" - ssh_to_fuel_master "scp node-${CONTROLLER_ID}:/root/rally/log.log ${WORK_FLDR}/log.log" - ssh_to_fuel_master "scp node-${CONTROLLER_ID}:/root/rally/tempest.conf ${WORK_FLDR}/tempest.conf" - ssh_to_fuel_master "scp node-${CONTROLLER_ID}:/root/rally/tempest.log ${WORK_FLDR}/tempest.log" - - scp_from_fuel_master "${WORK_FLDR}/verification.xml" ./ - scp_from_fuel_master "${WORK_FLDR}/log.log" ./log.log - scp_from_fuel_master "${WORK_FLDR}/tempest.conf" ./tempest.conf - scp_from_fuel_master "${WORK_FLDR}/tempest.log" ./tempest.log - set -e -else - echo "INVALID TEMPEST RUNNER '${TEMPEST_RUNNER}'" -fi set +e if [[ -n "${REPORT_PREFIX}" ]]; then mkdir -p "${REPORT_PREFIX}" - cp -f verification.xml "${REPORT_PREFIX}" + cp -f "${LOCAL_STORAGE}"/verification.xml "${REPORT_PREFIX}" fi source "${VENV_PATH}/bin/activate" @@ -220,4 +86,5 @@ SNAPSHOT_NAME="after-tempest-${BUILD_ID}-$(date +%d-%m-%Y_%Hh_%Mm)" dos.py snapshot "${ENV_NAME}" "${SNAPSHOT_NAME}" dos.py destroy "${ENV_NAME}" deactivate +docker rmi "${IMAGE_ID}"