X-Git-Url: https://review.fuel-infra.org/gitweb?a=blobdiff_plain;f=tests%2Fruntests.sh;h=f92d02cdf679f399bae4b550e3c940755d7489ac;hb=refs%2Fheads%2Fopenstack-ci%2Ffuel-8.0%2Fliberty;hp=50dd9f67c1851562c38158c15f42abd911fc7b3a;hpb=1f7aebf6adad8c8064ecc7f74bbb0c0d363f9872;p=openstack-build%2Fceilometer-build.git

diff --git a/tests/runtests.sh b/tests/runtests.sh
index 50dd9f6..163281e 100755
--- a/tests/runtests.sh
+++ b/tests/runtests.sh
@@ -1,262 +1,388 @@
-#!/bin/bash -x
+#!/bin/bash
 
+set -eux
+
+package_to_test=${1}
 RES=0
 
-package=$1
+mysql_pass="admin"
+token=$(openssl rand -hex 10)
+auth_string="--os-auth-url http://127.0.0.1:35357 --os-token ${token} --os-endpoint http://127.0.0.1:35357/v2.0/"
+ceilometer_auth_string="--debug --os-username ceilometer --os-password ceilometer --os-tenant-name ceilometer --os-auth-url http://127.0.0.1:35357"
+ceilometer_service=
+
+keystone_log_file="/var/log/keystone/keystone.log"
+ceilometer_log_dir="/var/log/ceilometer"
+mongodb_log="/var/log/mongodb/mongodb.log"
+
+if [[ `cat /etc/*-release | head -n 1 | awk '{print $1}'` =~ Ubuntu ]]; then
+   os="ubuntu"
+   packages_list="python-pip curl"
+   export DEBIAN_FRONTEND=noninteractive
+   command_to_install="apt-get install -y --force-yes"
+   keystone_package="keystone"
+   keystone_service="keystone"
+   mysql_service="mysql"
+   mongodb_service="mongodb"
+   ceilometer_packages=(
+      ceilometer-agent-central
+      ceilometer-agent-compute
+      ceilometer-collector
+      ceilometer-agent-notification
+      ceilometer-polling
+      ceilometer-api
+      ceilometer-alarm-notifier
+      ceilometer-alarm-evaluator
+   )
+
+   ceilometer_services=("${ceilometer_packages[@]}")
+
+   ceilometer_binaries=(
+     ceilometer-collector
+     ceilometer-agent-notification
+     ceilometer-polling
+     ceilometer-api
+     ceilometer-alarm-notifier
+     ceilometer-alarm-evaluator
+     ceilometer-dbsync
+     ceilometer-expirer
+     ceilometer-rootwrap
+     ceilometer-send-sample
+   )
+   config_files=(
+     api_paste.ini
+     ceilometer.conf
+     event_definitions.yaml
+     event_pipeline.yaml
+     pipeline.yaml
+     policy.json
+   )
+else
+   setenforce 0 || :
+   service firewalld stop || :
+   os="centos"
+   packages_list="python-pip curl"
+   command_to_install="yum install -y"
+   keystone_package="openstack-keystone"
+   keystone_service="openstack-keystone"
+   mysql_service="mysql"
+   mongodb_service="mongod"
+   ceilometer_packages=(
+      python-ceilometerclient
+      openstack-ceilometer-central
+      openstack-ceilometer-compute
+      openstack-ceilometer-collector
+      openstack-ceilometer-notification
+      openstack-ceilometer-polling
+      openstack-ceilometer-api
+      openstack-ceilometer-alarm
+   )
+   ceilometer_services=(
+      openstack-ceilometer-central
+      openstack-ceilometer-compute
+      openstack-ceilometer-collector
+      openstack-ceilometer-notification
+      openstack-ceilometer-polling
+      openstack-ceilometer-api
+      openstack-ceilometer-alarm-notifier
+      openstack-ceilometer-alarm-evaluator
+   )
+   ceilometer_binaries=(
+     ceilometer-agent-notification
+     ceilometer-alarm-evaluator
+     ceilometer-alarm-notifier
+     ceilometer-api
+     ceilometer-collector
+     ceilometer-dbsync
+     ceilometer-expirer
+     ceilometer-polling
+     ceilometer-send-sample
+   )
+   config_files=(
+     api_paste.ini
+     ceilometer.conf
+     event_definitions.yaml
+     event_pipeline.yaml
+     pipeline.yaml
+     policy.json
+   )
+fi
+
+list_commands=(
+   alarm-list
+   event-list
+   meter-list
+   resource-list
+   sample-list
+)
+
+install_packages() {
+   ${command_to_install} $@
+}
 
-prepare_script() {
-   echo '#!/usr/bin/env python
-import pika
-connection = pika.BlockingConnection(pika.ConnectionParameters(host="127.0.0.1"))
-channel = connection.channel()
-channel.queue_declare(queue="notifications.info")
-test=r'"'"'{"event_type": "compute.instance.update", "_unique_id": "fake", "payload": {"ephemeral_gb": 0, "instance_id": "fake", "user_id": "fake", "root_gb": 0, "tenant_id": "fake", "memory_mb": 64, "vcpus": 1}, "priority": "INFO"}'"'"'
-channel.basic_publish(exchange="nova", routing_key="notifications.info",
-                      properties=pika.BasicProperties(priority=0, delivery_mode=2, headers="", content_encoding="utf-8", content_type="application/json"),
-                      body=test)
-print " [x] Message was sent!"
-connection.close()' > $(pwd)/send.py
+prepare_vm() {
+   install_packages ${packages_list}
+}
+
+install_and_setup_keystone() {
+   install_packages ${keystone_package}
+   service ${keystone_service} stop
+
+   cat > /etc/keystone/keystone.conf << EOF
+[DEFAULT]
+admin_token=${token}
+log_file=${keystone_log_file}
+rabbit_password = guest
+rabbit_hosts = 127.0.0.1
+notification_driver = messaging
+notification_topics = notifications
+[database]
+connection=mysql://keystone:keystone@127.0.0.1/keystone
+[token]
+provider=keystone.token.providers.uuid.Provider
+EOF
+   [ -d "/var/log/keystone/" ] || mkdir /var/log/keystone/
+   touch ${keystone_log_file}
+   chown -R keystone:keystone /var/log/keystone/
+   chown -R keystone:keystone /etc/keystone
+   keystone-manage db_sync
+   service ${keystone_service} restart
+   sleep 5
+   if [ -z "$(curl -s http://127.0.0.1:5000/ | grep '^{"versions":')" ]; then
+      exit_on_error 1 "Failed to get to keystone API.\nTests failed because of keystone problem"
+   fi
+}
+
+install_and_setup_mysql() {
+   if [[ "$os" == "ubuntu" ]]; then
+     echo "mysql-server mysql-server/root_password select ${mysql_pass}" | debconf-set-selections
+     echo "mysql-server mysql-server/root_password_again select ${mysql_pass}" | debconf-set-selections
+     ${command_to_install} mysql-server mysql-client python-mysqldb
+     MYSQL_PASSWORD=''
+   else
+     ${command_to_install} mysql-client mysql-server mysql-libs MySQL-python
+     MYSQL_PASSWORD=$(cat /root/.mysql_secret | head -1 | awk -F': ' '{print $2}')
+   fi
+   service mysql restart
+   sleep 10
+   if [ ! -z "${MYSQL_PASSWORD}" ]; then
+       mysqladmin -u root -p${MYSQL_PASSWORD} password ${mysql_pass}
+   fi
+   if [ -z "$(service ${mysql_service} status | grep "running" )" ]; then
+     exit_on_error 1 "Mysql service has failed to start"
+   fi
+
+   mysql -uroot -p${mysql_pass} -Bse "drop database if exists keystone"
+   mysql -uroot -p${mysql_pass} -Bse "create database keystone"
+   mysql -uroot -p${mysql_pass} -Bse "GRANT ALL PRIVILEGES ON keystone.* TO 'keystone'@'localhost' IDENTIFIED BY 'keystone'"
+   mysql -uroot -p${mysql_pass} -Bse "drop database if exists ceilometer"
+   mysql -uroot -p${mysql_pass} -Bse "create database ceilometer"
+   mysql -uroot -p${mysql_pass} -Bse "GRANT ALL PRIVILEGES ON ceilometer.* TO 'ceilometer'@'localhost' IDENTIFIED BY 'ceilometer'"
+   mysql -uroot -p${mysql_pass} -Bse "flush privileges"
+}
+
+install_and_setup_rabbit() {
+   install_packages rabbitmq-server
+   echo "127.0.0.1 $(hostname)" >> /etc/hosts
+   if [[ "$os" == "ubuntu" ]]; then
+       update-rc.d rabbitmq-server defaults
+   else
+       systemctl enable rabbitmq-server
+   fi
+   service rabbitmq-server restart
+   sleep 5
 }
 
-prepare_config() {
-   mkdir -p /var/log/ceilometer
-   echo -e "[DEFAULT]
+ceilometer_configuration() {
+   cat > /etc/ceilometer/ceilometer.conf << EOF
+[DEFAULT]
 api_paste_config = /etc/ceilometer/api_paste.ini
 policy_file = /etc/ceilometer/policy.json
 debug = True
 verbose = True
-log_file = /var/log/ceilometer/$package.log
-rpc_backend = ceilometer.openstack.common.rpc.impl_kombu
+log_dir = ${ceilometer_log_dir}
+auth_strategy = keystone
+rabbit_password = guest
+rabbit_hosts = 127.0.0.1
+
+[keystone_authtoken]
+auth_host = 127.0.0.1
+auth_port = 35357
+auth_protocol = http
+admin_tenant_name = ceilometer
+admin_user = ceilometer
+admin_password = ceilometer
+auth_url=http://127.0.0.1:35357
 
 [database]
-connection=sqlite:////tmp/ceilometer" > /etc/ceilometer/ceilometer.conf
+connection=mysql://ceilometer:ceilometer@127.0.0.1/ceilometer
+
+[notification]
+store_events = True
+EOF
+   [ -e /etc/default/ceilometer-polling ] && rm -f /etc/default/ceilometer-polling
+   echo "DAEMON_ARGS=\"--polling-namespaces=central\"" > /etc/default/ceilometer-polling
+
+   [ -d "${ceilometer_log_dir}" ] || mkdir ${ceilometer_log_dir}
+   chown -R ceilometer:ceilometer ${ceilometer_log_dir}
+   chown -R ceilometer:ceilometer /etc/ceilometer
 
    ceilometer-dbsync --config-file /etc/ceilometer/ceilometer.conf
-   if [ $? != "0" ]; then
-      RES=1
-      exit $RES
+   if [ "$?" -ne '0' ]; then
+      cat ${ceilometer_log_dir}/ceilometer-dbsync.log
+      exit_on_error 1 "DB sync command has failures"
    fi
-   # Clean log file
-   rm -f /var/log/ceilometer/$package.log
+
+   keystone ${auth_string} user-create --name ceilometer --pass ceilometer
+   keystone ${auth_string} tenant-create --name ceilometer
+   keystone ${auth_string} role-create --name=admin
+   keystone ${auth_string} user-role-add --user ceilometer --role admin --tenant ceilometer
+   ceilometer_service=$(keystone ${auth_string} service-create --name=ceilometer\
+ --type=metering --description="ceilometer" | grep id | awk -F '|' '{print $3}' | tr -d ' ')
+   keystone ${auth_string} endpoint-create --region RegionOne --service-id=${ceilometer_service}\
+ --publicurl=http://127.0.0.1:8777/ --internalurl=http://127.0.0.1:8777/ --adminurl=http://127.0.0.1:8777/
 }
 
-prepare_vm() {
-   if [[ `cat /etc/*-release | head -n 1 | awk '{print $1}'` =~ Ubuntu ]]; then
-      apt-get install rabbitmq-server screen python-pip sqlite3 -y
-   else
-      yum install screen rabbitmq-server python-pip sqlite -y
+send_fake_instance_notification() {
+   if [ $os == "centos" ]; then
+     curl -LO https://bootstrap.pypa.io/get-pip.py
+     python get-pip.py
    fi
-   service rabbitmq-server restart
-
-   for (( i = 1; i <= 5; i++ ))
-   {
-      if echo "$(service rabbitmq-server status)" | grep -q "Error: unable to connect to node";
-         then
-            service rabbitmq-server start
-            if [ "$i" -eq 5 ]
-               then
-                  echo "rabbitmq-server is dead"
-                  exit 1
-            fi
-         else
-            echo "$(service rabbitmq-server status)"
-            break
-      fi
-   }
 
-   sleep 5
-}
-
-send_fake_notification() {
    pip install pika==0.9.8
-   prepare_script
+   echo '#!/usr/bin/env python
+import pika
+import uuid
+connection = pika.BlockingConnection(pika.ConnectionParameters(host="127.0.0.1"))
+channel = connection.channel()
+channel.queue_declare(queue="notifications.info")
+test='"'"'{"event_type": "compute.instance.create", "payload": {"instance_id": "fake", "memory_mb": 512, "disk_gb": 20, "root_gb": 20, "ephemeral_gb": 0, "vcpus": 1, "instance_type_id": 1, "state": "fake"}, "publisher_id": "fake", "priority":"INFO", "message_id": "%s"}'"'"' % uuid.uuid4()
+channel.basic_publish(exchange="nova", routing_key="notifications.info",
+                      properties=pika.BasicProperties(priority=0, delivery_mode=2, headers="", content_encoding="utf-8", content_type="application/json"),
+                      body=test)
+print " [x] Message was sent!"
+connection.close()' > $(pwd)/send.py
    python $(pwd)/send.py
 }
 
-check_fake_notification() {
-   sqlite3 /tmp/ceilometer 'select * from meter;'
-   local check=$(sqlite3 /tmp/ceilometer 'select * from meter;' | grep instance)
-   if [ -z "$check" ]; then
-      echo -e "Meter 'instance' is lost"
-      RES=1
-      echo $RES
+exit_on_error() {
+   # $1 - responce code
+   # $2 - message
+   if [ "${1}" -ne '0' ]; then
+      echo -e "${2}"
+      RES=${1}
+      exit ${RES}
    fi
 }
 
-check_pollsters() {
-   local test1=$(cat /var/log/ceilometer/$package.log | grep "Polling pollster $1")
-   local test2=$(cat /var/log/ceilometer/$package.log | grep "Polling pollster $2")
-   local test3=$(cat /var/log/ceilometer/$package.log | grep "Polling pollster $3")
-   if [ -z "$test1" -o -z "$test2" -o -z "$test3" ]; then
-      echo -e "Some pollsters aren't work"
-      RES=1
-      echo $RES
+found=false
+for i in ${ceilometer_packages[@]}; do
+   if [ "${i}" == "${package_to_test}" ]; then
+      found=true
    fi
-}
+done
 
-check_running() {
-   if [ $(ps aux | grep $1 | wc -l) -lt "2" ]; then
-      echo "Process $1 is not running"
-      RES=1
-      echo $RES
-   fi
-}
+if ${found}; then
+   echo "Test for package ${package_to_test} will be run with ceilometer-common package"
+   RES=0
+fi
 
-case $package in
-   ceilometer-api|openstack-ceilometer-api)
-      echo "Skip due to resolving oslo.middleware problem: ImportError: No module named middleware"
-      exit 0
-      prepare_config
+case ${package_to_test} in
+   ceilometer-common|openstack-ceilometer-common)
       prepare_vm
+      install_packages "${ceilometer_packages[@]}"
+      install_and_setup_rabbit
+      install_and_setup_mysql
+      install_and_setup_keystone
+      ceilometer_configuration
+
+      for binary in ${ceilometer_binaries[@]}; do
+         if [ ! -f "/usr/bin/${binary}" ]; then
+            exit_on_error 1 "Ceilometer binary ${binary} is missed"
+         fi
+      done
 
-      screen -dmS ceilometer-api
-      sleep 2
-      screen -S ceilometer-api -p 0 -X stuff 'ceilometer-api --config-file /etc/ceilometer/ceilometer.conf
-'
-      sleep 10
-      check_running ceilometer-api
+      for config in ${config_files[@]}; do
+         if [ ! -f "/etc/ceilometer/${config}" ]; then
+            exit_on_error 1 "Ceilometer configuration file ${config} is missed"
+         fi
+      done
+
+      for service in ${ceilometer_services[@]}; do
+         service ${service} restart
+         sleep 5
+         if [ -z "$(service ${service} status | grep "running")" ]; then
+            cat /var/log/ceilometer/${service}.log
+            exit_on_error 1 "Service ${service} has failed to start"
+         fi
+      done
 
-      repsonce=$(curl "http://localhost:8777" | grep "Authentication required")
+      repsonce=$(curl "http://127.0.0.1:8777" | grep "Authentication required")
       if [ -z "$repsonce" ]; then
-         echo -e "Ceilometer api doesn't work. No one process is listening on 8777 port"
-         RES=1
+         exit_on_error 1 "Ceilometer api doesn't work.\n No one process is listening on 8777 port"
       fi
 
-      killall -15 ceilometer-api
 
-      cat /var/log/ceilometer/$package.log
-   ;;
-   ceilometer-collector|openstack-ceilometer-collector|ceilometer-agent-notification|openstack-ceilometer-notification)
-      prepare_config
-      prepare_vm
+      for Command in ${list_commands[@]}; do
+         ceilometer ${ceilometer_auth_string} ${Command} || true
+         cat $keystone_log_file
+         if [ "$?" -ne '0' ]; then
+             exit_on_error 1 "Command ${Command} has failed"
+         fi
+      done
 
-      if [[ `cat /etc/*-release | head -n 1 | awk '{print $1}'` =~ Ubuntu ]]; then
-         apt-get install ceilometer-collector ceilometer-agent-notification -y
-      else
-         yum install openstack-ceilometer-collector openstack-ceilometer-notification -y
-      fi
+      send_fake_instance_notification
 
-      screen -dmS ceilometer-collector
-      screen -dmS ceilometer-agent-notification
-      sleep 2
-      screen -S ceilometer-collector -p 0 -X stuff 'ceilometer-collector --config-file /etc/ceilometer/ceilometer.conf --log-file /var/log/ceilometer/ceilometer-collector.log
-'
-      screen -S ceilometer-agent-notification -p 0 -X stuff 'ceilometer-agent-notification --config-file /etc/ceilometer/ceilometer.conf --log-file /var/log/ceilometer/ceilometer-anotification.log
-'
-      sleep 10
-      check_running ceilometer-collector
-      check_running ceilometer-agent-notification
-      send_fake_notification
-      sleep 2
-      check_fake_notification
-
-      killall -15 ceilometer-collector ceilometer-agent-notification
-
-      cat /var/log/ceilometer/ceilometer-collector.log
-      cat /var/log/ceilometer/ceilometer-anotification.log
-   ;;
-   ceilometer-alarm-notifier|ceilometer-alarm-evaluator|openstack-ceilometer-alarm)
-      if [[ `cat /etc/*-release | head -n 1 | awk '{print $1}'` =~ Ubuntu ]]; then
-         # Testing alarm-notifier and alarm-evaluator together
-         apt-get install ceilometer-alarm-evaluator ceilometer-alarm-notifier -y --force-yes
+      # Keystone notifications
+      if [ -z "$(ceilometer ${ceilometer_auth_string} meter-list | grep fake)" ]; then
+         exit_on_error 1 "Keystone notifications don't work"
       fi
 
-      echo "Testing 'ceilometer-alarm-notifier'"
-      prepare_config
-      prepare_vm
-
-      screen -dmS ceilometer-alarm-notifier
-      sleep 2
-      screen -S ceilometer-alarm-notifier -p 0 -X stuff 'ceilometer-alarm-notifier --config-file /etc/ceilometer/ceilometer.conf --log-file /var/log/ceilometer/ceilometer-alarm-notifier.log
-'
-      sleep 10
-      check_running ceilometer-alarm-notifier
-
-      if [ -z "$(cat /var/log/ceilometer/ceilometer-alarm-notifier.log | grep 'Connected to AMQP server on')" ]; then
-         echo -e "Start alarm-notifier process is failed"
-         RES=1
+      # Instance notification
+      resource_id=$(ceilometer ${ceilometer_auth_string} resource-list | grep fake)
+      if [ -z "${resource_id}" ]; then
+         exit_on_error 1 "Notification is not received"
       fi
 
-      killall -15 ceilometer-alarm-notifier
-
-      cat /var/log/ceilometer/ceilometer-alarm-notifier.log
-
-      echo "Testing 'ceilometer-alarm-evaluator'"
-      screen -dmS ceilometer-alarm-evaluator
-      sleep 2
-      screen -S ceilometer-alarm-evaluator -p 0 -X stuff 'ceilometer-alarm-evaluator --config-file /etc/ceilometer/ceilometer.conf --log-file /var/log/ceilometer/ceilometer-alarm-evaluator.log
-'
-      sleep 10
-      check_running ceilometer-alarm-evaluator
-
-      if [ -z "$(cat /var/log/ceilometer/ceilometer-alarm-evaluator.log | grep 'alarm evaluation cycle')" ]; then
-         echo -e "Start alarm-evalutor process is failed"
-         RES=1
+      # Events
+      if [ -z "$(ceilometer ${ceilometer_auth_string} event-list | grep fake)" ]; then
+         exit_on_error 1 "Events wasn't created"
       fi
 
-      killall -15 ceilometer-alarm-evaluator
-
-      cat /var/log/ceilometer/ceilometer-alarm-evaluator.log
-   ;;
-   ceilometer-agent-compute|openstack-ceilometer-compute)
-      prepare_vm
-      prepare_config
-
-      screen -dmS ceilometer-agent-compute
-      sleep 2
-      screen -S ceilometer-agent-compute -p 0 -X stuff 'ceilometer-agent-compute --config-file /etc/ceilometer/ceilometer.conf
-'
-      sleep 10
-      check_running ceilometer-agent-compute
-      check_pollsters "cpu" "memory" "disk"
-
-      killall -15 ceilometer-agent-compute
+      # Alarming
+      ceilometer -dv ${ceilometer_auth_string} alarm-threshold-create --name cpu_high --description 'instance running hot' \
+ --meter-name cpu_util  --threshold 70.0 --comparison-operator gt  --statistic avg \
+ --period 600 --evaluation-periods 3 --alarm-action 'log://' --query resource_id="${resource_id}"
 
-      cat /var/log/ceilometer/$package.log
-   ;;
-   ceilometer-agent-central|openstack-ceilometer-central)
-      prepare_vm
-      prepare_config
-
-      screen -dmS ceilometer-agent-central
-      sleep 2
-      screen -S ceilometer-agent-central -p 0 -X stuff 'ceilometer-agent-central --config-file /etc/ceilometer/ceilometer.conf
-'
-      sleep 10
-      check_running ceilometer-agent-central
-      check_pollsters "network" "switch" "hardware"
+      if [ "$?" -ne '0' -a -z "$(ceilometer ${ceilometer_auth_string} alarm-list | grep cpu_high)" ]; then
+         exit_on_error 1 "Alarm creation has failed"
+      fi
 
-      killall -15 ceilometer-agent-central
+      # Samples
+      ceilometer -dv ${ceilometer_auth_string} sample-create -m fake_sample --meter-type gauge --meter-unit '%' --sample-volume 100 -r "fake"
 
-      cat /var/log/ceilometer/$package.log
-   ;;
-   ceilometer-common|openstack-ceilometer-common)
-      if [ -z "$(cut -d: -f1 /etc/passwd | grep 'ceilometer')" ]; then
-         echo -e "User 'ceilometer' doesn't exist"
-         RES=1
+      if [ "$?" -ne '0' -a -z "$(ceilometer ${ceilometer_auth_string} sample-list | grep fake_sample)" ]; then
+         exit_on_error 1 "Sample creation has failed"
       fi
-      config_files="ceilometer.conf policy.json pipeline.yaml api_paste.ini event_definitions.yaml"
-      for i in $config_files; do
-         if [ ! -f "/etc/ceilometer/$i" ]; then
-            RES=1
-            echo "File /etc/ceilometer/$i doesn't exist"
-         fi
-      done
+   ;;
+   ceilometer-agent-ipmi|openstack-ceilometer-ipmi)
+      # Ceilometer-agent-ipmi is not included in common testing
+      # because it is not used in Fuel now
+      #if [ ! -f "/usr/bin/ceilometer-agent-ipmi" ]; then
+      #   exit_on_error 1 "Ceilometer ipmi binary is missed"
+      #fi
    ;;
    python-ceilometer)
       python -c 'import ceilometer'
       if [ $? -ne '0' ]; then
-         echo -e "Couldn't import module 'ceilometer'"
-         RES=1
-      fi
-      if [ -z "$(which ceilometer)" ]; then
-         RES=1
-         echo -e "Can't find command 'ceilometer'"
+         exit_on_error 1 "Couldn't import module 'ceilometer'"
       fi
    ;;
    *)
-      echo "test not defined, skipping..."
+      if ! ${found}; then
+         echo "Test not defined. Please create test for package ${package_to_test}"
+      fi
    ;;
 esac
 
-exit $RES
+exit ${RES}