From 111a056c0f212ed38ff6127f6c9b9dc13fd536f0 Mon Sep 17 00:00:00 2001 From: Kendall Nelson Date: Thu, 13 Aug 2015 10:17:36 -0500 Subject: [PATCH] Dynamically create cinder.conf.sample As it stands, the opts.py file that is passed into oslo-config-generator isn't being generated dynamically and the old way of generating the cinder.conf.sample is dependent on oslo-incubator which Cinder is trying to move away from. oslo-config-generator works differently than oslo-incubator so a number of changes had to be made in order to make this switch. This patch adds the config directory to Cinder and in it are two files: -generate_cinder_opts.py that will take the results of a grep command to create the opts.py file to be passed into oslo-config-generator. -cinder.conf which is the new configuration for oslo-config-generator. The file is inside the config directory to be consistent with other projects. Some changes were made to the generate_sample.sh file in order to give the base directories and target directories to the generate_cinder_opts.py program. tox.ini was edited to remove the checkonly option because all that needs to happen in check_uptodate.sh is a check to ensure that the cinder.conf.sample is actually being generated with no issues. All options were removed from the check_uptodate.sh because they were unnecessary given the new, more simple way of generating the cinder.conf.sample. setup.cfg was also edited in order to add information oslo-config-generator needs to run. Co-Authored By: Jay Bryant Co-Authored By: Jacob Gregor Change-Id: I643dbe5675ae9280e204f691781e617266f570d5 Closes-Bug: 1473768 Closes-Bug: 1437904 Closes-Bug: 1381563 --- cinder/config/cinder-config-generator.conf | 21 +++ cinder/config/generate_cinder_opts.py | 204 +++++++++++++++++++++ setup.cfg | 6 + tools/config/check_uptodate.sh | 38 +--- tools/config/generate_sample.sh | 168 +++++------------ tools/config/oslo.config.generator.rc | 1 - tox.ini | 4 +- 7 files changed, 289 insertions(+), 153 deletions(-) create mode 100644 cinder/config/cinder-config-generator.conf create mode 100644 cinder/config/generate_cinder_opts.py delete mode 100644 tools/config/oslo.config.generator.rc diff --git a/cinder/config/cinder-config-generator.conf b/cinder/config/cinder-config-generator.conf new file mode 100644 index 000000000..8a69dd790 --- /dev/null +++ b/cinder/config/cinder-config-generator.conf @@ -0,0 +1,21 @@ +[DEFAULT] +output_file = etc/cinder/cinder.conf.sample +wrap_width = 79 +namespace = cinder +namespace = keystonemiddleware.auth_token +namespace = oslo.config +namespace = oslo.concurrency +namespace = oslo.context +namespace = oslo.db +namesapce = oslo.i18n +namespace = oslo.log +namespace = oslo.messaging +namespace = oslo.middleware +namespace = oslo.policy +namespace = oslo.reports +namespace = oslo.rootwrap +namespace = oslo.serialization +namespace = oslo.service +namespace = oslo.utils +namespace = oslo.versionedobjects +namespace = oslo.vmware diff --git a/cinder/config/generate_cinder_opts.py b/cinder/config/generate_cinder_opts.py new file mode 100644 index 000000000..9fa8d13d7 --- /dev/null +++ b/cinder/config/generate_cinder_opts.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess + +if __name__ == "__main__": + opt_file = open("cinder/opts.py", 'a') + opt_dict = {} + dir_trees_list = [] + + opt_file.write("import copy\n") + opt_file.write("import itertools\n\n") + + targetdir = os.environ['TARGETDIR'] + basedir = os.environ['BASEDIRESC'] + + common_string = ('find ' + targetdir + ' -type f -name "*.py" ! ' + '-path "*/tests/*" -exec grep -l "%s" {} ' + '+ | sed -e "s/^' + basedir + + '\///g" | sort -u') + + cmd_opts = common_string % "CONF.register_opts(" + output_opts = subprocess.check_output('{}'.format(cmd_opts), shell = True) + dir_trees_list = output_opts.split() + + cmd_opt = common_string % "CONF.register_opt(" + output_opt = subprocess.check_output('{}'.format(cmd_opt), shell = True) + temp_list = output_opt.split() + + for item in temp_list: + dir_trees_list.append(item) + dir_trees_list.sort() + + flag = False + + for atree in dir_trees_list: + + if atree == "cinder/config/generate_cinder_opts.py": + continue + + dirs_list = atree.split('/') + + import_module = "from " + init_import_module = "" + import_name = "" + + for dir in dirs_list: + if dir.find(".py") == -1: + import_module += dir + "." + init_import_module += dir + "." + import_name += dir + "_" + else: + if dir[:-3] != "__init__": + import_name += dir[:-3].replace("_", "") + import_module = (import_module[:-1] + " import " + + dir[:-3] + " as " + import_name) + opt_file.write(import_module + "\n") + else: + import_name = import_name[:-1].replace('/', '.') + init_import = atree[:-12].replace('/', '.') + opt_file.write("import " + init_import + "\n") + flag = True + if flag is False: + opt_dict[import_name] = atree + else: + opt_dict[init_import_module.strip(".")] = atree + + flag = False + + registered_opts_dict = {'fc-zone-manager': [], + 'keymgr': [], + 'BRCD_FABRIC_EXAMPLE': [], + 'CISCO_FABRIC_EXAMPLE': [], + 'profiler': [], + 'DEFAULT': [], } + + def _write_item(opts): + list_name = opts[-3:] + if list_name.lower() == "opts": + opt_file.write(" [" + opts.strip("\n") + "],\n") + else: + opt_file.write(" " + opts.strip("\n") + ",\n") + + for key in opt_dict: + fd = os.open(opt_dict[key], os.O_RDONLY) + afile = os.fdopen(fd, "r") + + for aline in afile: + exists = aline.find("CONF.register_opts(") + if exists != -1: + # TODO(kjnelson) FIX THIS LATER. These are instances where + # CONF.register_opts is happening without actually registering + # real lists of opts + + exists = aline.find('base_san_opts') + if (exists != -1) or (key == 'cinder_volume_configuration'): + continue + + if aline.find("fc-zone-manager") != -1: + fc_zm_list = aline.replace("CONF.register_opts(", '') + fc_zm_list = fc_zm_list.replace(", 'fc-zone-manager')", '') + fc_zm_list.strip() + line = key + "." + fc_zm_list + registered_opts_dict['fc-zone-manager'].append(line) + elif aline.find("keymgr") != -1: + keymgr_list = aline.replace("CONF.register_opts(", '') + keymgr_list = keymgr_list.replace(", group='keymgr')", '') + keymgr_list = keymgr_list.replace(", 'keymgr')", '') + keymgr_list.strip() + line = key + "." + keymgr_list + registered_opts_dict['keymgr'].append(line) + elif aline.find("BRCD_FABRIC_EXAMPLE") != -1: + brcd_list = aline.replace("CONF.register_opts(", '') + replace_string = ", 'BRCD_FABRIC_EXAMPLE')" + brcd_list = brcd_list.replace(replace_string, '') + brcd_list.strip() + line = key + "." + brcd_list + registered_opts_dict['BRCD_FABRIC_EXAMPLE'].append(line) + elif aline.find("CISCO_FABRIC_EXAMPLE") != -1: + cisco_list = aline.replace("CONF.register_opts(", '') + replace_string = ", 'CISCO_FABRIC_EXAMPLE')" + cisco_list = cisco_list.replace(replace_string, '') + cisco_list.strip() + line = key + "." + cisco_list + registered_opts_dict['CISCO_FABRIC_EXAMPLE'].append(line) + elif aline.find("profiler") != -1: + profiler_list = aline.replace("CONF.register_opts(", '') + replace_string = ', group="profiler")' + profiler_list = profiler_list.replace(replace_string, '') + profiler_list.strip() + line = key + "." + profiler_list + registered_opts_dict['profiler'].append(line) + else: + default_list = aline.replace("CONF.register_opts(", '') + default_list = default_list.replace(')', '').strip() + line = key + "." + default_list + registered_opts_dict['DEFAULT'].append(line) + opt_dict[key] = registered_opts_dict + + list_str = ("def list_opts():\n" + " return [\n" + " ('DEFAULT',\n" + " itertools.chain(\n") + opt_file.write(list_str) + + for item in registered_opts_dict["DEFAULT"]: + _write_item(item) + + profiler_str = (" )),\n" + " ('profiler',\n" + " itertools.chain(\n") + opt_file.write(profiler_str) + + for item in registered_opts_dict["profiler"]: + _write_item(item) + + cisco_str = (" )),\n" + " ('CISCO_FABRIC_EXAMPLE',\n" + " itertools.chain(\n") + opt_file.write(cisco_str) + + for item in registered_opts_dict["CISCO_FABRIC_EXAMPLE"]: + _write_item(item) + + brcd_str = (" )),\n" + " ('BRCD_FABRIC_EXAMPLE',\n" + " itertools.chain(\n") + opt_file.write(brcd_str) + + for item in registered_opts_dict["BRCD_FABRIC_EXAMPLE"]: + _write_item(item) + + keymgr_str = (" )),\n" + " ('keymgr',\n" + " itertools.chain(\n") + opt_file.write(keymgr_str) + + for item in registered_opts_dict["keymgr"]: + _write_item(item) + + fczm_str = (" )),\n" + " ('fc-zone-manager',\n" + " itertools.chain(\n") + opt_file.write(fczm_str) + + for item in registered_opts_dict["fc-zone-manager"]: + _write_item(item) + + closing_str = (" )),\n" + "]\n\n\n") + opt_file.write(closing_str) + opt_file.close() diff --git a/setup.cfg b/setup.cfg index c7f7e5e26..d15b0967b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,6 +42,12 @@ cinder.scheduler.weights = ChanceWeigher = cinder.scheduler.weights.chance:ChanceWeigher GoodnessWeigher = cinder.scheduler.weights.goodness:GoodnessWeigher VolumeNumberWeigher = cinder.scheduler.weights.volume_number:VolumeNumberWeigher +oslo.config.opts = + cinder = cinder.opts:list_opts + keystonemiddleware = keystonemiddleware.auth_token:list_opts + oslo_concurrency = oslo_concurrency.opts:list_opts + oslo.messaging = oslo_messaging.opts:list_opts + oslo.db.concurrency = oslo.db.concurrency:list_opts console_scripts = cinder-all = cinder.cmd.all:main cinder-api = cinder.cmd.api:main diff --git a/tools/config/check_uptodate.sh b/tools/config/check_uptodate.sh index 89a4a3ec2..5b50e1fef 100755 --- a/tools/config/check_uptodate.sh +++ b/tools/config/check_uptodate.sh @@ -12,33 +12,15 @@ CFGFILE_NAME=${PROJECT_NAME}.conf.sample TEMPDIR=`mktemp -d /tmp/${PROJECT_NAME}.XXXXXX` trap "rm -rf $TEMPDIR" EXIT -tools/config/generate_sample.sh -b ./ -p ${PROJECT_NAME} -o ${TEMPDIR} - -# generate_sample.sh may return 0 even when it fails. - -if [ $CHECKONLY -eq 1 ]; then - # Check whether something was generated. - if [ ! -s ${TEMPDIR}/${CFGFILE_NAME} ]; then - echo "Failed to generate ${CFGFILE_NAME}." - exit 1 - fi +tools/config/generate_sample.sh from_tox + +if [ -e etc/${PROJECT_NAME}/${CFGFILE_NAME} ]; then + CFGFILE=etc/${PROJECT_NAME}/${CFGFILE_NAME} +elif [ -e cinder/opts.py]; then + echo -en "\n\nWARNING: Found cinder/opts.py file. \n" + echo -en "Check for generate_cinder_opts.py failure.\n\n" + exit 1 else - if [ -e etc/${PROJECT_NAME}/${CFGFILE_NAME} ]; then - CFGFILE=etc/${PROJECT_NAME}/${CFGFILE_NAME} - elif [ -e etc/${CFGFILE_NAME} ]; then - CFGFILE=etc/${CFGFILE_NAME} - else - echo "${0##*/}: can not find config file" - exit 1 - fi - - if ! diff -u ${TEMPDIR}/${CFGFILE_NAME} ${CFGFILE} - then - echo "${0##*/}: ${PROJECT_NAME}.conf.sample is not up to date." - echo "${0##*/}: Please run ${0%%${0##*/}}generate_sample.sh from within a VENV." - echo " \'source .venv/bin/activate; generate_sample.sh\'" - echo "OR simply run tox genconfig" - echo " \'tox -egenconfig\'" - exit 1 - fi + echo "${0##*/}: Can't find config file." + exit 1 fi diff --git a/tools/config/generate_sample.sh b/tools/config/generate_sample.sh index 842777b36..652c834c9 100755 --- a/tools/config/generate_sample.sh +++ b/tools/config/generate_sample.sh @@ -10,66 +10,30 @@ # CINDER_CONFIG_GENERATOR_EXTRA_LIBRARIES: list of libraries to discover. # CINDER_CONFIG_GENERATOR_EXCLUDED_FILES: list of files to remove from automatic listing. -print_hint() { - echo "Try \`${0##*/} --help' for more information." >&2 -} - -PARSED_OPTIONS=$(getopt -n "${0##*/}" -o hb:p:m:l:o: \ - --long help,base-dir:,package-name:,output-dir:,module:,library: -- "$@") +BASEDIR=${BASEDIR:-`pwd`} -if [ $? != 0 ] ; then print_hint ; exit 1 ; fi +print_error () +{ + echo -en "\n\n##########################################################" + echo -en "\nERROR: ${0} was not called from tox." + echo -en "\n Execute 'tox -e genconfig' for cinder.conf.sample" + echo -en "\n generation." + echo -en "\n##########################################################\n\n" +} -eval set -- "$PARSED_OPTIONS" +if [ -z ${1} ] ; then + print_error + exit 1 +fi -while true; do - case "$1" in - -h|--help) - echo "${0##*/} [options]" - echo "" - echo "options:" - echo "-h, --help show brief help" - echo "-b, --base-dir=DIR project base directory" - echo "-p, --package-name=NAME project package name" - echo "-o, --output-dir=DIR file output directory" - echo "-m, --module=MOD extra python module to interrogate for options" - echo "-l, --library=LIB extra library that registers options for discovery" - exit 0 - ;; - -b|--base-dir) - shift - BASEDIR=`echo $1 | sed -e 's/\/*$//g'` - shift - ;; - -p|--package-name) - shift - PACKAGENAME=`echo $1` - shift - ;; - -o|--output-dir) - shift - OUTPUTDIR=`echo $1 | sed -e 's/\/*$//g'` - shift - ;; - -m|--module) - shift - MODULES="$MODULES -m $1" - shift - ;; - -l|--library) - shift - LIBRARIES="$LIBRARIES -l $1" - shift - ;; - --) - break - ;; - esac -done +if [ ${1} != "from_tox" ] ; then + print_error + exit 1 +fi -BASEDIR=${BASEDIR:-`pwd`} if ! [ -d $BASEDIR ] then - echo "${0##*/}: missing project base directory" >&2 ; print_hint ; exit 1 + echo "${0##*/}: missing project base directory" >&2 ; exit 1 elif [[ $BASEDIR != /* ]] then BASEDIR=$(cd "$BASEDIR" && pwd) @@ -77,84 +41,44 @@ fi PACKAGENAME=${PACKAGENAME:-$(python setup.py --name)} TARGETDIR=$BASEDIR/$PACKAGENAME -if ! [ -d $TARGETDIR ] -then - echo "${0##*/}: invalid project package name" >&2 ; print_hint ; exit 1 -fi - -OUTPUTDIR=${OUTPUTDIR:-$BASEDIR/etc} -# NOTE(bnemec): Some projects put their sample config in etc/, -# some in etc/$PACKAGENAME/ -if [ -d $OUTPUTDIR/$PACKAGENAME ] -then - OUTPUTDIR=$OUTPUTDIR/$PACKAGENAME -elif ! [ -d $OUTPUTDIR ] -then - echo "${0##*/}: cannot access \`$OUTPUTDIR': No such file or directory" >&2 - exit 1 +if ! [ -d $TARGETDIR ] ; then + echo "${0##*/}: invalid project package name" >&2 ; exit 1 fi BASEDIRESC=`echo $BASEDIR | sed -e 's/\//\\\\\//g'` find $TARGETDIR -type f -name "*.pyc" -delete -FILES=$(find $TARGETDIR -type f -name "*.py" ! -path "*/tests/*" \ - -exec grep -l "Opt(" {} + | sed -e "s/^$BASEDIRESC\///g" | sort -u) - -RC_FILE="`dirname $0`/oslo.config.generator.rc" -if test -r "$RC_FILE" -then - source "$RC_FILE" -fi -for filename in ${CINDER_CONFIG_GENERATOR_EXCLUDED_FILES}; do - FILES="${FILES[@]/$filename/}" -done +export TARGETDIR=$TARGETDIR +export BASEDIRESC=$BASEDIRESC -for mod in ${CINDER_CONFIG_GENERATOR_EXTRA_MODULES}; do - MODULES="$MODULES -m $mod" -done +python cinder/config/generate_cinder_opts.py -for lib in ${CINDER_CONFIG_GENERATOR_EXTRA_LIBRARIES}; do - LIBRARIES="$LIBRARIES -l $lib" -done +if [ $? -ne 0 ] +then + echo -en "\n\n#################################################" + echo -en "\nERROR: Non-zero exit from generate_cinder_opts.py." + echo -en "\n See output above for details.\n" + echo -en "#################################################\n" + exit 1 +fi -export EVENTLET_NO_GREENDNS=yes +oslo-config-generator --config-file=cinder/config/cinder-config-generator.conf -OS_VARS=$(set | sed -n '/^OS_/s/=[^=]*$//gp' | xargs) -[ "$OS_VARS" ] && eval "unset \$OS_VARS" -DEFAULT_MODULEPATH=cinder.openstack.common.config.generator -MODULEPATH=${MODULEPATH:-$DEFAULT_MODULEPATH} -OUTPUTFILE=$OUTPUTDIR/$PACKAGENAME.conf.sample -python -m $MODULEPATH $MODULES $LIBRARIES $FILES > $OUTPUTFILE -if [ $? != 0 ] +if [ $? -ne 0 ] then - echo "Can not generate $OUTPUTFILE" + echo -en "\n\n#################################################" + echo -en "\nERROR: Non-zero exit from oslo-config-generator." + echo -en "\n See output above for details.\n" + echo -en "#################################################\n" + exit 1 +fi +if [ ! -s ./etc/cinder/cinder.conf.sample ] ; then + echo -en "\n\n#########################################################" + echo -en "\nERROR: etc/cinder/cinder.sample.conf not created properly." + echo -en "\n See above output for details.\n" + echo -en "###########################################################\n" exit 1 fi -# Hook to allow projects to append custom config file snippets -CONCAT_FILES=$(ls $BASEDIR/tools/config/*.conf.sample 2>/dev/null) -for CONCAT_FILE in $CONCAT_FILES; do - cat $CONCAT_FILE >> $OUTPUTFILE -done - -# NOTE(jsbryant): We collect the lib config options separately and -# append them. The generator requires that the name of the library be used -# as the entry point so we need to use oslo.*, not oslo_* . -oslo-config-generator \ ---namespace oslo.concurrency \ ---namespace oslo.config \ ---namespace oslo.context \ ---namespace oslo.log \ ---namespace oslo.serialization \ ---namespace oslo.utils \ ---namespace oslo.db \ ---namespace oslo.rootwrap \ ---namespace oslo.messaging \ ---namespace oslo.i18n \ ---namespace oslo.middleware \ ---namespace oslo.service.service \ ---namespace oslo.service.periodic_task \ ---namespace policy \ ---namespace oslo.policy \ ---namespace oslo.db.concurrency \ ---namespace keystonemiddleware.auth_token | grep -v '^\[DEFAULT\]' >> $OUTPUTFILE +rm -f cinder/opts.py +rm -f cinder/opts.pyc diff --git a/tools/config/oslo.config.generator.rc b/tools/config/oslo.config.generator.rc deleted file mode 100644 index 3d60f7eec..000000000 --- a/tools/config/oslo.config.generator.rc +++ /dev/null @@ -1 +0,0 @@ -export CINDER_CONFIG_GENERATOR_EXTRA_MODULES="keystonemiddleware.auth_token" diff --git a/tox.ini b/tox.ini index 4eea5df4c..9343669b4 100644 --- a/tox.ini +++ b/tox.ini @@ -111,7 +111,7 @@ commands = flake8 {posargs} . cinder/common # Check that .po and .pot files are valid: bash -c "find cinder -type f -regex '.*\.pot?' -print0|xargs -0 -n 1 msgfmt --check-format -o /dev/null" - {toxinidir}/tools/config/check_uptodate.sh --checkonly + {toxinidir}/tools/config/check_uptodate.sh {toxinidir}/tools/check_exec.py {toxinidir}/cinder [testenv:pylint] @@ -129,7 +129,7 @@ commands = [testenv:genconfig] sitepackages = False envdir = {toxworkdir}/venv -commands = {toxinidir}/tools/config/generate_sample.sh -b . -p cinder -o etc/cinder +commands = {toxinidir}/tools/config/generate_sample.sh from_tox [testenv:venv] commands = {posargs} -- 2.45.2