As it stands, the opts.py file that is passed into
oslo-config-generator isn't being generated dynamically
and the old way of generating the cinder.conf.sample is
dependent on oslo-incubator which Cinder is trying to
move away from. oslo-config-generator works differently
than oslo-incubator so a number of changes had to be made
in order to make this switch.
This patch adds the config directory to Cinder and in it
are two files:
-generate_cinder_opts.py that will take the
results of a grep command to create the opts.py
file to be passed into oslo-config-generator.
-cinder.conf which is the new configuration for
oslo-config-generator. The file is inside the config
directory to be consistent with other projects.
Some changes were made to the generate_sample.sh file in
order to give the base directories and target directories
to the generate_cinder_opts.py program.
tox.ini was edited to remove the checkonly option because
all that needs to happen in check_uptodate.sh is a check to
ensure that the cinder.conf.sample is actually being
generated with no issues.
All options were removed from the check_uptodate.sh
because they were unnecessary given the new, more simple
way of generating the cinder.conf.sample.
setup.cfg was also edited in order to add information
oslo-config-generator needs to run.
Co-Authored By: Jay Bryant <jsbryant@us.ibm.com>
Co-Authored By: Jacob Gregor <jgregor@us.ibm.com>
Change-Id: I643dbe5675ae9280e204f691781e617266f570d5
Closes-Bug:
1473768
Closes-Bug:
1437904
Closes-Bug:
1381563
--- /dev/null
+[DEFAULT]
+output_file = etc/cinder/cinder.conf.sample
+wrap_width = 79
+namespace = cinder
+namespace = keystonemiddleware.auth_token
+namespace = oslo.config
+namespace = oslo.concurrency
+namespace = oslo.context
+namespace = oslo.db
+namesapce = oslo.i18n
+namespace = oslo.log
+namespace = oslo.messaging
+namespace = oslo.middleware
+namespace = oslo.policy
+namespace = oslo.reports
+namespace = oslo.rootwrap
+namespace = oslo.serialization
+namespace = oslo.service
+namespace = oslo.utils
+namespace = oslo.versionedobjects
+namespace = oslo.vmware
--- /dev/null
+#!/usr/bin/env python
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+
+if __name__ == "__main__":
+ opt_file = open("cinder/opts.py", 'a')
+ opt_dict = {}
+ dir_trees_list = []
+
+ opt_file.write("import copy\n")
+ opt_file.write("import itertools\n\n")
+
+ targetdir = os.environ['TARGETDIR']
+ basedir = os.environ['BASEDIRESC']
+
+ common_string = ('find ' + targetdir + ' -type f -name "*.py" ! '
+ '-path "*/tests/*" -exec grep -l "%s" {} '
+ '+ | sed -e "s/^' + basedir +
+ '\///g" | sort -u')
+
+ cmd_opts = common_string % "CONF.register_opts("
+ output_opts = subprocess.check_output('{}'.format(cmd_opts), shell = True)
+ dir_trees_list = output_opts.split()
+
+ cmd_opt = common_string % "CONF.register_opt("
+ output_opt = subprocess.check_output('{}'.format(cmd_opt), shell = True)
+ temp_list = output_opt.split()
+
+ for item in temp_list:
+ dir_trees_list.append(item)
+ dir_trees_list.sort()
+
+ flag = False
+
+ for atree in dir_trees_list:
+
+ if atree == "cinder/config/generate_cinder_opts.py":
+ continue
+
+ dirs_list = atree.split('/')
+
+ import_module = "from "
+ init_import_module = ""
+ import_name = ""
+
+ for dir in dirs_list:
+ if dir.find(".py") == -1:
+ import_module += dir + "."
+ init_import_module += dir + "."
+ import_name += dir + "_"
+ else:
+ if dir[:-3] != "__init__":
+ import_name += dir[:-3].replace("_", "")
+ import_module = (import_module[:-1] + " import " +
+ dir[:-3] + " as " + import_name)
+ opt_file.write(import_module + "\n")
+ else:
+ import_name = import_name[:-1].replace('/', '.')
+ init_import = atree[:-12].replace('/', '.')
+ opt_file.write("import " + init_import + "\n")
+ flag = True
+ if flag is False:
+ opt_dict[import_name] = atree
+ else:
+ opt_dict[init_import_module.strip(".")] = atree
+
+ flag = False
+
+ registered_opts_dict = {'fc-zone-manager': [],
+ 'keymgr': [],
+ 'BRCD_FABRIC_EXAMPLE': [],
+ 'CISCO_FABRIC_EXAMPLE': [],
+ 'profiler': [],
+ 'DEFAULT': [], }
+
+ def _write_item(opts):
+ list_name = opts[-3:]
+ if list_name.lower() == "opts":
+ opt_file.write(" [" + opts.strip("\n") + "],\n")
+ else:
+ opt_file.write(" " + opts.strip("\n") + ",\n")
+
+ for key in opt_dict:
+ fd = os.open(opt_dict[key], os.O_RDONLY)
+ afile = os.fdopen(fd, "r")
+
+ for aline in afile:
+ exists = aline.find("CONF.register_opts(")
+ if exists != -1:
+ # TODO(kjnelson) FIX THIS LATER. These are instances where
+ # CONF.register_opts is happening without actually registering
+ # real lists of opts
+
+ exists = aline.find('base_san_opts')
+ if (exists != -1) or (key == 'cinder_volume_configuration'):
+ continue
+
+ if aline.find("fc-zone-manager") != -1:
+ fc_zm_list = aline.replace("CONF.register_opts(", '')
+ fc_zm_list = fc_zm_list.replace(", 'fc-zone-manager')", '')
+ fc_zm_list.strip()
+ line = key + "." + fc_zm_list
+ registered_opts_dict['fc-zone-manager'].append(line)
+ elif aline.find("keymgr") != -1:
+ keymgr_list = aline.replace("CONF.register_opts(", '')
+ keymgr_list = keymgr_list.replace(", group='keymgr')", '')
+ keymgr_list = keymgr_list.replace(", 'keymgr')", '')
+ keymgr_list.strip()
+ line = key + "." + keymgr_list
+ registered_opts_dict['keymgr'].append(line)
+ elif aline.find("BRCD_FABRIC_EXAMPLE") != -1:
+ brcd_list = aline.replace("CONF.register_opts(", '')
+ replace_string = ", 'BRCD_FABRIC_EXAMPLE')"
+ brcd_list = brcd_list.replace(replace_string, '')
+ brcd_list.strip()
+ line = key + "." + brcd_list
+ registered_opts_dict['BRCD_FABRIC_EXAMPLE'].append(line)
+ elif aline.find("CISCO_FABRIC_EXAMPLE") != -1:
+ cisco_list = aline.replace("CONF.register_opts(", '')
+ replace_string = ", 'CISCO_FABRIC_EXAMPLE')"
+ cisco_list = cisco_list.replace(replace_string, '')
+ cisco_list.strip()
+ line = key + "." + cisco_list
+ registered_opts_dict['CISCO_FABRIC_EXAMPLE'].append(line)
+ elif aline.find("profiler") != -1:
+ profiler_list = aline.replace("CONF.register_opts(", '')
+ replace_string = ', group="profiler")'
+ profiler_list = profiler_list.replace(replace_string, '')
+ profiler_list.strip()
+ line = key + "." + profiler_list
+ registered_opts_dict['profiler'].append(line)
+ else:
+ default_list = aline.replace("CONF.register_opts(", '')
+ default_list = default_list.replace(')', '').strip()
+ line = key + "." + default_list
+ registered_opts_dict['DEFAULT'].append(line)
+ opt_dict[key] = registered_opts_dict
+
+ list_str = ("def list_opts():\n"
+ " return [\n"
+ " ('DEFAULT',\n"
+ " itertools.chain(\n")
+ opt_file.write(list_str)
+
+ for item in registered_opts_dict["DEFAULT"]:
+ _write_item(item)
+
+ profiler_str = (" )),\n"
+ " ('profiler',\n"
+ " itertools.chain(\n")
+ opt_file.write(profiler_str)
+
+ for item in registered_opts_dict["profiler"]:
+ _write_item(item)
+
+ cisco_str = (" )),\n"
+ " ('CISCO_FABRIC_EXAMPLE',\n"
+ " itertools.chain(\n")
+ opt_file.write(cisco_str)
+
+ for item in registered_opts_dict["CISCO_FABRIC_EXAMPLE"]:
+ _write_item(item)
+
+ brcd_str = (" )),\n"
+ " ('BRCD_FABRIC_EXAMPLE',\n"
+ " itertools.chain(\n")
+ opt_file.write(brcd_str)
+
+ for item in registered_opts_dict["BRCD_FABRIC_EXAMPLE"]:
+ _write_item(item)
+
+ keymgr_str = (" )),\n"
+ " ('keymgr',\n"
+ " itertools.chain(\n")
+ opt_file.write(keymgr_str)
+
+ for item in registered_opts_dict["keymgr"]:
+ _write_item(item)
+
+ fczm_str = (" )),\n"
+ " ('fc-zone-manager',\n"
+ " itertools.chain(\n")
+ opt_file.write(fczm_str)
+
+ for item in registered_opts_dict["fc-zone-manager"]:
+ _write_item(item)
+
+ closing_str = (" )),\n"
+ "]\n\n\n")
+ opt_file.write(closing_str)
+ opt_file.close()
ChanceWeigher = cinder.scheduler.weights.chance:ChanceWeigher
GoodnessWeigher = cinder.scheduler.weights.goodness:GoodnessWeigher
VolumeNumberWeigher = cinder.scheduler.weights.volume_number:VolumeNumberWeigher
+oslo.config.opts =
+ cinder = cinder.opts:list_opts
+ keystonemiddleware = keystonemiddleware.auth_token:list_opts
+ oslo_concurrency = oslo_concurrency.opts:list_opts
+ oslo.messaging = oslo_messaging.opts:list_opts
+ oslo.db.concurrency = oslo.db.concurrency:list_opts
console_scripts =
cinder-all = cinder.cmd.all:main
cinder-api = cinder.cmd.api:main
TEMPDIR=`mktemp -d /tmp/${PROJECT_NAME}.XXXXXX`
trap "rm -rf $TEMPDIR" EXIT
-tools/config/generate_sample.sh -b ./ -p ${PROJECT_NAME} -o ${TEMPDIR}
-
-# generate_sample.sh may return 0 even when it fails.
-
-if [ $CHECKONLY -eq 1 ]; then
- # Check whether something was generated.
- if [ ! -s ${TEMPDIR}/${CFGFILE_NAME} ]; then
- echo "Failed to generate ${CFGFILE_NAME}."
- exit 1
- fi
+tools/config/generate_sample.sh from_tox
+
+if [ -e etc/${PROJECT_NAME}/${CFGFILE_NAME} ]; then
+ CFGFILE=etc/${PROJECT_NAME}/${CFGFILE_NAME}
+elif [ -e cinder/opts.py]; then
+ echo -en "\n\nWARNING: Found cinder/opts.py file. \n"
+ echo -en "Check for generate_cinder_opts.py failure.\n\n"
+ exit 1
else
- if [ -e etc/${PROJECT_NAME}/${CFGFILE_NAME} ]; then
- CFGFILE=etc/${PROJECT_NAME}/${CFGFILE_NAME}
- elif [ -e etc/${CFGFILE_NAME} ]; then
- CFGFILE=etc/${CFGFILE_NAME}
- else
- echo "${0##*/}: can not find config file"
- exit 1
- fi
-
- if ! diff -u ${TEMPDIR}/${CFGFILE_NAME} ${CFGFILE}
- then
- echo "${0##*/}: ${PROJECT_NAME}.conf.sample is not up to date."
- echo "${0##*/}: Please run ${0%%${0##*/}}generate_sample.sh from within a VENV."
- echo " \'source .venv/bin/activate; generate_sample.sh\'"
- echo "OR simply run tox genconfig"
- echo " \'tox -egenconfig\'"
- exit 1
- fi
+ echo "${0##*/}: Can't find config file."
+ exit 1
fi
# CINDER_CONFIG_GENERATOR_EXTRA_LIBRARIES: list of libraries to discover.
# CINDER_CONFIG_GENERATOR_EXCLUDED_FILES: list of files to remove from automatic listing.
-print_hint() {
- echo "Try \`${0##*/} --help' for more information." >&2
-}
-
-PARSED_OPTIONS=$(getopt -n "${0##*/}" -o hb:p:m:l:o: \
- --long help,base-dir:,package-name:,output-dir:,module:,library: -- "$@")
+BASEDIR=${BASEDIR:-`pwd`}
-if [ $? != 0 ] ; then print_hint ; exit 1 ; fi
+print_error ()
+{
+ echo -en "\n\n##########################################################"
+ echo -en "\nERROR: ${0} was not called from tox."
+ echo -en "\n Execute 'tox -e genconfig' for cinder.conf.sample"
+ echo -en "\n generation."
+ echo -en "\n##########################################################\n\n"
+}
-eval set -- "$PARSED_OPTIONS"
+if [ -z ${1} ] ; then
+ print_error
+ exit 1
+fi
-while true; do
- case "$1" in
- -h|--help)
- echo "${0##*/} [options]"
- echo ""
- echo "options:"
- echo "-h, --help show brief help"
- echo "-b, --base-dir=DIR project base directory"
- echo "-p, --package-name=NAME project package name"
- echo "-o, --output-dir=DIR file output directory"
- echo "-m, --module=MOD extra python module to interrogate for options"
- echo "-l, --library=LIB extra library that registers options for discovery"
- exit 0
- ;;
- -b|--base-dir)
- shift
- BASEDIR=`echo $1 | sed -e 's/\/*$//g'`
- shift
- ;;
- -p|--package-name)
- shift
- PACKAGENAME=`echo $1`
- shift
- ;;
- -o|--output-dir)
- shift
- OUTPUTDIR=`echo $1 | sed -e 's/\/*$//g'`
- shift
- ;;
- -m|--module)
- shift
- MODULES="$MODULES -m $1"
- shift
- ;;
- -l|--library)
- shift
- LIBRARIES="$LIBRARIES -l $1"
- shift
- ;;
- --)
- break
- ;;
- esac
-done
+if [ ${1} != "from_tox" ] ; then
+ print_error
+ exit 1
+fi
-BASEDIR=${BASEDIR:-`pwd`}
if ! [ -d $BASEDIR ]
then
- echo "${0##*/}: missing project base directory" >&2 ; print_hint ; exit 1
+ echo "${0##*/}: missing project base directory" >&2 ; exit 1
elif [[ $BASEDIR != /* ]]
then
BASEDIR=$(cd "$BASEDIR" && pwd)
PACKAGENAME=${PACKAGENAME:-$(python setup.py --name)}
TARGETDIR=$BASEDIR/$PACKAGENAME
-if ! [ -d $TARGETDIR ]
-then
- echo "${0##*/}: invalid project package name" >&2 ; print_hint ; exit 1
-fi
-
-OUTPUTDIR=${OUTPUTDIR:-$BASEDIR/etc}
-# NOTE(bnemec): Some projects put their sample config in etc/,
-# some in etc/$PACKAGENAME/
-if [ -d $OUTPUTDIR/$PACKAGENAME ]
-then
- OUTPUTDIR=$OUTPUTDIR/$PACKAGENAME
-elif ! [ -d $OUTPUTDIR ]
-then
- echo "${0##*/}: cannot access \`$OUTPUTDIR': No such file or directory" >&2
- exit 1
+if ! [ -d $TARGETDIR ] ; then
+ echo "${0##*/}: invalid project package name" >&2 ; exit 1
fi
BASEDIRESC=`echo $BASEDIR | sed -e 's/\//\\\\\//g'`
find $TARGETDIR -type f -name "*.pyc" -delete
-FILES=$(find $TARGETDIR -type f -name "*.py" ! -path "*/tests/*" \
- -exec grep -l "Opt(" {} + | sed -e "s/^$BASEDIRESC\///g" | sort -u)
-
-RC_FILE="`dirname $0`/oslo.config.generator.rc"
-if test -r "$RC_FILE"
-then
- source "$RC_FILE"
-fi
-for filename in ${CINDER_CONFIG_GENERATOR_EXCLUDED_FILES}; do
- FILES="${FILES[@]/$filename/}"
-done
+export TARGETDIR=$TARGETDIR
+export BASEDIRESC=$BASEDIRESC
-for mod in ${CINDER_CONFIG_GENERATOR_EXTRA_MODULES}; do
- MODULES="$MODULES -m $mod"
-done
+python cinder/config/generate_cinder_opts.py
-for lib in ${CINDER_CONFIG_GENERATOR_EXTRA_LIBRARIES}; do
- LIBRARIES="$LIBRARIES -l $lib"
-done
+if [ $? -ne 0 ]
+then
+ echo -en "\n\n#################################################"
+ echo -en "\nERROR: Non-zero exit from generate_cinder_opts.py."
+ echo -en "\n See output above for details.\n"
+ echo -en "#################################################\n"
+ exit 1
+fi
-export EVENTLET_NO_GREENDNS=yes
+oslo-config-generator --config-file=cinder/config/cinder-config-generator.conf
-OS_VARS=$(set | sed -n '/^OS_/s/=[^=]*$//gp' | xargs)
-[ "$OS_VARS" ] && eval "unset \$OS_VARS"
-DEFAULT_MODULEPATH=cinder.openstack.common.config.generator
-MODULEPATH=${MODULEPATH:-$DEFAULT_MODULEPATH}
-OUTPUTFILE=$OUTPUTDIR/$PACKAGENAME.conf.sample
-python -m $MODULEPATH $MODULES $LIBRARIES $FILES > $OUTPUTFILE
-if [ $? != 0 ]
+if [ $? -ne 0 ]
then
- echo "Can not generate $OUTPUTFILE"
+ echo -en "\n\n#################################################"
+ echo -en "\nERROR: Non-zero exit from oslo-config-generator."
+ echo -en "\n See output above for details.\n"
+ echo -en "#################################################\n"
+ exit 1
+fi
+if [ ! -s ./etc/cinder/cinder.conf.sample ] ; then
+ echo -en "\n\n#########################################################"
+ echo -en "\nERROR: etc/cinder/cinder.sample.conf not created properly."
+ echo -en "\n See above output for details.\n"
+ echo -en "###########################################################\n"
exit 1
fi
-# Hook to allow projects to append custom config file snippets
-CONCAT_FILES=$(ls $BASEDIR/tools/config/*.conf.sample 2>/dev/null)
-for CONCAT_FILE in $CONCAT_FILES; do
- cat $CONCAT_FILE >> $OUTPUTFILE
-done
-
-# NOTE(jsbryant): We collect the lib config options separately and
-# append them. The generator requires that the name of the library be used
-# as the entry point so we need to use oslo.*, not oslo_* .
-oslo-config-generator \
---namespace oslo.concurrency \
---namespace oslo.config \
---namespace oslo.context \
---namespace oslo.log \
---namespace oslo.serialization \
---namespace oslo.utils \
---namespace oslo.db \
---namespace oslo.rootwrap \
---namespace oslo.messaging \
---namespace oslo.i18n \
---namespace oslo.middleware \
---namespace oslo.service.service \
---namespace oslo.service.periodic_task \
---namespace policy \
---namespace oslo.policy \
---namespace oslo.db.concurrency \
---namespace keystonemiddleware.auth_token | grep -v '^\[DEFAULT\]' >> $OUTPUTFILE
+rm -f cinder/opts.py
+rm -f cinder/opts.pyc
+++ /dev/null
-export CINDER_CONFIG_GENERATOR_EXTRA_MODULES="keystonemiddleware.auth_token"
flake8 {posargs} . cinder/common
# Check that .po and .pot files are valid:
bash -c "find cinder -type f -regex '.*\.pot?' -print0|xargs -0 -n 1 msgfmt --check-format -o /dev/null"
- {toxinidir}/tools/config/check_uptodate.sh --checkonly
+ {toxinidir}/tools/config/check_uptodate.sh
{toxinidir}/tools/check_exec.py {toxinidir}/cinder
[testenv:pylint]
[testenv:genconfig]
sitepackages = False
envdir = {toxworkdir}/venv
-commands = {toxinidir}/tools/config/generate_sample.sh -b . -p cinder -o etc/cinder
+commands = {toxinidir}/tools/config/generate_sample.sh from_tox
[testenv:venv]
commands = {posargs}
Code Review / openstack-build / cinder-build.git / commitdiff