-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
osapi_compute_extension_opt = cfg.MultiStrOpt('osapi_compute_extension',
default=DEFAULT_EXTENSIONS)
-Option schemas are registered with with the config manager at runtime, but
-before the option is referenced::
+Option schemas are registered with the config manager at runtime, but before
+the option is referenced::
class ExtensionManager(object):
fix_path('~'),
os.path.join('/etc', project) if project else None,
'/etc'
- ]
+ ]
return filter(bool, cfg_dirs)
multi = False
def __init__(self, name, dest=None, short=None, default=None,
- metavar=None, help=None, secret=False, required=False):
+ metavar=None, help=None, secret=False, required=False,
+ deprecated_name=None):
"""Construct an Opt object.
The only required parameter is the option's name. However, it is
:param help: an explanation of how the option is used
:param secret: true iff the value should be obfuscated in log output
:param required: true iff a value must be supplied for this option
+ :param deprecated_name: deprecated name option. Acts like an alias
"""
self.name = name
if dest is None:
self.help = help
self.secret = secret
self.required = required
+ if deprecated_name is not None:
+ self.deprecated_name = deprecated_name.replace('-', '_')
+ else:
+ self.deprecated_name = None
def _get_from_config_parser(self, cparser, section):
"""Retrieves the option value from a MultiConfigParser object.
:param cparser: a ConfigParser object
:param section: a section name
"""
- return cparser.get(section, self.dest)
+ return self._cparser_get_with_deprecated(cparser, section)
+
+ def _cparser_get_with_deprecated(self, cparser, section):
+ """If cannot find option as dest try deprecated_name alias."""
+ if self.deprecated_name is not None:
+ return cparser.get(section, [self.dest, self.deprecated_name])
+ return cparser.get(section, [self.dest])
def _add_to_cli(self, parser, group=None):
"""Makes the option available in the command line interface.
container = self._get_optparse_container(parser, group)
kwargs = self._get_optparse_kwargs(group)
prefix = self._get_optparse_prefix('', group)
- self._add_to_optparse(container, self.name, self.short, kwargs, prefix)
+ self._add_to_optparse(container, self.name, self.short, kwargs, prefix,
+ self.deprecated_name)
- def _add_to_optparse(self, container, name, short, kwargs, prefix=''):
+ def _add_to_optparse(self, container, name, short, kwargs, prefix='',
+ deprecated_name=None):
"""Add an option to an optparse parser or group.
:param container: an optparse.OptionContainer object
args = ['--' + prefix + name]
if short:
args += ['-' + short]
+ if deprecated_name:
+ args += ['--' + prefix + deprecated_name]
for a in args:
if container.has_option(a):
raise DuplicateOptError(a)
dest = self.dest
if group is not None:
dest = group.name + '_' + dest
- kwargs.update({
- 'dest': dest,
- 'metavar': self.metavar,
- 'help': self.help,
- })
+ kwargs.update({'dest': dest,
+ 'metavar': self.metavar,
+ 'help': self.help, })
return kwargs
def _get_optparse_prefix(self, prefix, group):
return value
- return [convert_bool(v) for v in cparser.get(section, self.dest)]
+ return [convert_bool(v) for v in
+ self._cparser_get_with_deprecated(cparser, section)]
def _add_to_cli(self, parser, group=None):
"""Extends the base class method to add the --nooptname option."""
kwargs = self._get_optparse_kwargs(group, action='store_false')
prefix = self._get_optparse_prefix('no', group)
kwargs["help"] = "The inverse of --" + self.name
- self._add_to_optparse(container, self.name, None, kwargs, prefix)
+ self._add_to_optparse(container, self.name, None, kwargs, prefix,
+ self.deprecated_name)
def _get_optparse_kwargs(self, group, action='store_true', **kwargs):
"""Extends the base optparse keyword dict for boolean options."""
def _get_from_config_parser(self, cparser, section):
"""Retrieve the opt value as a integer from ConfigParser."""
- return [int(v) for v in cparser.get(section, self.dest)]
+ return [int(v) for v in self._cparser_get_with_deprecated(cparser,
+ section)]
def _get_optparse_kwargs(self, group, **kwargs):
"""Extends the base optparse keyword dict for integer options."""
def _get_from_config_parser(self, cparser, section):
"""Retrieve the opt value as a float from ConfigParser."""
- return [float(v) for v in cparser.get(section, self.dest)]
+ return [float(v) for v in
+ self._cparser_get_with_deprecated(cparser, section)]
def _get_optparse_kwargs(self, group, **kwargs):
"""Extends the base optparse keyword dict for float options."""
def _get_from_config_parser(self, cparser, section):
"""Retrieve the opt value as a list from ConfigParser."""
- return [v.split(',') for v in cparser.get(section, self.dest)]
+ return [v.split(',') for v in
+ self._cparser_get_with_deprecated(cparser, section)]
def _get_optparse_kwargs(self, group, **kwargs):
"""Extends the base optparse keyword dict for list options."""
return super(MultiStrOpt,
self)._get_optparse_kwargs(group, action='append')
+ def _cparser_get_with_deprecated(self, cparser, section):
+ """If cannot find option as dest try deprecated_name alias."""
+ if self.deprecated_name is not None:
+ return cparser.get(section, [self.dest, self.deprecated_name],
+ multi=True)
+ return cparser.get(section, [self.dest], multi=True)
+
class OptGroup(object):
class MultiConfigParser(object):
def __init__(self):
- self.sections = {}
+ self.parsed = []
def read(self, config_files):
read_ok = []
for filename in config_files:
- parser = ConfigParser(filename, self.sections)
+ sections = {}
+ parser = ConfigParser(filename, sections)
try:
parser.parse()
except IOError:
continue
-
+ self.parsed.insert(0, sections)
read_ok.append(filename)
return read_ok
- def get(self, section, name):
- return self.sections[section][name]
+ def get(self, section, names, multi=False):
+ rvalue = []
+ for sections in self.parsed:
+ if section not in sections:
+ continue
+ for name in names:
+ if name in sections[section]:
+ if multi:
+ rvalue = sections[section][name] + rvalue
+ else:
+ return sections[section][name]
+ if multi and rvalue != []:
+ return rvalue
+ raise KeyError
class ConfigOpts(collections.Mapping):
self._oparser.disable_interspersed_args()
self._config_opts = [
- MultiStrOpt('config-file',
- default=default_config_files,
- metavar='PATH',
- help='Path to a config file to use. Multiple config '
- 'files can be specified, with values in later '
- 'files taking precedence. The default files '
- ' used are: %s' % (default_config_files, )),
+ MultiStrOpt('config-file',
+ default=default_config_files,
+ metavar='PATH',
+ help='Path to a config file to use. Multiple config '
+ 'files can be specified, with values in later '
+ 'files taking precedence. The default files '
+ ' used are: %s' % (default_config_files, )),
StrOpt('config-dir',
metavar='DIR',
help='Path to a config directory to pull *.conf '
'the file(s), if any, specified via --config-file, '
'hence over-ridden options in the directory take '
'precedence.'),
- ]
+ ]
self.register_cli_opts(self._config_opts)
self.project = project
def _substitute(self, value):
"""Perform string template substitution.
- Substititue any template variables (e.g. $foo, ${bar}) in the supplied
+ Substitute any template variables (e.g. $foo, ${bar}) in the supplied
string value(s) with opt values.
:param value: the string value, or list of string values
default, opt, override = [info[k] for k in sorted(info.keys())]
if opt.required:
- if (default is not None or
- override is not None):
+ if (default is not None or override is not None):
continue
if self._get(opt.name, group) is None:
short='v',
default=False,
help='Print more verbose output'),
- ]
+ ]
logging_cli_opts = [
StrOpt('log-config',
StrOpt('syslog-log-facility',
default='LOG_USER',
help='syslog facility to receive log lines')
- ]
+ ]
def __init__(self):
super(CommonConfigOpts, self).__init__()
Exceptions common to OpenStack projects
"""
+import itertools
import logging
--- /dev/null
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2012 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+gettext for openstack-common modules.
+
+Usual usage in an openstack.common module:
+
+ from openstack.common.gettextutils import _
+"""
+
+import gettext
+
+
+t = gettext.translation('openstack-common', 'locale', fallback=True)
+
+
+def _(msg):
+ return t.ugettext(msg)
"""
import sys
+import traceback
def import_class(import_str):
return getattr(sys.modules[mod_str], class_str)
except (ImportError, ValueError, AttributeError), exc:
raise ImportError('Class %s cannot be found (%s)' %
- (class_str, str(exc)))
+ (class_str,
+ traceback.format_exception(*sys.exc_info())))
def import_object(import_str, *args, **kwargs):
return import_class(import_str)(*args, **kwargs)
+def import_object_ns(name_space, import_str, *args, **kwargs):
+ """
+ Import a class and return an instance of it, first by trying
+ to find the class in a default namespace, then failing back to
+ a full path if not found in the default namespace.
+ """
+ import_value = "%s.%s" % (name_space, import_str)
+ try:
+ return import_class(import_value)(*args, **kwargs)
+ except ImportError:
+ return import_class(import_str)(*args, **kwargs)
+
+
def import_module(import_str):
"""Import a module."""
__import__(import_str)
else:
key, value = line[:colon], line[colon + 1:]
- return key.strip(), [value.strip()]
+ value = value.strip()
+ if value[0] == value[-1] and value[0] == "\"" or value[0] == "'":
+ value = value[1:-1]
+ return key.strip(), [value]
def parse(self, lineiter):
key = None
Utilities with minimum-depends for use in setup.py
"""
+import datetime
import os
import re
import subprocess
+import sys
from setuptools.command import sdist
# -f lines are for index locations, and don't get used here
elif re.match(r'\s*-f\s+', line):
pass
+ # argparse is part of the standard library starting with 2.7
+ # adding it to the requirements list screws distro installs
+ elif line == 'argparse' and sys.version_info >= (2, 7):
+ pass
else:
requirements.append(line)
def _run_shell_command(cmd):
output = subprocess.Popen(["/bin/sh", "-c", cmd],
stdout=subprocess.PIPE)
- return output.communicate()[0].strip()
-
-
-def write_vcsversion(location):
- """Produce a vcsversion dict that mimics the old one produced by bzr.
- """
- if os.path.isdir('.git'):
- branch_nick_cmd = 'git branch | grep -Ei "\* (.*)" | cut -f2 -d" "'
- branch_nick = _run_shell_command(branch_nick_cmd)
- revid_cmd = "git rev-parse HEAD"
- revid = _run_shell_command(revid_cmd).split()[0]
- revno_cmd = "git log --oneline | wc -l"
- revno = _run_shell_command(revno_cmd)
- with open(location, 'w') as version_file:
- version_file.write("""
-# This file is automatically generated by setup.py, So don't edit it. :)
-version_info = {
- 'branch_nick': '%s',
- 'revision_id': '%s',
- 'revno': %s
-}
-""" % (branch_nick, revid, revno))
+ out = output.communicate()
+ if len(out) == 0:
+ return None
+ if len(out[0].strip()) == 0:
+ return None
+ return out[0].strip()
+
+
+def _get_git_next_version_suffix(branch_name):
+ datestamp = datetime.datetime.now().strftime('%Y%m%d')
+ if branch_name == 'milestone-proposed':
+ revno_prefix = "r"
+ else:
+ revno_prefix = ""
+ _run_shell_command("git fetch origin +refs/meta/*:refs/remotes/meta/*")
+ milestone_cmd = "git show meta/openstack/release:%s" % branch_name
+ milestonever = _run_shell_command(milestone_cmd)
+ if not milestonever:
+ milestonever = ""
+ post_version = _get_git_post_version()
+ # post version should look like:
+ # 0.1.1.4.gcc9e28a
+ # where the bit after the last . is the short sha, and the bit between
+ # the last and second to last is the revno count
+ (revno, sha) = post_version.split(".")[-2:]
+ first_half = "%(milestonever)s~%(datestamp)s" % locals()
+ second_half = "%(revno_prefix)s%(revno)s.%(sha)s" % locals()
+ return ".".join((first_half, second_half))
+
+
+def _get_git_current_tag():
+ return _run_shell_command("git tag --contains HEAD")
+
+
+def _get_git_tag_info():
+ return _run_shell_command("git describe --tags")
+
+
+def _get_git_post_version():
+ current_tag = _get_git_current_tag()
+ if current_tag is not None:
+ return current_tag
+ else:
+ tag_info = _get_git_tag_info()
+ if tag_info is None:
+ base_version = "0.0"
+ cmd = "git --no-pager log --oneline"
+ out = _run_shell_command(cmd)
+ revno = len(out.split("\n"))
+ sha = _run_shell_command("git describe --always")
+ else:
+ tag_infos = tag_info.split("-")
+ base_version = "-".join(tag_infos[:-2])
+ (revno, sha) = tag_infos[-2:]
+ return "%s.%s.%s" % (base_version, revno, sha)
def write_git_changelog():
"""Write a changelog based on the git changelog."""
- if os.path.isdir('.git'):
- git_log_cmd = 'git log --stat'
- changelog = _run_shell_command(git_log_cmd)
- mailmap = parse_mailmap()
- with open("ChangeLog", "w") as changelog_file:
- changelog_file.write(canonicalize_emails(changelog, mailmap))
+ new_changelog = 'ChangeLog'
+ if not os.getenv('SKIP_WRITE_GIT_CHANGELOG'):
+ if os.path.isdir('.git'):
+ git_log_cmd = 'git log --stat'
+ changelog = _run_shell_command(git_log_cmd)
+ mailmap = parse_mailmap()
+ with open(new_changelog, "w") as changelog_file:
+ changelog_file.write(canonicalize_emails(changelog, mailmap))
+ else:
+ open(new_changelog, 'w').close()
def generate_authors():
jenkins_email = 'jenkins@review.openstack.org'
old_authors = 'AUTHORS.in'
new_authors = 'AUTHORS'
- if os.path.isdir('.git'):
- # don't include jenkins email address in AUTHORS file
- git_log_cmd = ("git log --format='%aN <%aE>' | sort -u | "
- "grep -v " + jenkins_email)
- changelog = _run_shell_command(git_log_cmd)
- mailmap = parse_mailmap()
- with open(new_authors, 'w') as new_authors_fh:
- new_authors_fh.write(canonicalize_emails(changelog, mailmap))
- if os.path.exists(old_authors):
- with open(old_authors, "r") as old_authors_fh:
- new_authors_fh.write('\n' + old_authors_fh.read())
+ if not os.getenv('SKIP_GENERATE_AUTHORS'):
+ if os.path.isdir('.git'):
+ # don't include jenkins email address in AUTHORS file
+ git_log_cmd = ("git log --format='%aN <%aE>' | sort -u | "
+ "grep -v " + jenkins_email)
+ changelog = _run_shell_command(git_log_cmd)
+ mailmap = parse_mailmap()
+ with open(new_authors, 'w') as new_authors_fh:
+ new_authors_fh.write(canonicalize_emails(changelog, mailmap))
+ if os.path.exists(old_authors):
+ with open(old_authors, "r") as old_authors_fh:
+ new_authors_fh.write('\n' + old_authors_fh.read())
+ else:
+ open(new_authors, 'w').close()
+
+
+_rst_template = """%(heading)s
+%(underline)s
+
+.. automodule:: %(module)s
+ :members:
+ :undoc-members:
+ :show-inheritance:
+"""
+
+
+def read_versioninfo(project):
+ """Read the versioninfo file. If it doesn't exist, we're in a github
+ zipball, and there's really no way to know what version we really
+ are, but that should be ok, because the utility of that should be
+ just about nil if this code path is in use in the first place."""
+ versioninfo_path = os.path.join(project, 'versioninfo')
+ if os.path.exists(versioninfo_path):
+ with open(versioninfo_path, 'r') as vinfo:
+ version = vinfo.read().strip()
+ else:
+ version = "0.0.0"
+ return version
+
+
+def write_versioninfo(project, version):
+ """Write a simple file containing the version of the package."""
+ open(os.path.join(project, 'versioninfo'), 'w').write("%s\n" % version)
def get_cmdclass():
cmdclass = dict()
+ def _find_modules(arg, dirname, files):
+ for filename in files:
+ if filename.endswith('.py') and filename != '__init__.py':
+ arg["%s.%s" % (dirname.replace('/', '.'),
+ filename[:-3])] = True
+
class LocalSDist(sdist.sdist):
"""Builds the ChangeLog and Authors files from VC first."""
from sphinx.setup_command import BuildDoc
class LocalBuildDoc(BuildDoc):
+ def generate_autoindex(self):
+ print "**Autodocumenting from %s" % os.path.abspath(os.curdir)
+ modules = {}
+ option_dict = self.distribution.get_option_dict('build_sphinx')
+ source_dir = os.path.join(option_dict['source_dir'][1], 'api')
+ if not os.path.exists(source_dir):
+ os.makedirs(source_dir)
+ for pkg in self.distribution.packages:
+ if '.' not in pkg:
+ os.path.walk(pkg, _find_modules, modules)
+ module_list = modules.keys()
+ module_list.sort()
+ autoindex_filename = os.path.join(source_dir, 'autoindex.rst')
+ with open(autoindex_filename, 'w') as autoindex:
+ autoindex.write(""".. toctree::
+ :maxdepth: 1
+
+""")
+ for module in module_list:
+ output_filename = os.path.join(source_dir,
+ "%s.rst" % module)
+ heading = "The :mod:`%s` Module" % module
+ underline = "=" * len(heading)
+ values = dict(module=module, heading=heading,
+ underline=underline)
+
+ print "Generating %s" % output_filename
+ with open(output_filename, 'w') as output_file:
+ output_file.write(_rst_template % values)
+ autoindex.write(" %s.rst\n" % module)
+
def run(self):
+ if not os.getenv('SPHINX_DEBUG'):
+ self.generate_autoindex()
+
for builder in ['html', 'man']:
self.builder = builder
self.finalize_options()
+ self.project = self.distribution.get_name()
+ self.version = self.distribution.get_version()
+ self.release = self.distribution.get_version()
BuildDoc.run(self)
cmdclass['build_sphinx'] = LocalBuildDoc
except ImportError:
pass
return cmdclass
+
+
+def get_git_branchname():
+ for branch in _run_shell_command("git branch --color=never").split("\n"):
+ if branch.startswith('*'):
+ _branch_name = branch.split()[1].strip()
+ if _branch_name == "(no":
+ _branch_name = "no-branch"
+ return _branch_name
+
+
+def get_pre_version(projectname, base_version):
+ """Return a version which is leading up to a version that will
+ be released in the future."""
+ if os.path.isdir('.git'):
+ current_tag = _get_git_current_tag()
+ if current_tag is not None:
+ version = current_tag
+ else:
+ branch_name = os.getenv('BRANCHNAME',
+ os.getenv('GERRIT_REFNAME',
+ get_git_branchname()))
+ version_suffix = _get_git_next_version_suffix(branch_name)
+ version = "%s~%s" % (base_version, version_suffix)
+ write_versioninfo(projectname, version)
+ return version
+ else:
+ version = read_versioninfo(projectname)
+ return version
+
+
+def get_post_version(projectname):
+ """Return a version which is equal to the tag that's on the current
+ revision if there is one, or tag plus number of additional revisions
+ if the current revision has no tag."""
+
+ if os.path.isdir('.git'):
+ version = _get_git_post_version()
+ write_versioninfo(projectname, version)
+ return version
+ return read_versioninfo(projectname)
Time related utilities and helper functions.
"""
+import calendar
import datetime
+import time
import iso8601
TIME_FORMAT = "%Y-%m-%dT%H:%M:%S"
+PERFECT_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%f"
def isotime(at=None):
raise ValueError(e.message)
+def strtime(at=None, fmt=PERFECT_TIME_FORMAT):
+ """Returns formatted utcnow."""
+ if not at:
+ at = utcnow()
+ return at.strftime(fmt)
+
+
+def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT):
+ """Turn a formatted time back into a datetime."""
+ return datetime.datetime.strptime(timestr, fmt)
+
+
def normalize_time(timestamp):
"""Normalize time in arbitrary timezone to UTC"""
offset = timestamp.utcoffset()
return timestamp.replace(tzinfo=None) - offset if offset else timestamp
+def is_older_than(before, seconds):
+ """Return True if before is older than seconds."""
+ return utcnow() - before > datetime.timedelta(seconds=seconds)
+
+
+def utcnow_ts():
+ """Timestamp version of our utcnow function."""
+ return calendar.timegm(utcnow().timetuple())
+
+
def utcnow():
"""Overridable version of utils.utcnow."""
if utcnow.override_time:
utcnow.override_time = override_time
+def advance_time_delta(timedelta):
+ """Advance overriden time using a datetime.timedelta."""
+ assert(not utcnow.override_time is None)
+ utcnow.override_time += timedelta
+
+
+def advance_time_seconds(seconds):
+ """Advance overriden time by seconds."""
+ advance_time_delta(datetime.timedelta(0, seconds))
+
+
def clear_time_override():
"""Remove the overridden time."""
utcnow.override_time = None
from eventlet.green import subprocess
from heat.openstack.common import exception
+from heat.openstack.common.gettextutils import _
LOG = logging.getLogger(__name__)
LOG.debug(_('Result was %s') % _returncode)
if (isinstance(check_exit_code, int) and
not isinstance(check_exit_code, bool) and
- _returncode != check_exit_code):
+ _returncode != check_exit_code):
(stdout, stderr) = result
raise exception.ProcessExecutionError(
- exit_code=_returncode,
- stdout=stdout,
- stderr=stderr,
- cmd=' '.join(cmd))
+ exit_code=_returncode,
+ stdout=stdout,
+ stderr=stderr,
+ cmd=' '.join(cmd))
return result
except exception.ProcessExecutionError:
if not attempts:
[DEFAULT]
# The list of modules to copy from openstack-common
-modules=cfg,local,iniparser,utils,exception,timeutils,importutils,setup
+modules=gettextutils,cfg,local,iniparser,utils,exception,timeutils,importutils,setup
# The base module to hold the copy of openstack.common
base=heat
from heat.openstack.common import setup
-setup.write_vcsversion('heat/vcsversion.py')
-
# import this after write_vcsversion because version imports vcsversion
from heat import version
Code Review / openstack-build / heat-build.git / commitdiff