From b3e5647339105883d178470a7700700199c13772 Mon Sep 17 00:00:00 2001 From: Zane Bitter Date: Fri, 30 Nov 2012 14:54:07 +0100 Subject: [PATCH] Update openstack-common Now at oslo-incubator version 1fd7694e96da4c7e461d3c966fab1e81ee57c315 Change-Id: If6ca594185cdaf6f2b279185cb4a4f62391aa828 Signed-off-by: Zane Bitter --- heat/openstack/common/cfg.py | 353 ++++++++++-------- heat/openstack/common/eventlet_backdoor.py | 10 +- heat/openstack/common/exception.py | 4 +- heat/openstack/common/excutils.py | 6 +- heat/openstack/common/importutils.py | 2 +- heat/openstack/common/jsonutils.py | 2 +- heat/openstack/common/log.py | 4 +- heat/openstack/common/loopingcall.py | 9 +- heat/openstack/common/notifier/api.py | 7 +- .../openstack/common/notifier/rpc_notifier.py | 46 +++ heat/openstack/common/rpc/amqp.py | 13 +- heat/openstack/common/rpc/common.py | 6 +- heat/openstack/common/rpc/dispatcher.py | 10 +- heat/openstack/common/rpc/impl_qpid.py | 25 +- heat/openstack/common/rpc/matchmaker.py | 2 +- heat/openstack/common/rpc/service.py | 5 + heat/openstack/common/service.py | 33 +- heat/openstack/common/threadgroup.py | 25 +- heat/openstack/common/timeutils.py | 29 +- heat/openstack/common/utils.py | 13 +- tools/pip-requires | 2 + 21 files changed, 361 insertions(+), 245 deletions(-) create mode 100644 heat/openstack/common/notifier/rpc_notifier.py diff --git a/heat/openstack/common/cfg.py b/heat/openstack/common/cfg.py index 2f196776..f792c977 100644 --- a/heat/openstack/common/cfg.py +++ b/heat/openstack/common/cfg.py @@ -205,27 +205,13 @@ Option values may reference other values using PEP 292 string substitution:: Note that interpolation can be avoided by using '$$'. -For command line utilities that dispatch to other command line utilities, the -disable_interspersed_args() method is available. If this this method is called, -then parsing e.g.:: - - script --verbose cmd --debug /tmp/mything - -will no longer return:: - - ['cmd', '/tmp/mything'] - -as the leftover arguments, but will instead return:: - - ['cmd', '--debug', '/tmp/mything'] - -i.e. argument parsing is stopped at the first non-option argument. +FIXME(markmc): document add_cli_subparsers() Options may be declared as required so that an error is raised if the user does not supply a value for the option. Options may be declared as secret so that their values are not leaked into -log files: +log files:: opts = [ cfg.StrOpt('s3_store_access_key', secret=True), @@ -234,28 +220,28 @@ log files: ] This module also contains a global instance of the CommonConfigOpts class -in order to support a common usage pattern in OpenStack: +in order to support a common usage pattern in OpenStack:: - from heat.openstack.common import cfg + from heat.openstack.common import cfg - opts = [ - cfg.StrOpt('bind_host', default='0.0.0.0'), - cfg.IntOpt('bind_port', default=9292), - ] + opts = [ + cfg.StrOpt('bind_host', default='0.0.0.0'), + cfg.IntOpt('bind_port', default=9292), + ] - CONF = cfg.CONF - CONF.register_opts(opts) + CONF = cfg.CONF + CONF.register_opts(opts) - def start(server, app): - server.start(app, CONF.bind_port, CONF.bind_host) + def start(server, app): + server.start(app, CONF.bind_port, CONF.bind_host) """ +import argparse import collections import copy import functools import glob -import optparse import os import string import sys @@ -489,6 +475,8 @@ class Opt(object): a single character CLI option name default: the default value of the option + positional: + True if the option is a positional CLI argument metavar: the name shown as the argument to a CLI option in --help output help: @@ -497,8 +485,8 @@ class Opt(object): multi = False def __init__(self, name, dest=None, short=None, default=None, - metavar=None, help=None, secret=False, required=False, - deprecated_name=None): + positional=False, metavar=None, help=None, + secret=False, required=False, deprecated_name=None): """Construct an Opt object. The only required parameter is the option's name. However, it is @@ -508,6 +496,7 @@ class Opt(object): :param dest: the name of the corresponding ConfigOpts property :param short: a single character CLI option name :param default: the default value of the option + :param positional: True if the option is a positional CLI argument :param metavar: the option argument to show in --help :param help: an explanation of how the option is used :param secret: true iff the value should be obfuscated in log output @@ -521,6 +510,7 @@ class Opt(object): self.dest = dest self.short = short self.default = default + self.positional = positional self.metavar = metavar self.help = help self.secret = secret @@ -561,64 +551,72 @@ class Opt(object): :param parser: the CLI option parser :param group: an optional OptGroup object """ - container = self._get_optparse_container(parser, group) - kwargs = self._get_optparse_kwargs(group) - prefix = self._get_optparse_prefix('', group) - self._add_to_optparse(container, self.name, self.short, kwargs, prefix, - self.deprecated_name) + container = self._get_argparse_container(parser, group) + kwargs = self._get_argparse_kwargs(group) + prefix = self._get_argparse_prefix('', group) + self._add_to_argparse(container, self.name, self.short, kwargs, prefix, + self.positional, self.deprecated_name) - def _add_to_optparse(self, container, name, short, kwargs, prefix='', - deprecated_name=None): - """Add an option to an optparse parser or group. + def _add_to_argparse(self, container, name, short, kwargs, prefix='', + positional=False, deprecated_name=None): + """Add an option to an argparse parser or group. - :param container: an optparse.OptionContainer object + :param container: an argparse._ArgumentGroup object :param name: the opt name :param short: the short opt name - :param kwargs: the keyword arguments for add_option() + :param kwargs: the keyword arguments for add_argument() :param prefix: an optional prefix to prepend to the opt name + :param position: whether the optional is a positional CLI argument :raises: DuplicateOptError if a naming confict is detected """ - args = ['--' + prefix + name] + def hyphen(arg): + return arg if not positional else '' + + args = [hyphen('--') + prefix + name] if short: - args += ['-' + short] + args.append(hyphen('-') + short) if deprecated_name: - args += ['--' + prefix + deprecated_name] - for a in args: - if container.has_option(a): - raise DuplicateOptError(a) - container.add_option(*args, **kwargs) + args.append(hyphen('--') + prefix + deprecated_name) + + try: + container.add_argument(*args, **kwargs) + except argparse.ArgumentError as e: + raise DuplicateOptError(e) - def _get_optparse_container(self, parser, group): - """Returns an optparse.OptionContainer. + def _get_argparse_container(self, parser, group): + """Returns an argparse._ArgumentGroup. - :param parser: an optparse.OptionParser + :param parser: an argparse.ArgumentParser :param group: an (optional) OptGroup object - :returns: an optparse.OptionGroup if a group is given, else the parser + :returns: an argparse._ArgumentGroup if group is given, else parser """ if group is not None: - return group._get_optparse_group(parser) + return group._get_argparse_group(parser) else: return parser - def _get_optparse_kwargs(self, group, **kwargs): - """Build a dict of keyword arguments for optparse's add_option(). + def _get_argparse_kwargs(self, group, **kwargs): + """Build a dict of keyword arguments for argparse's add_argument(). Most opt types extend this method to customize the behaviour of the - options added to optparse. + options added to argparse. :param group: an optional group :param kwargs: optional keyword arguments to add to :returns: a dict of keyword arguments """ - dest = self.dest - if group is not None: - dest = group.name + '_' + dest - kwargs.update({'dest': dest, - 'metavar': self.metavar, + if not self.positional: + dest = self.dest + if group is not None: + dest = group.name + '_' + dest + kwargs['dest'] = dest + else: + kwargs['nargs'] = '?' + kwargs.update({'metavar': self.metavar, 'help': self.help, }) return kwargs - def _get_optparse_prefix(self, prefix, group): + def _get_argparse_prefix(self, prefix, group): """Build a prefix for the CLI option name, if required. CLI options in a group are prefixed with the group's name in order @@ -656,6 +654,11 @@ class BoolOpt(Opt): _boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True, '0': False, 'no': False, 'false': False, 'off': False} + def __init__(self, *args, **kwargs): + if 'positional' in kwargs: + raise ValueError('positional boolean args not supported') + super(BoolOpt, self).__init__(*args, **kwargs) + def _get_from_config_parser(self, cparser, section): """Retrieve the opt value as a boolean from ConfigParser.""" def convert_bool(v): @@ -671,21 +674,32 @@ class BoolOpt(Opt): def _add_to_cli(self, parser, group=None): """Extends the base class method to add the --nooptname option.""" super(BoolOpt, self)._add_to_cli(parser, group) - self._add_inverse_to_optparse(parser, group) + self._add_inverse_to_argparse(parser, group) - def _add_inverse_to_optparse(self, parser, group): + def _add_inverse_to_argparse(self, parser, group): """Add the --nooptname option to the option parser.""" - container = self._get_optparse_container(parser, group) - kwargs = self._get_optparse_kwargs(group, action='store_false') - prefix = self._get_optparse_prefix('no', group) + container = self._get_argparse_container(parser, group) + kwargs = self._get_argparse_kwargs(group, action='store_false') + prefix = self._get_argparse_prefix('no', group) kwargs["help"] = "The inverse of --" + self.name - self._add_to_optparse(container, self.name, None, kwargs, prefix, - self.deprecated_name) + self._add_to_argparse(container, self.name, None, kwargs, prefix, + self.positional, self.deprecated_name) - def _get_optparse_kwargs(self, group, action='store_true', **kwargs): - """Extends the base optparse keyword dict for boolean options.""" - return super(BoolOpt, - self)._get_optparse_kwargs(group, action=action, **kwargs) + def _get_argparse_kwargs(self, group, action='store_true', **kwargs): + """Extends the base argparse keyword dict for boolean options.""" + + kwargs = super(BoolOpt, self)._get_argparse_kwargs(group, **kwargs) + + # metavar has no effect for BoolOpt + if 'metavar' in kwargs: + del kwargs['metavar'] + + if action != 'store_true': + action = 'store_false' + + kwargs['action'] = action + + return kwargs class IntOpt(Opt): @@ -697,10 +711,10 @@ class IntOpt(Opt): return [int(v) for v in self._cparser_get_with_deprecated(cparser, section)] - def _get_optparse_kwargs(self, group, **kwargs): - """Extends the base optparse keyword dict for integer options.""" + def _get_argparse_kwargs(self, group, **kwargs): + """Extends the base argparse keyword dict for integer options.""" return super(IntOpt, - self)._get_optparse_kwargs(group, type='int', **kwargs) + self)._get_argparse_kwargs(group, type=int, **kwargs) class FloatOpt(Opt): @@ -712,10 +726,10 @@ class FloatOpt(Opt): return [float(v) for v in self._cparser_get_with_deprecated(cparser, section)] - def _get_optparse_kwargs(self, group, **kwargs): - """Extends the base optparse keyword dict for float options.""" - return super(FloatOpt, - self)._get_optparse_kwargs(group, type='float', **kwargs) + def _get_argparse_kwargs(self, group, **kwargs): + """Extends the base argparse keyword dict for float options.""" + return super(FloatOpt, self)._get_argparse_kwargs(group, + type=float, **kwargs) class ListOpt(Opt): @@ -725,23 +739,26 @@ class ListOpt(Opt): is a list containing these strings. """ + class _StoreListAction(argparse.Action): + """ + An argparse action for parsing an option value into a list. + """ + def __call__(self, parser, namespace, values, option_string=None): + if values is not None: + values = [a.strip() for a in values.split(',')] + setattr(namespace, self.dest, values) + def _get_from_config_parser(self, cparser, section): """Retrieve the opt value as a list from ConfigParser.""" return [v.split(',') for v in self._cparser_get_with_deprecated(cparser, section)] - def _get_optparse_kwargs(self, group, **kwargs): - """Extends the base optparse keyword dict for list options.""" - return super(ListOpt, - self)._get_optparse_kwargs(group, - type='string', - action='callback', - callback=self._parse_list, - **kwargs) - - def _parse_list(self, option, opt, value, parser): - """An optparse callback for parsing an option value into a list.""" - setattr(parser.values, self.dest, value.split(',')) + def _get_argparse_kwargs(self, group, **kwargs): + """Extends the base argparse keyword dict for list options.""" + return Opt._get_argparse_kwargs(self, + group, + action=ListOpt._StoreListAction, + **kwargs) class MultiStrOpt(Opt): @@ -752,10 +769,14 @@ class MultiStrOpt(Opt): """ multi = True - def _get_optparse_kwargs(self, group, **kwargs): - """Extends the base optparse keyword dict for multi str options.""" - return super(MultiStrOpt, - self)._get_optparse_kwargs(group, action='append') + def _get_argparse_kwargs(self, group, **kwargs): + """Extends the base argparse keyword dict for multi str options.""" + kwargs = super(MultiStrOpt, self)._get_argparse_kwargs(group) + if not self.positional: + kwargs['action'] = 'append' + else: + kwargs['nargs'] = '*' + return kwargs def _cparser_get_with_deprecated(self, cparser, section): """If cannot find option as dest try deprecated_name alias.""" @@ -800,19 +821,20 @@ class OptGroup(object): self.help = help self._opts = {} # dict of dicts of (opt:, override:, default:) - self._optparse_group = None + self._argparse_group = None - def _register_opt(self, opt): + def _register_opt(self, opt, cli=False): """Add an opt to this group. :param opt: an Opt object + :param cli: whether this is a CLI option :returns: False if previously registered, True otherwise :raises: DuplicateOptError if a naming conflict is detected """ if _is_opt_registered(self._opts, opt): return False - self._opts[opt.dest] = {'opt': opt} + self._opts[opt.dest] = {'opt': opt, 'cli': cli} return True @@ -824,16 +846,16 @@ class OptGroup(object): if opt.dest in self._opts: del self._opts[opt.dest] - def _get_optparse_group(self, parser): - """Build an optparse.OptionGroup for this group.""" - if self._optparse_group is None: - self._optparse_group = optparse.OptionGroup(parser, self.title, - self.help) - return self._optparse_group + def _get_argparse_group(self, parser): + if self._argparse_group is None: + """Build an argparse._ArgumentGroup for this group.""" + self._argparse_group = parser.add_argument_group(self.title, + self.help) + return self._argparse_group def _clear(self): """Clear this group's option parsing state.""" - self._optparse_group = None + self._argparse_group = None class ParseError(iniparser.ParseError): @@ -928,26 +950,45 @@ class ConfigOpts(collections.Mapping): self._groups = {} self._args = None + + # for subparser support, a root parser should be initialized earlier + # and conserved for later use + self._pre_init_parser = None self._oparser = None self._cparser = None self._cli_values = {} self.__cache = {} self._config_opts = [] - self._disable_interspersed_args = False - def _setup(self, project, prog, version, usage, default_config_files): - """Initialize a ConfigOpts object for option parsing.""" + def _pre_setup(self, project, prog, version, usage, default_config_files): + """Initialize a ConfigCliParser object for option parsing.""" + if prog is None: prog = os.path.basename(sys.argv[0]) if default_config_files is None: default_config_files = find_config_files(project, prog) - self._oparser = optparse.OptionParser(prog=prog, - version=version, - usage=usage) - if self._disable_interspersed_args: - self._oparser.disable_interspersed_args() + # if _pre_init_parser does not exist, create one + if self._pre_init_parser is None: + self._oparser = argparse.ArgumentParser(prog=prog, usage=usage) + # otherwise, use the pre-initialized parser with subparsers + # and re-initialize parser + else: + self._oparser = self._pre_init_parser + self._oparser.prog = prog + self._oparser.version = version + self._oparser.usage = usage + self._pre_init_parser = None + + self._oparser.add_argument('--version', + action='version', + version=version) + + return prog, default_config_files + + def _setup(self, project, prog, version, usage, default_config_files): + """Initialize a ConfigOpts object for option parsing.""" self._config_opts = [ MultiStrOpt('config-file', @@ -1017,18 +1058,23 @@ class ConfigOpts(collections.Mapping): :raises: SystemExit, ConfigFilesNotFoundError, ConfigFileParseError, RequiredOptError, DuplicateOptError """ + self.clear() + prog, default_config_files = self._pre_setup(project, + prog, + version, + usage, + default_config_files) + self._setup(project, prog, version, usage, default_config_files) - self._cli_values, leftovers = self._parse_cli_opts(args) + self._cli_values = self._parse_cli_opts(args) self._parse_config_files() self._check_required_opts() - return leftovers - def __getattr__(self, name): """Look up an option value and perform string substitution. @@ -1055,6 +1101,13 @@ class ConfigOpts(collections.Mapping): """Return the number of options and option groups.""" return len(self._opts) + len(self._groups) + def add_cli_subparsers(self, **kwargs): + # only add subparsers to pre-initialized root parser + # to avoid cleared by self.clear() + if self._pre_init_parser is None: + self._pre_init_parser = argparse.ArgumentParser() + return self._pre_init_parser.add_subparsers(**kwargs) + def reset(self): """Clear the object state and unset overrides and defaults.""" self._unset_defaults_and_overrides() @@ -1072,7 +1125,7 @@ class ConfigOpts(collections.Mapping): group._clear() @__clear_cache - def register_opt(self, opt, group=None): + def register_opt(self, opt, group=None, cli=False): """Register an option schema. Registering an option schema makes any option value which is previously @@ -1080,17 +1133,19 @@ class ConfigOpts(collections.Mapping): as an attribute of this object. :param opt: an instance of an Opt sub-class + :param cli: whether this is a CLI option :param group: an optional OptGroup object or group name :return: False if the opt was already register, True otherwise :raises: DuplicateOptError """ if group is not None: - return self._get_group(group, autocreate=True)._register_opt(opt) + group = self._get_group(group, autocreate=True) + return group._register_opt(opt, cli) if _is_opt_registered(self._opts, opt): return False - self._opts[opt.dest] = {'opt': opt} + self._opts[opt.dest] = {'opt': opt, 'cli': cli} return True @@ -1116,7 +1171,7 @@ class ConfigOpts(collections.Mapping): if self._args is not None: raise ArgsAlreadyParsedError("cannot register CLI option") - return self.register_opt(opt, group, clear_cache=False) + return self.register_opt(opt, group, cli=True, clear_cache=False) @__clear_cache def register_cli_opts(self, opts, group=None): @@ -1243,10 +1298,11 @@ class ConfigOpts(collections.Mapping): for info in group._opts.values(): yield info, group - def _all_opts(self): - """A generator function for iteration opts.""" + def _all_cli_opts(self): + """A generator function for iterating CLI opts.""" for info, group in self._all_opt_infos(): - yield info['opt'], group + if info['cli']: + yield info['opt'], group def _unset_defaults_and_overrides(self): """Unset any default or override on all options.""" @@ -1254,31 +1310,6 @@ class ConfigOpts(collections.Mapping): info.pop('default', None) info.pop('override', None) - def disable_interspersed_args(self): - """Set parsing to stop on the first non-option. - - If this this method is called, then parsing e.g. - - script --verbose cmd --debug /tmp/mything - - will no longer return: - - ['cmd', '/tmp/mything'] - - as the leftover arguments, but will instead return: - - ['cmd', '--debug', '/tmp/mything'] - - i.e. argument parsing is stopped at the first non-option argument. - """ - self._disable_interspersed_args = True - - def enable_interspersed_args(self): - """Set parsing to not stop on the first non-option. - - This it the default behaviour.""" - self._disable_interspersed_args = False - def find_file(self, name): """Locate a file located alongside the config files. @@ -1401,6 +1432,10 @@ class ConfigOpts(collections.Mapping): if not opt.multi: return value + # argparse ignores default=None for nargs='*' + if opt.positional and not value: + value = opt.default + return value + values if values: @@ -1523,12 +1558,10 @@ class ConfigOpts(collections.Mapping): """ self._args = args - for opt, group in self._all_opts(): + for opt, group in self._all_cli_opts(): opt._add_to_cli(self._oparser, group) - values, leftovers = self._oparser.parse_args(args) - - return vars(values), leftovers + return vars(self._oparser.parse_args(args)) class GroupAttr(collections.Mapping): @@ -1543,12 +1576,12 @@ class ConfigOpts(collections.Mapping): :param conf: a ConfigOpts object :param group: an OptGroup object """ - self.conf = conf - self.group = group + self._conf = conf + self._group = group def __getattr__(self, name): """Look up an option value and perform template substitution.""" - return self.conf._get(name, self.group) + return self._conf._get(name, self._group) def __getitem__(self, key): """Look up an option value and perform string substitution.""" @@ -1556,16 +1589,16 @@ class ConfigOpts(collections.Mapping): def __contains__(self, key): """Return True if key is the name of a registered opt or group.""" - return key in self.group._opts + return key in self._group._opts def __iter__(self): """Iterate over all registered opt and group names.""" - for key in self.group._opts.keys(): + for key in self._group._opts.keys(): yield key def __len__(self): """Return the number of options and option groups.""" - return len(self.group._opts) + return len(self._group._opts) class StrSubWrapper(object): @@ -1623,12 +1656,12 @@ class CommonConfigOpts(ConfigOpts): metavar='FORMAT', help='A logging.Formatter log message format string which may ' 'use any of the available logging.LogRecord attributes. ' - 'Default: %default'), + 'Default: %(default)s'), StrOpt('log-date-format', default=DEFAULT_LOG_DATE_FORMAT, metavar='DATE_FORMAT', - help='Format string for %(asctime)s in log records. ' - 'Default: %default'), + help='Format string for %%(asctime)s in log records. ' + 'Default: %(default)s'), StrOpt('log-file', metavar='PATH', help='(Optional) Name of log file to output to. ' diff --git a/heat/openstack/common/eventlet_backdoor.py b/heat/openstack/common/eventlet_backdoor.py index 589d2ee4..28c922f9 100644 --- a/heat/openstack/common/eventlet_backdoor.py +++ b/heat/openstack/common/eventlet_backdoor.py @@ -46,7 +46,7 @@ def _find_objects(t): def _print_greenthreads(): - for i, gt in enumerate(find_objects(greenlet.greenlet)): + for i, gt in enumerate(_find_objects(greenlet.greenlet)): print i, gt traceback.print_stack(gt.gr_frame) print @@ -61,7 +61,7 @@ def initialize_if_enabled(): } if CONF.backdoor_port is None: - return + return None # NOTE(johannes): The standard sys.displayhook will print the value of # the last expression and set it to __builtin__._, which overwrites @@ -73,6 +73,8 @@ def initialize_if_enabled(): pprint.pprint(val) sys.displayhook = displayhook - eventlet.spawn_n(eventlet.backdoor.backdoor_server, - eventlet.listen(('localhost', CONF.backdoor_port)), + sock = eventlet.listen(('localhost', CONF.backdoor_port)) + port = sock.getsockname()[1] + eventlet.spawn_n(eventlet.backdoor.backdoor_server, sock, locals=backdoor_locals) + return port diff --git a/heat/openstack/common/exception.py b/heat/openstack/common/exception.py index 4866de2f..01e7b084 100644 --- a/heat/openstack/common/exception.py +++ b/heat/openstack/common/exception.py @@ -21,6 +21,8 @@ Exceptions common to OpenStack projects import logging +from heat.openstack.common.gettextutils import _ + class Error(Exception): def __init__(self, message=None): @@ -97,7 +99,7 @@ def wrap_exception(f): except Exception, e: if not isinstance(e, Error): #exc_type, exc_value, exc_traceback = sys.exc_info() - logging.exception('Uncaught exception') + logging.exception(_('Uncaught exception')) #logging.error(traceback.extract_stack(exc_traceback)) raise Error(str(e)) raise diff --git a/heat/openstack/common/excutils.py b/heat/openstack/common/excutils.py index 5dd48301..890b466a 100644 --- a/heat/openstack/common/excutils.py +++ b/heat/openstack/common/excutils.py @@ -24,6 +24,8 @@ import logging import sys import traceback +from heat.openstack.common.gettextutils import _ + @contextlib.contextmanager def save_and_reraise_exception(): @@ -43,7 +45,7 @@ def save_and_reraise_exception(): try: yield except Exception: - logging.error('Original exception being dropped: %s' % - (traceback.format_exception(type_, value, tb))) + logging.error(_('Original exception being dropped: %s'), + traceback.format_exception(type_, value, tb)) raise raise type_, value, tb diff --git a/heat/openstack/common/importutils.py b/heat/openstack/common/importutils.py index f45372b4..2a28b455 100644 --- a/heat/openstack/common/importutils.py +++ b/heat/openstack/common/importutils.py @@ -29,7 +29,7 @@ def import_class(import_str): try: __import__(mod_str) return getattr(sys.modules[mod_str], class_str) - except (ValueError, AttributeError), exc: + except (ValueError, AttributeError): raise ImportError('Class %s cannot be found (%s)' % (class_str, traceback.format_exception(*sys.exc_info()))) diff --git a/heat/openstack/common/jsonutils.py b/heat/openstack/common/jsonutils.py index 2566fd11..4109af48 100644 --- a/heat/openstack/common/jsonutils.py +++ b/heat/openstack/common/jsonutils.py @@ -120,7 +120,7 @@ def to_primitive(value, convert_instances=False, level=0): level=level + 1) else: return value - except TypeError, e: + except TypeError: # Class objects are tricky since they may define something like # __iter__ defined but it isn't callable as list(). return unicode(value) diff --git a/heat/openstack/common/log.py b/heat/openstack/common/log.py index 723ab54d..7c1faeee 100644 --- a/heat/openstack/common/log.py +++ b/heat/openstack/common/log.py @@ -50,7 +50,7 @@ from heat.openstack.common import notifier log_opts = [ cfg.StrOpt('logging_context_format_string', default='%(asctime)s %(levelname)s %(name)s [%(request_id)s ' - '%(user_id)s %(project_id)s] %(instance)s' + '%(user)s %(tenant)s] %(instance)s' '%(message)s', help='format string to use for log messages with context'), cfg.StrOpt('logging_default_format_string', @@ -174,7 +174,7 @@ class ContextAdapter(logging.LoggerAdapter): self.log(logging.AUDIT, msg, *args, **kwargs) def deprecated(self, msg, *args, **kwargs): - stdmsg = _("Deprecated Config: %s") % msg + stdmsg = _("Deprecated: %s") % msg if CONF.fatal_deprecations: self.critical(stdmsg, *args, **kwargs) raise DeprecatedConfig(msg=stdmsg) diff --git a/heat/openstack/common/loopingcall.py b/heat/openstack/common/loopingcall.py index a58b02f2..3d1b4a13 100644 --- a/heat/openstack/common/loopingcall.py +++ b/heat/openstack/common/loopingcall.py @@ -24,6 +24,7 @@ from eventlet import greenthread from heat.openstack.common.gettextutils import _ from heat.openstack.common import log as logging +from heat.openstack.common import timeutils LOG = logging.getLogger(__name__) @@ -62,10 +63,16 @@ class LoopingCall(object): try: while self._running: + start = timeutils.utcnow() self.f(*self.args, **self.kw) + end = timeutils.utcnow() if not self._running: break - greenthread.sleep(interval) + delay = interval - timeutils.delta_seconds(start, end) + if delay <= 0: + LOG.warn(_('task run outlasted interval by %s sec') % + -delay) + greenthread.sleep(delay if delay > 0 else 0) except LoopingCallDone, e: self.stop() done.send(e.retvalue) diff --git a/heat/openstack/common/notifier/api.py b/heat/openstack/common/notifier/api.py index d1ea0f2f..2fc32eb2 100644 --- a/heat/openstack/common/notifier/api.py +++ b/heat/openstack/common/notifier/api.py @@ -137,10 +137,11 @@ def notify(context, publisher_id, event_type, priority, payload): for driver in _get_drivers(): try: driver.notify(context, msg) - except Exception, e: + except Exception as e: LOG.exception(_("Problem '%(e)s' attempting to " "send to notification system. " - "Payload=%(payload)s") % locals()) + "Payload=%(payload)s") + % dict(e=e, payload=payload)) _drivers = None @@ -166,7 +167,7 @@ def add_driver(notification_driver): try: driver = importutils.import_module(notification_driver) _drivers[notification_driver] = driver - except ImportError as e: + except ImportError: LOG.exception(_("Failed to load notifier %s. " "These notifications will not be sent.") % notification_driver) diff --git a/heat/openstack/common/notifier/rpc_notifier.py b/heat/openstack/common/notifier/rpc_notifier.py new file mode 100644 index 00000000..1a37f243 --- /dev/null +++ b/heat/openstack/common/notifier/rpc_notifier.py @@ -0,0 +1,46 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from heat.openstack.common import cfg +from heat.openstack.common import context as req_context +from heat.openstack.common.gettextutils import _ +from heat.openstack.common import log as logging +from heat.openstack.common import rpc + +LOG = logging.getLogger(__name__) + +notification_topic_opt = cfg.ListOpt( + 'notification_topics', default=['notifications', ], + help='AMQP topic used for openstack notifications') + +CONF = cfg.CONF +CONF.register_opt(notification_topic_opt) + + +def notify(context, message): + """Sends a notification via RPC""" + if not context: + context = req_context.get_admin_context() + priority = message.get('priority', + CONF.default_notification_level) + priority = priority.lower() + for topic in CONF.notification_topics: + topic = '%s.%s' % (topic, priority) + try: + rpc.notify(context, topic, message) + except Exception: + LOG.exception(_("Could not send notification to %(topic)s. " + "Payload=%(message)s"), locals()) diff --git a/heat/openstack/common/rpc/amqp.py b/heat/openstack/common/rpc/amqp.py index 2374a13b..af661920 100644 --- a/heat/openstack/common/rpc/amqp.py +++ b/heat/openstack/common/rpc/amqp.py @@ -26,7 +26,6 @@ AMQP, but is deprecated and predates this code. """ import inspect -import logging import sys import uuid @@ -38,6 +37,7 @@ from heat.openstack.common import cfg from heat.openstack.common import excutils from heat.openstack.common.gettextutils import _ from heat.openstack.common import local +from heat.openstack.common import log as logging from heat.openstack.common.rpc import common as rpc_common @@ -55,7 +55,7 @@ class Pool(pools.Pool): # TODO(comstud): Timeout connections not used in a while def create(self): - LOG.debug('Pool creating new connection') + LOG.debug(_('Pool creating new connection')) return self.connection_cls(self.conf) def empty(self): @@ -282,8 +282,8 @@ class ProxyCallback(object): ctxt.reply(rval, None, connection_pool=self.connection_pool) # This final None tells multicall that it is done. ctxt.reply(ending=True, connection_pool=self.connection_pool) - except Exception as e: - LOG.exception('Exception during message handling') + except Exception: + LOG.exception(_('Exception during message handling')) ctxt.reply(None, sys.exc_info(), connection_pool=self.connection_pool) @@ -407,8 +407,9 @@ def fanout_cast_to_server(conf, context, server_params, topic, msg, def notify(conf, context, topic, msg, connection_pool): """Sends a notification event on a topic.""" - event_type = msg.get('event_type') - LOG.debug(_('Sending %(event_type)s on %(topic)s'), locals()) + LOG.debug(_('Sending %(event_type)s on %(topic)s'), + dict(event_type=msg.get('event_type'), + topic=topic)) pack_context(msg, context) with ConnectionContext(conf, connection_pool) as conn: conn.notify_send(topic, msg) diff --git a/heat/openstack/common/rpc/common.py b/heat/openstack/common/rpc/common.py index 7ff4e317..b56533dc 100644 --- a/heat/openstack/common/rpc/common.py +++ b/heat/openstack/common/rpc/common.py @@ -18,13 +18,13 @@ # under the License. import copy -import logging import traceback from heat.openstack.common.gettextutils import _ from heat.openstack.common import importutils from heat.openstack.common import jsonutils from heat.openstack.common import local +from heat.openstack.common import log as logging LOG = logging.getLogger(__name__) @@ -40,7 +40,7 @@ class RPCException(Exception): try: message = self.message % kwargs - except Exception as e: + except Exception: # kwargs doesn't match a variable in the message # log the issue and the kwargs LOG.exception(_('Exception in string format operation')) @@ -258,7 +258,7 @@ def deserialize_remote_exception(conf, data): # we cannot necessarily change an exception message so we must override # the __str__ method. failure.__class__ = new_ex_type - except TypeError as e: + except TypeError: # NOTE(ameade): If a core exception then just add the traceback to the # first exception argument. failure.args = (message,) + failure.args[1:] diff --git a/heat/openstack/common/rpc/dispatcher.py b/heat/openstack/common/rpc/dispatcher.py index 7d1a5a55..72d2ae07 100644 --- a/heat/openstack/common/rpc/dispatcher.py +++ b/heat/openstack/common/rpc/dispatcher.py @@ -41,8 +41,8 @@ server side of the API at the same time. However, as the code stands today, there can be both versioned and unversioned APIs implemented in the same code base. - -EXAMPLES: +EXAMPLES +======== Nova was the first project to use versioned rpc APIs. Consider the compute rpc API as an example. The client side is in nova/compute/rpcapi.py and the server @@ -50,12 +50,13 @@ side is in nova/compute/manager.py. Example 1) Adding a new method. +------------------------------- Adding a new method is a backwards compatible change. It should be added to nova/compute/manager.py, and RPC_API_VERSION should be bumped from X.Y to X.Y+1. On the client side, the new method in nova/compute/rpcapi.py should have a specific version specified to indicate the minimum API version that must -be implemented for the method to be supported. For example: +be implemented for the method to be supported. For example:: def get_host_uptime(self, ctxt, host): topic = _compute_topic(self.topic, ctxt, host, None) @@ -67,10 +68,11 @@ get_host_uptime() method. Example 2) Adding a new parameter. +---------------------------------- Adding a new parameter to an rpc method can be made backwards compatible. The RPC_API_VERSION on the server side (nova/compute/manager.py) should be bumped. -The implementation of the method must not expect the parameter to be present. +The implementation of the method must not expect the parameter to be present.:: def some_remote_method(self, arg1, arg2, newarg=None): # The code needs to deal with newarg=None for cases diff --git a/heat/openstack/common/rpc/impl_qpid.py b/heat/openstack/common/rpc/impl_qpid.py index c7afcc48..20715d32 100644 --- a/heat/openstack/common/rpc/impl_qpid.py +++ b/heat/openstack/common/rpc/impl_qpid.py @@ -17,7 +17,6 @@ import functools import itertools -import logging import time import uuid @@ -29,6 +28,7 @@ import qpid.messaging.exceptions from heat.openstack.common import cfg from heat.openstack.common.gettextutils import _ from heat.openstack.common import jsonutils +from heat.openstack.common import log as logging from heat.openstack.common.rpc import amqp as rpc_amqp from heat.openstack.common.rpc import common as rpc_common @@ -41,6 +41,9 @@ qpid_opts = [ cfg.StrOpt('qpid_port', default='5672', help='Qpid broker port'), + cfg.ListOpt('qpid_hosts', + default=['$qpid_hostname:$qpid_port'], + help='Qpid HA cluster host:port pairs'), cfg.StrOpt('qpid_username', default='', help='Username for qpid connection'), @@ -277,22 +280,21 @@ class Connection(object): self.conf = conf params = { - 'hostname': self.conf.qpid_hostname, - 'port': self.conf.qpid_port, + 'qpid_hosts': self.conf.qpid_hosts, 'username': self.conf.qpid_username, 'password': self.conf.qpid_password, } params.update(server_params or {}) - self.broker = params['hostname'] + ":" + str(params['port']) + self.brokers = params['qpid_hosts'] self.username = params['username'] self.password = params['password'] - self.connection_create() + self.connection_create(self.brokers[0]) self.reconnect() - def connection_create(self): + def connection_create(self, broker): # Create the connection - this does not open the connection - self.connection = qpid.messaging.Connection(self.broker) + self.connection = qpid.messaging.Connection(broker) # Check if flags are set and if so set them for the connection # before we call open @@ -320,10 +322,14 @@ class Connection(object): except qpid.messaging.exceptions.ConnectionError: pass + attempt = 0 delay = 1 while True: + broker = self.brokers[attempt % len(self.brokers)] + attempt += 1 + try: - self.connection_create() + self.connection_create(broker) self.connection.open() except qpid.messaging.exceptions.ConnectionError, e: msg_dict = dict(e=e, delay=delay) @@ -333,10 +339,9 @@ class Connection(object): time.sleep(delay) delay = min(2 * delay, 60) else: + LOG.info(_('Connected to AMQP server on %s'), broker) break - LOG.info(_('Connected to AMQP server on %s'), self.broker) - self.session = self.connection.session() if self.consumers: diff --git a/heat/openstack/common/rpc/matchmaker.py b/heat/openstack/common/rpc/matchmaker.py index 05534e3f..3a9437c8 100644 --- a/heat/openstack/common/rpc/matchmaker.py +++ b/heat/openstack/common/rpc/matchmaker.py @@ -21,10 +21,10 @@ return keys for direct exchanges, per (approximate) AMQP parlance. import contextlib import itertools import json -import logging from heat.openstack.common import cfg from heat.openstack.common.gettextutils import _ +from heat.openstack.common import log as logging matchmaker_opts = [ diff --git a/heat/openstack/common/rpc/service.py b/heat/openstack/common/rpc/service.py index 44a9cdc8..b292a247 100644 --- a/heat/openstack/common/rpc/service.py +++ b/heat/openstack/common/rpc/service.py @@ -57,6 +57,11 @@ class Service(service.Service): self.conn.create_consumer(self.topic, dispatcher, fanout=True) + # Hook to allow the manager to do other initializations after + # the rpc connection is created. + if callable(getattr(self.manager, 'initialize_service_hook', None)): + self.manager.initialize_service_hook(self) + # Consume from all consumers in a thread self.conn.consume_in_thread() diff --git a/heat/openstack/common/service.py b/heat/openstack/common/service.py index e2fbae04..8b63785f 100644 --- a/heat/openstack/common/service.py +++ b/heat/openstack/common/service.py @@ -27,7 +27,7 @@ import sys import time import eventlet -import greenlet +import extras import logging as std_logging from heat.openstack.common import cfg @@ -36,11 +36,8 @@ from heat.openstack.common.gettextutils import _ from heat.openstack.common import log as logging from heat.openstack.common import threadgroup -try: - from heat.openstack.common import rpc -except ImportError: - rpc = None +rpc = extras.try_import('heat.openstack.common.rpc') CONF = cfg.CONF LOG = logging.getLogger(__name__) @@ -54,7 +51,7 @@ class Launcher(object): :returns: None """ - self._services = [] + self._services = threadgroup.ThreadGroup('launcher') eventlet_backdoor.initialize_if_enabled() @staticmethod @@ -75,8 +72,7 @@ class Launcher(object): :returns: None """ - gt = eventlet.spawn(self.run_service, service) - self._services.append(gt) + self._services.add_thread(self.run_service, service) def stop(self): """Stop all services which are currently running. @@ -84,8 +80,7 @@ class Launcher(object): :returns: None """ - for service in self._services: - service.kill() + self._services.stop() def wait(self): """Waits until all services have been stopped, and then returns. @@ -93,11 +88,7 @@ class Launcher(object): :returns: None """ - for service in self._services: - try: - service.wait() - except greenlet.GreenletExit: - pass + self._services.wait() class SignalExit(SystemExit): @@ -132,9 +123,9 @@ class ServiceLauncher(Launcher): except SystemExit as exc: status = exc.code finally: - self.stop() if rpc: rpc.cleanup() + self.stop() return status @@ -260,10 +251,12 @@ class ProcessLauncher(object): if os.WIFSIGNALED(status): sig = os.WTERMSIG(status) - LOG.info(_('Child %(pid)d killed by signal %(sig)d'), locals()) + LOG.info(_('Child %(pid)d killed by signal %(sig)d'), + dict(pid=pid, sig=sig)) else: code = os.WEXITSTATUS(status) - LOG.info(_('Child %(pid)d exited with status %(code)d'), locals()) + LOG.info(_('Child %(pid)s exited with status %(code)d'), + dict(pid=pid, code=code)) if pid not in self.children: LOG.warning(_('pid %d not in child list'), pid) @@ -309,8 +302,8 @@ class ProcessLauncher(object): class Service(object): """Service object for binaries running on hosts.""" - def __init__(self): - self.tg = threadgroup.ThreadGroup('service') + def __init__(self, threads=1000): + self.tg = threadgroup.ThreadGroup('service', threads) def start(self): pass diff --git a/heat/openstack/common/threadgroup.py b/heat/openstack/common/threadgroup.py index 09d9b398..618e3c70 100644 --- a/heat/openstack/common/threadgroup.py +++ b/heat/openstack/common/threadgroup.py @@ -18,7 +18,6 @@ from eventlet import greenlet from eventlet import greenpool from eventlet import greenthread -from heat.openstack.common.gettextutils import _ from heat.openstack.common import log as logging from heat.openstack.common import loopingcall @@ -27,19 +26,17 @@ LOG = logging.getLogger(__name__) def _thread_done(gt, *args, **kwargs): - ''' - Callback function to be passed to GreenThread.link() when we spawn() - Calls the ThreadGroup to notify if. - ''' + """ Callback function to be passed to GreenThread.link() when we spawn() + Calls the :class:`ThreadGroup` to notify if. + + """ kwargs['group'].thread_done(kwargs['thread']) class Thread(object): - """ - Wrapper around a greenthread, that holds a reference to - the ThreadGroup. The Thread will notify the ThreadGroup - when it has done so it can be removed from the threads - list. + """ Wrapper around a greenthread, that holds a reference to the + :class:`ThreadGroup`. The Thread will notify the :class:`ThreadGroup` when + it has done so it can be removed from the threads list. """ def __init__(self, name, thread, group): self.name = name @@ -54,11 +51,11 @@ class Thread(object): class ThreadGroup(object): - """ - The point of this class is to: - - keep track of timers and greenthreads (making it easier to stop them + """ The point of the ThreadGroup classis to: + + * keep track of timers and greenthreads (making it easier to stop them when need be). - - provide an easy API to add timers. + * provide an easy API to add timers. """ def __init__(self, name, thread_pool_size=10): self.name = name diff --git a/heat/openstack/common/timeutils.py b/heat/openstack/common/timeutils.py index 86004391..ea691642 100644 --- a/heat/openstack/common/timeutils.py +++ b/heat/openstack/common/timeutils.py @@ -87,7 +87,10 @@ def utcnow_ts(): def utcnow(): """Overridable version of utils.utcnow.""" if utcnow.override_time: - return utcnow.override_time + try: + return utcnow.override_time.pop(0) + except AttributeError: + return utcnow.override_time return datetime.datetime.utcnow() @@ -95,14 +98,21 @@ utcnow.override_time = None def set_time_override(override_time=datetime.datetime.utcnow()): - """Override utils.utcnow to return a constant time.""" + """ + Override utils.utcnow to return a constant time or a list thereof, + one at a time. + """ utcnow.override_time = override_time def advance_time_delta(timedelta): """Advance overridden time using a datetime.timedelta.""" assert(not utcnow.override_time is None) - utcnow.override_time += timedelta + try: + for dt in utcnow.override_time: + dt += timedelta + except TypeError: + utcnow.override_time += timedelta def advance_time_seconds(seconds): @@ -135,3 +145,16 @@ def unmarshall_time(tyme): minute=tyme['minute'], second=tyme['second'], microsecond=tyme['microsecond']) + + +def delta_seconds(before, after): + """ + Compute the difference in seconds between two date, time, or + datetime objects (as a float, to microsecond resolution). + """ + delta = after - before + try: + return delta.total_seconds() + except AttributeError: + return ((delta.days * 24 * 3600) + delta.seconds + + float(delta.microseconds) / (10 ** 6)) diff --git a/heat/openstack/common/utils.py b/heat/openstack/common/utils.py index 36990676..05f0e9f7 100644 --- a/heat/openstack/common/utils.py +++ b/heat/openstack/common/utils.py @@ -20,15 +20,6 @@ System-level utilities and helper functions. """ import logging -import random -import shlex - -from eventlet.green import subprocess -from eventlet import greenthread - -from heat.openstack.common import exception -from heat.openstack.common.gettextutils import _ - LOG = logging.getLogger(__name__) @@ -38,7 +29,9 @@ def int_from_bool_as_string(subject): Interpret a string as a boolean and return either 1 or 0. Any string value in: + ('True', 'true', 'On', 'on', '1') + is interpreted as a boolean True. Useful for JSON-decoded stuff and config file parsing @@ -51,7 +44,9 @@ def bool_from_string(subject): Interpret a string as a boolean. Any string value in: + ('True', 'true', 'On', 'on', 'Yes', 'yes', '1') + is interpreted as a boolean True. Useful for JSON-decoded stuff and config file parsing diff --git a/tools/pip-requires b/tools/pip-requires index 197497e0..44265099 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -12,10 +12,12 @@ PyCrypto boto>=2.4.0 eventlet>=0.9.17 +extras greenlet>=0.3.1 httplib2 iso8601>=0.1.4 kombu==1.0.4 +argparse lxml>=2.3,<=2.3.5 sqlalchemy-migrate>=0.7.2 python-novaclient -- 2.45.2