From: Eric Harney Date: Thu, 16 Jan 2014 19:08:53 +0000 (-0500) Subject: Sync common modules from Oslo X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=2c39b37625dd094b2a7b63821aab8d71801f26a0;p=openstack-build%2Fcinder-build.git Sync common modules from Oslo Identified as dependencies for RPC sync by update.py Contains code from: 12bcdb7 Remove vim header e6494c2 Use six.iteritems to make dict work on Python2/3 6d0a6c3 Correct invalid docstrings fe18bfa Remove keystone from default_log_levels default 0228b81 Logging excepthook: print exception info if debug=True 8b2b0b7 Use hacking import_exceptions for gettextutils._ 6d0a6c3 Correct invalid docstrings fe18bfa Remove keystone from default_log_levels default 13eb01c Adding domain to context and log 27b21bc Unify different names between Python2/3 with six.moves a6f40ac Don't log to stdout when log_dir is set 4c22556 Use py3kcompat urlutils functions instead of urlparse 0bf03b7 Add network_utils.urlsplit e456727 Remove useless logging in networks_utils 8b2b0b7 Use hacking import_exceptions for gettextutils._ 37e46bb disable SIGHUP restart behavior in foreground f08f6ba service: replace eventlet event by threading 32cd4c0 Allow configurable ProcessLauncher liveness check 25ff65e Make wait & stop methods work on all threads 7013471 Fix typo in parameter documentation (timeutils) ae931a9 Avoid TypeError in is_older_than, is_newer_than ad17a69 Fix filter() usage due to python 3 compability Oslo version: 7a51572 Merge "Implement cache abstraction layer" Date: Wed Jan 15 19:31:16 2014 +0000 Change-Id: If83fb203ecbae912e365118861477a9be834fde3 --- diff --git a/cinder/openstack/common/eventlet_backdoor.py b/cinder/openstack/common/eventlet_backdoor.py index d530cda37..f1526f6b0 100644 --- a/cinder/openstack/common/eventlet_backdoor.py +++ b/cinder/openstack/common/eventlet_backdoor.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright (c) 2012 OpenStack Foundation. # Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. @@ -31,7 +29,7 @@ import eventlet.backdoor import greenlet from oslo.config import cfg -from cinder.openstack.common.gettextutils import _ # noqa +from cinder.openstack.common.gettextutils import _ from cinder.openstack.common import log as logging help_for_backdoor_port = ( @@ -66,7 +64,7 @@ def _dont_use_this(): def _find_objects(t): - return filter(lambda o: isinstance(o, t), gc.get_objects()) + return [o for o in gc.get_objects() if isinstance(o, t)] def _print_greenthreads(): diff --git a/cinder/openstack/common/importutils.py b/cinder/openstack/common/importutils.py index 7a303f93f..4fd9ae2bc 100644 --- a/cinder/openstack/common/importutils.py +++ b/cinder/openstack/common/importutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # diff --git a/cinder/openstack/common/jsonutils.py b/cinder/openstack/common/jsonutils.py index b06d855bf..205f710f0 100644 --- a/cinder/openstack/common/jsonutils.py +++ b/cinder/openstack/common/jsonutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # Copyright 2011 Justin Santa Barbara @@ -41,8 +39,12 @@ import json try: import xmlrpclib except ImportError: - # NOTE(jd): xmlrpclib is not shipped with Python 3 - xmlrpclib = None + # NOTE(jaypipes): xmlrpclib was renamed to xmlrpc.client in Python3 + # however the function and object call signatures + # remained the same. This whole try/except block should + # be removed and replaced with a call to six.moves once + # six 1.4.2 is released. See http://bit.ly/1bqrVzu + import xmlrpc.client as xmlrpclib import six @@ -124,14 +126,14 @@ def to_primitive(value, convert_instances=False, convert_datetime=True, level=level, max_depth=max_depth) if isinstance(value, dict): - return dict((k, recursive(v)) for k, v in value.iteritems()) + return dict((k, recursive(v)) for k, v in six.iteritems(value)) elif isinstance(value, (list, tuple)): return [recursive(lv) for lv in value] # It's not clear why xmlrpclib created their own DateTime type, but # for our purposes, make it a datetime type which is explicitly # handled - if xmlrpclib and isinstance(value, xmlrpclib.DateTime): + if isinstance(value, xmlrpclib.DateTime): value = datetime.datetime(*tuple(value.timetuple())[:6]) if convert_datetime and isinstance(value, datetime.datetime): diff --git a/cinder/openstack/common/local.py b/cinder/openstack/common/local.py index e82f17d0f..0819d5b97 100644 --- a/cinder/openstack/common/local.py +++ b/cinder/openstack/common/local.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # diff --git a/cinder/openstack/common/log.py b/cinder/openstack/common/log.py index d844880cd..70d9c4c46 100644 --- a/cinder/openstack/common/log.py +++ b/cinder/openstack/common/log.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. @@ -43,7 +41,7 @@ from oslo.config import cfg import six from six import moves -from cinder.openstack.common.gettextutils import _ # noqa +from cinder.openstack.common.gettextutils import _ from cinder.openstack.common import importutils from cinder.openstack.common import jsonutils from cinder.openstack.common import local @@ -132,7 +130,7 @@ generic_log_opts = [ log_opts = [ cfg.StrOpt('logging_context_format_string', default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s ' - '%(name)s [%(request_id)s %(user)s %(tenant)s] ' + '%(name)s [%(request_id)s %(user_identity)s] ' '%(instance)s%(message)s', help='format string to use for log messages with context'), cfg.StrOpt('logging_default_format_string', @@ -151,7 +149,6 @@ log_opts = [ 'amqp=WARN', 'amqplib=WARN', 'boto=WARN', - 'keystone=INFO', 'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO', @@ -238,10 +235,11 @@ def mask_password(message, secret="***"): """Replace password with 'secret' in message. :param message: The string which includes security information. - :param secret: value with which to replace passwords, defaults to "***". + :param secret: value with which to replace passwords. :returns: The unicode value of message with the password fields masked. For example: + >>> mask_password("'adminPass' : 'aaaaa'") "'adminPass' : '***'" >>> mask_password("'admin_pass' : 'aaaaa'") @@ -334,10 +332,12 @@ class ContextAdapter(BaseLoggerAdapter): elif instance_uuid: instance_extra = (CONF.instance_uuid_format % {'uuid': instance_uuid}) - extra.update({'instance': instance_extra}) + extra['instance'] = instance_extra + + extra.setdefault('user_identity', kwargs.pop('user_identity', None)) - extra.update({"project": self.project}) - extra.update({"version": self.version}) + extra['project'] = self.project + extra['version'] = self.version extra['extra'] = extra.copy() return msg, kwargs @@ -351,7 +351,7 @@ class JSONFormatter(logging.Formatter): def formatException(self, ei, strip_newlines=True): lines = traceback.format_exception(*ei) if strip_newlines: - lines = [itertools.ifilter( + lines = [moves.filter( lambda x: x, line.rstrip().splitlines()) for line in lines] lines = list(itertools.chain(*lines)) @@ -391,7 +391,7 @@ class JSONFormatter(logging.Formatter): def _create_logging_excepthook(product_name): def logging_excepthook(exc_type, value, tb): extra = {} - if CONF.verbose: + if CONF.verbose or CONF.debug: extra['exc_info'] = (exc_type, value, tb) getLogger(product_name).critical(str(value), **extra) return logging_excepthook @@ -477,7 +477,7 @@ def _setup_logging_from_conf(): streamlog = ColorHandler() log_root.addHandler(streamlog) - elif not CONF.log_file: + elif not logpath: # pass sys.stdout as a positional argument # python2.6 calls the argument strm, in 2.7 it's stream streamlog = logging.StreamHandler(sys.stdout) diff --git a/cinder/openstack/common/loopingcall.py b/cinder/openstack/common/loopingcall.py index 9d5a057ec..eabb605ff 100644 --- a/cinder/openstack/common/loopingcall.py +++ b/cinder/openstack/common/loopingcall.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # Copyright 2011 Justin Santa Barbara @@ -22,7 +20,7 @@ import sys from eventlet import event from eventlet import greenthread -from cinder.openstack.common.gettextutils import _ # noqa +from cinder.openstack.common.gettextutils import _ from cinder.openstack.common import log as logging from cinder.openstack.common import timeutils diff --git a/cinder/openstack/common/network_utils.py b/cinder/openstack/common/network_utils.py index c94cae6e0..5807b67b5 100644 --- a/cinder/openstack/common/network_utils.py +++ b/cinder/openstack/common/network_utils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2012 OpenStack Foundation. # All Rights Reserved. # @@ -19,10 +17,7 @@ Network-related utilities and helper functions. """ -from cinder.openstack.common import log as logging - - -LOG = logging.getLogger(__name__) +from cinder.openstack.common.py3kcompat import urlutils def parse_host_port(address, default_port=None): @@ -67,3 +62,18 @@ def parse_host_port(address, default_port=None): port = default_port return (host, None if port is None else int(port)) + + +def urlsplit(url, scheme='', allow_fragments=True): + """Parse a URL using urlparse.urlsplit(), splitting query and fragments. + This function papers over Python issue9374 when needed. + + The parameters are the same as urlparse.urlsplit. + """ + scheme, netloc, path, query, fragment = urlutils.urlsplit( + url, scheme, allow_fragments) + if allow_fragments and '#' in path: + path, fragment = path.split('#', 1) + if '?' in path: + path, query = path.split('?', 1) + return urlutils.SplitResult(scheme, netloc, path, query, fragment) diff --git a/cinder/openstack/common/service.py b/cinder/openstack/common/service.py index 605dbf380..7632625d4 100644 --- a/cinder/openstack/common/service.py +++ b/cinder/openstack/common/service.py @@ -23,6 +23,7 @@ import os import random import signal import sys +import threading import time try: @@ -34,7 +35,6 @@ except ImportError: UnsupportedOperation = None import eventlet -from eventlet import event from oslo.config import cfg from cinder.openstack.common import eventlet_backdoor @@ -206,10 +206,16 @@ class ServiceWrapper(object): class ProcessLauncher(object): - def __init__(self): + def __init__(self, wait_interval=0.01): + """Constructor. + + :param wait_interval: The interval to sleep for between checks + of child process exit. + """ self.children = {} self.sigcaught = None self.running = True + self.wait_interval = wait_interval rfd, self.writepipe = os.pipe() self.readpipe = eventlet.greenio.GreenPipe(rfd, 'r') self.handle_signal() @@ -367,7 +373,7 @@ class ProcessLauncher(object): # Yield to other threads if no children have exited # Sleep for a short time to avoid excessive CPU usage # (see bug #1095346) - eventlet.greenthread.sleep(.01) + eventlet.greenthread.sleep(self.wait_interval) continue while self.running and len(wrap.children) < wrap.workers: self._start_child(wrap) @@ -413,11 +419,10 @@ class Service(object): self.tg = threadgroup.ThreadGroup(threads) # signal that the service is done shutting itself down: - self._done = event.Event() + self._done = threading.Event() def reset(self): - # NOTE(Fengqian): docs for Event.reset() recommend against using it - self._done = event.Event() + self._done = threading.Event() def start(self): pass @@ -426,8 +431,7 @@ class Service(object): self.tg.stop() self.tg.wait() # Signal that service cleanup is done: - if not self._done.ready(): - self._done.send() + self._done.set() def wait(self): self._done.wait() @@ -438,7 +442,7 @@ class Services(object): def __init__(self): self.services = [] self.tg = threadgroup.ThreadGroup() - self.done = event.Event() + self.done = threading.Event() def add(self, service): self.services.append(service) @@ -452,8 +456,7 @@ class Services(object): # Each service has performed cleanup, now signal that the run_service # wrapper threads can now die: - if not self.done.ready(): - self.done.send() + self.done.set() # reap threads: self.tg.stop() @@ -463,7 +466,7 @@ class Services(object): def restart(self): self.stop() - self.done = event.Event() + self.done = threading.Event() for restart_service in self.services: restart_service.reset() self.tg.add_thread(self.run_service, restart_service, self.done) diff --git a/cinder/openstack/common/threadgroup.py b/cinder/openstack/common/threadgroup.py index 36e05e9cc..edeaf3070 100644 --- a/cinder/openstack/common/threadgroup.py +++ b/cinder/openstack/common/threadgroup.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2012 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -89,7 +87,10 @@ class ThreadGroup(object): def stop(self): current = greenthread.getcurrent() - for x in self.threads: + + # Iterate over a copy of self.threads so thread_done doesn't + # modify the list while we're iterating + for x in self.threads[:]: if x is current: # don't kill the current thread. continue @@ -114,7 +115,10 @@ class ThreadGroup(object): except Exception as ex: LOG.exception(ex) current = greenthread.getcurrent() - for x in self.threads: + + # Iterate over a copy of self.threads so thread_done doesn't + # modify the list while we're iterating + for x in self.threads[:]: if x is current: continue try: diff --git a/cinder/openstack/common/timeutils.py b/cinder/openstack/common/timeutils.py index db471ea51..d5ed81d3e 100644 --- a/cinder/openstack/common/timeutils.py +++ b/cinder/openstack/common/timeutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # @@ -79,6 +77,9 @@ def is_older_than(before, seconds): """Return True if before is older than seconds.""" if isinstance(before, six.string_types): before = parse_strtime(before).replace(tzinfo=None) + else: + before = before.replace(tzinfo=None) + return utcnow() - before > datetime.timedelta(seconds=seconds) @@ -86,6 +87,9 @@ def is_newer_than(after, seconds): """Return True if after is newer than seconds.""" if isinstance(after, six.string_types): after = parse_strtime(after).replace(tzinfo=None) + else: + after = after.replace(tzinfo=None) + return after - utcnow() > datetime.timedelta(seconds=seconds) @@ -197,8 +201,8 @@ def total_seconds(delta): def is_soon(dt, window): """Determines if time is going to happen in the next window seconds. - :params dt: the time - :params window: minimum seconds to remain to consider the time not soon + :param dt: the time + :param window: minimum seconds to remain to consider the time not soon :return: True if expiration is within the given duration """ diff --git a/etc/cinder/cinder.conf.sample b/etc/cinder/cinder.conf.sample index 01213a6a7..d290361a2 100644 --- a/etc/cinder/cinder.conf.sample +++ b/etc/cinder/cinder.conf.sample @@ -524,7 +524,7 @@ # format string to use for log messages with context (string # value) -#logging_context_format_string=%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user)s %(tenant)s] %(instance)s%(message)s +#logging_context_format_string=%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s # format string to use for log messages without context # (string value) @@ -539,7 +539,7 @@ #logging_exception_prefix=%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s %(instance)s # list of logger=LEVEL pairs (list value) -#default_log_levels=amqp=WARN,amqplib=WARN,boto=WARN,keystone=INFO,qpid=WARN,sqlalchemy=WARN,suds=INFO,iso8601=WARN +#default_log_levels=amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,iso8601=WARN # publish error events (boolean value) #publish_errors=false