From: Matt Riedemann Date: Fri, 27 Jun 2014 16:50:56 +0000 (-0700) Subject: Sync processutils from oslo with deps X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=4edc71d51ec03d5e0211cb637f580bbe20c9ce73;p=openstack-build%2Fcinder-build.git Sync processutils from oslo with deps The main target for the sync is to pick up the get_worker_count() method in commit 85f178489a128a04a7ee3ed44018403caa109ef0 so that we can set osapi_volume_workers equal to the number of CPUs on the host. Changes: processutils ------------ 85f1784 Move nova.utils.cpu_count() to processutils module cdcc19c Mask passwords that are included in commands 8a0f567 Remove str() from LOG.* and exceptions 51778f9 Allow passing environment variables to execute() fcf517d Update oslo log messages with translation domains af41592 Catch OSError in processutils f773ea2 Fix i18n problem in processutils module 8b2b0b7 Use hacking import_exceptions for gettextutils._ 3b71f46 Fixed misspellings of common words 12bcdb7 Remove vim header a4dab73 Correct execute() to check 0 in check_exit_code importutils ----------- 1173e46 Remove ValueError when accessing sys.modules jsonutils --------- 0d7296f Add kwargs to jsonutils.load(s) functions log --- 109e325 Use oslo.messaging to publish log errors de4adbc pep8: fixed multiple violations eac71f5 Fix common.log.ContextFormatter for Python 3 d78b633 Fixes a simple spelling mistake 621d831 always log a traceback in the sys.excepthook 90ae24b Remove redundant default=None for config options af36c2a Fix logging setup for Python 3.4 cdcc19c Mask passwords that are included in commands strutils -------- 8a0f567 Remove str() from LOG.* and exceptions fd18c28 Fix safe_encode(): return bytes on Python 3 302c7c8 strutils: Allow safe_{encode,decode} to take bytes as input timeutils --------- 250cd88 Fixed a new pep8 error and a small typo Partial-Bug: #1333370 Change-Id: I7115cb1ae341a995502ca8ba988c12e9168259e0 --- diff --git a/cinder/openstack/common/gettextutils.py b/cinder/openstack/common/gettextutils.py index 025731985..4957a7470 100644 --- a/cinder/openstack/common/gettextutils.py +++ b/cinder/openstack/common/gettextutils.py @@ -373,8 +373,8 @@ def get_available_languages(domain): 'zh_Hant_HK': 'zh_HK', 'zh_Hant': 'zh_TW', 'fil': 'tl_PH'} - for (locale, alias) in six.iteritems(aliases): - if locale in language_list and alias not in language_list: + for (locale_, alias) in six.iteritems(aliases): + if locale_ in language_list and alias not in language_list: language_list.append(alias) _AVAILABLE_LANGUAGES[domain] = language_list diff --git a/cinder/openstack/common/importutils.py b/cinder/openstack/common/importutils.py index edc38050a..9183a9468 100644 --- a/cinder/openstack/common/importutils.py +++ b/cinder/openstack/common/importutils.py @@ -24,10 +24,10 @@ import traceback def import_class(import_str): """Returns a class from a string including module and class.""" mod_str, _sep, class_str = import_str.rpartition('.') + __import__(mod_str) try: - __import__(mod_str) return getattr(sys.modules[mod_str], class_str) - except (ValueError, AttributeError): + except AttributeError: raise ImportError('Class %s cannot be found (%s)' % (class_str, traceback.format_exception(*sys.exc_info()))) diff --git a/cinder/openstack/common/jsonutils.py b/cinder/openstack/common/jsonutils.py index 0aaa138db..c34596202 100644 --- a/cinder/openstack/common/jsonutils.py +++ b/cinder/openstack/common/jsonutils.py @@ -168,12 +168,12 @@ def dumps(value, default=to_primitive, **kwargs): return json.dumps(value, default=default, **kwargs) -def loads(s, encoding='utf-8'): - return json.loads(strutils.safe_decode(s, encoding)) +def loads(s, encoding='utf-8', **kwargs): + return json.loads(strutils.safe_decode(s, encoding), **kwargs) -def load(fp, encoding='utf-8'): - return json.load(codecs.getreader(encoding)(fp)) +def load(fp, encoding='utf-8', **kwargs): + return json.load(codecs.getreader(encoding)(fp), **kwargs) try: diff --git a/cinder/openstack/common/log.py b/cinder/openstack/common/log.py index e62c8fa3c..3e1e96849 100644 --- a/cinder/openstack/common/log.py +++ b/cinder/openstack/common/log.py @@ -59,7 +59,10 @@ _SANITIZE_PATTERNS = [] _FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])', r'(<%(key)s>).*?()', r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])', - r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])'] + r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])', + r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?[\'"])' + '.*?([\'"])', + r'(%(key)s\s*--?[A-z]+\s*).*?([\s])'] for key in _SANITIZE_KEYS: for pattern in _FORMAT_PATTERNS: @@ -89,7 +92,6 @@ logging_cli_opts = [ 'files. For details about logging configuration files, ' 'see the Python logging module documentation.'), cfg.StrOpt('log-format', - default=None, metavar='FORMAT', help='DEPRECATED. ' 'A logging.Formatter log message format string which may ' @@ -115,7 +117,7 @@ logging_cli_opts = [ default=False, help='Use syslog for logging. ' 'Existing syslog format is DEPRECATED during I, ' - 'and will chang in J to honor RFC5424.'), + 'and will change in J to honor RFC5424.'), cfg.BoolOpt('use-syslog-rfc-format', # TODO(bogdando) remove or use True after existing # syslog format deprecation in J @@ -422,9 +424,7 @@ class JSONFormatter(logging.Formatter): def _create_logging_excepthook(product_name): def logging_excepthook(exc_type, value, tb): - extra = {} - if CONF.verbose or CONF.debug: - extra['exc_info'] = (exc_type, value, tb) + extra = {'exc_info': (exc_type, value, tb)} getLogger(product_name).critical( "".join(traceback.format_exception_only(exc_type, value)), **extra) @@ -462,9 +462,8 @@ def setup(product_name, version='unknown'): def set_defaults(logging_context_format_string): - cfg.set_defaults(log_opts, - logging_context_format_string= - logging_context_format_string) + cfg.set_defaults( + log_opts, logging_context_format_string=logging_context_format_string) def _find_facility_from_conf(): @@ -541,9 +540,14 @@ def _setup_logging_from_conf(project, version): log_root.addHandler(streamlog) if CONF.publish_errors: - handler = importutils.import_object( - "cinder.openstack.common.log_handler.PublishErrorsHandler", - logging.ERROR) + try: + handler = importutils.import_object( + "cinder.openstack.common.log_handler.PublishErrorsHandler", + logging.ERROR) + except ImportError: + handler = importutils.import_object( + "oslo.messaging.notify.log_handler.PublishErrorsHandler", + logging.ERROR) log_root.addHandler(handler) datefmt = CONF.log_date_format @@ -569,9 +573,15 @@ def _setup_logging_from_conf(project, version): for pair in CONF.default_log_levels: mod, _sep, level_name = pair.partition('=') - level = logging.getLevelName(level_name) logger = logging.getLogger(mod) - logger.setLevel(level) + # NOTE(AAzza) in python2.6 Logger.setLevel doesn't convert string name + # to integer code. + if sys.version_info < (2, 7): + level = logging.getLevelName(level_name) + logger.setLevel(level) + else: + logger.setLevel(level_name) + _loggers = {} @@ -660,14 +670,19 @@ class ContextFormatter(logging.Formatter): record.__dict__[key] = '' if record.__dict__.get('request_id'): - self._fmt = CONF.logging_context_format_string + fmt = CONF.logging_context_format_string else: - self._fmt = CONF.logging_default_format_string + fmt = CONF.logging_default_format_string if (record.levelno == logging.DEBUG and CONF.logging_debug_format_suffix): - self._fmt += " " + CONF.logging_debug_format_suffix + fmt += " " + CONF.logging_debug_format_suffix + if sys.version_info < (3, 2): + self._fmt = fmt + else: + self._style = logging.PercentStyle(fmt) + self._fmt = self._style._fmt # Cache this on the record, Logger will respect our formatted copy if record.exc_info: record.exc_text = self.formatException(record.exc_info, record) diff --git a/cinder/openstack/common/processutils.py b/cinder/openstack/common/processutils.py index 059f570b1..1743106aa 100644 --- a/cinder/openstack/common/processutils.py +++ b/cinder/openstack/common/processutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # @@ -19,6 +17,9 @@ System-level utilities and helper functions. """ +import errno +import logging as stdlib_logging +import multiprocessing import os import random import shlex @@ -26,6 +27,7 @@ import signal from eventlet.green import subprocess from eventlet import greenthread +import six from cinder.openstack.common.gettextutils import _ from cinder.openstack.common import log as logging @@ -54,11 +56,18 @@ class ProcessExecutionError(Exception): self.description = description if description is None: - description = "Unexpected error while running command." + description = _("Unexpected error while running command.") if exit_code is None: exit_code = '-' - message = ("%s\nCommand: %s\nExit code: %s\nStdout: %r\nStderr: %r" - % (description, cmd, exit_code, stdout, stderr)) + message = _('%(description)s\n' + 'Command: %(cmd)s\n' + 'Exit code: %(exit_code)s\n' + 'Stdout: %(stdout)r\n' + 'Stderr: %(stderr)r') % {'description': description, + 'cmd': cmd, + 'exit_code': exit_code, + 'stdout': stdout, + 'stderr': stderr} super(ProcessExecutionError, self).__init__(message) @@ -81,7 +90,10 @@ def execute(*cmd, **kwargs): :param cmd: Passed to subprocess.Popen. :type cmd: string :param process_input: Send to opened process. - :type proces_input: string + :type process_input: string + :param env_variables: Environment variables and their values that + will be set for the process. + :type env_variables: dict :param check_exit_code: Single bool, int, or list of allowed exit codes. Defaults to [0]. Raise :class:`ProcessExecutionError` unless @@ -102,6 +114,9 @@ def execute(*cmd, **kwargs): :param shell: whether or not there should be a shell used to execute this command. Defaults to false. :type shell: boolean + :param loglevel: log level for execute commands. + :type loglevel: int. (Should be stdlib_logging.DEBUG or + stdlib_logging.INFO) :returns: (stdout, stderr) from process execution :raises: :class:`UnknownArgumentError` on receiving unknown arguments @@ -109,6 +124,7 @@ def execute(*cmd, **kwargs): """ process_input = kwargs.pop('process_input', None) + env_variables = kwargs.pop('env_variables', None) check_exit_code = kwargs.pop('check_exit_code', [0]) ignore_exit_code = False delay_on_retry = kwargs.pop('delay_on_retry', True) @@ -116,6 +132,7 @@ def execute(*cmd, **kwargs): run_as_root = kwargs.pop('run_as_root', False) root_helper = kwargs.pop('root_helper', '') shell = kwargs.pop('shell', False) + loglevel = kwargs.pop('loglevel', stdlib_logging.DEBUG) if isinstance(check_exit_code, bool): ignore_exit_code = not check_exit_code @@ -130,8 +147,8 @@ def execute(*cmd, **kwargs): if run_as_root and hasattr(os, 'geteuid') and os.geteuid() != 0: if not root_helper: raise NoRootWrapSpecified( - message=('Command requested root, but did not specify a root ' - 'helper.')) + message=_('Command requested root, but did not ' + 'specify a root helper.')) cmd = shlex.split(root_helper) + list(cmd) cmd = map(str, cmd) @@ -139,7 +156,8 @@ def execute(*cmd, **kwargs): while attempts > 0: attempts -= 1 try: - LOG.debug(_('Running cmd (subprocess): %s'), ' '.join(cmd)) + LOG.log(loglevel, 'Running cmd (subprocess): %s', + ' '.join(logging.mask_password(cmd))) _PIPE = subprocess.PIPE # pylint: disable=E1101 if os.name == 'nt': @@ -155,28 +173,37 @@ def execute(*cmd, **kwargs): stderr=_PIPE, close_fds=close_fds, preexec_fn=preexec_fn, - shell=shell) + shell=shell, + env=env_variables) result = None - if process_input is not None: - result = obj.communicate(process_input) - else: - result = obj.communicate() + for _i in six.moves.range(20): + # NOTE(russellb) 20 is an arbitrary number of retries to + # prevent any chance of looping forever here. + try: + if process_input is not None: + result = obj.communicate(process_input) + else: + result = obj.communicate() + except OSError as e: + if e.errno in (errno.EAGAIN, errno.EINTR): + continue + raise + break obj.stdin.close() # pylint: disable=E1101 _returncode = obj.returncode # pylint: disable=E1101 - if _returncode: - LOG.debug(_('Result was %s') % _returncode) - if not ignore_exit_code and _returncode not in check_exit_code: - (stdout, stderr) = result - raise ProcessExecutionError(exit_code=_returncode, - stdout=stdout, - stderr=stderr, - cmd=' '.join(cmd)) + LOG.log(loglevel, 'Result was %s' % _returncode) + if not ignore_exit_code and _returncode not in check_exit_code: + (stdout, stderr) = result + raise ProcessExecutionError(exit_code=_returncode, + stdout=stdout, + stderr=stderr, + cmd=' '.join(cmd)) return result except ProcessExecutionError: if not attempts: raise else: - LOG.debug(_('%r failed. Retrying.'), cmd) + LOG.log(loglevel, '%r failed. Retrying.', cmd) if delay_on_retry: greenthread.sleep(random.randint(20, 200) / 100.0) finally: @@ -203,7 +230,7 @@ def trycmd(*args, **kwargs): out, err = execute(*args, **kwargs) failed = False except ProcessExecutionError as exn: - out, err = '', str(exn) + out, err = '', six.text_type(exn) failed = True if not failed and discard_warnings and err: @@ -215,7 +242,7 @@ def trycmd(*args, **kwargs): def ssh_execute(ssh, cmd, process_input=None, addl_env=None, check_exit_code=True): - LOG.debug(_('Running cmd (SSH): %s'), cmd) + LOG.debug('Running cmd (SSH): %s', cmd) if addl_env: raise InvalidArgumentError(_('Environment not supported over SSH')) @@ -236,7 +263,7 @@ def ssh_execute(ssh, cmd, process_input=None, # exit_status == -1 if no exit code was returned if exit_status != -1: - LOG.debug(_('Result was %s') % exit_status) + LOG.debug('Result was %s' % exit_status) if check_exit_code and exit_status != 0: raise ProcessExecutionError(exit_code=exit_status, stdout=stdout, @@ -244,3 +271,15 @@ def ssh_execute(ssh, cmd, process_input=None, cmd=cmd) return (stdout, stderr) + + +def get_worker_count(): + """Utility to get the default worker count. + + @return: The number of CPUs if that can be determined, else a default + worker count of 1 is returned. + """ + try: + return multiprocessing.cpu_count() + except NotImplementedError: + return 1 diff --git a/cinder/openstack/common/strutils.py b/cinder/openstack/common/strutils.py index a1d127578..c8dd132d9 100644 --- a/cinder/openstack/common/strutils.py +++ b/cinder/openstack/common/strutils.py @@ -78,7 +78,7 @@ def bool_from_string(subject, strict=False, default=False): Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'. """ if not isinstance(subject, six.string_types): - subject = str(subject) + subject = six.text_type(subject) lowered = subject.strip().lower() @@ -98,7 +98,8 @@ def bool_from_string(subject, strict=False, default=False): def safe_decode(text, incoming=None, errors='strict'): - """Decodes incoming str using `incoming` if they're not already unicode. + """Decodes incoming text/bytes string using `incoming` if they're not + already unicode. :param incoming: Text's current encoding :param errors: Errors handling policy. See here for valid @@ -107,7 +108,7 @@ def safe_decode(text, incoming=None, errors='strict'): representation of it. :raises TypeError: If text is not an instance of str """ - if not isinstance(text, six.string_types): + if not isinstance(text, (six.string_types, six.binary_type)): raise TypeError("%s can't be decoded" % type(text)) if isinstance(text, six.text_type): @@ -137,7 +138,7 @@ def safe_decode(text, incoming=None, errors='strict'): def safe_encode(text, incoming=None, encoding='utf-8', errors='strict'): - """Encodes incoming str/unicode using `encoding`. + """Encodes incoming text/bytes string using `encoding`. If incoming is not specified, text is expected to be encoded with current python's default encoding. (`sys.getdefaultencoding`) @@ -150,7 +151,7 @@ def safe_encode(text, incoming=None, representation of it. :raises TypeError: If text is not an instance of str """ - if not isinstance(text, six.string_types): + if not isinstance(text, (six.string_types, six.binary_type)): raise TypeError("%s can't be encoded" % type(text)) if not incoming: @@ -158,19 +159,13 @@ def safe_encode(text, incoming=None, sys.getdefaultencoding()) if isinstance(text, six.text_type): - if six.PY3: - return text.encode(encoding, errors).decode(incoming) - else: - return text.encode(encoding, errors) + return text.encode(encoding, errors) elif text and encoding != incoming: # Decode text before encoding it with `encoding` text = safe_decode(text, incoming, errors) - if six.PY3: - return text.encode(encoding, errors).decode(incoming) - else: - return text.encode(encoding, errors) - - return text + return text.encode(encoding, errors) + else: + return text def string_to_bytes(text, unit_system='IEC', return_int=False): diff --git a/cinder/openstack/common/timeutils.py b/cinder/openstack/common/timeutils.py index 52688a026..c48da95f1 100644 --- a/cinder/openstack/common/timeutils.py +++ b/cinder/openstack/common/timeutils.py @@ -114,7 +114,7 @@ def utcnow(): def iso8601_from_timestamp(timestamp): - """Returns a iso8601 formatted date from timestamp.""" + """Returns an iso8601 formatted date from timestamp.""" return isotime(datetime.datetime.utcfromtimestamp(timestamp)) @@ -134,7 +134,7 @@ def set_time_override(override_time=None): def advance_time_delta(timedelta): """Advance overridden time using a datetime.timedelta.""" - assert(not utcnow.override_time is None) + assert utcnow.override_time is not None try: for dt in utcnow.override_time: dt += timedelta diff --git a/etc/cinder/cinder.conf.sample b/etc/cinder/cinder.conf.sample index f5a8cc005..dcb45f0be 100644 --- a/etc/cinder/cinder.conf.sample +++ b/etc/cinder/cinder.conf.sample @@ -800,7 +800,7 @@ #log_dir= # Use syslog for logging. Existing syslog format is DEPRECATED -# during I, and will chang in J to honor RFC5424. (boolean +# during I, and will change in J to honor RFC5424. (boolean # value) #use_syslog=false