'zh_Hant_HK': 'zh_HK',
'zh_Hant': 'zh_TW',
'fil': 'tl_PH'}
- for (locale, alias) in six.iteritems(aliases):
- if locale in language_list and alias not in language_list:
+ for (locale_, alias) in six.iteritems(aliases):
+ if locale_ in language_list and alias not in language_list:
language_list.append(alias)
_AVAILABLE_LANGUAGES[domain] = language_list
def import_class(import_str):
"""Returns a class from a string including module and class."""
mod_str, _sep, class_str = import_str.rpartition('.')
+ __import__(mod_str)
try:
- __import__(mod_str)
return getattr(sys.modules[mod_str], class_str)
- except (ValueError, AttributeError):
+ except AttributeError:
raise ImportError('Class %s cannot be found (%s)' %
(class_str,
traceback.format_exception(*sys.exc_info())))
return json.dumps(value, default=default, **kwargs)
-def loads(s, encoding='utf-8'):
- return json.loads(strutils.safe_decode(s, encoding))
+def loads(s, encoding='utf-8', **kwargs):
+ return json.loads(strutils.safe_decode(s, encoding), **kwargs)
-def load(fp, encoding='utf-8'):
- return json.load(codecs.getreader(encoding)(fp))
+def load(fp, encoding='utf-8', **kwargs):
+ return json.load(codecs.getreader(encoding)(fp), **kwargs)
try:
_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
r'(<%(key)s>).*?(</%(key)s>)',
r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])',
- r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])']
+ r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])',
+ r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?[\'"])'
+ '.*?([\'"])',
+ r'(%(key)s\s*--?[A-z]+\s*).*?([\s])']
for key in _SANITIZE_KEYS:
for pattern in _FORMAT_PATTERNS:
'files. For details about logging configuration files, '
'see the Python logging module documentation.'),
cfg.StrOpt('log-format',
- default=None,
metavar='FORMAT',
help='DEPRECATED. '
'A logging.Formatter log message format string which may '
default=False,
help='Use syslog for logging. '
'Existing syslog format is DEPRECATED during I, '
- 'and will chang in J to honor RFC5424.'),
+ 'and will change in J to honor RFC5424.'),
cfg.BoolOpt('use-syslog-rfc-format',
# TODO(bogdando) remove or use True after existing
# syslog format deprecation in J
def _create_logging_excepthook(product_name):
def logging_excepthook(exc_type, value, tb):
- extra = {}
- if CONF.verbose or CONF.debug:
- extra['exc_info'] = (exc_type, value, tb)
+ extra = {'exc_info': (exc_type, value, tb)}
getLogger(product_name).critical(
"".join(traceback.format_exception_only(exc_type, value)),
**extra)
def set_defaults(logging_context_format_string):
- cfg.set_defaults(log_opts,
- logging_context_format_string=
- logging_context_format_string)
+ cfg.set_defaults(
+ log_opts, logging_context_format_string=logging_context_format_string)
def _find_facility_from_conf():
log_root.addHandler(streamlog)
if CONF.publish_errors:
- handler = importutils.import_object(
- "cinder.openstack.common.log_handler.PublishErrorsHandler",
- logging.ERROR)
+ try:
+ handler = importutils.import_object(
+ "cinder.openstack.common.log_handler.PublishErrorsHandler",
+ logging.ERROR)
+ except ImportError:
+ handler = importutils.import_object(
+ "oslo.messaging.notify.log_handler.PublishErrorsHandler",
+ logging.ERROR)
log_root.addHandler(handler)
datefmt = CONF.log_date_format
for pair in CONF.default_log_levels:
mod, _sep, level_name = pair.partition('=')
- level = logging.getLevelName(level_name)
logger = logging.getLogger(mod)
- logger.setLevel(level)
+ # NOTE(AAzza) in python2.6 Logger.setLevel doesn't convert string name
+ # to integer code.
+ if sys.version_info < (2, 7):
+ level = logging.getLevelName(level_name)
+ logger.setLevel(level)
+ else:
+ logger.setLevel(level_name)
+
_loggers = {}
record.__dict__[key] = ''
if record.__dict__.get('request_id'):
- self._fmt = CONF.logging_context_format_string
+ fmt = CONF.logging_context_format_string
else:
- self._fmt = CONF.logging_default_format_string
+ fmt = CONF.logging_default_format_string
if (record.levelno == logging.DEBUG and
CONF.logging_debug_format_suffix):
- self._fmt += " " + CONF.logging_debug_format_suffix
+ fmt += " " + CONF.logging_debug_format_suffix
+ if sys.version_info < (3, 2):
+ self._fmt = fmt
+ else:
+ self._style = logging.PercentStyle(fmt)
+ self._fmt = self._style._fmt
# Cache this on the record, Logger will respect our formatted copy
if record.exc_info:
record.exc_text = self.formatException(record.exc_info, record)
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
System-level utilities and helper functions.
"""
+import errno
+import logging as stdlib_logging
+import multiprocessing
import os
import random
import shlex
from eventlet.green import subprocess
from eventlet import greenthread
+import six
from cinder.openstack.common.gettextutils import _
from cinder.openstack.common import log as logging
self.description = description
if description is None:
- description = "Unexpected error while running command."
+ description = _("Unexpected error while running command.")
if exit_code is None:
exit_code = '-'
- message = ("%s\nCommand: %s\nExit code: %s\nStdout: %r\nStderr: %r"
- % (description, cmd, exit_code, stdout, stderr))
+ message = _('%(description)s\n'
+ 'Command: %(cmd)s\n'
+ 'Exit code: %(exit_code)s\n'
+ 'Stdout: %(stdout)r\n'
+ 'Stderr: %(stderr)r') % {'description': description,
+ 'cmd': cmd,
+ 'exit_code': exit_code,
+ 'stdout': stdout,
+ 'stderr': stderr}
super(ProcessExecutionError, self).__init__(message)
:param cmd: Passed to subprocess.Popen.
:type cmd: string
:param process_input: Send to opened process.
- :type proces_input: string
+ :type process_input: string
+ :param env_variables: Environment variables and their values that
+ will be set for the process.
+ :type env_variables: dict
:param check_exit_code: Single bool, int, or list of allowed exit
codes. Defaults to [0]. Raise
:class:`ProcessExecutionError` unless
:param shell: whether or not there should be a shell used to
execute this command. Defaults to false.
:type shell: boolean
+ :param loglevel: log level for execute commands.
+ :type loglevel: int. (Should be stdlib_logging.DEBUG or
+ stdlib_logging.INFO)
:returns: (stdout, stderr) from process execution
:raises: :class:`UnknownArgumentError` on
receiving unknown arguments
"""
process_input = kwargs.pop('process_input', None)
+ env_variables = kwargs.pop('env_variables', None)
check_exit_code = kwargs.pop('check_exit_code', [0])
ignore_exit_code = False
delay_on_retry = kwargs.pop('delay_on_retry', True)
run_as_root = kwargs.pop('run_as_root', False)
root_helper = kwargs.pop('root_helper', '')
shell = kwargs.pop('shell', False)
+ loglevel = kwargs.pop('loglevel', stdlib_logging.DEBUG)
if isinstance(check_exit_code, bool):
ignore_exit_code = not check_exit_code
if run_as_root and hasattr(os, 'geteuid') and os.geteuid() != 0:
if not root_helper:
raise NoRootWrapSpecified(
- message=('Command requested root, but did not specify a root '
- 'helper.'))
+ message=_('Command requested root, but did not '
+ 'specify a root helper.'))
cmd = shlex.split(root_helper) + list(cmd)
cmd = map(str, cmd)
while attempts > 0:
attempts -= 1
try:
- LOG.debug(_('Running cmd (subprocess): %s'), ' '.join(cmd))
+ LOG.log(loglevel, 'Running cmd (subprocess): %s',
+ ' '.join(logging.mask_password(cmd)))
_PIPE = subprocess.PIPE # pylint: disable=E1101
if os.name == 'nt':
stderr=_PIPE,
close_fds=close_fds,
preexec_fn=preexec_fn,
- shell=shell)
+ shell=shell,
+ env=env_variables)
result = None
- if process_input is not None:
- result = obj.communicate(process_input)
- else:
- result = obj.communicate()
+ for _i in six.moves.range(20):
+ # NOTE(russellb) 20 is an arbitrary number of retries to
+ # prevent any chance of looping forever here.
+ try:
+ if process_input is not None:
+ result = obj.communicate(process_input)
+ else:
+ result = obj.communicate()
+ except OSError as e:
+ if e.errno in (errno.EAGAIN, errno.EINTR):
+ continue
+ raise
+ break
obj.stdin.close() # pylint: disable=E1101
_returncode = obj.returncode # pylint: disable=E1101
- if _returncode:
- LOG.debug(_('Result was %s') % _returncode)
- if not ignore_exit_code and _returncode not in check_exit_code:
- (stdout, stderr) = result
- raise ProcessExecutionError(exit_code=_returncode,
- stdout=stdout,
- stderr=stderr,
- cmd=' '.join(cmd))
+ LOG.log(loglevel, 'Result was %s' % _returncode)
+ if not ignore_exit_code and _returncode not in check_exit_code:
+ (stdout, stderr) = result
+ raise ProcessExecutionError(exit_code=_returncode,
+ stdout=stdout,
+ stderr=stderr,
+ cmd=' '.join(cmd))
return result
except ProcessExecutionError:
if not attempts:
raise
else:
- LOG.debug(_('%r failed. Retrying.'), cmd)
+ LOG.log(loglevel, '%r failed. Retrying.', cmd)
if delay_on_retry:
greenthread.sleep(random.randint(20, 200) / 100.0)
finally:
out, err = execute(*args, **kwargs)
failed = False
except ProcessExecutionError as exn:
- out, err = '', str(exn)
+ out, err = '', six.text_type(exn)
failed = True
if not failed and discard_warnings and err:
def ssh_execute(ssh, cmd, process_input=None,
addl_env=None, check_exit_code=True):
- LOG.debug(_('Running cmd (SSH): %s'), cmd)
+ LOG.debug('Running cmd (SSH): %s', cmd)
if addl_env:
raise InvalidArgumentError(_('Environment not supported over SSH'))
# exit_status == -1 if no exit code was returned
if exit_status != -1:
- LOG.debug(_('Result was %s') % exit_status)
+ LOG.debug('Result was %s' % exit_status)
if check_exit_code and exit_status != 0:
raise ProcessExecutionError(exit_code=exit_status,
stdout=stdout,
cmd=cmd)
return (stdout, stderr)
+
+
+def get_worker_count():
+ """Utility to get the default worker count.
+
+ @return: The number of CPUs if that can be determined, else a default
+ worker count of 1 is returned.
+ """
+ try:
+ return multiprocessing.cpu_count()
+ except NotImplementedError:
+ return 1
Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'.
"""
if not isinstance(subject, six.string_types):
- subject = str(subject)
+ subject = six.text_type(subject)
lowered = subject.strip().lower()
def safe_decode(text, incoming=None, errors='strict'):
- """Decodes incoming str using `incoming` if they're not already unicode.
+ """Decodes incoming text/bytes string using `incoming` if they're not
+ already unicode.
:param incoming: Text's current encoding
:param errors: Errors handling policy. See here for valid
representation of it.
:raises TypeError: If text is not an instance of str
"""
- if not isinstance(text, six.string_types):
+ if not isinstance(text, (six.string_types, six.binary_type)):
raise TypeError("%s can't be decoded" % type(text))
if isinstance(text, six.text_type):
def safe_encode(text, incoming=None,
encoding='utf-8', errors='strict'):
- """Encodes incoming str/unicode using `encoding`.
+ """Encodes incoming text/bytes string using `encoding`.
If incoming is not specified, text is expected to be encoded with
current python's default encoding. (`sys.getdefaultencoding`)
representation of it.
:raises TypeError: If text is not an instance of str
"""
- if not isinstance(text, six.string_types):
+ if not isinstance(text, (six.string_types, six.binary_type)):
raise TypeError("%s can't be encoded" % type(text))
if not incoming:
sys.getdefaultencoding())
if isinstance(text, six.text_type):
- if six.PY3:
- return text.encode(encoding, errors).decode(incoming)
- else:
- return text.encode(encoding, errors)
+ return text.encode(encoding, errors)
elif text and encoding != incoming:
# Decode text before encoding it with `encoding`
text = safe_decode(text, incoming, errors)
- if six.PY3:
- return text.encode(encoding, errors).decode(incoming)
- else:
- return text.encode(encoding, errors)
-
- return text
+ return text.encode(encoding, errors)
+ else:
+ return text
def string_to_bytes(text, unit_system='IEC', return_int=False):
def iso8601_from_timestamp(timestamp):
- """Returns a iso8601 formatted date from timestamp."""
+ """Returns an iso8601 formatted date from timestamp."""
return isotime(datetime.datetime.utcfromtimestamp(timestamp))
def advance_time_delta(timedelta):
"""Advance overridden time using a datetime.timedelta."""
- assert(not utcnow.override_time is None)
+ assert utcnow.override_time is not None
try:
for dt in utcnow.override_time:
dt += timedelta
#log_dir=<None>
# Use syslog for logging. Existing syslog format is DEPRECATED
-# during I, and will chang in J to honor RFC5424. (boolean
+# during I, and will change in J to honor RFC5424. (boolean
# value)
#use_syslog=false
Code Review / openstack-build / cinder-build.git / commitdiff