"""
import itertools
-import uuid
+
+from cinder.openstack.common import uuidutils
def generate_request_id():
- return 'req-' + str(uuid.uuid4())
+ return 'req-%s' % uuidutils.generate_uuid()
class RequestContext(object):
- """
+ """Helper class to represent useful information about a request context.
+
Stores information about the security context under which the user
accesses the system, as well as additional request information.
"""
# License for the specific language governing permissions and limitations
# under the License.
+from __future__ import print_function
+
import gc
import pprint
import sys
def _dont_use_this():
- print "Don't use this, just disconnect instead"
+ print("Don't use this, just disconnect instead")
def _find_objects(t):
def _print_greenthreads():
for i, gt in enumerate(_find_objects(greenlet.greenlet)):
- print i, gt
+ print(i, gt)
traceback.print_stack(gt.gr_frame)
- print
+ print()
def _print_nativethreads():
for threadId, stack in sys._current_frames().items():
- print threadId
+ print(threadId)
traceback.print_stack(stack)
- print
+ print()
def initialize_if_enabled():
def _wrap(*args, **kw):
try:
return f(*args, **kw)
- except Exception, e:
+ except Exception as e:
if not isinstance(e, Error):
#exc_type, exc_value, exc_traceback = sys.exc_info()
logging.exception(_('Uncaught exception'))
class OpenstackException(Exception):
- """
- Base Exception
+ """Base Exception class.
To correctly use this class, inherit from it and define
a 'message' property. That message will get printf'd
def import_class(import_str):
- """Returns a class from a string including module and class"""
+ """Returns a class from a string including module and class."""
mod_str, _sep, class_str = import_str.rpartition('.')
try:
__import__(mod_str)
def import_object_ns(name_space, import_str, *args, **kwargs):
- """
- Import a class and return an instance of it, first by trying
+ """Tries to import object from default namespace.
+
+ Imports a class and return an instance of it, first by trying
to find the class in a default namespace, then failing back to
a full path if not found in the default namespace.
"""
This way only one of either foo or bar can be executing at a time.
- The lock_file_prefix argument is used to provide lock files on disk with a
- meaningful prefix. The prefix should end with a hyphen ('-') if specified.
-
- The external keyword argument denotes whether this lock should work across
- multiple processes. This means that if two different workers both run a
- a method decorated with @synchronized('mylock', external=True), only one
- of them will execute at a time.
-
- The lock_path keyword argument is used to specify a special location for
- external lock files to live. If nothing is set, then CONF.lock_path is
- used as a default.
+ :param lock_file_prefix: The lock_file_prefix argument is used to provide
+ lock files on disk with a meaningful prefix. The prefix should end with a
+ hyphen ('-') if specified.
+
+ :param external: The external keyword argument denotes whether this lock
+ should work across multiple processes. This means that if two different
+ workers both run a a method decorated with @synchronized('mylock',
+ external=True), only one of them will execute at a time.
+
+ :param lock_path: The lock_path keyword argument is used to specify a
+ special location for external lock files to live. If nothing is set, then
+ CONF.lock_path is used as a default.
"""
def wrap(f):
def getLazyLogger(name='unknown', version='unknown'):
- """
- create a pass-through logger that does not create the real logger
+ """Returns lazy logger.
+
+ Creates a pass-through logger that does not create the real logger
until it is really needed and delegates all calls to the real logger
- once it is created
+ once it is created.
"""
return LazyAdapter(name, version)
LOG.warn(_('task run outlasted interval by %s sec') %
-delay)
greenthread.sleep(delay if delay > 0 else 0)
- except LoopingCallDone, e:
+ except LoopingCallDone as e:
self.stop()
done.send(e.retvalue)
except Exception:
LOG.debug(_('Dynamic looping call sleeping for %.02f '
'seconds'), idle)
greenthread.sleep(idle)
- except LoopingCallDone, e:
+ except LoopingCallDone as e:
self.stop()
done.send(e.retvalue)
except Exception:
Network-related utilities and helper functions.
"""
-import logging
+from cinder.openstack.common import log as logging
+
LOG = logging.getLogger(__name__)
def parse_host_port(address, default_port=None):
- """
- Interpret a string as a host:port pair.
+ """Interpret a string as a host:port pair.
+
An IPv6 address MUST be escaped if accompanied by a port,
because otherwise ambiguity ensues: 2001:db8:85a3::8a2e:370:7334
means both [2001:db8:85a3::8a2e:370:7334] and
def notify_decorator(name, fn):
- """ decorator for notify which is used from utils.monkey_patch()
+ """Decorator for notify which is used from utils.monkey_patch().
:param name: name of the function
:param function: - object of the function
def notify(_context, message):
"""Notifies the recipient of the desired event given the model.
- Log notifications using openstack's default logging system"""
+
+ Log notifications using openstack's default logging system.
+ """
priority = message.get('priority',
CONF.default_notification_level)
def notify(_context, message):
- """Notifies the recipient of the desired event given the model"""
+ """Notifies the recipient of the desired event given the model."""
pass
def notify(context, message):
- """Sends a notification via RPC"""
+ """Sends a notification via RPC."""
if not context:
context = req_context.get_admin_context()
priority = message.get('priority',
def notify(context, message):
- """Sends a notification via RPC"""
+ """Sends a notification via RPC."""
if not context:
context = req_context.get_admin_context()
priority = message.get('priority',
LOG = logging.getLogger(__name__)
+class InvalidArgumentError(Exception):
+ def __init__(self, message=None):
+ super(InvalidArgumentError, self).__init__(message)
+
+
class UnknownArgumentError(Exception):
def __init__(self, message=None):
super(UnknownArgumentError, self).__init__(message)
def execute(*cmd, **kwargs):
- """
- Helper method to shell out and execute a command through subprocess with
- optional retry.
+ """Helper method to shell out and execute a command through subprocess.
+
+ Allows optional retry.
:param cmd: Passed to subprocess.Popen.
:type cmd: string
elif isinstance(check_exit_code, int):
check_exit_code = [check_exit_code]
- if len(kwargs):
+ if kwargs:
raise UnknownArgumentError(_('Got unknown keyword args '
'to utils.execute: %r') % kwargs)
# call clean something up in between calls, without
# it two execute calls in a row hangs the second one
greenthread.sleep(0)
+
+
+def trycmd(*args, **kwargs):
+ """A wrapper around execute() to more easily handle warnings and errors.
+
+ Returns an (out, err) tuple of strings containing the output of
+ the command's stdout and stderr. If 'err' is not empty then the
+ command can be considered to have failed.
+
+ :discard_warnings True | False. Defaults to False. If set to True,
+ then for succeeding commands, stderr is cleared
+
+ """
+ discard_warnings = kwargs.pop('discard_warnings', False)
+
+ try:
+ out, err = execute(*args, **kwargs)
+ failed = False
+ except ProcessExecutionError as exn:
+ out, err = '', str(exn)
+ failed = True
+
+ if not failed and discard_warnings and err:
+ # Handle commands that output to stderr but otherwise succeed
+ err = ''
+
+ return out, err
+
+
+def ssh_execute(ssh, cmd, process_input=None,
+ addl_env=None, check_exit_code=True):
+ LOG.debug(_('Running cmd (SSH): %s'), cmd)
+ if addl_env:
+ raise InvalidArgumentError(_('Environment not supported over SSH'))
+
+ if process_input:
+ # This is (probably) fixable if we need it...
+ raise InvalidArgumentError(_('process_input not supported over SSH'))
+
+ stdin_stream, stdout_stream, stderr_stream = ssh.exec_command(cmd)
+ channel = stdout_stream.channel
+
+ # NOTE(justinsb): This seems suspicious...
+ # ...other SSH clients have buffering issues with this approach
+ stdout = stdout_stream.read()
+ stderr = stderr_stream.read()
+ stdin_stream.close()
+
+ exit_status = channel.recv_exit_status()
+
+ # exit_status == -1 if no exit code was returned
+ if exit_status != -1:
+ LOG.debug(_('Result was %s') % exit_status)
+ if check_exit_code and exit_status != 0:
+ raise ProcessExecutionError(exit_code=exit_status,
+ stdout=stdout,
+ stderr=stderr,
+ cmd=cmd)
+
+ return (stdout, stderr)
they are needed, to avoid allowing more than is necessary.
"""
+from __future__ import print_function
+
import ConfigParser
import logging
import os
def _exit_error(execname, message, errorcode, log=True):
- print "%s: %s" % (execname, message)
+ print("%s: %s" % (execname, message))
if log:
logging.error(message)
sys.exit(errorcode)
class CommandFilter(object):
- """Command filter only checking that the 1st argument matches exec_path"""
+ """Command filter only checking that the 1st argument matches exec_path."""
def __init__(self, exec_path, run_as, *args):
self.name = ''
self.real_exec = None
def get_exec(self, exec_dirs=[]):
- """Returns existing executable, or empty string if none found"""
+ """Returns existing executable, or empty string if none found."""
if self.real_exec is not None:
return self.real_exec
self.real_exec = ""
return self.real_exec
def match(self, userargs):
- """Only check that the first argument (command) matches exec_path"""
- if (os.path.basename(self.exec_path) == userargs[0]):
- return True
- return False
+ """Only check that the first argument (command) matches exec_path."""
+ return os.path.basename(self.exec_path) == userargs[0]
def get_command(self, userargs, exec_dirs=[]):
"""Returns command to execute (with sudo -u if run_as != root)."""
return [to_exec] + userargs[1:]
def get_environment(self, userargs):
- """Returns specific environment to set, None if none"""
+ """Returns specific environment to set, None if none."""
return None
class RegExpFilter(CommandFilter):
- """Command filter doing regexp matching for every argument"""
+ """Command filter doing regexp matching for every argument."""
def match(self, userargs):
# Early skip if command or number of args don't match
class DnsmasqFilter(CommandFilter):
- """Specific filter for the dnsmasq call (which includes env)"""
+ """Specific filter for the dnsmasq call (which includes env)."""
CONFIG_FILE_ARG = 'CONFIG_FILE'
class DeprecatedDnsmasqFilter(DnsmasqFilter):
- """Variant of dnsmasq filter to support old-style FLAGFILE"""
+ """Variant of dnsmasq filter to support old-style FLAGFILE."""
CONFIG_FILE_ARG = 'FLAGFILE'
return False
try:
command = os.readlink("/proc/%d/exe" % int(args[1]))
+ # NOTE(yufang521247): /proc/PID/exe may have '\0' on the
+ # end, because python doen't stop at '\0' when read the
+ # target path.
+ command = command.split('\0')[0]
# NOTE(dprince): /proc/PID/exe may have ' (deleted)' on
# the end if an executable is updated or deleted
if command.endswith(" (deleted)"):
class ReadFileFilter(CommandFilter):
- """Specific filter for the utils.read_file_as_root call"""
+ """Specific filter for the utils.read_file_as_root call."""
def __init__(self, file_path, *args):
self.file_path = file_path
class FilterMatchNotExecutable(Exception):
- """
- This exception is raised when a filter matched but no executable was
- found.
- """
+ """Raised when a filter matched but no executable was found."""
def __init__(self, match=None, **kwargs):
self.match = match
def build_filter(class_name, *args):
- """Returns a filter object of class class_name"""
+ """Returns a filter object of class class_name."""
if not hasattr(filters, class_name):
logging.warning("Skipping unknown filter class (%s) specified "
"in filter definitions" % class_name)
def load_filters(filters_path):
- """Load filters from a list of directories"""
+ """Load filters from a list of directories."""
filterlist = []
for filterdir in filters_path:
if not os.path.isdir(filterdir):
return filterlist
-def match_filter(filters, userargs, exec_dirs=[]):
- """
- Checks user command and arguments through command filters and
- returns the first matching filter.
+def match_filter(filter_list, userargs, exec_dirs=[]):
+ """Checks user command and arguments through command filters.
+
+ Returns the first matching filter.
+
Raises NoFilterMatched if no filter matched.
Raises FilterMatchNotExecutable if no executable was found for the
best filter match.
"""
first_not_executable_filter = None
- for f in filters:
+ for f in filter_list:
if f.match(userargs):
# Try other filters if executable is absent
if not f.get_exec(exec_dirs=exec_dirs):
"""
self._services = threadgroup.ThreadGroup()
- eventlet_backdoor.initialize_if_enabled()
+ self.backdoor_port = eventlet_backdoor.initialize_if_enabled()
@staticmethod
def run_service(service):
:returns: None
"""
+ service.backdoor_port = self.backdoor_port
self._services.add_thread(self.run_service, service)
def stop(self):
return wrap
def wait(self):
- """Loop waiting on children to die and respawning as necessary"""
+ """Loop waiting on children to die and respawning as necessary."""
LOG.debug(_('Full set of CONF:'))
CONF.log_opt_values(LOG, std_logging.DEBUG)
System-level utilities and helper functions.
"""
+import re
import sys
+import unicodedata
from cinder.openstack.common.gettextutils import _
+# Used for looking up extensions of text
+# to their 'multiplied' byte amount
+BYTE_MULTIPLIERS = {
+ '': 1,
+ 't': 1024 ** 4,
+ 'g': 1024 ** 3,
+ 'm': 1024 ** 2,
+ 'k': 1024,
+}
+
+
TRUE_STRINGS = ('1', 't', 'true', 'on', 'y', 'yes')
FALSE_STRINGS = ('0', 'f', 'false', 'off', 'n', 'no')
+SLUGIFY_STRIP_RE = re.compile(r"[^\w\s-]")
+SLUGIFY_HYPHENATE_RE = re.compile(r"[-\s]+")
+
def int_from_bool_as_string(subject):
- """
- Interpret a string as a boolean and return either 1 or 0.
+ """Interpret a string as a boolean and return either 1 or 0.
Any string value in:
def bool_from_string(subject, strict=False):
- """
- Interpret a string as a boolean.
+ """Interpret a string as a boolean.
A case-insensitive match is performed such that strings matching 't',
'true', 'on', 'y', 'yes', or '1' are considered True and, when
def safe_decode(text, incoming=None, errors='strict'):
- """
- Decodes incoming str using `incoming` if they're
- not already unicode.
+ """Decodes incoming str using `incoming` if they're not already unicode.
:param incoming: Text's current encoding
:param errors: Errors handling policy. See here for valid
def safe_encode(text, incoming=None,
encoding='utf-8', errors='strict'):
- """
- Encodes incoming str/unicode using `encoding`. If
- incoming is not specified, text is expected to
- be encoded with current python's default encoding.
- (`sys.getdefaultencoding`)
+ """Encodes incoming str/unicode using `encoding`.
+
+ If incoming is not specified, text is expected to be encoded with
+ current python's default encoding. (`sys.getdefaultencoding`)
:param incoming: Text's current encoding
:param encoding: Expected encoding for text (Default UTF-8)
return text.encode(encoding, errors)
return text
+
+
+def to_bytes(text, default=0):
+ """Try to turn a string into a number of bytes. Looks at the last
+ characters of the text to determine what conversion is needed to
+ turn the input text into a byte number.
+
+ Supports: B/b, K/k, M/m, G/g, T/t (or the same with b/B on the end)
+
+ """
+ # Take off everything not number 'like' (which should leave
+ # only the byte 'identifier' left)
+ mult_key_org = text.lstrip('-1234567890')
+ mult_key = mult_key_org.lower()
+ mult_key_len = len(mult_key)
+ if mult_key.endswith("b"):
+ mult_key = mult_key[0:-1]
+ try:
+ multiplier = BYTE_MULTIPLIERS[mult_key]
+ if mult_key_len:
+ # Empty cases shouldn't cause text[0:-0]
+ text = text[0:-mult_key_len]
+ return int(text) * multiplier
+ except KeyError:
+ msg = _('Unknown byte multiplier: %s') % mult_key_org
+ raise TypeError(msg)
+ except ValueError:
+ return default
+
+
+def to_slug(value, incoming=None, errors="strict"):
+ """Normalize string.
+
+ Convert to lowercase, remove non-word characters, and convert spaces
+ to hyphens.
+
+ Inspired by Django's `slugify` filter.
+
+ :param value: Text to slugify
+ :param incoming: Text's current encoding
+ :param errors: Errors handling policy. See here for valid
+ values http://docs.python.org/2/library/codecs.html
+ :returns: slugified unicode representation of `value`
+ :raises TypeError: If text is not an instance of basestring
+ """
+ value = safe_decode(value, incoming, errors)
+ # NOTE(aababilov): no need to use safe_(encode|decode) here:
+ # encodings are always "ascii", error handling is always "ignore"
+ # and types are always known (first: unicode; second: str)
+ value = unicodedata.normalize("NFKD", value).encode(
+ "ascii", "ignore").decode("ascii")
+ value = SLUGIFY_STRIP_RE.sub("", value).strip().lower()
+ return SLUGIFY_HYPHENATE_RE.sub("-", value)
def _thread_done(gt, *args, **kwargs):
- """ Callback function to be passed to GreenThread.link() when we spawn()
+ """Callback function to be passed to GreenThread.link() when we spawn()
Calls the :class:`ThreadGroup` to notify if.
"""
class Thread(object):
- """ Wrapper around a greenthread, that holds a reference to the
+ """Wrapper around a greenthread, that holds a reference to the
:class:`ThreadGroup`. The Thread will notify the :class:`ThreadGroup` when
it has done so it can be removed from the threads list.
"""
class ThreadGroup(object):
- """ The point of the ThreadGroup classis to:
+ """The point of the ThreadGroup classis to:
* keep track of timers and greenthreads (making it easier to stop them
when need be).
self.threads = []
self.timers = []
+ def add_dynamic_timer(self, callback, initial_delay=None,
+ periodic_interval_max=None, *args, **kwargs):
+ timer = loopingcall.DynamicLoopingCall(callback, *args, **kwargs)
+ timer.start(initial_delay=initial_delay,
+ periodic_interval_max=periodic_interval_max)
+ self.timers.append(timer)
+
def add_timer(self, interval, callback, initial_delay=None,
*args, **kwargs):
pulse = loopingcall.FixedIntervalLoopingCall(callback, *args, **kwargs)
def isotime(at=None, subsecond=False):
- """Stringify time in ISO 8601 format"""
+ """Stringify time in ISO 8601 format."""
if not at:
at = utcnow()
st = at.strftime(_ISO8601_TIME_FORMAT
def parse_isotime(timestr):
- """Parse time from ISO 8601 format"""
+ """Parse time from ISO 8601 format."""
try:
return iso8601.parse_date(timestr)
except iso8601.ParseError as e:
def normalize_time(timestamp):
- """Normalize time in arbitrary timezone to UTC naive object"""
+ """Normalize time in arbitrary timezone to UTC naive object."""
offset = timestamp.utcoffset()
if offset is None:
return timestamp
def iso8601_from_timestamp(timestamp):
- """Returns a iso8601 formated date from timestamp"""
+ """Returns a iso8601 formated date from timestamp."""
return isotime(datetime.datetime.utcfromtimestamp(timestamp))
def set_time_override(override_time=datetime.datetime.utcnow()):
- """
- Override utils.utcnow to return a constant time or a list thereof,
- one at a time.
+ """Overrides utils.utcnow.
+
+ Make it return a constant time or a list thereof, one at a time.
"""
utcnow.override_time = override_time
def marshall_now(now=None):
"""Make an rpc-safe datetime with microseconds.
- Note: tzinfo is stripped, but not required for relative times."""
+ Note: tzinfo is stripped, but not required for relative times.
+ """
if not now:
now = utcnow()
return dict(day=now.day, month=now.month, year=now.year, hour=now.hour,
def delta_seconds(before, after):
- """
+ """Return the difference between two timing objects.
+
Compute the difference in seconds between two date, time, or
datetime objects (as a float, to microsecond resolution).
"""
def is_soon(dt, window):
- """
- Determines if time is going to happen in the next window seconds.
+ """Determines if time is going to happen in the next window seconds.
:params dt: the time
:params window: minimum seconds to remain to consider the time not soon
"""Provides methods needed by installation script for OpenStack development
virtual environments.
+Since this script is used to bootstrap a virtualenv from the system's Python
+environment, it should be kept strictly compatible with Python 2.6.
+
Synced in from openstack-common
"""
-import argparse
+from __future__ import print_function
+
+import optparse
import os
import subprocess
import sys
self.project = project
def die(self, message, *args):
- print >> sys.stderr, message % args
+ print(message % args, file=sys.stderr)
sys.exit(1)
def check_python_version(self):
virtual environment.
"""
if not os.path.isdir(self.venv):
- print 'Creating venv...',
+ print('Creating venv...', end=' ')
if no_site_packages:
self.run_command(['virtualenv', '-q', '--no-site-packages',
self.venv])
else:
self.run_command(['virtualenv', '-q', self.venv])
- print 'done.'
- print 'Installing pip in venv...',
+ print('done.')
+ print('Installing pip in venv...', end=' ')
if not self.run_command(['tools/with_venv.sh', 'easy_install',
'pip>1.0']).strip():
self.die("Failed to install pip.")
- print 'done.'
+ print('done.')
else:
- print "venv already exists..."
+ print("venv already exists...")
pass
def pip_install(self, *args):
redirect_output=False)
def install_dependencies(self):
- print 'Installing dependencies with pip (this can take a while)...'
+ print('Installing dependencies with pip (this can take a while)...')
# First things first, make sure our venv has the latest pip and
# distribute.
def parse_args(self, argv):
"""Parses command-line arguments."""
- parser = argparse.ArgumentParser()
- parser.add_argument('-n', '--no-site-packages',
- action='store_true',
- help="Do not inherit packages from global Python "
- "install")
- return parser.parse_args(argv[1:])
+ parser = optparse.OptionParser()
+ parser.add_option('-n', '--no-site-packages',
+ action='store_true',
+ help="Do not inherit packages from global Python "
+ "install")
+ return parser.parse_args(argv[1:])[0]
class Distro(InstallVenv):
return
if self.check_cmd('easy_install'):
- print 'Installing virtualenv via easy_install...',
+ print('Installing virtualenv via easy_install...', end=' ')
if self.run_command(['easy_install', 'virtualenv']):
- print 'Succeeded'
+ print('Succeeded')
return
else:
- print 'Failed'
+ print('Failed')
self.die('ERROR: virtualenv not found.\n\n%s development'
' requires virtualenv, please install it using your'
return self.run_command_with_code(['rpm', '-q', pkg],
check_exit_code=False)[1] == 0
- def yum_install(self, pkg, **kwargs):
- print "Attempting to install '%s' via yum" % pkg
- self.run_command(['sudo', 'yum', 'install', '-y', pkg], **kwargs)
-
def apply_patch(self, originalfile, patchfile):
self.run_command(['patch', '-N', originalfile, patchfile],
check_exit_code=False)
return
if not self.check_pkg('python-virtualenv'):
- self.yum_install('python-virtualenv', check_exit_code=False)
+ self.die("Please install 'python-virtualenv'.")
super(Fedora, self).install_virtualenv()
This can be removed when the fix is applied upstream.
Nova: https://bugs.launchpad.net/nova/+bug/884915
- Upstream: https://bitbucket.org/which_linden/eventlet/issue/89
+ Upstream: https://bitbucket.org/eventlet/eventlet/issue/89
+ RHEL: https://bugzilla.redhat.com/958868
"""
# Install "patch" program if it's not there
if not self.check_pkg('patch'):
- self.yum_install('patch')
+ self.die("Please install 'patch'.")
# Apply the eventlet patch
self.apply_patch(os.path.join(self.venv, 'lib', self.py_version,
import install_venv_common as install_venv
+def first_file(file_list):
+ for candidate in file_list:
+ if os.path.exists(candidate):
+ return candidate
+
+
def main(argv):
root = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
venv = os.environ['VIRTUAL_ENV']
- pip_requires = os.path.join(root, 'requirements.txt')
- test_requires = os.path.join(root, 'test-requirements.txt')
+ pip_requires = first_file([
+ os.path.join(root, 'requirements.txt'),
+ os.path.join(root, 'tools', 'pip-requires'),
+ ])
+ test_requires = first_file([
+ os.path.join(root, 'test-requirements.txt'),
+ os.path.join(root, 'tools', 'test-requires'),
+ ])
py_version = "python%s.%s" % (sys.version_info[0], sys.version_info[1])
project = 'cinder'
install = install_venv.InstallVenv(root, venv, pip_requires, test_requires,
Code Review / openstack-build / cinder-build.git / commitdiff