From 9a8d41e0132d4676b611f940de552dd450b7cac7 Mon Sep 17 00:00:00 2001 From: Zhongyue Luo Date: Mon, 13 Jan 2014 09:44:56 +0800 Subject: [PATCH] Sync oslo imageutils, strutils to cinder Changes in imageutils: =============================== bec3a5e Implements SI/IEC unit system conversion to bytes 8b2b0b7 Use hacking import_exceptions for gettextutils._ aad179d Fixing misspelled encryption key in QemuImgInfo 12bcdb7 Remove vim header 2bd46eb Refactors byte size extraction logic Changes in strutils: ==================== bec3a5e Implements SI/IEC unit system conversion to bytes e53fe85 strutils bool_from_string, allow specified default 8b2b0b7 Use hacking import_exceptions for gettextutils._ 84d461e Fix a bug in safe_encode where it returns a bytes object in py3 12bcdb7 Remove vim header 3970d46 Fix typos in oslo 1a2df89 Enable H302 hacking check 67bd769 python3: Fix traceback while running python3. b0c51ec Refactors to_bytes Change-Id: I765bcd40f959162874bfda412b69e673c6e5f732 Related-bug: #1189635 Related-bug: #1193765 Related-Bug: #1257829 --- cinder/openstack/common/imageutils.py | 33 +++++--- cinder/openstack/common/strutils.py | 116 ++++++++++++++++---------- cinder/volume/utils.py | 12 +-- 3 files changed, 98 insertions(+), 63 deletions(-) diff --git a/cinder/openstack/common/imageutils.py b/cinder/openstack/common/imageutils.py index 312ffb5d1..babffceec 100644 --- a/cinder/openstack/common/imageutils.py +++ b/cinder/openstack/common/imageutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. @@ -23,7 +21,7 @@ Helper methods to deal with images. import re -from cinder.openstack.common.gettextutils import _ # noqa +from cinder.openstack.common.gettextutils import _ from cinder.openstack.common import strutils @@ -31,7 +29,7 @@ class QemuImgInfo(object): BACKING_FILE_RE = re.compile((r"^(.*?)\s*\(actual\s+path\s*:" r"\s+(.*?)\)\s*$"), re.I) TOP_LEVEL_RE = re.compile(r"^([\w\d\s\_\-]+):(.*)$") - SIZE_RE = re.compile(r"\(\s*(\d+)\s+bytes\s*\)", re.I) + SIZE_RE = re.compile(r"(\d+)(\w+)?(\s*\(\s*(\d+)\s+bytes\s*\))?", re.I) def __init__(self, cmd_output=None): details = self._parse(cmd_output or '') @@ -42,7 +40,7 @@ class QemuImgInfo(object): self.cluster_size = details.get('cluster_size') self.disk_size = details.get('disk_size') self.snapshots = details.get('snapshot_list', []) - self.encryption = details.get('encryption') + self.encrypted = details.get('encrypted') def __str__(self): lines = [ @@ -55,6 +53,8 @@ class QemuImgInfo(object): ] if self.snapshots: lines.append("snapshots: %s" % self.snapshots) + if self.encrypted: + lines.append("encrypted: %s" % self.encrypted) return "\n".join(lines) def _canonicalize(self, field): @@ -70,13 +70,17 @@ class QemuImgInfo(object): def _extract_bytes(self, details): # Replace it with the byte amount real_size = self.SIZE_RE.search(details) - if real_size: - details = real_size.group(1) - try: - details = strutils.to_bytes(details) - except TypeError: - pass - return details + if not real_size: + raise ValueError(_('Invalid input value "%s".') % details) + magnitude = real_size.group(1) + unit_of_measure = real_size.group(2) + bytes_info = real_size.group(3) + if bytes_info: + return int(real_size.group(4)) + elif not unit_of_measure: + return int(magnitude) + return strutils.string_to_bytes('%s%sB' % (magnitude, unit_of_measure), + return_int=True) def _extract_details(self, root_cmd, root_details, lines_after): real_details = root_details @@ -87,7 +91,10 @@ class QemuImgInfo(object): real_details = backing_match.group(2).strip() elif root_cmd in ['virtual_size', 'cluster_size', 'disk_size']: # Replace it with the byte amount (if we can convert it) - real_details = self._extract_bytes(root_details) + if root_details == 'None': + real_details = 0 + else: + real_details = self._extract_bytes(root_details) elif root_cmd == 'file_format': real_details = real_details.strip().lower() elif root_cmd == 'snapshot_list': diff --git a/cinder/openstack/common/strutils.py b/cinder/openstack/common/strutils.py index 8cd996599..a1d127578 100644 --- a/cinder/openstack/common/strutils.py +++ b/cinder/openstack/common/strutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # @@ -19,23 +17,31 @@ System-level utilities and helper functions. """ +import math import re import sys import unicodedata +import six + from cinder.openstack.common.gettextutils import _ -# Used for looking up extensions of text -# to their 'multiplied' byte amount -BYTE_MULTIPLIERS = { - '': 1, - 't': 1024 ** 4, - 'g': 1024 ** 3, - 'm': 1024 ** 2, - 'k': 1024, +UNIT_PREFIX_EXPONENT = { + 'k': 1, + 'K': 1, + 'Ki': 1, + 'M': 2, + 'Mi': 2, + 'G': 3, + 'Gi': 3, + 'T': 4, + 'Ti': 4, +} +UNIT_SYSTEM_INFO = { + 'IEC': (1024, re.compile(r'(^[-+]?\d*\.?\d+)([KMGT]i?)?(b|bit|B)$')), + 'SI': (1000, re.compile(r'(^[-+]?\d*\.?\d+)([kMGT])?(b|bit|B)$')), } -BYTE_REGEX = re.compile(r'(^-?\d+)(\D*)') TRUE_STRINGS = ('1', 't', 'true', 'on', 'y', 'yes') FALSE_STRINGS = ('0', 'f', 'false', 'off', 'n', 'no') @@ -58,12 +64,12 @@ def int_from_bool_as_string(subject): return bool_from_string(subject) and 1 or 0 -def bool_from_string(subject, strict=False): +def bool_from_string(subject, strict=False, default=False): """Interpret a string as a boolean. A case-insensitive match is performed such that strings matching 't', 'true', 'on', 'y', 'yes', or '1' are considered True and, when - `strict=False`, anything else is considered False. + `strict=False`, anything else returns the value specified by 'default'. Useful for JSON-decoded stuff and config file parsing. @@ -71,7 +77,7 @@ def bool_from_string(subject, strict=False): ValueError which is useful when parsing values passed in from an API call. Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'. """ - if not isinstance(subject, basestring): + if not isinstance(subject, six.string_types): subject = str(subject) lowered = subject.strip().lower() @@ -88,7 +94,7 @@ def bool_from_string(subject, strict=False): 'acceptable': acceptable} raise ValueError(msg) else: - return False + return default def safe_decode(text, incoming=None, errors='strict'): @@ -99,12 +105,12 @@ def safe_decode(text, incoming=None, errors='strict'): values http://docs.python.org/2/library/codecs.html :returns: text or a unicode `incoming` encoded representation of it. - :raises TypeError: If text is not an isntance of basestring + :raises TypeError: If text is not an instance of str """ - if not isinstance(text, basestring): + if not isinstance(text, six.string_types): raise TypeError("%s can't be decoded" % type(text)) - if isinstance(text, unicode): + if isinstance(text, six.text_type): return text if not incoming: @@ -142,53 +148,75 @@ def safe_encode(text, incoming=None, values http://docs.python.org/2/library/codecs.html :returns: text or a bytestring `encoding` encoded representation of it. - :raises TypeError: If text is not an isntance of basestring + :raises TypeError: If text is not an instance of str """ - if not isinstance(text, basestring): + if not isinstance(text, six.string_types): raise TypeError("%s can't be encoded" % type(text)) if not incoming: incoming = (sys.stdin.encoding or sys.getdefaultencoding()) - if isinstance(text, unicode): - return text.encode(encoding, errors) + if isinstance(text, six.text_type): + if six.PY3: + return text.encode(encoding, errors).decode(incoming) + else: + return text.encode(encoding, errors) elif text and encoding != incoming: # Decode text before encoding it with `encoding` text = safe_decode(text, incoming, errors) - return text.encode(encoding, errors) + if six.PY3: + return text.encode(encoding, errors).decode(incoming) + else: + return text.encode(encoding, errors) return text -def to_bytes(text, default=0): - """Converts a string into an integer of bytes. +def string_to_bytes(text, unit_system='IEC', return_int=False): + """Converts a string into an float representation of bytes. + + The units supported for IEC :: + + Kb(it), Kib(it), Mb(it), Mib(it), Gb(it), Gib(it), Tb(it), Tib(it) + KB, KiB, MB, MiB, GB, GiB, TB, TiB - Looks at the last characters of the text to determine - what conversion is needed to turn the input text into a byte number. - Supports "B, K(B), M(B), G(B), and T(B)". (case insensitive) + The units supported for SI :: + + kb(it), Mb(it), Gb(it), Tb(it) + kB, MB, GB, TB + + Note that the SI unit system does not support capital letter 'K' :param text: String input for bytes size conversion. - :param default: Default return value when text is blank. + :param unit_system: Unit system for byte size conversion. + :param return_int: If True, returns integer representation of text + in bytes. (default: decimal) + :returns: Numerical representation of text in bytes. + :raises ValueError: If text has an invalid value. """ - match = BYTE_REGEX.search(text) + try: + base, reg_ex = UNIT_SYSTEM_INFO[unit_system] + except KeyError: + msg = _('Invalid unit system: "%s"') % unit_system + raise ValueError(msg) + match = reg_ex.match(text) if match: - magnitude = int(match.group(1)) - mult_key_org = match.group(2) - if not mult_key_org: - return magnitude - elif text: + magnitude = float(match.group(1)) + unit_prefix = match.group(2) + if match.group(3) in ['b', 'bit']: + magnitude /= 8 + else: msg = _('Invalid string format: %s') % text - raise TypeError(msg) + raise ValueError(msg) + if not unit_prefix: + res = magnitude else: - return default - mult_key = mult_key_org.lower().replace('b', '', 1) - multiplier = BYTE_MULTIPLIERS.get(mult_key) - if multiplier is None: - msg = _('Unknown byte multiplier: %s') % mult_key_org - raise TypeError(msg) - return magnitude * multiplier + res = magnitude * pow(base, UNIT_PREFIX_EXPONENT[unit_prefix]) + if return_int: + return int(math.ceil(res)) + return res def to_slug(value, incoming=None, errors="strict"): @@ -204,7 +232,7 @@ def to_slug(value, incoming=None, errors="strict"): :param errors: Errors handling policy. See here for valid values http://docs.python.org/2/library/codecs.html :returns: slugified unicode representation of `value` - :raises TypeError: If text is not an instance of basestring + :raises TypeError: If text is not an instance of str """ value = safe_decode(value, incoming, errors) # NOTE(aababilov): no need to use safe_(encode|decode) here: diff --git a/cinder/volume/utils.py b/cinder/volume/utils.py index 6429d5590..4827f8833 100644 --- a/cinder/volume/utils.py +++ b/cinder/volume/utils.py @@ -136,12 +136,12 @@ def _calculate_count(size_in_m, blocksize): # Check if volume_dd_blocksize is valid try: - # Rule out zero-sized/negative dd blocksize which + # Rule out zero-sized/negative/float dd blocksize which # cannot be caught by strutils - if blocksize.startswith(('-', '0')): + if blocksize.startswith(('-', '0')) or '.' in blocksize: raise ValueError - bs = strutils.to_bytes(blocksize) - except (ValueError, TypeError): + bs = strutils.string_to_bytes('%sB' % blocksize) + except ValueError: msg = (_("Incorrect value error: %(blocksize)s, " "it may indicate that \'volume_dd_blocksize\' " "was configured incorrectly. Fall back to default.") @@ -150,9 +150,9 @@ def _calculate_count(size_in_m, blocksize): # Fall back to default blocksize CONF.clear_override('volume_dd_blocksize') blocksize = CONF.volume_dd_blocksize - bs = strutils.to_bytes(blocksize) + bs = strutils.string_to_bytes('%sB' % blocksize) - count = math.ceil(size_in_m * units.MiB / float(bs)) + count = math.ceil(size_in_m * units.MiB / bs) return blocksize, int(count) -- 2.45.2