From 2b4c8e79ecf912025a59b5f853abe1e54115746c Mon Sep 17 00:00:00 2001 From: Thang Pham Date: Mon, 10 Nov 2014 17:02:02 -0500 Subject: [PATCH] Cinder objects base This patch is to insert the objects code base, currently based on nova.objects. It will be later based on oslo_versionedobjects once it is available. This is done to make progress on changing cinder internals to use objects, since nova.objects and oslo.versionedobjects are similar. Inserted objects code base into cinder and modified cinder services to use object-aware serializer. Change-Id: I42a6cc43da19ca29db4cfb33cbf794afed18f758 Implements: blueprint cinder-objects --- cinder/cmd/api.py | 3 + cinder/cmd/manage.py | 5 +- cinder/cmd/volume.py | 3 + cinder/exception.py | 24 + cinder/hacking/checks.py | 3 +- cinder/objects/__init__.py | 27 + cinder/objects/base.py | 867 ++++++++++++++++++++++++ cinder/objects/fields.py | 607 +++++++++++++++++ cinder/scheduler/rpcapi.py | 5 +- cinder/service.py | 4 +- cinder/tests/objects/__init__.py | 0 cinder/tests/objects/test_fields.py | 316 +++++++++ cinder/tests/objects/test_objects.py | 968 +++++++++++++++++++++++++++ cinder/tests/test_cmd.py | 6 +- cinder/tests/test_utils.py | 16 + cinder/utils.py | 26 + cinder/volume/rpcapi.py | 4 +- tools/lintstack.py | 10 + 18 files changed, 2887 insertions(+), 7 deletions(-) create mode 100644 cinder/objects/__init__.py create mode 100644 cinder/objects/base.py create mode 100644 cinder/objects/fields.py create mode 100644 cinder/tests/objects/__init__.py create mode 100644 cinder/tests/objects/test_fields.py create mode 100644 cinder/tests/objects/test_objects.py diff --git a/cinder/cmd/api.py b/cinder/cmd/api.py index da7262279..c3eb509e7 100644 --- a/cinder/cmd/api.py +++ b/cinder/cmd/api.py @@ -23,6 +23,8 @@ eventlet.monkey_patch() import sys import warnings +from cinder import objects + warnings.simplefilter('once', DeprecationWarning) from oslo_config import cfg @@ -43,6 +45,7 @@ CONF = cfg.CONF def main(): + objects.register_all() CONF(sys.argv[1:], project='cinder', version=version.version_string()) logging.setup("cinder") diff --git a/cinder/cmd/manage.py b/cinder/cmd/manage.py index d920203a3..6b433471c 100644 --- a/cinder/cmd/manage.py +++ b/cinder/cmd/manage.py @@ -76,6 +76,7 @@ from cinder import db from cinder.db import migration as db_migration from cinder.db.sqlalchemy import api as db_api from cinder.i18n import _ +from cinder.objects import base as objects_base from cinder.openstack.common import log as logging from cinder import rpc from cinder import utils @@ -271,7 +272,9 @@ class VolumeCommands(object): if not rpc.initialized(): rpc.init(CONF) target = messaging.Target(topic=CONF.volume_topic) - self._client = rpc.get_client(target) + serializer = objects_base.CinderObjectSerializer() + self._client = rpc.get_client(target, serializer=serializer) + return self._client @args('volume_id', diff --git a/cinder/cmd/volume.py b/cinder/cmd/volume.py index 0eed7c7e7..ba50ea5eb 100644 --- a/cinder/cmd/volume.py +++ b/cinder/cmd/volume.py @@ -21,6 +21,8 @@ import os import eventlet +from cinder import objects + if os.name == 'nt': # eventlet monkey patching the os module causes subprocess.Popen to fail # on Windows when using pipes due to missing non-blocking IO support. @@ -54,6 +56,7 @@ CONF = cfg.CONF def main(): + objects.register_all() CONF(sys.argv[1:], project='cinder', version=version.version_string()) logging.setup("cinder") diff --git a/cinder/exception.py b/cinder/exception.py index c5b3fd81b..d308d3288 100644 --- a/cinder/exception.py +++ b/cinder/exception.py @@ -657,6 +657,30 @@ class EvaluatorParseException(Exception): message = _("Error during evaluator parsing: %(reason)s") +class ObjectActionError(CinderException): + msg_fmt = _('Object action %(action)s failed because: %(reason)s') + + +class ObjectFieldInvalid(CinderException): + msg_fmt = _('Field %(field)s of %(objname)s is not an instance of Field') + + +class UnsupportedObjectError(CinderException): + msg_fmt = _('Unsupported object type %(objtype)s') + + +class OrphanedObjectError(CinderException): + msg_fmt = _('Cannot call %(method)s on orphaned %(objtype)s object') + + +class IncompatibleObjectVersion(CinderException): + msg_fmt = _('Version %(objver)s of %(objname)s is not supported') + + +class ReadOnlyFieldError(CinderException): + msg_fmt = _('Cannot modify readonly field %(field)s') + + # Driver specific exceptions # Coraid class CoraidException(VolumeDriverException): diff --git a/cinder/hacking/checks.py b/cinder/hacking/checks.py index 231d65999..5f44b7f47 100644 --- a/cinder/hacking/checks.py +++ b/cinder/hacking/checks.py @@ -29,7 +29,8 @@ Guidelines for writing new hacking checks """ -UNDERSCORE_IMPORT_FILES = [] +# NOTE(thangp): Ignore N323 pep8 error caused by importing cinder objects +UNDERSCORE_IMPORT_FILES = ['./cinder/objects/__init__.py'] translated_log = re.compile( r"(.)*LOG\.(audit|error|info|warn|warning|critical|exception)" diff --git a/cinder/objects/__init__.py b/cinder/objects/__init__.py new file mode 100644 index 000000000..4330fb346 --- /dev/null +++ b/cinder/objects/__init__.py @@ -0,0 +1,27 @@ +# Copyright 2015 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# NOTE(comstud): You may scratch your head as you see code that imports +# this module and then accesses attributes for objects such as Instance, +# etc, yet you do not see these attributes in here. Never fear, there is +# a little bit of magic. When objects are registered, an attribute is set +# on this module automatically, pointing to the newest/latest version of +# the object. + + +def register_all(): + # NOTE(danms): You must make sure your object gets imported in this + # function in order for it to be registered by services that may + # need to receive it via RPC. + pass diff --git a/cinder/objects/base.py b/cinder/objects/base.py new file mode 100644 index 000000000..5bf74ded5 --- /dev/null +++ b/cinder/objects/base.py @@ -0,0 +1,867 @@ +# Copyright 2015 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Cinder common internal object model""" + +import collections +import contextlib +import copy +import datetime +import functools +import traceback + +import netaddr +from oslo import messaging +from oslo_utils import timeutils +import six + +from cinder import context +from cinder import exception +from cinder.i18n import _, _LE +from cinder import objects +from cinder.objects import fields +from cinder.openstack.common import log as logging +from cinder.openstack.common import versionutils +from cinder import utils + + +LOG = logging.getLogger('object') + + +class NotSpecifiedSentinel(object): + pass + + +def get_attrname(name): + """Return the mangled name of the attribute's underlying storage.""" + return '_' + name + + +def make_class_properties(cls): + # NOTE(danms/comstud): Inherit fields from super classes. + # mro() returns the current class first and returns 'object' last, so + # those can be skipped. Also be careful to not overwrite any fields + # that already exist. And make sure each cls has its own copy of + # fields and that it is not sharing the dict with a super class. + cls.fields = dict(cls.fields) + for supercls in cls.mro()[1:-1]: + if not hasattr(supercls, 'fields'): + continue + for name, field in supercls.fields.items(): + if name not in cls.fields: + cls.fields[name] = field + for name, field in cls.fields.iteritems(): + if not isinstance(field, fields.Field): + raise exception.ObjectFieldInvalid( + field=name, objname=cls.obj_name()) + + def getter(self, name=name): + attrname = get_attrname(name) + if not hasattr(self, attrname): + self.obj_load_attr(name) + return getattr(self, attrname) + + def setter(self, value, name=name, field=field): + attrname = get_attrname(name) + field_value = field.coerce(self, name, value) + if field.read_only and hasattr(self, attrname): + # Note(yjiang5): _from_db_object() may iterate + # every field and write, no exception in such situation. + if getattr(self, attrname) != field_value: + raise exception.ReadOnlyFieldError(field=name) + else: + return + + self._changed_fields.add(name) + try: + return setattr(self, attrname, field_value) + except Exception: + attr = "%s.%s" % (self.obj_name(), name) + LOG.exception(_LE('Error setting %(attr)s'), {'attr': attr}) + raise + + setattr(cls, name, property(getter, setter)) + + +class CinderObjectMetaclass(type): + """Metaclass that allows tracking of object classes.""" + + # NOTE(danms): This is what controls whether object operations are + # remoted. If this is not None, use it to remote things over RPC. + indirection_api = None + + def __init__(cls, names, bases, dict_): + if not hasattr(cls, '_obj_classes'): + # This means this is a base class using the metaclass. I.e., + # the 'CinderObject' class. + cls._obj_classes = collections.defaultdict(list) + return + + def _vers_tuple(obj): + return tuple([int(x) for x in obj.VERSION.split(".")]) + + # Add the subclass to CinderObject._obj_classes. If the + # same version already exists, replace it. Otherwise, + # keep the list with newest version first. + make_class_properties(cls) + obj_name = cls.obj_name() + for i, obj in enumerate(cls._obj_classes[obj_name]): + if cls.VERSION == obj.VERSION: + cls._obj_classes[obj_name][i] = cls + # Update cinder.objects with this newer class. + setattr(objects, obj_name, cls) + break + if _vers_tuple(cls) > _vers_tuple(obj): + # Insert before. + cls._obj_classes[obj_name].insert(i, cls) + if i == 0: + # Later version than we've seen before. Update + # cinder.objects. + setattr(objects, obj_name, cls) + break + else: + cls._obj_classes[obj_name].append(cls) + # Either this is the first time we've seen the object or it's + # an older version than anything we'e seen. Update cinder.objects + # only if it's the first time we've seen this object name. + if not hasattr(objects, obj_name): + setattr(objects, obj_name, cls) + + +# These are decorators that mark an object's method as remotable. +# If the metaclass is configured to forward object methods to an +# indirection service, these will result in making an RPC call +# instead of directly calling the implementation in the object. Instead, +# the object implementation on the remote end will perform the +# requested action and the result will be returned here. +def remotable_classmethod(fn): + """Decorator for remotable classmethods.""" + @functools.wraps(fn) + def wrapper(cls, context, *args, **kwargs): + if CinderObject.indirection_api: + result = CinderObject.indirection_api.object_class_action( + context, cls.obj_name(), fn.__name__, cls.VERSION, + args, kwargs) + else: + result = fn(cls, context, *args, **kwargs) + if isinstance(result, CinderObject): + result._context = context + return result + + # NOTE(danms): Make this discoverable + wrapper.remotable = True + wrapper.original_fn = fn + return classmethod(wrapper) + + +# See comment above for remotable_classmethod() +# +# Note that this will use either the provided context, or the one +# stashed in the object. If neither are present, the object is +# "orphaned" and remotable methods cannot be called. +def remotable(fn): + """Decorator for remotable object methods.""" + @functools.wraps(fn) + def wrapper(self, *args, **kwargs): + ctxt = self._context + try: + if isinstance(args[0], (context.RequestContext)): + ctxt = args[0] + args = args[1:] + except IndexError: + pass + if ctxt is None: + raise exception.OrphanedObjectError(method=fn.__name__, + objtype=self.obj_name()) + # Force this to be set if it wasn't before. + self._context = ctxt + if CinderObject.indirection_api: + updates, result = CinderObject.indirection_api.object_action( + ctxt, self, fn.__name__, args, kwargs) + for key, value in updates.iteritems(): + if key in self.fields: + field = self.fields[key] + # NOTE(ndipanov): Since CinderObjectSerializer will have + # deserialized any object fields into objects already, + # we do not try to deserialize them again here. + if isinstance(value, CinderObject): + self[key] = value + else: + self[key] = field.from_primitive(self, key, value) + self.obj_reset_changes() + self._changed_fields = set(updates.get('obj_what_changed', [])) + return result + else: + return fn(self, ctxt, *args, **kwargs) + + wrapper.remotable = True + wrapper.original_fn = fn + return wrapper + + +@six.add_metaclass(CinderObjectMetaclass) +class CinderObject(object): + """Base class and object factory. + + This forms the base of all objects that can be remoted or instantiated + via RPC. Simply defining a class that inherits from this base class + will make it remotely instantiatable. Objects should implement the + necessary "get" classmethod routines as well as "save" object methods + as appropriate. + """ + + # Object versioning rules + # + # Each service has its set of objects, each with a version attached. When + # a client attempts to call an object method, the server checks to see if + # the version of that object matches (in a compatible way) its object + # implementation. If so, cool, and if not, fail. + # + # This version is allowed to have three parts, X.Y.Z, where the .Z element + # is reserved for stable branch backports. The .Z is ignored for the + # purposes of triggering a backport, which means anything changed under + # a .Z must be additive and non-destructive such that a node that knows + # about X.Y can consider X.Y.Z equivalent. + VERSION = '1.0' + + # The fields present in this object as key:field pairs. For example: + # + # fields = { 'foo': fields.IntegerField(), + # 'bar': fields.StringField(), + # } + fields = {} + obj_extra_fields = [] + + # Table of sub-object versioning information + # + # This contains a list of version mappings, by the field name of + # the subobject. The mappings must be in order of oldest to + # newest, and are tuples of (my_version, subobject_version). A + # request to backport this object to $my_version will cause the + # subobject to be backported to $subobject_version. + # + # obj_relationships = { + # 'subobject1': [('1.2', '1.1'), ('1.4', '1.2')], + # 'subobject2': [('1.2', '1.0')], + # } + # + # In the above example: + # + # - If we are asked to backport our object to version 1.3, + # subobject1 will be backported to version 1.1, since it was + # bumped to version 1.2 when our version was 1.4. + # - If we are asked to backport our object to version 1.5, + # no changes will be made to subobject1 or subobject2, since + # they have not changed since version 1.4. + # - If we are asked to backlevel our object to version 1.1, we + # will remove both subobject1 and subobject2 from the primitive, + # since they were not added until version 1.2. + obj_relationships = {} + + def __init__(self, context=None, **kwargs): + self._changed_fields = set() + self._context = context + for key in kwargs.keys(): + setattr(self, key, kwargs[key]) + + def __repr__(self): + return '%s(%s)' % ( + self.obj_name(), + ','.join(['%s=%s' % (name, + (self.obj_attr_is_set(name) and + field.stringify(getattr(self, name)) or + '')) + for name, field in sorted(self.fields.items())])) + + @classmethod + def obj_name(cls): + """Return a canonical name for this object. + + The canonical name will be used over the wire for remote hydration. + """ + return cls.__name__ + + @classmethod + def obj_class_from_name(cls, objname, objver): + """Returns a class from the registry based on a name and version.""" + if objname not in cls._obj_classes: + LOG.error(_LE('Unable to instantiate unregistered object type ' + '%(objtype)s'), dict(objtype=objname)) + raise exception.UnsupportedObjectError(objtype=objname) + + # NOTE(comstud): If there's not an exact match, return the highest + # compatible version. The objects stored in the class are sorted + # such that highest version is first, so only set compatible_match + # once below. + compatible_match = None + + for objclass in cls._obj_classes[objname]: + if objclass.VERSION == objver: + return objclass + if (not compatible_match and + versionutils.is_compatible(objver, objclass.VERSION)): + compatible_match = objclass + + if compatible_match: + return compatible_match + + # As mentioned above, latest version is always first in the list. + latest_ver = cls._obj_classes[objname][0].VERSION + raise exception.IncompatibleObjectVersion(objname=objname, + objver=objver, + supported=latest_ver) + + @classmethod + def _obj_from_primitive(cls, context, objver, primitive): + self = cls() + self._context = context + self.VERSION = objver + objdata = primitive['cinder_object.data'] + changes = primitive.get('cinder_object.changes', []) + for name, field in self.fields.items(): + if name in objdata: + setattr(self, name, field.from_primitive(self, name, + objdata[name])) + self._changed_fields = set([x for x in changes if x in self.fields]) + return self + + @classmethod + def obj_from_primitive(cls, primitive, context=None): + """Object field-by-field hydration.""" + if primitive['cinder_object.namespace'] != 'cinder': + # NOTE(danms): We don't do anything with this now, but it's + # there for "the future" + raise exception.UnsupportedObjectError( + objtype='%s.%s' % (primitive['cinder_object.namespace'], + primitive['cinder_object.name'])) + objname = primitive['cinder_object.name'] + objver = primitive['cinder_object.version'] + objclass = cls.obj_class_from_name(objname, objver) + return objclass._obj_from_primitive(context, objver, primitive) + + def __deepcopy__(self, memo): + """Efficiently make a deep copy of this object.""" + + # NOTE(danms): A naive deepcopy would copy more than we need, + # and since we have knowledge of the volatile bits of the + # object, we can be smarter here. Also, nested entities within + # some objects may be uncopyable, so we can avoid those sorts + # of issues by copying only our field data. + + nobj = self.__class__() + nobj._context = self._context + for name in self.fields: + if self.obj_attr_is_set(name): + nval = copy.deepcopy(getattr(self, name), memo) + setattr(nobj, name, nval) + nobj._changed_fields = set(self._changed_fields) + return nobj + + def obj_clone(self): + """Create a copy.""" + return copy.deepcopy(self) + + def _obj_make_obj_compatible(self, primitive, target_version, field): + """Backlevel a sub-object based on our versioning rules. + + This is responsible for backporting objects contained within + this object's primitive according to a set of rules we + maintain about version dependencies between objects. This + requires that the obj_relationships table in this object is + correct and up-to-date. + + :param:primitive: The primitive version of this object + :param:target_version: The version string requested for this object + :param:field: The name of the field in this object containing the + sub-object to be backported + """ + + def _do_backport(to_version): + obj = getattr(self, field) + if not obj: + return + if isinstance(obj, CinderObject): + obj.obj_make_compatible( + primitive[field]['cinder_object.data'], + to_version) + primitive[field]['cinder_object.version'] = to_version + elif isinstance(obj, list): + for i, element in enumerate(obj): + element.obj_make_compatible( + primitive[field][i]['cinder_object.data'], + to_version) + primitive[field][i]['cinder_object.version'] = to_version + + target_version = utils.convert_version_to_tuple(target_version) + for index, versions in enumerate(self.obj_relationships[field]): + my_version, child_version = versions + my_version = utils.convert_version_to_tuple(my_version) + if target_version < my_version: + if index == 0: + # We're backporting to a version from before this + # subobject was added: delete it from the primitive. + del primitive[field] + else: + # We're in the gap between index-1 and index, so + # backport to the older version + last_child_version = \ + self.obj_relationships[field][index - 1][1] + _do_backport(last_child_version) + return + elif target_version == my_version: + # This is the first mapping that satisfies the + # target_version request: backport the object. + _do_backport(child_version) + return + + def obj_make_compatible(self, primitive, target_version): + """Make an object representation compatible with a target version. + + This is responsible for taking the primitive representation of + an object and making it suitable for the given target_version. + This may mean converting the format of object attributes, removing + attributes that have been added since the target version, etc. In + general: + + - If a new version of an object adds a field, this routine + should remove it for older versions. + - If a new version changed or restricted the format of a field, this + should convert it back to something a client knowing only of the + older version will tolerate. + - If an object that this object depends on is bumped, then this + object should also take a version bump. Then, this routine should + backlevel the dependent object (by calling its obj_make_compatible()) + if the requested version of this object is older than the version + where the new dependent object was added. + + :param:primitive: The result of self.obj_to_primitive() + :param:target_version: The version string requested by the recipient + of the object + :raises: cinder.exception.UnsupportedObjectError if conversion + is not possible for some reason + """ + for key, field in self.fields.items(): + if not isinstance(field, (fields.ObjectField, + fields.ListOfObjectsField)): + continue + if not self.obj_attr_is_set(key): + continue + if key not in self.obj_relationships: + # NOTE(danms): This is really a coding error and shouldn't + # happen unless we miss something + raise exception.ObjectActionError( + action='obj_make_compatible', + reason='No rule for %s' % key) + self._obj_make_obj_compatible(primitive, target_version, key) + + def obj_to_primitive(self, target_version=None): + """Simple base-case dehydration. + + This calls to_primitive() for each item in fields. + """ + primitive = dict() + for name, field in self.fields.items(): + if self.obj_attr_is_set(name): + primitive[name] = field.to_primitive(self, name, + getattr(self, name)) + if target_version: + self.obj_make_compatible(primitive, target_version) + obj = {'cinder_object.name': self.obj_name(), + 'cinder_object.namespace': 'cinder', + 'cinder_object.version': target_version or self.VERSION, + 'cinder_object.data': primitive} + if self.obj_what_changed(): + obj['cinder_object.changes'] = list(self.obj_what_changed()) + return obj + + def obj_set_defaults(self, *attrs): + if not attrs: + attrs = [name for name, field in self.fields.items() + if field.default != fields.UnspecifiedDefault] + + for attr in attrs: + default = self.fields[attr].default + if default is fields.UnspecifiedDefault: + raise exception.ObjectActionError( + action='set_defaults', + reason='No default set for field %s' % attr) + setattr(self, attr, default) + + def obj_load_attr(self, attrname): + """Load an additional attribute from the real object.""" + raise NotImplementedError( + _("Cannot load '%s' in the base class") % attrname) + + def save(self, context): + """Save the changed fields back to the store. + + This is optional for subclasses, but is presented here in the base + class for consistency among those that do. + """ + raise NotImplementedError('Cannot save anything in the base class') + + def obj_what_changed(self): + """Returns a set of fields that have been modified.""" + changes = set(self._changed_fields) + for field in self.fields: + if (self.obj_attr_is_set(field) and + isinstance(getattr(self, field), CinderObject) and + getattr(self, field).obj_what_changed()): + changes.add(field) + return changes + + def obj_get_changes(self): + """Returns a dict of changed fields and their new values.""" + changes = {} + for key in self.obj_what_changed(): + changes[key] = getattr(self, key) + return changes + + def obj_reset_changes(self, fields=None): + """Reset the list of fields that have been changed. + + Note that this is NOT "revert to previous values" + """ + if fields: + self._changed_fields -= set(fields) + else: + self._changed_fields.clear() + + def obj_attr_is_set(self, attrname): + """Test object to see if attrname is present. + + Returns True if the named attribute has a value set, or + False if not. Raises AttributeError if attrname is not + a valid attribute for this object. + """ + if attrname not in self.obj_fields: + raise AttributeError( + _("%(objname)s object has no attribute '%(attrname)s'") % + {'objname': self.obj_name(), 'attrname': attrname}) + return hasattr(self, get_attrname(attrname)) + + @property + def obj_fields(self): + return self.fields.keys() + self.obj_extra_fields + + +class CinderObjectDictCompat(object): + """Mix-in to provide dictionary key access compat + + If an object needs to support attribute access using + dictionary items instead of object attributes, inherit + from this class. This should only be used as a temporary + measure until all callers are converted to use modern + attribute access. + + NOTE(berrange) This class will eventually be deleted. + """ + + # dictish syntactic sugar + def iteritems(self): + """For backwards-compatibility with dict-based objects. + + NOTE(danms): May be removed in the future. + """ + for name in self.obj_fields: + if (self.obj_attr_is_set(name) or + name in self.obj_extra_fields): + yield name, getattr(self, name) + + items = lambda self: list(self.iteritems()) + + def __getitem__(self, name): + """For backwards-compatibility with dict-based objects. + + NOTE(danms): May be removed in the future. + """ + return getattr(self, name) + + def __setitem__(self, name, value): + """For backwards-compatibility with dict-based objects. + + NOTE(danms): May be removed in the future. + """ + setattr(self, name, value) + + def __contains__(self, name): + """For backwards-compatibility with dict-based objects. + + NOTE(danms): May be removed in the future. + """ + try: + return self.obj_attr_is_set(name) + except AttributeError: + return False + + def get(self, key, value=NotSpecifiedSentinel): + """For backwards-compatibility with dict-based objects. + + NOTE(danms): May be removed in the future. + """ + if key not in self.obj_fields: + raise AttributeError("'%s' object has no attribute '%s'" % ( + self.__class__, key)) + if value != NotSpecifiedSentinel and not self.obj_attr_is_set(key): + return value + else: + return getattr(self, key) + + def update(self, updates): + """For backwards-compatibility with dict-base objects. + + NOTE(danms): May be removed in the future. + """ + for key, value in updates.items(): + setattr(self, key, value) + + +class CinderPersistentObject(object): + """Mixin class for Persistent objects. + This adds the fields that we use in common for all persistent objects. + """ + fields = { + 'created_at': fields.DateTimeField(nullable=True), + 'updated_at': fields.DateTimeField(nullable=True), + 'deleted_at': fields.DateTimeField(nullable=True), + 'deleted': fields.BooleanField(default=False), + } + + @contextlib.contextmanager + def obj_as_admin(self): + """Context manager to make an object call as an admin. + + This temporarily modifies the context embedded in an object to + be elevated() and restores it after the call completes. Example + usage: + + with obj.obj_as_admin(): + obj.save() + + """ + if self._context is None: + raise exception.OrphanedObjectError(method='obj_as_admin', + objtype=self.obj_name()) + + original_context = self._context + self._context = self._context.elevated() + try: + yield + finally: + self._context = original_context + + +class ObjectListBase(object): + """Mixin class for lists of objects. + + This mixin class can be added as a base class for an object that + is implementing a list of objects. It adds a single field of 'objects', + which is the list store, and behaves like a list itself. It supports + serialization of the list of objects automatically. + """ + fields = { + 'objects': fields.ListOfObjectsField('CinderObject'), + } + + # This is a dictionary of my_version:child_version mappings so that + # we can support backleveling our contents based on the version + # requested of the list object. + child_versions = {} + + def __init__(self, *args, **kwargs): + super(ObjectListBase, self).__init__(*args, **kwargs) + if 'objects' not in kwargs: + self.objects = [] + self._changed_fields.discard('objects') + + def __iter__(self): + """List iterator interface.""" + return iter(self.objects) + + def __len__(self): + """List length.""" + return len(self.objects) + + def __getitem__(self, index): + """List index access.""" + if isinstance(index, slice): + new_obj = self.__class__() + new_obj.objects = self.objects[index] + # NOTE(danms): We must be mixed in with a CinderObject! + new_obj.obj_reset_changes() + new_obj._context = self._context + return new_obj + return self.objects[index] + + def __contains__(self, value): + """List membership test.""" + return value in self.objects + + def count(self, value): + """List count of value occurrences.""" + return self.objects.count(value) + + def index(self, value): + """List index of value.""" + return self.objects.index(value) + + def sort(self, cmp=None, key=None, reverse=False): + self.objects.sort(cmp=cmp, key=key, reverse=reverse) + + def obj_make_compatible(self, primitive, target_version): + primitives = primitive['objects'] + child_target_version = self.child_versions.get(target_version, '1.0') + for index, item in enumerate(self.objects): + self.objects[index].obj_make_compatible( + primitives[index]['cinder_object.data'], + child_target_version) + primitives[index]['cinder_object.version'] = child_target_version + + def obj_what_changed(self): + changes = set(self._changed_fields) + for child in self.objects: + if child.obj_what_changed(): + changes.add('objects') + return changes + + +class CinderObjectSerializer(messaging.NoOpSerializer): + """A CinderObject-aware Serializer. + + This implements the Oslo Serializer interface and provides the + ability to serialize and deserialize CinderObject entities. Any service + that needs to accept or return CinderObjects as arguments or result values + should pass this to its RPCClient and RPCServer objects. + """ + + def _process_object(self, context, objprim): + try: + objinst = CinderObject.obj_from_primitive(objprim, context=context) + except exception.IncompatibleObjectVersion: + objver = objprim['cinder_object.version'] + if objver.count('.') == 2: + # NOTE(danms): For our purposes, the .z part of the version + # should be safe to accept without requiring a backport + objprim['cinder_object.version'] = \ + '.'.join(objver.split('.')[:2]) + return self._process_object(context, objprim) + raise + + return objinst + + def _process_iterable(self, context, action_fn, values): + """Process an iterable, taking an action on each value. + :param:context: Request context + :param:action_fn: Action to take on each item in values + :param:values: Iterable container of things to take action on + :returns: A new container of the same type (except set) with + items from values having had action applied. + """ + iterable = values.__class__ + if issubclass(iterable, dict): + return iterable(**{k: action_fn(context, v) + for k, v in six.iteritems(values)}) + else: + # NOTE(danms): A set can't have an unhashable value inside, such as + # a dict. Convert sets to tuples, which is fine, since we can't + # send them over RPC anyway. + if iterable == set: + iterable = tuple + return iterable([action_fn(context, value) for value in values]) + + def serialize_entity(self, context, entity): + if isinstance(entity, (tuple, list, set, dict)): + entity = self._process_iterable(context, self.serialize_entity, + entity) + elif (hasattr(entity, 'obj_to_primitive') and + callable(entity.obj_to_primitive)): + entity = entity.obj_to_primitive() + return entity + + def deserialize_entity(self, context, entity): + if isinstance(entity, dict) and 'cinder_object.name' in entity: + entity = self._process_object(context, entity) + elif isinstance(entity, (tuple, list, set, dict)): + entity = self._process_iterable(context, self.deserialize_entity, + entity) + return entity + + +def obj_to_primitive(obj): + """Recursively turn an object into a python primitive. + + A CinderObject becomes a dict, and anything that implements ObjectListBase + becomes a list. + """ + if isinstance(obj, ObjectListBase): + return [obj_to_primitive(x) for x in obj] + elif isinstance(obj, CinderObject): + result = {} + for key in obj.obj_fields: + if obj.obj_attr_is_set(key) or key in obj.obj_extra_fields: + result[key] = obj_to_primitive(getattr(obj, key)) + return result + elif isinstance(obj, netaddr.IPAddress): + return str(obj) + elif isinstance(obj, netaddr.IPNetwork): + return str(obj) + else: + return obj + + +def obj_make_list(context, list_obj, item_cls, db_list, **extra_args): + """Construct an object list from a list of primitives. + + This calls item_cls._from_db_object() on each item of db_list, and + adds the resulting object to list_obj. + + :param:context: Request contextr + :param:list_obj: An ObjectListBase object + :param:item_cls: The CinderObject class of the objects within the list + :param:db_list: The list of primitives to convert to objects + :param:extra_args: Extra arguments to pass to _from_db_object() + :returns: list_obj + """ + list_obj.objects = [] + for db_item in db_list: + item = item_cls._from_db_object(context, item_cls(), db_item, + **extra_args) + list_obj.objects.append(item) + list_obj._context = context + list_obj.obj_reset_changes() + return list_obj + + +def serialize_args(fn): + """Decorator that will do the arguments serialization before remoting.""" + def wrapper(obj, *args, **kwargs): + for kw in kwargs: + value_arg = kwargs.get(kw) + if kw == 'exc_val' and value_arg: + kwargs[kw] = str(value_arg) + elif kw == 'exc_tb' and ( + not isinstance(value_arg, six.string_types) and value_arg): + kwargs[kw] = ''.join(traceback.format_tb(value_arg)) + elif isinstance(value_arg, datetime.datetime): + kwargs[kw] = timeutils.isotime(value_arg) + if hasattr(fn, '__call__'): + return fn(obj, *args, **kwargs) + # NOTE(danms): We wrap a descriptor, so use that protocol + return fn.__get__(None, obj)(*args, **kwargs) + + # NOTE(danms): Make this discoverable + wrapper.remotable = getattr(fn, 'remotable', False) + wrapper.original_fn = fn + return (functools.wraps(fn)(wrapper) if hasattr(fn, '__call__') + else classmethod(wrapper)) diff --git a/cinder/objects/fields.py b/cinder/objects/fields.py new file mode 100644 index 000000000..168b8677e --- /dev/null +++ b/cinder/objects/fields.py @@ -0,0 +1,607 @@ +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc +import datetime + +import iso8601 +import netaddr +from oslo_utils import timeutils +import six + +from cinder.i18n import _ + + +class KeyTypeError(TypeError): + def __init__(self, expected, value): + super(KeyTypeError, self).__init__( + _('Key %(key)s must be of type %(expected)s not %(actual)s' + ) % {'key': repr(value), + 'expected': expected.__name__, + 'actual': value.__class__.__name__, + }) + + +class ElementTypeError(TypeError): + def __init__(self, expected, key, value): + super(ElementTypeError, self).__init__( + _('Element %(key)s:%(val)s must be of type %(expected)s' + ' not %(actual)s' + ) % {'key': key, + 'val': repr(value), + 'expected': expected, + 'actual': value.__class__.__name__, + }) + + +@six.add_metaclass(abc.ABCMeta) +class AbstractFieldType(object): + @abc.abstractmethod + def coerce(self, obj, attr, value): + """This is called to coerce (if possible) a value on assignment. + + This method should convert the value given into the designated type, + or throw an exception if this is not possible. + + :param:obj: The CinderObject on which an attribute is being set + :param:attr: The name of the attribute being set + :param:value: The value being set + :returns: A properly-typed value + """ + pass + + @abc.abstractmethod + def from_primitive(self, obj, attr, value): + """This is called to deserialize a value. + + This method should deserialize a value from the form given by + to_primitive() to the designated type. + + :param:obj: The CinderObject on which the value is to be set + :param:attr: The name of the attribute which will hold the value + :param:value: The serialized form of the value + :returns: The natural form of the value + """ + pass + + @abc.abstractmethod + def to_primitive(self, obj, attr, value): + """This is called to serialize a value. + + This method should serialize a value to the form expected by + from_primitive(). + + :param:obj: The CinderObject on which the value is set + :param:attr: The name of the attribute holding the value + :param:value: The natural form of the value + :returns: The serialized form of the value + """ + pass + + @abc.abstractmethod + def describe(self): + """Returns a string describing the type of the field.""" + pass + + @abc.abstractmethod + def stringify(self, value): + """Returns a short stringified version of a value.""" + pass + + +class FieldType(AbstractFieldType): + @staticmethod + def coerce(obj, attr, value): + return value + + @staticmethod + def from_primitive(obj, attr, value): + return value + + @staticmethod + def to_primitive(obj, attr, value): + return value + + def describe(self): + return self.__class__.__name__ + + def stringify(self, value): + return str(value) + + +class UnspecifiedDefault(object): + pass + + +class Field(object): + def __init__(self, field_type, nullable=False, + default=UnspecifiedDefault, read_only=False): + self._type = field_type + self._nullable = nullable + self._default = default + self._read_only = read_only + + def __repr__(self): + args = { + 'nullable': self._nullable, + 'default': self._default, + } + return '%s(%s)' % (self._type.__class__.__name__, + ','.join(['%s=%s' % (k, v) + for k, v in args.items()])) + + @property + def nullable(self): + return self._nullable + + @property + def default(self): + return self._default + + @property + def read_only(self): + return self._read_only + + def _null(self, obj, attr): + if self.nullable: + return None + elif self._default != UnspecifiedDefault: + # NOTE(danms): We coerce the default value each time the field + # is set to None as our contract states that we'll let the type + # examine the object and attribute name at that time. + return self._type.coerce(obj, attr, self._default) + else: + raise ValueError(_("Field `%s' cannot be None") % attr) + + def coerce(self, obj, attr, value): + """Coerce a value to a suitable type. + + This is called any time you set a value on an object, like: + + foo.myint = 1 + + and is responsible for making sure that the value (1 here) is of + the proper type, or can be sanely converted. + + This also handles the potentially nullable or defaultable + nature of the field and calls the coerce() method on a + FieldType to actually do the coercion. + + :param:obj: The object being acted upon + :param:attr: The name of the attribute/field being set + :param:value: The value being set + :returns: The properly-typed value + """ + if value is None: + return self._null(obj, attr) + else: + return self._type.coerce(obj, attr, value) + + def from_primitive(self, obj, attr, value): + """Deserialize a value from primitive form. + + This is responsible for deserializing a value from primitive + into regular form. It calls the from_primitive() method on a + FieldType to do the actual deserialization. + + :param:obj: The object being acted upon + :param:attr: The name of the attribute/field being deserialized + :param:value: The value to be deserialized + :returns: The deserialized value + """ + if value is None: + return None + else: + return self._type.from_primitive(obj, attr, value) + + def to_primitive(self, obj, attr, value): + """Serialize a value to primitive form. + + This is responsible for serializing a value to primitive + form. It calls to_primitive() on a FieldType to do the actual + serialization. + + :param:obj: The object being acted upon + :param:attr: The name of the attribute/field being serialized + :param:value: The value to be serialized + :returns: The serialized value + """ + if value is None: + return None + else: + return self._type.to_primitive(obj, attr, value) + + def describe(self): + """Return a short string describing the type of this field.""" + name = self._type.describe() + prefix = self.nullable and 'Nullable' or '' + return prefix + name + + def stringify(self, value): + if value is None: + return 'None' + else: + return self._type.stringify(value) + + +class String(FieldType): + @staticmethod + def coerce(obj, attr, value): + # FIXME(danms): We should really try to avoid the need to do this + if isinstance(value, (six.string_types, int, long, float, + datetime.datetime)): + return unicode(value) + else: + raise ValueError(_('A string is required here, not %s') % + value.__class__.__name__) + + @staticmethod + def stringify(value): + return "'%s'" % value + + +class UUID(FieldType): + @staticmethod + def coerce(obj, attr, value): + # FIXME(danms): We should actually verify the UUIDness here + return six.text_type(value) + + +class Integer(FieldType): + @staticmethod + def coerce(obj, attr, value): + return int(value) + + +class Float(FieldType): + def coerce(self, obj, attr, value): + return float(value) + + +class Boolean(FieldType): + @staticmethod + def coerce(obj, attr, value): + return bool(value) + + +class DateTime(FieldType): + @staticmethod + def coerce(obj, attr, value): + if isinstance(value, six.string_types): + # NOTE(danms): Being tolerant of isotime strings here will help us + # during our objects transition + value = timeutils.parse_isotime(value) + elif not isinstance(value, datetime.datetime): + raise ValueError(_('A datetime.datetime is required here')) + + if value.utcoffset() is None: + # NOTE(danms): Legacy objects from sqlalchemy are stored in UTC, + # but are returned without a timezone attached. + # As a transitional aid, assume a tz-naive object is in UTC. + value = value.replace(tzinfo=iso8601.iso8601.Utc()) + return value + + def from_primitive(self, obj, attr, value): + return self.coerce(obj, attr, timeutils.parse_isotime(value)) + + @staticmethod + def to_primitive(obj, attr, value): + return timeutils.isotime(value) + + @staticmethod + def stringify(value): + return timeutils.isotime(value) + + +class IPAddress(FieldType): + @staticmethod + def coerce(obj, attr, value): + try: + return netaddr.IPAddress(value) + except netaddr.AddrFormatError as e: + raise ValueError(six.text_type(e)) + + def from_primitive(self, obj, attr, value): + return self.coerce(obj, attr, value) + + @staticmethod + def to_primitive(obj, attr, value): + return six.text_type(value) + + +class IPV4Address(IPAddress): + @staticmethod + def coerce(obj, attr, value): + result = IPAddress.coerce(obj, attr, value) + if result.version != 4: + raise ValueError(_('Network "%s" is not valid') % value) + return result + + +class IPV6Address(IPAddress): + @staticmethod + def coerce(obj, attr, value): + result = IPAddress.coerce(obj, attr, value) + if result.version != 6: + raise ValueError(_('Network "%s" is not valid') % value) + return result + + +class IPV4AndV6Address(IPAddress): + @staticmethod + def coerce(obj, attr, value): + result = IPAddress.coerce(obj, attr, value) + if result.version != 4 and result.version != 6: + raise ValueError(_('Network "%s" is not valid') % value) + return result + + +class IPNetwork(IPAddress): + @staticmethod + def coerce(obj, attr, value): + try: + return netaddr.IPNetwork(value) + except netaddr.AddrFormatError as e: + raise ValueError(six.text_type(e)) + + +class IPV4Network(IPNetwork): + @staticmethod + def coerce(obj, attr, value): + try: + return netaddr.IPNetwork(value, version=4) + except netaddr.AddrFormatError as e: + raise ValueError(six.text_type(e)) + + +class IPV6Network(IPNetwork): + @staticmethod + def coerce(obj, attr, value): + try: + return netaddr.IPNetwork(value, version=6) + except netaddr.AddrFormatError as e: + raise ValueError(six.text_type(e)) + + +class CompoundFieldType(FieldType): + def __init__(self, element_type, **field_args): + self._element_type = Field(element_type, **field_args) + + +class List(CompoundFieldType): + def coerce(self, obj, attr, value): + if not isinstance(value, list): + raise ValueError(_('A list is required here')) + for index, element in enumerate(list(value)): + value[index] = self._element_type.coerce( + obj, '%s[%i]' % (attr, index), element) + return value + + def to_primitive(self, obj, attr, value): + return [self._element_type.to_primitive(obj, attr, x) for x in value] + + def from_primitive(self, obj, attr, value): + return [self._element_type.from_primitive(obj, attr, x) for x in value] + + def stringify(self, value): + return '[%s]' % ( + ','.join([self._element_type.stringify(x) for x in value])) + + +class Dict(CompoundFieldType): + def coerce(self, obj, attr, value): + if not isinstance(value, dict): + raise ValueError(_('A dict is required here')) + for key, element in value.items(): + if not isinstance(key, six.string_types): + # NOTE(guohliu) In order to keep compatibility with python3 + # we need to use six.string_types rather than basestring here, + # since six.string_types is a tuple, so we need to pass the + # real type in. + raise KeyTypeError(six.string_types[0], key) + value[key] = self._element_type.coerce( + obj, '%s["%s"]' % (attr, key), element) + return value + + def to_primitive(self, obj, attr, value): + primitive = {} + for key, element in value.items(): + primitive[key] = self._element_type.to_primitive( + obj, '%s["%s"]' % (attr, key), element) + return primitive + + def from_primitive(self, obj, attr, value): + concrete = {} + for key, element in value.items(): + concrete[key] = self._element_type.from_primitive( + obj, '%s["%s"]' % (attr, key), element) + return concrete + + def stringify(self, value): + return '{%s}' % ( + ','.join(['%s=%s' % (key, self._element_type.stringify(val)) + for key, val in sorted(value.items())])) + + +class DictProxyField(object): + """Descriptor allowing us to assign pinning data as a dict of key_types. + + This allows us to have an object field that will be a dict of key_type + keys, allowing that will convert back to string-keyed dict. + + This will take care of the conversion while the dict field will make sure + that we store the raw json-serializable data on the object. + + key_type should return a type that unambiguously responds to six.text_type + so that calling key_type on it yields the same thing. + """ + def __init__(self, dict_field_name, key_type=int): + self._fld_name = dict_field_name + self._key_type = key_type + + def __get__(self, obj, obj_type=None): + if obj is None: + return self + if getattr(obj, self._fld_name) is None: + return + return {self._key_type(k): v + for k, v in six.iteritems(getattr(obj, self._fld_name))} + + def __set__(self, obj, val): + if val is None: + setattr(obj, self._fld_name, val) + else: + setattr(obj, self._fld_name, {six.text_type(k): v + for k, v in six.iteritems(val)}) + + +class Set(CompoundFieldType): + def coerce(self, obj, attr, value): + if not isinstance(value, set): + raise ValueError(_('A set is required here')) + + coerced = set() + for element in value: + coerced.add(self._element_type.coerce( + obj, '%s["%s"]' % (attr, element), element)) + return coerced + + def to_primitive(self, obj, attr, value): + return tuple( + self._element_type.to_primitive(obj, attr, x) for x in value) + + def from_primitive(self, obj, attr, value): + return set([self._element_type.from_primitive(obj, attr, x) + for x in value]) + + def stringify(self, value): + return 'set([%s])' % ( + ','.join([self._element_type.stringify(x) for x in value])) + + +class Object(FieldType): + def __init__(self, obj_name, **kwargs): + self._obj_name = obj_name + super(Object, self).__init__(**kwargs) + + def coerce(self, obj, attr, value): + try: + obj_name = value.obj_name() + except AttributeError: + obj_name = "" + + if obj_name != self._obj_name: + raise ValueError(_('An object of type %s is required here') % + self._obj_name) + return value + + @staticmethod + def to_primitive(obj, attr, value): + return value.obj_to_primitive() + + @staticmethod + def from_primitive(obj, attr, value): + # FIXME(danms): Avoid circular import from base.py + from cinder.objects import base as obj_base + # NOTE (ndipanov): If they already got hydrated by the serializer, just + # pass them back unchanged + if isinstance(value, obj_base.CinderObject): + return value + return obj_base.CinderObject.obj_from_primitive(value, obj._context) + + def describe(self): + return "Object<%s>" % self._obj_name + + def stringify(self, value): + if 'uuid' in value.fields: + ident = '(%s)' % (value.obj_attr_is_set('uuid') and value.uuid or + 'UNKNOWN') + elif 'id' in value.fields: + ident = '(%s)' % (value.obj_attr_is_set('id') and value.id or + 'UNKNOWN') + else: + ident = '' + + return '%s%s' % (self._obj_name, ident) + + +class AutoTypedField(Field): + AUTO_TYPE = None + + def __init__(self, **kwargs): + super(AutoTypedField, self).__init__(self.AUTO_TYPE, **kwargs) + + +class StringField(AutoTypedField): + AUTO_TYPE = String() + + +class UUIDField(AutoTypedField): + AUTO_TYPE = UUID() + + +class IntegerField(AutoTypedField): + AUTO_TYPE = Integer() + + +class FloatField(AutoTypedField): + AUTO_TYPE = Float() + + +class BooleanField(AutoTypedField): + AUTO_TYPE = Boolean() + + +class DateTimeField(AutoTypedField): + AUTO_TYPE = DateTime() + + +class DictOfStringsField(AutoTypedField): + AUTO_TYPE = Dict(String()) + + +class DictOfNullableStringsField(AutoTypedField): + AUTO_TYPE = Dict(String(), nullable=True) + + +class DictOfIntegersField(AutoTypedField): + AUTO_TYPE = Dict(Integer()) + + +class ListOfStringsField(AutoTypedField): + AUTO_TYPE = List(String()) + + +class SetOfIntegersField(AutoTypedField): + AUTO_TYPE = Set(Integer()) + + +class ListOfSetsOfIntegersField(AutoTypedField): + AUTO_TYPE = List(Set(Integer())) + + +class ListOfDictOfNullableStringsField(AutoTypedField): + AUTO_TYPE = List(Dict(String(), nullable=True)) + + +class ObjectField(AutoTypedField): + def __init__(self, objtype, **kwargs): + self.AUTO_TYPE = Object(objtype) + super(ObjectField, self).__init__(**kwargs) + + +class ListOfObjectsField(AutoTypedField): + def __init__(self, objtype, **kwargs): + self.AUTO_TYPE = List(Object(objtype)) + super(ListOfObjectsField, self).__init__(**kwargs) diff --git a/cinder/scheduler/rpcapi.py b/cinder/scheduler/rpcapi.py index 2e5a1a177..bd61dbd4c 100644 --- a/cinder/scheduler/rpcapi.py +++ b/cinder/scheduler/rpcapi.py @@ -20,6 +20,7 @@ from oslo import messaging from oslo_config import cfg from oslo_serialization import jsonutils +from cinder.objects import base as objects_base from cinder import rpc @@ -48,7 +49,9 @@ class SchedulerAPI(object): super(SchedulerAPI, self).__init__() target = messaging.Target(topic=CONF.scheduler_topic, version=self.RPC_API_VERSION) - self.client = rpc.get_client(target, version_cap='1.7') + serializer = objects_base.CinderObjectSerializer() + self.client = rpc.get_client(target, version_cap='1.7', + serializer=serializer) def create_consistencygroup(self, ctxt, topic, group_id, request_spec_list=None, diff --git a/cinder/service.py b/cinder/service.py index 5918bafa2..89ece8a32 100644 --- a/cinder/service.py +++ b/cinder/service.py @@ -35,6 +35,7 @@ from cinder import context from cinder import db from cinder import exception from cinder.i18n import _ +from cinder.objects import base as objects_base from cinder.openstack.common import log as logging from cinder.openstack.common import loopingcall from cinder.openstack.common import service @@ -154,7 +155,8 @@ class Service(service.Service): target = messaging.Target(topic=self.topic, server=self.host) endpoints = [self.manager] endpoints.extend(self.manager.additional_endpoints) - self.rpcserver = rpc.get_server(target, endpoints) + serializer = objects_base.CinderObjectSerializer() + self.rpcserver = rpc.get_server(target, endpoints, serializer) self.rpcserver.start() if self.report_interval: diff --git a/cinder/tests/objects/__init__.py b/cinder/tests/objects/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cinder/tests/objects/test_fields.py b/cinder/tests/objects/test_fields.py new file mode 100644 index 000000000..1edd58479 --- /dev/null +++ b/cinder/tests/objects/test_fields.py @@ -0,0 +1,316 @@ +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import datetime + +import iso8601 +from oslo_utils import timeutils + +from cinder.objects import base as obj_base +from cinder.objects import fields +from cinder import test + + +class FakeFieldType(fields.FieldType): + def coerce(self, obj, attr, value): + return '*%s*' % value + + def to_primitive(self, obj, attr, value): + return '!%s!' % value + + def from_primitive(self, obj, attr, value): + return value[1:-1] + + +class TestField(test.TestCase): + def setUp(self): + super(TestField, self).setUp() + self.field = fields.Field(FakeFieldType()) + self.coerce_good_values = [('foo', '*foo*')] + self.coerce_bad_values = [] + self.to_primitive_values = [('foo', '!foo!')] + self.from_primitive_values = [('!foo!', 'foo')] + + def test_coerce_good_values(self): + for in_val, out_val in self.coerce_good_values: + self.assertEqual(out_val, self.field.coerce('obj', 'attr', in_val)) + + def test_coerce_bad_values(self): + for in_val in self.coerce_bad_values: + self.assertRaises((TypeError, ValueError), + self.field.coerce, 'obj', 'attr', in_val) + + def test_to_primitive(self): + for in_val, prim_val in self.to_primitive_values: + self.assertEqual(prim_val, self.field.to_primitive('obj', 'attr', + in_val)) + + def test_from_primitive(self): + class ObjectLikeThing(object): + _context = 'context' + + for prim_val, out_val in self.from_primitive_values: + self.assertEqual(out_val, self.field.from_primitive( + ObjectLikeThing, 'attr', prim_val)) + + def test_stringify(self): + self.assertEqual('123', self.field.stringify(123)) + + +class TestString(TestField): + def setUp(self): + super(TestField, self).setUp() + self.field = fields.StringField() + self.coerce_good_values = [('foo', 'foo'), (1, '1'), (1L, '1'), + (True, 'True')] + self.coerce_bad_values = [None] + self.to_primitive_values = self.coerce_good_values[0:1] + self.from_primitive_values = self.coerce_good_values[0:1] + + def test_stringify(self): + self.assertEqual("'123'", self.field.stringify(123)) + + +class TestInteger(TestField): + def setUp(self): + super(TestField, self).setUp() + self.field = fields.IntegerField() + self.coerce_good_values = [(1, 1), ('1', 1)] + self.coerce_bad_values = ['foo', None] + self.to_primitive_values = self.coerce_good_values[0:1] + self.from_primitive_values = self.coerce_good_values[0:1] + + +class TestFloat(TestField): + def setUp(self): + super(TestFloat, self).setUp() + self.field = fields.FloatField() + self.coerce_good_values = [(1.1, 1.1), ('1.1', 1.1)] + self.coerce_bad_values = ['foo', None] + self.to_primitive_values = self.coerce_good_values[0:1] + self.from_primitive_values = self.coerce_good_values[0:1] + + +class TestBoolean(TestField): + def setUp(self): + super(TestField, self).setUp() + self.field = fields.BooleanField() + self.coerce_good_values = [(True, True), (False, False), (1, True), + ('foo', True), (0, False), ('', False)] + self.coerce_bad_values = [] + self.to_primitive_values = self.coerce_good_values[0:2] + self.from_primitive_values = self.coerce_good_values[0:2] + + +class TestDateTime(TestField): + def setUp(self): + super(TestDateTime, self).setUp() + self.dt = datetime.datetime(1955, 11, 5, tzinfo=iso8601.iso8601.Utc()) + self.field = fields.DateTimeField() + self.coerce_good_values = [(self.dt, self.dt), + (timeutils.isotime(self.dt), self.dt)] + self.coerce_bad_values = [1, 'foo'] + self.to_primitive_values = [(self.dt, timeutils.isotime(self.dt))] + self.from_primitive_values = [(timeutils.isotime(self.dt), self.dt)] + + def test_stringify(self): + self.assertEqual( + '1955-11-05T18:00:00Z', + self.field.stringify( + datetime.datetime(1955, 11, 5, 18, 0, 0, + tzinfo=iso8601.iso8601.Utc()))) + + +class TestDict(TestField): + def setUp(self): + super(TestDict, self).setUp() + self.field = fields.Field(fields.Dict(FakeFieldType())) + self.coerce_good_values = [({'foo': 'bar'}, {'foo': '*bar*'}), + ({'foo': 1}, {'foo': '*1*'})] + self.coerce_bad_values = [{1: 'bar'}, 'foo'] + self.to_primitive_values = [({'foo': 'bar'}, {'foo': '!bar!'})] + self.from_primitive_values = [({'foo': '!bar!'}, {'foo': 'bar'})] + + def test_stringify(self): + self.assertEqual("{key=val}", self.field.stringify({'key': 'val'})) + + +class TestDictOfStrings(TestField): + def setUp(self): + super(TestDictOfStrings, self).setUp() + self.field = fields.DictOfStringsField() + self.coerce_good_values = [({'foo': 'bar'}, {'foo': 'bar'}), + ({'foo': 1}, {'foo': '1'})] + self.coerce_bad_values = [{1: 'bar'}, {'foo': None}, 'foo'] + self.to_primitive_values = [({'foo': 'bar'}, {'foo': 'bar'})] + self.from_primitive_values = [({'foo': 'bar'}, {'foo': 'bar'})] + + def test_stringify(self): + self.assertEqual("{key='val'}", self.field.stringify({'key': 'val'})) + + +class TestDictOfIntegers(TestField): + def setUp(self): + super(TestDictOfIntegers, self).setUp() + self.field = fields.DictOfIntegersField() + self.coerce_good_values = [({'foo': '42'}, {'foo': 42}), + ({'foo': 4.2}, {'foo': 4})] + self.coerce_bad_values = [{1: 'bar'}, {'foo': 'boo'}, + 'foo', {'foo': None}] + self.to_primitive_values = [({'foo': 42}, {'foo': 42})] + self.from_primitive_values = [({'foo': 42}, {'foo': 42})] + + def test_stringify(self): + self.assertEqual("{key=42}", self.field.stringify({'key': 42})) + + +class TestDictOfStringsNone(TestField): + def setUp(self): + super(TestDictOfStringsNone, self).setUp() + self.field = fields.DictOfNullableStringsField() + self.coerce_good_values = [({'foo': 'bar'}, {'foo': 'bar'}), + ({'foo': 1}, {'foo': '1'}), + ({'foo': None}, {'foo': None})] + self.coerce_bad_values = [{1: 'bar'}, 'foo'] + self.to_primitive_values = [({'foo': 'bar'}, {'foo': 'bar'})] + self.from_primitive_values = [({'foo': 'bar'}, {'foo': 'bar'})] + + def test_stringify(self): + self.assertEqual("{k2=None,key='val'}", + self.field.stringify({'k2': None, + 'key': 'val'})) + + +class TestListOfDictOfNullableStringsField(TestField): + def setUp(self): + super(TestListOfDictOfNullableStringsField, self).setUp() + self.field = fields.ListOfDictOfNullableStringsField() + self.coerce_good_values = [([{'f': 'b', 'f1': 'b1'}, {'f2': 'b2'}], + [{'f': 'b', 'f1': 'b1'}, {'f2': 'b2'}]), + ([{'f': 1}, {'f1': 'b1'}], + [{'f': '1'}, {'f1': 'b1'}]), + ([{'foo': None}], [{'foo': None}])] + self.coerce_bad_values = [[{1: 'a'}], ['ham', 1], ['eggs']] + self.to_primitive_values = [([{'f': 'b'}, {'f1': 'b1'}, {'f2': None}], + [{'f': 'b'}, {'f1': 'b1'}, {'f2': None}])] + self.from_primitive_values = [([{'f': 'b'}, {'f1': 'b1'}, + {'f2': None}], + [{'f': 'b'}, {'f1': 'b1'}, + {'f2': None}])] + + def test_stringify(self): + self.assertEqual("[{f=None,f1='b1'},{f2='b2'}]", + self.field.stringify( + [{'f': None, 'f1': 'b1'}, {'f2': 'b2'}])) + + +class TestList(TestField): + def setUp(self): + super(TestList, self).setUp() + self.field = fields.Field(fields.List(FakeFieldType())) + self.coerce_good_values = [(['foo', 'bar'], ['*foo*', '*bar*'])] + self.coerce_bad_values = ['foo'] + self.to_primitive_values = [(['foo'], ['!foo!'])] + self.from_primitive_values = [(['!foo!'], ['foo'])] + + def test_stringify(self): + self.assertEqual('[123]', self.field.stringify([123])) + + +class TestListOfStrings(TestField): + def setUp(self): + super(TestListOfStrings, self).setUp() + self.field = fields.ListOfStringsField() + self.coerce_good_values = [(['foo', 'bar'], ['foo', 'bar'])] + self.coerce_bad_values = ['foo'] + self.to_primitive_values = [(['foo'], ['foo'])] + self.from_primitive_values = [(['foo'], ['foo'])] + + def test_stringify(self): + self.assertEqual("['abc']", self.field.stringify(['abc'])) + + +class TestSet(TestField): + def setUp(self): + super(TestSet, self).setUp() + self.field = fields.Field(fields.Set(FakeFieldType())) + self.coerce_good_values = [(set(['foo', 'bar']), + set(['*foo*', '*bar*']))] + self.coerce_bad_values = [['foo'], {'foo': 'bar'}] + self.to_primitive_values = [(set(['foo']), tuple(['!foo!']))] + self.from_primitive_values = [(tuple(['!foo!']), set(['foo']))] + + def test_stringify(self): + self.assertEqual('set([123])', self.field.stringify(set([123]))) + + +class TestSetOfIntegers(TestField): + def setUp(self): + super(TestSetOfIntegers, self).setUp() + self.field = fields.SetOfIntegersField() + self.coerce_good_values = [(set(['1', 2]), + set([1, 2]))] + self.coerce_bad_values = [set(['foo'])] + self.to_primitive_values = [(set([1]), tuple([1]))] + self.from_primitive_values = [(tuple([1]), set([1]))] + + def test_stringify(self): + self.assertEqual('set([1,2])', self.field.stringify(set([1, 2]))) + + +class TestListOfSetsOfIntegers(TestField): + def setUp(self): + super(TestListOfSetsOfIntegers, self).setUp() + self.field = fields.ListOfSetsOfIntegersField() + self.coerce_good_values = [([set(['1', 2]), set([3, '4'])], + [set([1, 2]), set([3, 4])])] + self.coerce_bad_values = [[set(['foo'])]] + self.to_primitive_values = [([set([1])], [tuple([1])])] + self.from_primitive_values = [([tuple([1])], [set([1])])] + + def test_stringify(self): + self.assertEqual('[set([1,2])]', self.field.stringify([set([1, 2])])) + + +class TestObject(TestField): + def setUp(self): + super(TestObject, self).setUp() + + class TestableObject(obj_base.CinderObject): + fields = { + 'uuid': fields.StringField(), + } + + def __eq__(self, value): + # NOTE(danms): Be rather lax about this equality thing to + # satisfy the assertEqual() in test_from_primitive(). We + # just want to make sure the right type of object is re-created + return value.__class__.__name__ == TestableObject.__name__ + + class OtherTestableObject(obj_base.CinderObject): + pass + + test_inst = TestableObject() + self._test_cls = TestableObject + self.field = fields.Field(fields.Object('TestableObject')) + self.coerce_good_values = [(test_inst, test_inst)] + self.coerce_bad_values = [OtherTestableObject(), 1, 'foo'] + self.to_primitive_values = [(test_inst, test_inst.obj_to_primitive())] + self.from_primitive_values = [(test_inst.obj_to_primitive(), + test_inst), (test_inst, test_inst)] + + def test_stringify(self): + obj = self._test_cls(uuid='fake-uuid') + self.assertEqual('TestableObject(fake-uuid)', + self.field.stringify(obj)) diff --git a/cinder/tests/objects/test_objects.py b/cinder/tests/objects/test_objects.py new file mode 100644 index 000000000..1eacc4c88 --- /dev/null +++ b/cinder/tests/objects/test_objects.py @@ -0,0 +1,968 @@ +# Copyright 2015 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import contextlib +import copy +import datetime + +import mock +from oslo_serialization import jsonutils +from oslo_utils import timeutils +import six +from testtools import matchers + +from cinder import context +from cinder import exception +from cinder import objects +from cinder.objects import base +from cinder.objects import fields +from cinder import test +from cinder.tests import fake_notifier + + +class MyOwnedObject(base.CinderPersistentObject, base.CinderObject): + VERSION = '1.0' + fields = {'baz': fields.Field(fields.Integer())} + + +class MyObj(base.CinderPersistentObject, base.CinderObject, + base.CinderObjectDictCompat): + VERSION = '1.6' + fields = {'foo': fields.Field(fields.Integer(), default=1), + 'bar': fields.Field(fields.String()), + 'missing': fields.Field(fields.String()), + 'readonly': fields.Field(fields.Integer(), read_only=True), + 'rel_object': fields.ObjectField('MyOwnedObject', nullable=True), + 'rel_objects': fields.ListOfObjectsField('MyOwnedObject', + nullable=True), + } + + @staticmethod + def _from_db_object(context, obj, db_obj): + self = MyObj() + self.foo = db_obj['foo'] + self.bar = db_obj['bar'] + self.missing = db_obj['missing'] + self.readonly = 1 + return self + + def obj_load_attr(self, attrname): + setattr(self, attrname, 'loaded!') + + @base.remotable_classmethod + def query(cls, context): + obj = cls(context=context, foo=1, bar='bar') + obj.obj_reset_changes() + return obj + + @base.remotable + def marco(self, context): + return 'polo' + + @base.remotable + def _update_test(self, context): + if context.project_id == 'alternate': + self.bar = 'alternate-context' + else: + self.bar = 'updated' + + @base.remotable + def save(self, context): + self.obj_reset_changes() + + @base.remotable + def refresh(self, context): + self.foo = 321 + self.bar = 'refreshed' + self.obj_reset_changes() + + @base.remotable + def modify_save_modify(self, context): + self.bar = 'meow' + self.save() + self.foo = 42 + self.rel_object = MyOwnedObject(baz=42) + + def obj_make_compatible(self, primitive, target_version): + super(MyObj, self).obj_make_compatible(primitive, target_version) + # NOTE(danms): Simulate an older version that had a different + # format for the 'bar' attribute + if target_version == '1.1' and 'bar' in primitive: + primitive['bar'] = 'old%s' % primitive['bar'] + + +class MyObjDiffVers(MyObj): + VERSION = '1.5' + + @classmethod + def obj_name(cls): + return 'MyObj' + + +class MyObj2(object): + @classmethod + def obj_name(cls): + return 'MyObj' + + @base.remotable_classmethod + def query(cls, *args, **kwargs): + pass + + +class RandomMixInWithNoFields(object): + """Used to test object inheritance using a mixin that has no fields.""" + pass + + +class TestSubclassedObject(RandomMixInWithNoFields, MyObj): + fields = {'new_field': fields.Field(fields.String())} + + +class TestMetaclass(test.TestCase): + def test_obj_tracking(self): + + @six.add_metaclass(base.CinderObjectMetaclass) + class NewBaseClass(object): + VERSION = '1.0' + fields = {} + + @classmethod + def obj_name(cls): + return cls.__name__ + + class Fake1TestObj1(NewBaseClass): + @classmethod + def obj_name(cls): + return 'fake1' + + class Fake1TestObj2(Fake1TestObj1): + pass + + class Fake1TestObj3(Fake1TestObj1): + VERSION = '1.1' + + class Fake2TestObj1(NewBaseClass): + @classmethod + def obj_name(cls): + return 'fake2' + + class Fake1TestObj4(Fake1TestObj3): + VERSION = '1.2' + + class Fake2TestObj2(Fake2TestObj1): + VERSION = '1.1' + + class Fake1TestObj5(Fake1TestObj1): + VERSION = '1.1' + + # Newest versions first in the list. Duplicate versions take the + # newest object. + expected = {'fake1': [Fake1TestObj4, Fake1TestObj5, Fake1TestObj2], + 'fake2': [Fake2TestObj2, Fake2TestObj1]} + self.assertEqual(expected, NewBaseClass._obj_classes) + # The following should work, also. + self.assertEqual(expected, Fake1TestObj1._obj_classes) + self.assertEqual(expected, Fake1TestObj2._obj_classes) + self.assertEqual(expected, Fake1TestObj3._obj_classes) + self.assertEqual(expected, Fake1TestObj4._obj_classes) + self.assertEqual(expected, Fake1TestObj5._obj_classes) + self.assertEqual(expected, Fake2TestObj1._obj_classes) + self.assertEqual(expected, Fake2TestObj2._obj_classes) + + def test_field_checking(self): + def create_class(field): + class TestField(base.CinderObject): + VERSION = '1.5' + fields = {'foo': field()} + return TestField + + create_class(fields.BooleanField) + self.assertRaises(exception.ObjectFieldInvalid, + create_class, fields.Boolean) + self.assertRaises(exception.ObjectFieldInvalid, + create_class, int) + + +class TestObjToPrimitive(test.TestCase): + + def test_obj_to_primitive_list(self): + class MyObjElement(base.CinderObject): + fields = {'foo': fields.IntegerField()} + + def __init__(self, foo): + super(MyObjElement, self).__init__() + self.foo = foo + + class MyList(base.ObjectListBase, base.CinderObject): + fields = {'objects': fields.ListOfObjectsField('MyObjElement')} + + mylist = MyList() + mylist.objects = [MyObjElement(1), MyObjElement(2), MyObjElement(3)] + self.assertEqual([1, 2, 3], + [x['foo'] for x in base.obj_to_primitive(mylist)]) + + def test_obj_to_primitive_dict(self): + myobj = MyObj(foo=1, bar='foo') + self.assertEqual({'foo': 1, 'bar': 'foo'}, + base.obj_to_primitive(myobj)) + + def test_obj_to_primitive_recursive(self): + class MyList(base.ObjectListBase, base.CinderObject): + fields = {'objects': fields.ListOfObjectsField('MyObj')} + + mylist = MyList(objects=[MyObj(), MyObj()]) + for i, value in enumerate(mylist): + value.foo = i + self.assertEqual([{'foo': 0}, {'foo': 1}], + base.obj_to_primitive(mylist)) + + +class TestObjMakeList(test.TestCase): + + def test_obj_make_list(self): + class MyList(base.ObjectListBase, base.CinderObject): + pass + + db_objs = [{'foo': 1, 'bar': 'baz', 'missing': 'banana'}, + {'foo': 2, 'bar': 'bat', 'missing': 'apple'}, + ] + mylist = base.obj_make_list('ctxt', MyList(), MyObj, db_objs) + self.assertEqual(2, len(mylist)) + self.assertEqual('ctxt', mylist._context) + for index, item in enumerate(mylist): + self.assertEqual(db_objs[index]['foo'], item.foo) + self.assertEqual(db_objs[index]['bar'], item.bar) + self.assertEqual(db_objs[index]['missing'], item.missing) + + +def compare_obj(test, obj, db_obj, subs=None, allow_missing=None, + comparators=None): + """Compare a CinderObject and a dict-like database object. + + This automatically converts TZ-aware datetimes and iterates over + the fields of the object. + + :param:test: The TestCase doing the comparison + :param:obj: The CinderObject to examine + :param:db_obj: The dict-like database object to use as reference + :param:subs: A dict of objkey=dbkey field substitutions + :param:allow_missing: A list of fields that may not be in db_obj + :param:comparators: Map of comparator functions to use for certain fields + """ + + if subs is None: + subs = {} + if allow_missing is None: + allow_missing = [] + if comparators is None: + comparators = {} + + for key in obj.fields: + if key in allow_missing and not obj.obj_attr_is_set(key): + continue + obj_val = getattr(obj, key) + db_key = subs.get(key, key) + db_val = db_obj[db_key] + if isinstance(obj_val, datetime.datetime): + obj_val = obj_val.replace(tzinfo=None) + + if key in comparators: + comparator = comparators[key] + comparator(db_val, obj_val) + else: + test.assertEqual(db_val, obj_val) + + +class _BaseTestCase(test.TestCase): + def setUp(self): + super(_BaseTestCase, self).setUp() + self.remote_object_calls = list() + self.user_id = 'fake-user' + self.project_id = 'fake-project' + self.context = context.RequestContext(self.user_id, self.project_id) + fake_notifier.stub_notifier(self.stubs) + self.addCleanup(fake_notifier.reset) + + def compare_obj(self, obj, db_obj, subs=None, allow_missing=None, + comparators=None): + compare_obj(self, obj, db_obj, subs=subs, allow_missing=allow_missing, + comparators=comparators) + + def json_comparator(self, expected, obj_val): + # json-ify an object field for comparison with its db str + # equivalent + self.assertEqual(expected, jsonutils.dumps(obj_val)) + + def str_comparator(self, expected, obj_val): + """Compare an object field to a string in the db by performing + a simple coercion on the object field value. + """ + self.assertEqual(expected, str(obj_val)) + + def assertNotIsInstance(self, obj, cls, msg=None): + """Python < v2.7 compatibility. Assert 'not isinstance(obj, cls).""" + try: + f = super(_BaseTestCase, self).assertNotIsInstance + except AttributeError: + self.assertThat(obj, + matchers.Not(matchers.IsInstance(cls)), + message=msg or '') + else: + f(obj, cls, msg=msg) + + +class _LocalTest(_BaseTestCase): + def setUp(self): + super(_LocalTest, self).setUp() + # Just in case + base.CinderObject.indirection_api = None + + def assertRemotes(self): + self.assertEqual(self.remote_object_calls, []) + + +@contextlib.contextmanager +def things_temporarily_local(): + _api = base.CinderObject.indirection_api + base.CinderObject.indirection_api = None + yield + base.CinderObject.indirection_api = _api + + +class _TestObject(object): + def test_object_attrs_in_init(self): + # Now check the test one in this file. Should be newest version + self.assertEqual('1.6', objects.MyObj.VERSION) + + def test_hydration_type_error(self): + primitive = {'cinder_object.name': 'MyObj', + 'cinder_object.namespace': 'cinder', + 'cinder_object.version': '1.5', + 'cinder_object.data': {'foo': 'a'}} + self.assertRaises(ValueError, MyObj.obj_from_primitive, primitive) + + def test_hydration(self): + primitive = {'cinder_object.name': 'MyObj', + 'cinder_object.namespace': 'cinder', + 'cinder_object.version': '1.5', + 'cinder_object.data': {'foo': 1}} + real_method = MyObj._obj_from_primitive + + def _obj_from_primitive(*args): + return real_method(*args) + + with mock.patch.object(MyObj, '_obj_from_primitive') as ofp: + ofp.side_effect = _obj_from_primitive + obj = MyObj.obj_from_primitive(primitive) + ofp.assert_called_once_with(None, '1.5', primitive) + self.assertEqual(obj.foo, 1) + + def test_hydration_version_different(self): + primitive = {'cinder_object.name': 'MyObj', + 'cinder_object.namespace': 'cinder', + 'cinder_object.version': '1.2', + 'cinder_object.data': {'foo': 1}} + obj = MyObj.obj_from_primitive(primitive) + self.assertEqual(obj.foo, 1) + self.assertEqual('1.2', obj.VERSION) + + def test_hydration_bad_ns(self): + primitive = {'cinder_object.name': 'MyObj', + 'cinder_object.namespace': 'foo', + 'cinder_object.version': '1.5', + 'cinder_object.data': {'foo': 1}} + self.assertRaises(exception.UnsupportedObjectError, + MyObj.obj_from_primitive, primitive) + + def test_hydration_additional_unexpected_stuff(self): + primitive = {'cinder_object.name': 'MyObj', + 'cinder_object.namespace': 'cinder', + 'cinder_object.version': '1.5.1', + 'cinder_object.data': { + 'foo': 1, + 'unexpected_thing': 'foobar'}} + obj = MyObj.obj_from_primitive(primitive) + self.assertEqual(1, obj.foo) + self.assertFalse(hasattr(obj, 'unexpected_thing')) + # NOTE(danms): If we call obj_from_primitive() directly + # with a version containing .z, we'll get that version + # in the resulting object. In reality, when using the + # serializer, we'll get that snipped off (tested + # elsewhere) + self.assertEqual('1.5.1', obj.VERSION) + + def test_dehydration(self): + expected = {'cinder_object.name': 'MyObj', + 'cinder_object.namespace': 'cinder', + 'cinder_object.version': '1.6', + 'cinder_object.data': {'foo': 1}} + obj = MyObj(foo=1) + obj.obj_reset_changes() + self.assertEqual(obj.obj_to_primitive(), expected) + + def test_object_property(self): + obj = MyObj(foo=1) + self.assertEqual(obj.foo, 1) + + def test_object_property_type_error(self): + obj = MyObj() + + def fail(): + obj.foo = 'a' + self.assertRaises(ValueError, fail) + + def test_object_dict_syntax(self): + obj = MyObj(foo=123, bar='bar') + self.assertEqual(obj['foo'], 123) + self.assertEqual(sorted(obj.items(), key=lambda x: x[0]), + [('bar', 'bar'), ('foo', 123)]) + self.assertEqual(sorted(list(obj.iteritems()), key=lambda x: x[0]), + [('bar', 'bar'), ('foo', 123)]) + + def test_load(self): + obj = MyObj() + self.assertEqual(obj.bar, 'loaded!') + + def test_load_in_base(self): + class Foo(base.CinderObject): + fields = {'foobar': fields.Field(fields.Integer())} + obj = Foo() + with self.assertRaisesRegex(NotImplementedError, ".*foobar.*"): + obj.foobar + + def test_loaded_in_primitive(self): + obj = MyObj(foo=1) + obj.obj_reset_changes() + self.assertEqual(obj.bar, 'loaded!') + expected = {'cinder_object.name': 'MyObj', + 'cinder_object.namespace': 'cinder', + 'cinder_object.version': '1.6', + 'cinder_object.changes': ['bar'], + 'cinder_object.data': {'foo': 1, + 'bar': 'loaded!'}} + self.assertEqual(obj.obj_to_primitive(), expected) + + def test_changes_in_primitive(self): + obj = MyObj(foo=123) + self.assertEqual(obj.obj_what_changed(), set(['foo'])) + primitive = obj.obj_to_primitive() + self.assertIn('cinder_object.changes', primitive) + obj2 = MyObj.obj_from_primitive(primitive) + self.assertEqual(obj2.obj_what_changed(), set(['foo'])) + obj2.obj_reset_changes() + self.assertEqual(obj2.obj_what_changed(), set()) + + def test_obj_class_from_name(self): + obj = base.CinderObject.obj_class_from_name('MyObj', '1.5') + self.assertEqual('1.5', obj.VERSION) + + def test_obj_class_from_name_latest_compatible(self): + obj = base.CinderObject.obj_class_from_name('MyObj', '1.1') + self.assertEqual('1.6', obj.VERSION) + + def test_unknown_objtype(self): + self.assertRaises(exception.UnsupportedObjectError, + base.CinderObject.obj_class_from_name, 'foo', '1.0') + + def test_obj_class_from_name_supported_version(self): + error = None + try: + base.CinderObject.obj_class_from_name('MyObj', '1.25') + except exception.IncompatibleObjectVersion as error: + pass + + self.assertIsNotNone(error) + self.assertEqual('1.6', error.kwargs['supported']) + + def test_with_alternate_context(self): + ctxt1 = context.RequestContext('foo', 'foo') + ctxt2 = context.RequestContext('bar', 'alternate') + obj = MyObj.query(ctxt1) + obj._update_test(ctxt2) + self.assertEqual(obj.bar, 'alternate-context') + self.assertRemotes() + + def test_orphaned_object(self): + obj = MyObj.query(self.context) + obj._context = None + self.assertRaises(exception.OrphanedObjectError, + obj._update_test) + self.assertRemotes() + + def test_changed_1(self): + obj = MyObj.query(self.context) + obj.foo = 123 + self.assertEqual(obj.obj_what_changed(), set(['foo'])) + obj._update_test(self.context) + self.assertEqual(obj.obj_what_changed(), set(['foo', 'bar'])) + self.assertEqual(obj.foo, 123) + self.assertRemotes() + + def test_changed_2(self): + obj = MyObj.query(self.context) + obj.foo = 123 + self.assertEqual(obj.obj_what_changed(), set(['foo'])) + obj.save() + self.assertEqual(obj.obj_what_changed(), set([])) + self.assertEqual(obj.foo, 123) + self.assertRemotes() + + def test_changed_3(self): + obj = MyObj.query(self.context) + obj.foo = 123 + self.assertEqual(obj.obj_what_changed(), set(['foo'])) + obj.refresh() + self.assertEqual(obj.obj_what_changed(), set([])) + self.assertEqual(obj.foo, 321) + self.assertEqual(obj.bar, 'refreshed') + self.assertRemotes() + + def test_changed_4(self): + obj = MyObj.query(self.context) + obj.bar = 'something' + self.assertEqual(obj.obj_what_changed(), set(['bar'])) + obj.modify_save_modify(self.context) + self.assertEqual(obj.obj_what_changed(), set(['foo', 'rel_object'])) + self.assertEqual(obj.foo, 42) + self.assertEqual(obj.bar, 'meow') + self.assertIsInstance(obj.rel_object, MyOwnedObject) + self.assertRemotes() + + def test_changed_with_sub_object(self): + class ParentObject(base.CinderObject): + fields = {'foo': fields.IntegerField(), + 'bar': fields.ObjectField('MyObj'), + } + obj = ParentObject() + self.assertEqual(set(), obj.obj_what_changed()) + obj.foo = 1 + self.assertEqual(set(['foo']), obj.obj_what_changed()) + bar = MyObj() + obj.bar = bar + self.assertEqual(set(['foo', 'bar']), obj.obj_what_changed()) + obj.obj_reset_changes() + self.assertEqual(set(), obj.obj_what_changed()) + bar.foo = 1 + self.assertEqual(set(['bar']), obj.obj_what_changed()) + + def test_static_result(self): + obj = MyObj.query(self.context) + self.assertEqual(obj.bar, 'bar') + result = obj.marco() + self.assertEqual(result, 'polo') + self.assertRemotes() + + def test_updates(self): + obj = MyObj.query(self.context) + self.assertEqual(obj.foo, 1) + obj._update_test() + self.assertEqual(obj.bar, 'updated') + self.assertRemotes() + + def test_base_attributes(self): + dt = datetime.datetime(1955, 11, 5) + obj = MyObj(created_at=dt, updated_at=dt, deleted_at=None, + deleted=False) + expected = {'cinder_object.name': 'MyObj', + 'cinder_object.namespace': 'cinder', + 'cinder_object.version': '1.6', + 'cinder_object.changes': + ['deleted', 'created_at', 'deleted_at', 'updated_at'], + 'cinder_object.data': + {'created_at': timeutils.isotime(dt), + 'updated_at': timeutils.isotime(dt), + 'deleted_at': None, + 'deleted': False, + } + } + self.assertEqual(obj.obj_to_primitive(), expected) + + def test_contains(self): + obj = MyObj() + self.assertNotIn('foo', obj) + obj.foo = 1 + self.assertIn('foo', obj) + self.assertNotIn('does_not_exist', obj) + + def test_obj_attr_is_set(self): + obj = MyObj(foo=1) + self.assertTrue(obj.obj_attr_is_set('foo')) + self.assertFalse(obj.obj_attr_is_set('bar')) + self.assertRaises(AttributeError, obj.obj_attr_is_set, 'bang') + + def test_get(self): + obj = MyObj(foo=1) + # Foo has value, should not get the default + self.assertEqual(1, obj.get('foo', 2)) + # Foo has value, should return the value without error + self.assertEqual(1, obj.get('foo')) + # Bar is not loaded, so we should get the default + self.assertEqual('not-loaded', obj.get('bar', 'not-loaded')) + # Bar without a default should lazy-load + self.assertEqual('loaded!', obj.get('bar')) + # Bar now has a default, but loaded value should be returned + self.assertEqual('loaded!', obj.get('bar', 'not-loaded')) + # Invalid attribute should raise AttributeError + self.assertRaises(AttributeError, obj.get, 'nothing') + # ...even with a default + self.assertRaises(AttributeError, obj.get, 'nothing', 3) + + def test_object_inheritance(self): + base_fields = base.CinderPersistentObject.fields.keys() + myobj_fields = (['foo', 'bar', 'missing', + 'readonly', 'rel_object', 'rel_objects'] + + base_fields) + myobj3_fields = ['new_field'] + self.assertTrue(issubclass(TestSubclassedObject, MyObj)) + self.assertEqual(len(myobj_fields), len(MyObj.fields)) + self.assertEqual(set(myobj_fields), set(MyObj.fields.keys())) + self.assertEqual(len(myobj_fields) + len(myobj3_fields), + len(TestSubclassedObject.fields)) + self.assertEqual(set(myobj_fields) | set(myobj3_fields), + set(TestSubclassedObject.fields.keys())) + + def test_obj_as_admin(self): + obj = MyObj(context=self.context) + + def fake(*args, **kwargs): + self.assertTrue(obj._context.is_admin) + + with mock.patch.object(obj, 'obj_reset_changes') as mock_fn: + mock_fn.side_effect = fake + with obj.obj_as_admin(): + obj.save() + self.assertTrue(mock_fn.called) + + self.assertFalse(obj._context.is_admin) + + def test_obj_as_admin_orphaned(self): + def testme(): + obj = MyObj() + with obj.obj_as_admin(): + pass + self.assertRaises(exception.OrphanedObjectError, testme) + + def test_get_changes(self): + obj = MyObj() + self.assertEqual({}, obj.obj_get_changes()) + obj.foo = 123 + self.assertEqual({'foo': 123}, obj.obj_get_changes()) + obj.bar = 'test' + self.assertEqual({'foo': 123, 'bar': 'test'}, obj.obj_get_changes()) + obj.obj_reset_changes() + self.assertEqual({}, obj.obj_get_changes()) + + def test_obj_fields(self): + class TestObj(base.CinderObject): + fields = {'foo': fields.Field(fields.Integer())} + obj_extra_fields = ['bar'] + + @property + def bar(self): + return 'this is bar' + + obj = TestObj() + self.assertEqual(['foo', 'bar'], obj.obj_fields) + + def test_obj_constructor(self): + obj = MyObj(context=self.context, foo=123, bar='abc') + self.assertEqual(123, obj.foo) + self.assertEqual('abc', obj.bar) + self.assertEqual(set(['foo', 'bar']), obj.obj_what_changed()) + + def test_obj_read_only(self): + obj = MyObj(context=self.context, foo=123, bar='abc') + obj.readonly = 1 + self.assertRaises(exception.ReadOnlyFieldError, setattr, + obj, 'readonly', 2) + + def test_obj_repr(self): + obj = MyObj(foo=123) + self.assertEqual('MyObj(bar=,created_at=,deleted=,' + 'deleted_at=,foo=123,missing=,readonly=,' + 'rel_object=,rel_objects=,updated_at=)', + repr(obj)) + + def test_obj_make_obj_compatible(self): + subobj = MyOwnedObject(baz=1) + obj = MyObj(rel_object=subobj) + obj.obj_relationships = { + 'rel_object': [('1.5', '1.1'), ('1.7', '1.2')], + } + primitive = obj.obj_to_primitive()['cinder_object.data'] + with mock.patch.object(subobj, 'obj_make_compatible') as mock_compat: + obj._obj_make_obj_compatible(copy.copy(primitive), '1.8', + 'rel_object') + self.assertFalse(mock_compat.called) + + with mock.patch.object(subobj, 'obj_make_compatible') as mock_compat: + obj._obj_make_obj_compatible(copy.copy(primitive), + '1.7', 'rel_object') + mock_compat.assert_called_once_with( + primitive['rel_object']['cinder_object.data'], '1.2') + self.assertEqual('1.2', + primitive['rel_object']['cinder_object.version']) + + with mock.patch.object(subobj, 'obj_make_compatible') as mock_compat: + obj._obj_make_obj_compatible(copy.copy(primitive), + '1.6', 'rel_object') + mock_compat.assert_called_once_with( + primitive['rel_object']['cinder_object.data'], '1.1') + self.assertEqual('1.1', + primitive['rel_object']['cinder_object.version']) + + with mock.patch.object(subobj, 'obj_make_compatible') as mock_compat: + obj._obj_make_obj_compatible(copy.copy(primitive), '1.5', + 'rel_object') + mock_compat.assert_called_once_with( + primitive['rel_object']['cinder_object.data'], '1.1') + self.assertEqual('1.1', + primitive['rel_object']['cinder_object.version']) + + with mock.patch.object(subobj, 'obj_make_compatible') as mock_compat: + _prim = copy.copy(primitive) + obj._obj_make_obj_compatible(_prim, '1.4', 'rel_object') + self.assertFalse(mock_compat.called) + self.assertNotIn('rel_object', _prim) + + def test_obj_make_compatible_hits_sub_objects(self): + subobj = MyOwnedObject(baz=1) + obj = MyObj(foo=123, rel_object=subobj) + obj.obj_relationships = {'rel_object': [('1.0', '1.0')]} + with mock.patch.object(obj, '_obj_make_obj_compatible') as mock_compat: + obj.obj_make_compatible({'rel_object': 'foo'}, '1.10') + mock_compat.assert_called_once_with({'rel_object': 'foo'}, '1.10', + 'rel_object') + + def test_obj_make_compatible_skips_unset_sub_objects(self): + obj = MyObj(foo=123) + obj.obj_relationships = {'rel_object': [('1.0', '1.0')]} + with mock.patch.object(obj, '_obj_make_obj_compatible') as mock_compat: + obj.obj_make_compatible({'rel_object': 'foo'}, '1.10') + self.assertFalse(mock_compat.called) + + def test_obj_make_compatible_complains_about_missing_rules(self): + subobj = MyOwnedObject(baz=1) + obj = MyObj(foo=123, rel_object=subobj) + obj.obj_relationships = {} + self.assertRaises(exception.ObjectActionError, + obj.obj_make_compatible, {}, '1.0') + + def test_obj_make_compatible_handles_list_of_objects(self): + subobj = MyOwnedObject(baz=1) + obj = MyObj(rel_objects=[subobj]) + obj.obj_relationships = {'rel_objects': [('1.0', '1.123')]} + + def fake_make_compat(primitive, version): + self.assertEqual('1.123', version) + self.assertIn('baz', primitive) + + with mock.patch.object(subobj, 'obj_make_compatible') as mock_mc: + mock_mc.side_effect = fake_make_compat + obj.obj_to_primitive('1.0') + self.assertTrue(mock_mc.called) + + +class TestObject(_LocalTest, _TestObject): + def test_set_defaults(self): + obj = MyObj() + obj.obj_set_defaults('foo') + self.assertTrue(obj.obj_attr_is_set('foo')) + self.assertEqual(1, obj.foo) + + def test_set_defaults_no_default(self): + obj = MyObj() + self.assertRaises(exception.ObjectActionError, + obj.obj_set_defaults, 'bar') + + def test_set_all_defaults(self): + obj = MyObj() + obj.obj_set_defaults() + self.assertEqual(set(['deleted', 'foo']), obj.obj_what_changed()) + self.assertEqual(1, obj.foo) + + +class TestObjectListBase(test.TestCase): + def test_list_like_operations(self): + class MyElement(base.CinderObject): + fields = {'foo': fields.IntegerField()} + + def __init__(self, foo): + super(MyElement, self).__init__() + self.foo = foo + + class Foo(base.ObjectListBase, base.CinderObject): + fields = {'objects': fields.ListOfObjectsField('MyElement')} + + objlist = Foo(context='foo', + objects=[MyElement(1), MyElement(2), MyElement(3)]) + self.assertEqual(list(objlist), objlist.objects) + self.assertEqual(len(objlist), 3) + self.assertIn(objlist.objects[0], objlist) + self.assertEqual(list(objlist[:1]), [objlist.objects[0]]) + self.assertEqual(objlist[:1]._context, 'foo') + self.assertEqual(objlist[2], objlist.objects[2]) + self.assertEqual(objlist.count(objlist.objects[0]), 1) + self.assertEqual(objlist.index(objlist.objects[1]), 1) + objlist.sort(key=lambda x: x.foo, reverse=True) + self.assertEqual([3, 2, 1], + [x.foo for x in objlist]) + + def test_serialization(self): + class Foo(base.ObjectListBase, base.CinderObject): + fields = {'objects': fields.ListOfObjectsField('Bar')} + + class Bar(base.CinderObject): + fields = {'foo': fields.Field(fields.String())} + + obj = Foo(objects=[]) + for i in 'abc': + bar = Bar(foo=i) + obj.objects.append(bar) + + obj2 = base.CinderObject.obj_from_primitive(obj.obj_to_primitive()) + self.assertFalse(obj is obj2) + self.assertEqual([x.foo for x in obj], + [y.foo for y in obj2]) + + def test_list_changes(self): + class Foo(base.ObjectListBase, base.CinderObject): + fields = {'objects': fields.ListOfObjectsField('Bar')} + + class Bar(base.CinderObject): + fields = {'foo': fields.StringField()} + + obj = Foo(objects=[]) + self.assertEqual(set(['objects']), obj.obj_what_changed()) + obj.objects.append(Bar(foo='test')) + self.assertEqual(set(['objects']), obj.obj_what_changed()) + obj.obj_reset_changes() + # This should still look dirty because the child is dirty + self.assertEqual(set(['objects']), obj.obj_what_changed()) + obj.objects[0].obj_reset_changes() + # This should now look clean because the child is clean + self.assertEqual(set(), obj.obj_what_changed()) + + def test_initialize_objects(self): + class Foo(base.ObjectListBase, base.CinderObject): + fields = {'objects': fields.ListOfObjectsField('Bar')} + + class Bar(base.CinderObject): + fields = {'foo': fields.StringField()} + + obj = Foo() + self.assertEqual([], obj.objects) + self.assertEqual(set(), obj.obj_what_changed()) + + def test_obj_repr(self): + class Foo(base.ObjectListBase, base.CinderObject): + fields = {'objects': fields.ListOfObjectsField('Bar')} + + class Bar(base.CinderObject): + fields = {'uuid': fields.StringField()} + + obj = Foo(objects=[Bar(uuid='fake-uuid')]) + self.assertEqual('Foo(objects=[Bar(fake-uuid)])', repr(obj)) + + +class TestObjectSerializer(_BaseTestCase): + def test_serialize_entity_primitive(self): + ser = base.CinderObjectSerializer() + for thing in (1, 'foo', [1, 2], {'foo': 'bar'}): + self.assertEqual(thing, ser.serialize_entity(None, thing)) + + def test_deserialize_entity_primitive(self): + ser = base.CinderObjectSerializer() + for thing in (1, 'foo', [1, 2], {'foo': 'bar'}): + self.assertEqual(thing, ser.deserialize_entity(None, thing)) + + def _test_deserialize_entity_newer(self, obj_version, backported_to, + my_version='1.6'): + ser = base.CinderObjectSerializer() + ser._conductor = mock.Mock() + ser._conductor.object_backport.return_value = 'backported' + + class MyTestObj(MyObj): + VERSION = my_version + + obj = MyTestObj() + obj.VERSION = obj_version + primitive = obj.obj_to_primitive() + ser.deserialize_entity(self.context, primitive) + if backported_to is None: + self.assertFalse(ser._conductor.object_backport.called) + + def test_deserialize_entity_newer_revision_does_not_backport_zero(self): + self._test_deserialize_entity_newer('1.6.0', None) + + def test_deserialize_entity_newer_revision_does_not_backport(self): + self._test_deserialize_entity_newer('1.6.1', None) + + def test_deserialize_dot_z_with_extra_stuff(self): + primitive = {'cinder_object.name': 'MyObj', + 'cinder_object.namespace': 'cinder', + 'cinder_object.version': '1.6.1', + 'cinder_object.data': { + 'foo': 1, + 'unexpected_thing': 'foobar'}} + ser = base.CinderObjectSerializer() + obj = ser.deserialize_entity(self.context, primitive) + self.assertEqual(1, obj.foo) + self.assertFalse(hasattr(obj, 'unexpected_thing')) + # NOTE(danms): The serializer is where the logic lives that + # avoids backports for cases where only a .z difference in + # the received object version is detected. As a result, we + # end up with a version of what we expected, effectively the + # .0 of the object. + self.assertEqual('1.6', obj.VERSION) + + def test_object_serialization(self): + ser = base.CinderObjectSerializer() + obj = MyObj() + primitive = ser.serialize_entity(self.context, obj) + self.assertIn('cinder_object.name', primitive) + obj2 = ser.deserialize_entity(self.context, primitive) + self.assertIsInstance(obj2, MyObj) + self.assertEqual(self.context, obj2._context) + + def test_object_serialization_iterables(self): + ser = base.CinderObjectSerializer() + obj = MyObj() + for iterable in (list, tuple, set): + thing = iterable([obj]) + primitive = ser.serialize_entity(self.context, thing) + self.assertEqual(1, len(primitive)) + for item in primitive: + self.assertNotIsInstance(item, base.CinderObject) + thing2 = ser.deserialize_entity(self.context, primitive) + self.assertEqual(1, len(thing2)) + for item in thing2: + self.assertIsInstance(item, MyObj) + # dict case + thing = {'key': obj} + primitive = ser.serialize_entity(self.context, thing) + self.assertEqual(1, len(primitive)) + for item in primitive.itervalues(): + self.assertNotIsInstance(item, base.CinderObject) + thing2 = ser.deserialize_entity(self.context, primitive) + self.assertEqual(1, len(thing2)) + for item in thing2.itervalues(): + self.assertIsInstance(item, MyObj) + + # object-action updates dict case + thing = {'foo': obj.obj_to_primitive()} + primitive = ser.serialize_entity(self.context, thing) + self.assertEqual(thing, primitive) + thing2 = ser.deserialize_entity(self.context, thing) + self.assertIsInstance(thing2['foo'], base.CinderObject) diff --git a/cinder/tests/test_cmd.py b/cinder/tests/test_cmd.py index 1000ba269..b0abc57f4 100644 --- a/cinder/tests/test_cmd.py +++ b/cinder/tests/test_cmd.py @@ -398,12 +398,13 @@ class TestCinderManageCmd(test.TestCase): service_get_all.assert_called_once_with(mock.sentinel.ctxt) self.assertEqual(expected_out, fake_out.getvalue()) + @mock.patch('cinder.objects.base.CinderObjectSerializer') @mock.patch('cinder.rpc.get_client') @mock.patch('cinder.rpc.init') @mock.patch('cinder.rpc.initialized', return_value=False) @mock.patch('oslo.messaging.Target') def test_volume_commands_init(self, messaging_target, rpc_initialized, - rpc_init, get_client): + rpc_init, get_client, object_serializer): CONF.set_override('volume_topic', 'fake-topic') mock_target = messaging_target.return_value mock_rpc_client = get_client.return_value @@ -414,7 +415,8 @@ class TestCinderManageCmd(test.TestCase): rpc_initialized.assert_called_once_with() rpc_init.assert_called_once_with(CONF) messaging_target.assert_called_once_with(topic='fake-topic') - get_client.assert_called_once_with(mock_target) + get_client.assert_called_once_with(mock_target, + serializer=object_serializer()) self.assertEqual(mock_rpc_client, rpc_client) @mock.patch('cinder.db.volume_get') diff --git a/cinder/tests/test_utils.py b/cinder/tests/test_utils.py index 3429f4070..d733a57fa 100644 --- a/cinder/tests/test_utils.py +++ b/cinder/tests/test_utils.py @@ -1477,3 +1477,19 @@ class TestRetryDecorator(test.TestCase): self.assertRaises(WrongException, raise_unexpected_error) self.assertFalse(mock_sleep.called) + + +class VersionTestCase(test.TestCase): + def test_convert_version_to_int(self): + self.assertEqual(utils.convert_version_to_int('6.2.0'), 6002000) + self.assertEqual(utils.convert_version_to_int((6, 4, 3)), 6004003) + self.assertEqual(utils.convert_version_to_int((5, )), 5) + self.assertRaises(exception.CinderException, + utils.convert_version_to_int, '5a.6b') + + def test_convert_version_to_string(self): + self.assertEqual(utils.convert_version_to_str(6007000), '6.7.0') + self.assertEqual(utils.convert_version_to_str(4), '4') + + def test_convert_version_to_tuple(self): + self.assertEqual(utils.convert_version_to_tuple('6.7.0'), (6, 7, 0)) diff --git a/cinder/utils.py b/cinder/utils.py index ccbf325cb..c7b17eb26 100644 --- a/cinder/utils.py +++ b/cinder/utils.py @@ -781,3 +781,29 @@ def retry(exceptions, interval=1, retries=3, backoff_rate=2): return _wrapper return _decorator + + +def convert_version_to_int(version): + try: + if isinstance(version, six.string_types): + version = convert_version_to_tuple(version) + if isinstance(version, tuple): + return reduce(lambda x, y: (x * 1000) + y, version) + except Exception: + msg = _("Version %s is invalid.") % version + raise exception.CinderException(msg) + + +def convert_version_to_str(version_int): + version_numbers = [] + factor = 1000 + while version_int != 0: + version_number = version_int - (version_int // factor * factor) + version_numbers.insert(0, six.text_type(version_number)) + version_int = version_int / factor + + return reduce(lambda x, y: "%s.%s" % (x, y), version_numbers) + + +def convert_version_to_tuple(version_str): + return tuple(int(part) for part in version_str.split('.')) diff --git a/cinder/volume/rpcapi.py b/cinder/volume/rpcapi.py index e82692992..5f73f1417 100644 --- a/cinder/volume/rpcapi.py +++ b/cinder/volume/rpcapi.py @@ -20,6 +20,7 @@ from oslo import messaging from oslo_config import cfg from oslo_serialization import jsonutils +from cinder.objects import base as objects_base from cinder import rpc from cinder.volume import utils @@ -66,7 +67,8 @@ class VolumeAPI(object): super(VolumeAPI, self).__init__() target = messaging.Target(topic=CONF.volume_topic, version=self.BASE_RPC_API_VERSION) - self.client = rpc.get_client(target, '1.19') + serializer = objects_base.CinderObjectSerializer() + self.client = rpc.get_client(target, '1.19', serializer=serializer) def create_consistencygroup(self, ctxt, group, host): new_host = utils.extract_host(host) diff --git a/tools/lintstack.py b/tools/lintstack.py index d8459ff0e..b5da2a19c 100755 --- a/tools/lintstack.py +++ b/tools/lintstack.py @@ -46,6 +46,12 @@ ignore_messages = ["An attribute affected in cinder.tests", # checked elsewhere. We also ignore cinder.tests for now due to high false # positive rate. ignore_modules = ["cinder/openstack/common/", "cinder/tests/"] +# Note(thangp): E1101 should be ignored for only cinder.object modules. +# E1101 is error code related to accessing a non-existent member of an +# object, but should be ignored because the object member is created +# dynamically. +objects_ignore_codes = ["E1101"] +objects_ignore_modules = ["cinder/objects/"] KNOWN_PYLINT_EXCEPTIONS_FILE = "tools/pylint_exceptions" @@ -101,6 +107,10 @@ class LintOutput(object): return True if any(msg in self.message for msg in ignore_messages): return True + if (self.code in objects_ignore_codes and + any(self.filename.startswith(name) + for name in objects_ignore_modules)): + return True return False def key(self): -- 2.45.2