From: Russell Bryant Date: Tue, 17 Jul 2012 04:10:29 +0000 (+0100) Subject: Make use of openstack.common.jsonutils. X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=64306e87b38483c00e5055f246294f081e1f0a5b;p=openstack-build%2Fcinder-build.git Make use of openstack.common.jsonutils. This patch imports jsonutils from openstack-common. It removes the equivalent code from cinder.utils and then converts the code base to use jsonutils. The primary motivator for this change was to remove the rest of the dependencies from cinder.rpc on cinder.utils. Change-Id: I4b16577f774194a628bf84d5352e63ec1f868698 --- diff --git a/cinder/api/openstack/wsgi.py b/cinder/api/openstack/wsgi.py index bb309056e..2e2f40192 100644 --- a/cinder/api/openstack/wsgi.py +++ b/cinder/api/openstack/wsgi.py @@ -26,7 +26,7 @@ import webob from cinder import exception from cinder import log as logging -from cinder import utils +from cinder.openstack.common import jsonutils from cinder import wsgi @@ -129,7 +129,7 @@ class JSONDeserializer(TextDeserializer): def _from_json(self, datastring): try: - return utils.loads(datastring) + return jsonutils.loads(datastring) except ValueError: msg = _("cannot understand JSON") raise exception.MalformedRequestBody(reason=msg) @@ -241,7 +241,7 @@ class JSONDictSerializer(DictSerializer): """Default JSON request body serialization""" def default(self, data): - return utils.dumps(data) + return jsonutils.dumps(data) class XMLDictSerializer(DictSerializer): @@ -532,7 +532,7 @@ def action_peek_json(body): """Determine action to invoke.""" try: - decoded = utils.loads(body) + decoded = jsonutils.loads(body) except ValueError: msg = _("cannot understand JSON") raise exception.MalformedRequestBody(reason=msg) diff --git a/cinder/notifier/api.py b/cinder/notifier/api.py index f144b1536..26498585d 100644 --- a/cinder/notifier/api.py +++ b/cinder/notifier/api.py @@ -20,6 +20,7 @@ from cinder import utils from cinder import log as logging from cinder.openstack.common import cfg from cinder.openstack.common import importutils +from cinder.openstack.common import jsonutils from cinder.openstack.common import timeutils @@ -117,7 +118,7 @@ def notify(publisher_id, event_type, priority, payload): _('%s not in valid priorities') % priority) # Ensure everything is JSON serializable. - payload = utils.to_primitive(payload, convert_instances=True) + payload = jsonutils.to_primitive(payload, convert_instances=True) driver = importutils.import_module(FLAGS.notification_driver) msg = dict(message_id=str(uuid.uuid4()), diff --git a/cinder/openstack/common/jsonutils.py b/cinder/openstack/common/jsonutils.py new file mode 100644 index 000000000..7cf8bbda2 --- /dev/null +++ b/cinder/openstack/common/jsonutils.py @@ -0,0 +1,146 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +''' +JSON related utilities. + +This module provides a few things: + + 1) A handy function for getting an object down to something that can be + JSON serialized. See to_primitive(). + + 2) Wrappers around loads() and dumps(). The dumps() wrapper will + automatically use to_primitive() for you if needed. + + 3) This sets up anyjson to use the loads() and dumps() wrappers if anyjson + is available. +''' + + +import datetime +import inspect +import itertools +import json +import xmlrpclib + +from cinder.openstack.common import timeutils + + +def to_primitive(value, convert_instances=False, level=0): + """Convert a complex object into primitives. + + Handy for JSON serialization. We can optionally handle instances, + but since this is a recursive function, we could have cyclical + data structures. + + To handle cyclical data structures we could track the actual objects + visited in a set, but not all objects are hashable. Instead we just + track the depth of the object inspections and don't go too deep. + + Therefore, convert_instances=True is lossy ... be aware. + + """ + nasty = [inspect.ismodule, inspect.isclass, inspect.ismethod, + inspect.isfunction, inspect.isgeneratorfunction, + inspect.isgenerator, inspect.istraceback, inspect.isframe, + inspect.iscode, inspect.isbuiltin, inspect.isroutine, + inspect.isabstract] + for test in nasty: + if test(value): + return unicode(value) + + # value of itertools.count doesn't get caught by inspects + # above and results in infinite loop when list(value) is called. + if type(value) == itertools.count: + return unicode(value) + + # FIXME(vish): Workaround for LP bug 852095. Without this workaround, + # tests that raise an exception in a mocked method that + # has a @wrap_exception with a notifier will fail. If + # we up the dependency to 0.5.4 (when it is released) we + # can remove this workaround. + if getattr(value, '__module__', None) == 'mox': + return 'mock' + + if level > 3: + return '?' + + # The try block may not be necessary after the class check above, + # but just in case ... + try: + # It's not clear why xmlrpclib created their own DateTime type, but + # for our purposes, make it a datetime type which is explicitly + # handled + if isinstance(value, xmlrpclib.DateTime): + value = datetime.datetime(*tuple(value.timetuple())[:6]) + + if isinstance(value, (list, tuple)): + o = [] + for v in value: + o.append(to_primitive(v, convert_instances=convert_instances, + level=level)) + return o + elif isinstance(value, dict): + o = {} + for k, v in value.iteritems(): + o[k] = to_primitive(v, convert_instances=convert_instances, + level=level) + return o + elif isinstance(value, datetime.datetime): + return timeutils.strtime(value) + elif hasattr(value, 'iteritems'): + return to_primitive(dict(value.iteritems()), + convert_instances=convert_instances, + level=level) + elif hasattr(value, '__iter__'): + return to_primitive(list(value), level) + elif convert_instances and hasattr(value, '__dict__'): + # Likely an instance of something. Watch for cycles. + # Ignore class member vars. + return to_primitive(value.__dict__, + convert_instances=convert_instances, + level=level + 1) + else: + return value + except TypeError, e: + # Class objects are tricky since they may define something like + # __iter__ defined but it isn't callable as list(). + return unicode(value) + + +def dumps(value, default=to_primitive, **kwargs): + return json.dumps(value, default=default, **kwargs) + + +def loads(s): + return json.loads(s) + + +def load(s): + return json.load(s) + + +try: + import anyjson +except ImportError: + pass +else: + anyjson._modules.append((__name__, 'dumps', TypeError, + 'loads', ValueError, 'load')) + anyjson.force_implementation(__name__) diff --git a/cinder/rpc/common.py b/cinder/rpc/common.py index 0c370e90a..e5403cb1e 100644 --- a/cinder/rpc/common.py +++ b/cinder/rpc/common.py @@ -24,7 +24,7 @@ import traceback from cinder import log as logging from cinder.openstack.common import cfg from cinder.openstack.common import importutils -from cinder import utils +from cinder.openstack.common import jsonutils LOG = logging.getLogger(__name__) @@ -204,13 +204,13 @@ def serialize_remote_exception(failure_info): 'kwargs': kwargs } - json_data = utils.dumps(data) + json_data = jsonutils.dumps(data) return json_data def deserialize_remote_exception(conf, data): - failure = utils.loads(str(data)) + failure = jsonutils.loads(str(data)) trace = failure.get('tb', []) message = failure.get('message', "") + "\n" + "\n".join(trace) diff --git a/cinder/tests/test_utils.py b/cinder/tests/test_utils.py index d305d26f6..09a4798fe 100644 --- a/cinder/tests/test_utils.py +++ b/cinder/tests/test_utils.py @@ -444,93 +444,6 @@ class IsUUIDLikeTestCase(test.TestCase): self.assertUUIDLike(str(utils.gen_uuid()), True) -class ToPrimitiveTestCase(test.TestCase): - def test_list(self): - self.assertEquals(utils.to_primitive([1, 2, 3]), [1, 2, 3]) - - def test_empty_list(self): - self.assertEquals(utils.to_primitive([]), []) - - def test_tuple(self): - self.assertEquals(utils.to_primitive((1, 2, 3)), [1, 2, 3]) - - def test_dict(self): - self.assertEquals(utils.to_primitive(dict(a=1, b=2, c=3)), - dict(a=1, b=2, c=3)) - - def test_empty_dict(self): - self.assertEquals(utils.to_primitive({}), {}) - - def test_datetime(self): - x = datetime.datetime(1, 2, 3, 4, 5, 6, 7) - self.assertEquals(utils.to_primitive(x), "0001-02-03 04:05:06.000007") - - def test_iter(self): - class IterClass(object): - def __init__(self): - self.data = [1, 2, 3, 4, 5] - self.index = 0 - - def __iter__(self): - return self - - def next(self): - if self.index == len(self.data): - raise StopIteration - self.index = self.index + 1 - return self.data[self.index - 1] - - x = IterClass() - self.assertEquals(utils.to_primitive(x), [1, 2, 3, 4, 5]) - - def test_iteritems(self): - class IterItemsClass(object): - def __init__(self): - self.data = dict(a=1, b=2, c=3).items() - self.index = 0 - - def __iter__(self): - return self - - def next(self): - if self.index == len(self.data): - raise StopIteration - self.index = self.index + 1 - return self.data[self.index - 1] - - x = IterItemsClass() - ordered = utils.to_primitive(x) - ordered.sort() - self.assertEquals(ordered, [['a', 1], ['b', 2], ['c', 3]]) - - def test_instance(self): - class MysteryClass(object): - a = 10 - - def __init__(self): - self.b = 1 - - x = MysteryClass() - self.assertEquals(utils.to_primitive(x, convert_instances=True), - dict(b=1)) - - self.assertEquals(utils.to_primitive(x), x) - - def test_typeerror(self): - x = bytearray # Class, not instance - self.assertEquals(utils.to_primitive(x), u"") - - def test_nasties(self): - def foo(): - pass - x = [datetime, foo, dir] - ret = utils.to_primitive(x) - self.assertEquals(len(ret), 3) - self.assertTrue(ret[0].startswith(u"') - - class MonkeyPatchTestCase(test.TestCase): """Unit test for utils.monkey_patch().""" def setUp(self): diff --git a/cinder/utils.py b/cinder/utils.py index 640de4f85..8ea2df17b 100644 --- a/cinder/utils.py +++ b/cinder/utils.py @@ -25,8 +25,6 @@ import errno import functools import hashlib import inspect -import itertools -import json import os import pyclbr import random @@ -613,104 +611,6 @@ def utf8(value): return value -def to_primitive(value, convert_instances=False, level=0): - """Convert a complex object into primitives. - - Handy for JSON serialization. We can optionally handle instances, - but since this is a recursive function, we could have cyclical - data structures. - - To handle cyclical data structures we could track the actual objects - visited in a set, but not all objects are hashable. Instead we just - track the depth of the object inspections and don't go too deep. - - Therefore, convert_instances=True is lossy ... be aware. - - """ - nasty = [inspect.ismodule, inspect.isclass, inspect.ismethod, - inspect.isfunction, inspect.isgeneratorfunction, - inspect.isgenerator, inspect.istraceback, inspect.isframe, - inspect.iscode, inspect.isbuiltin, inspect.isroutine, - inspect.isabstract] - for test in nasty: - if test(value): - return unicode(value) - - # value of itertools.count doesn't get caught by inspects - # above and results in infinite loop when list(value) is called. - if type(value) == itertools.count: - return unicode(value) - - # FIXME(vish): Workaround for LP bug 852095. Without this workaround, - # tests that raise an exception in a mocked method that - # has a @wrap_exception with a notifier will fail. If - # we up the dependency to 0.5.4 (when it is released) we - # can remove this workaround. - if getattr(value, '__module__', None) == 'mox': - return 'mock' - - if level > 3: - return '?' - - # The try block may not be necessary after the class check above, - # but just in case ... - try: - if isinstance(value, (list, tuple)): - o = [] - for v in value: - o.append(to_primitive(v, convert_instances=convert_instances, - level=level)) - return o - elif isinstance(value, dict): - o = {} - for k, v in value.iteritems(): - o[k] = to_primitive(v, convert_instances=convert_instances, - level=level) - return o - elif isinstance(value, datetime.datetime): - return str(value) - elif hasattr(value, 'iteritems'): - return to_primitive(dict(value.iteritems()), - convert_instances=convert_instances, - level=level) - elif hasattr(value, '__iter__'): - return to_primitive(list(value), level) - elif convert_instances and hasattr(value, '__dict__'): - # Likely an instance of something. Watch for cycles. - # Ignore class member vars. - return to_primitive(value.__dict__, - convert_instances=convert_instances, - level=level + 1) - else: - return value - except TypeError, e: - # Class objects are tricky since they may define something like - # __iter__ defined but it isn't callable as list(). - return unicode(value) - - -def dumps(value): - try: - return json.dumps(value) - except TypeError: - pass - return json.dumps(to_primitive(value)) - - -def loads(s): - return json.loads(s) - - -try: - import anyjson -except ImportError: - pass -else: - anyjson._modules.append(("cinder.utils", "dumps", TypeError, - "loads", ValueError)) - anyjson.force_implementation("cinder.utils") - - def delete_if_exists(pathname): """delete a file, but ignore file not found error""" diff --git a/openstack-common.conf b/openstack-common.conf index e61aff5b2..c0e470fa0 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -1,7 +1,7 @@ [DEFAULT] # The list of modules to copy from openstack-common -modules=cfg,exception,local,importutils,iniparser,timeutils +modules=cfg,exception,importutils,iniparser,jsonutils,local,timeutils # The base module to hold the copy of openstack.common base=cinder