From 2f5360753308eb8b10581fc3c026c1b66f42ebdc Mon Sep 17 00:00:00 2001 From: Unmesh Gurjar Date: Sat, 11 Aug 2012 10:31:51 -0700 Subject: [PATCH] Adds new volume API extensions Adds following extensions: 1. Create volume from image 2. Copy volume to image Added unit tests. Implements: blueprint create-volume-from-image Change-Id: I9c73bd3fa2fa2e0648c01ff3f4fc66f757d7bc3f --- cinder/api/openstack/__init__.py | 4 +- cinder/api/openstack/extensions.py | 3 + cinder/api/openstack/volume/__init__.py | 4 +- .../openstack/volume/contrib/image_create.py | 31 ++ .../volume/contrib/volume_actions.py | 84 ++- cinder/api/openstack/volume/volumes.py | 46 +- cinder/flags.py | 3 + cinder/image/__init__.py | 16 + cinder/image/glance.py | 492 ++++++++++++++++++ cinder/tests/api/openstack/fakes.py | 13 +- .../volume/contrib/test_volume_actions.py | 149 +++++- .../tests/api/openstack/volume/test_router.py | 9 +- .../api/openstack/volume/test_volumes.py | 100 +++- cinder/tests/image/__init__.py | 20 + cinder/tests/image/fake.py | 248 +++++++++ cinder/tests/policy.json | 1 + cinder/tests/test_volume.py | 198 ++++++- cinder/utils.py | 15 +- cinder/volume/api.py | 68 ++- cinder/volume/driver.py | 22 + cinder/volume/manager.py | 65 ++- etc/cinder/rootwrap.d/volume.filters | 3 + 22 files changed, 1553 insertions(+), 41 deletions(-) create mode 100644 cinder/api/openstack/volume/contrib/image_create.py create mode 100644 cinder/image/__init__.py create mode 100644 cinder/image/glance.py create mode 100644 cinder/tests/image/__init__.py create mode 100644 cinder/tests/image/fake.py diff --git a/cinder/api/openstack/__init__.py b/cinder/api/openstack/__init__.py index 2bfaab778..2963bac41 100644 --- a/cinder/api/openstack/__init__.py +++ b/cinder/api/openstack/__init__.py @@ -95,7 +95,7 @@ class APIRouter(base_wsgi.Router): mapper = ProjectMapper() self.resources = {} - self._setup_routes(mapper) + self._setup_routes(mapper, ext_mgr) self._setup_ext_routes(mapper, ext_mgr) self._setup_extensions(ext_mgr) super(APIRouter, self).__init__(mapper) @@ -139,5 +139,5 @@ class APIRouter(base_wsgi.Router): resource.register_actions(controller) resource.register_extensions(controller) - def _setup_routes(self, mapper): + def _setup_routes(self, mapper, ext_mgr): raise NotImplementedError diff --git a/cinder/api/openstack/extensions.py b/cinder/api/openstack/extensions.py index 7f443637c..9c397e45b 100644 --- a/cinder/api/openstack/extensions.py +++ b/cinder/api/openstack/extensions.py @@ -180,6 +180,9 @@ class ExtensionManager(object): """ + def is_loaded(self, alias): + return alias in self.extensions + def register(self, ext): # Do nothing if the extension doesn't check out if not self._check_extension(ext): diff --git a/cinder/api/openstack/volume/__init__.py b/cinder/api/openstack/volume/__init__.py index 89bf9c067..542477ebf 100644 --- a/cinder/api/openstack/volume/__init__.py +++ b/cinder/api/openstack/volume/__init__.py @@ -39,7 +39,7 @@ class APIRouter(cinder.api.openstack.APIRouter): """ ExtensionManager = extensions.ExtensionManager - def _setup_routes(self, mapper): + def _setup_routes(self, mapper, ext_mgr): self.resources['versions'] = versions.create_resource() mapper.connect("versions", "/", controller=self.resources['versions'], @@ -47,7 +47,7 @@ class APIRouter(cinder.api.openstack.APIRouter): mapper.redirect("", "/") - self.resources['volumes'] = volumes.create_resource() + self.resources['volumes'] = volumes.create_resource(ext_mgr) mapper.resource("volume", "volumes", controller=self.resources['volumes'], collection={'detail': 'GET'}, diff --git a/cinder/api/openstack/volume/contrib/image_create.py b/cinder/api/openstack/volume/contrib/image_create.py new file mode 100644 index 000000000..51cf6bcbd --- /dev/null +++ b/cinder/api/openstack/volume/contrib/image_create.py @@ -0,0 +1,31 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2012 NTT. +# Copyright (c) 2012 OpenStack, LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""The Create Volume from Image extension.""" + + +from cinder.api.openstack import extensions + + +class Image_create(extensions.ExtensionDescriptor): + """Allow creating a volume from an image in the Create Volume v1 API""" + + name = "CreateVolumeExtension" + alias = "os-image-create" + namespace = "http://docs.openstack.org/volume/ext/image-create/api/v1" + updated = "2012-08-13T00:00:00+00:00" diff --git a/cinder/api/openstack/volume/contrib/volume_actions.py b/cinder/api/openstack/volume/contrib/volume_actions.py index a06d7aa85..2afa244b4 100644 --- a/cinder/api/openstack/volume/contrib/volume_actions.py +++ b/cinder/api/openstack/volume/contrib/volume_actions.py @@ -12,19 +12,19 @@ # License for the specific language governing permissions and limitations # under the License. -import os.path -import traceback - import webob from webob import exc +from xml.dom import minidom -from cinder.api.openstack import common from cinder.api.openstack import extensions from cinder.api.openstack import wsgi +from cinder.api.openstack import xmlutil from cinder import volume from cinder import exception from cinder import flags +from cinder import utils from cinder.openstack.common import log as logging +from cinder.openstack.common.rpc import common as rpc_common FLAGS = flags.FLAGS @@ -36,6 +36,40 @@ def authorize(context, action_name): extensions.extension_authorizer('volume', action)(context) +class VolumeToImageSerializer(xmlutil.TemplateBuilder): + def construct(self): + root = xmlutil.TemplateElement('os-volume_upload_image', + selector='os-volume_upload_image') + root.set('id') + root.set('updated_at') + root.set('status') + root.set('display_description') + root.set('size') + root.set('volume_type') + root.set('image_id') + root.set('container_format') + root.set('disk_format') + root.set('image_name') + return xmlutil.MasterTemplate(root, 1) + + +class VolumeToImageDeserializer(wsgi.XMLDeserializer): + """Deserializer to handle xml-formatted requests""" + def default(self, string): + dom = minidom.parseString(string) + action_node = dom.childNodes[0] + action_name = action_node.tagName + + action_data = {} + attributes = ["force", "image_name", "container_format", "disk_format"] + for attr in attributes: + if action_node.hasAttribute(attr): + action_data[attr] = action_node.getAttribute(attr) + if 'force' in action_data and action_data['force'] == 'True': + action_data['force'] = True + return {'body': {action_name: action_data}} + + class VolumeActionsController(wsgi.Controller): def __init__(self, *args, **kwargs): super(VolumeActionsController, self).__init__(*args, **kwargs) @@ -98,6 +132,48 @@ class VolumeActionsController(wsgi.Controller): self.volume_api.terminate_connection(context, volume, connector) return webob.Response(status_int=202) + @wsgi.response(202) + @wsgi.action('os-volume_upload_image') + @wsgi.serializers(xml=VolumeToImageSerializer) + @wsgi.deserializers(xml=VolumeToImageDeserializer) + def _volume_upload_image(self, req, id, body): + """Uploads the specified volume to image service.""" + context = req.environ['cinder.context'] + try: + params = body['os-volume_upload_image'] + except (TypeError, KeyError): + msg = _("Invalid request body") + raise webob.exc.HTTPBadRequest(explanation=msg) + + if not params.get("image_name"): + msg = _("No image_name was specified in request.") + raise webob.exc.HTTPBadRequest(explanation=msg) + + force = params.get('force', False) + try: + volume = self.volume_api.get(context, id) + except exception.VolumeNotFound, error: + raise webob.exc.HTTPNotFound(explanation=unicode(error)) + authorize(context, "upload_image") + image_metadata = {"container_format": params.get("container_format", + "bare"), + "disk_format": params.get("disk_format", "raw"), + "name": params["image_name"]} + try: + response = self.volume_api.copy_volume_to_image(context, + volume, + image_metadata, + force) + except exception.InvalidVolume, error: + raise webob.exc.HTTPBadRequest(explanation=unicode(error)) + except ValueError, error: + raise webob.exc.HTTPBadRequest(explanation=unicode(error)) + except rpc_common.RemoteError as error: + msg = "%(err_type)s: %(err_msg)s" % {'err_type': error.exc_type, + 'err_msg': error.value} + raise webob.exc.HTTPBadRequest(explanation=msg) + return {'os-volume_upload_image': response} + class Volume_actions(extensions.ExtensionDescriptor): """Enable volume actions diff --git a/cinder/api/openstack/volume/volumes.py b/cinder/api/openstack/volume/volumes.py index b620b3044..82dbe461b 100644 --- a/cinder/api/openstack/volume/volumes.py +++ b/cinder/api/openstack/volume/volumes.py @@ -24,6 +24,7 @@ from cinder.api.openstack import xmlutil from cinder import exception from cinder import flags from cinder.openstack.common import log as logging +from cinder import utils from cinder import volume from cinder.volume import volume_types @@ -61,17 +62,17 @@ def _translate_attachment_summary_view(_context, vol): return d -def _translate_volume_detail_view(context, vol): +def _translate_volume_detail_view(context, vol, image_id=None): """Maps keys for volumes details view.""" - d = _translate_volume_summary_view(context, vol) + d = _translate_volume_summary_view(context, vol, image_id) # No additional data / lookups at the moment return d -def _translate_volume_summary_view(context, vol): +def _translate_volume_summary_view(context, vol, image_id=None): """Maps keys for volumes summary view.""" d = {} @@ -97,6 +98,9 @@ def _translate_volume_summary_view(context, vol): d['snapshot_id'] = vol['snapshot_id'] + if image_id: + d['image_id'] = image_id + LOG.audit(_("vol=%s"), vol, context=context) if vol.get('volume_metadata'): @@ -158,8 +162,9 @@ class VolumesTemplate(xmlutil.TemplateBuilder): class VolumeController(object): """The Volumes API controller for the OpenStack API.""" - def __init__(self): + def __init__(self, ext_mgr): self.volume_api = volume.API() + self.ext_mgr = ext_mgr super(VolumeController, self).__init__() @wsgi.serializers(xml=VolumeTemplate) @@ -212,6 +217,21 @@ class VolumeController(object): res = [entity_maker(context, vol) for vol in limited_list] return {'volumes': res} + def _image_uuid_from_href(self, image_href): + # If the image href was generated by nova api, strip image_href + # down to an id. + try: + image_uuid = image_href.split('/').pop() + except (TypeError, AttributeError): + msg = _("Invalid imageRef provided.") + raise exc.HTTPBadRequest(explanation=msg) + + if not utils.is_uuid_like(image_uuid): + msg = _("Invalid imageRef provided.") + raise exc.HTTPBadRequest(explanation=msg) + + return image_uuid + @wsgi.serializers(xml=VolumeTemplate) def create(self, req, body): """Creates a new volume.""" @@ -253,6 +273,17 @@ class VolumeController(object): else: kwargs['snapshot'] = None + image_href = None + image_uuid = None + if self.ext_mgr.is_loaded('os-image-create'): + image_href = volume.get('imageRef') + if snapshot_id and image_href: + msg = _("Snapshot and image cannot be specified together.") + raise exc.HTTPBadRequest(explanation=msg) + if image_href: + image_uuid = self._image_uuid_from_href(image_href) + kwargs['image_id'] = image_uuid + kwargs['availability_zone'] = volume.get('availability_zone', None) new_volume = self.volume_api.create(context, @@ -264,7 +295,8 @@ class VolumeController(object): # TODO(vish): Instance should be None at db layer instead of # trying to lazy load, but for now we turn it into # a dict to avoid an error. - retval = _translate_volume_detail_view(context, dict(new_volume)) + retval = _translate_volume_detail_view(context, dict(new_volume), + image_uuid) return {'volume': retval} @@ -273,8 +305,8 @@ class VolumeController(object): return ('name', 'status') -def create_resource(): - return wsgi.Resource(VolumeController()) +def create_resource(ext_mgr): + return wsgi.Resource(VolumeController(ext_mgr)) def remove_invalid_options(context, search_options, allowed_search_options): diff --git a/cinder/flags.py b/cinder/flags.py index ae65c60c6..89b4bfa27 100644 --- a/cinder/flags.py +++ b/cinder/flags.py @@ -130,6 +130,9 @@ global_opts = [ default=['$glance_host:$glance_port'], help='A list of the glance api servers available to cinder ' '([hostname|ip]:port)'), + cfg.IntOpt('glance_num_retries', + default=0, + help='Number retries when downloading an image from glance'), cfg.StrOpt('scheduler_topic', default='cinder-scheduler', help='the topic scheduler nodes listen on'), diff --git a/cinder/image/__init__.py b/cinder/image/__init__.py new file mode 100644 index 000000000..11af4932e --- /dev/null +++ b/cinder/image/__init__.py @@ -0,0 +1,16 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2012 OpenStack, LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/cinder/image/glance.py b/cinder/image/glance.py new file mode 100644 index 000000000..024127437 --- /dev/null +++ b/cinder/image/glance.py @@ -0,0 +1,492 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Implementation of an image service that uses Glance as the backend""" + +from __future__ import absolute_import + +import copy +import itertools +import random +import sys +import time +import urlparse + +import glance.client +from glance.common import exception as glance_exception + +from cinder import exception +from cinder import flags +from cinder.openstack.common import jsonutils +from cinder.openstack.common import log as logging +from cinder.openstack.common import timeutils + + +LOG = logging.getLogger(__name__) +FLAGS = flags.FLAGS + + +def _parse_image_ref(image_href): + """Parse an image href into composite parts. + + :param image_href: href of an image + :returns: a tuple of the form (image_id, host, port) + :raises ValueError + + """ + o = urlparse.urlparse(image_href) + port = o.port or 80 + host = o.netloc.split(':', 1)[0] + image_id = o.path.split('/')[-1] + return (image_id, host, port) + + +def _create_glance_client(context, host, port): + params = {} + if FLAGS.auth_strategy == 'keystone': + params['creds'] = { + 'strategy': 'keystone', + 'username': context.user_id, + 'tenant': context.project_id, + } + params['auth_tok'] = context.auth_token + + return glance.client.Client(host, port, **params) + + +def get_api_servers(): + """ + Shuffle a list of FLAGS.glance_api_servers and return an iterator + that will cycle through the list, looping around to the beginning + if necessary. + """ + api_servers = [] + for api_server in FLAGS.glance_api_servers: + host, port_str = api_server.split(':') + api_servers.append((host, int(port_str))) + random.shuffle(api_servers) + return itertools.cycle(api_servers) + + +class GlanceClientWrapper(object): + """Glance client wrapper class that implements retries.""" + + def __init__(self, context=None, host=None, port=None): + if host is not None: + self._create_static_client(context, host, port) + else: + self.client = None + self.api_servers = None + + def _create_static_client(self, context, host, port): + """Create a client that we'll use for every call.""" + self.host = host + self.port = port + self.client = _create_glance_client(context, self.host, self.port) + + def _create_onetime_client(self, context): + """Create a client that will be used for one call.""" + if self.api_servers is None: + self.api_servers = get_api_servers() + self.host, self.port = self.api_servers.next() + return _create_glance_client(context, self.host, self.port) + + def call(self, context, method, *args, **kwargs): + """ + Call a glance client method. If we get a connection error, + retry the request according to FLAGS.glance_num_retries. + """ + + retry_excs = (glance_exception.ClientConnectionError, + glance_exception.ServiceUnavailable) + + num_attempts = 1 + FLAGS.glance_num_retries + + for attempt in xrange(1, num_attempts + 1): + if self.client: + client = self.client + else: + client = self._create_onetime_client(context) + try: + return getattr(client, method)(*args, **kwargs) + except retry_excs as e: + host = self.host + port = self.port + extra = "retrying" + error_msg = _("Error contacting glance server " + "'%(host)s:%(port)s' for '%(method)s', %(extra)s.") + if attempt == num_attempts: + extra = 'done trying' + LOG.exception(error_msg, locals()) + raise exception.GlanceConnectionFailed( + host=host, port=port, reason=str(e)) + LOG.exception(error_msg, locals()) + time.sleep(1) + # Not reached + + +class GlanceImageService(object): + """Provides storage and retrieval of disk image objects within Glance.""" + + def __init__(self, client=None): + if client is None: + client = GlanceClientWrapper() + self._client = client + + def detail(self, context, **kwargs): + """Calls out to Glance for a list of detailed image information.""" + params = self._extract_query_params(kwargs) + image_metas = self._get_images(context, **params) + + images = [] + for image_meta in image_metas: + if self._is_image_available(context, image_meta): + base_image_meta = self._translate_from_glance(image_meta) + images.append(base_image_meta) + return images + + def _extract_query_params(self, params): + _params = {} + accepted_params = ('filters', 'marker', 'limit', + 'sort_key', 'sort_dir') + for param in accepted_params: + if param in params: + _params[param] = params.get(param) + + return _params + + def _get_images(self, context, **kwargs): + """Get image entitites from images service""" + + # ensure filters is a dict + kwargs['filters'] = kwargs.get('filters') or {} + # NOTE(vish): don't filter out private images + kwargs['filters'].setdefault('is_public', 'none') + + return self._fetch_images(context, 'get_images_detailed', **kwargs) + + def _fetch_images(self, context, fetch_method, **kwargs): + """Paginate through results from glance server""" + try: + images = self._client.call(context, fetch_method, **kwargs) + except Exception: + _reraise_translated_exception() + + if not images: + # break out of recursive loop to end pagination + return + + for image in images: + yield image + + try: + # attempt to advance the marker in order to fetch next page + kwargs['marker'] = images[-1]['id'] + except KeyError: + raise exception.ImagePaginationFailed() + + try: + kwargs['limit'] = kwargs['limit'] - len(images) + # break if we have reached a provided limit + if kwargs['limit'] <= 0: + return + except KeyError: + # ignore missing limit, just proceed without it + pass + + for image in self._fetch_images(context, fetch_method, **kwargs): + yield image + + def show(self, context, image_id): + """Returns a dict with image data for the given opaque image id.""" + try: + image_meta = self._client.call(context, 'get_image_meta', + image_id) + except Exception: + _reraise_translated_image_exception(image_id) + + if not self._is_image_available(context, image_meta): + raise exception.ImageNotFound(image_id=image_id) + + base_image_meta = self._translate_from_glance(image_meta) + return base_image_meta + + def download(self, context, image_id, data): + """Calls out to Glance for metadata and data and writes data.""" + try: + image_meta, image_chunks = self._client.call(context, + 'get_image', image_id) + except Exception: + _reraise_translated_image_exception(image_id) + + for chunk in image_chunks: + data.write(chunk) + + def create(self, context, image_meta, data=None): + """Store the image data and return the new image id. + + :raises: AlreadyExists if the image already exist. + + """ + # Translate Base -> Service + LOG.debug(_('Creating image in Glance. Metadata passed in %s'), + image_meta) + sent_service_image_meta = self._translate_to_glance(image_meta) + LOG.debug(_('Metadata after formatting for Glance %s'), + sent_service_image_meta) + + recv_service_image_meta = self._client.call(context, + 'add_image', sent_service_image_meta, data) + + # Translate Service -> Base + base_image_meta = self._translate_from_glance(recv_service_image_meta) + LOG.debug(_('Metadata returned from Glance formatted for Base %s'), + base_image_meta) + return base_image_meta + + def update(self, context, image_id, image_meta, data=None, features=None): + """Replace the contents of the given image with the new data. + + :raises: ImageNotFound if the image does not exist. + + """ + # NOTE(vish): show is to check if image is available + self.show(context, image_id) + image_meta = self._translate_to_glance(image_meta) + try: + image_meta = self._client.call(context, 'update_image', + image_id, image_meta, data, features) + except Exception: + _reraise_translated_image_exception(image_id) + + base_image_meta = self._translate_from_glance(image_meta) + return base_image_meta + + def delete(self, context, image_id): + """Delete the given image. + + :raises: ImageNotFound if the image does not exist. + :raises: NotAuthorized if the user is not an owner. + + """ + # NOTE(vish): show is to check if image is available + self.show(context, image_id) + try: + result = self._client.call(context, 'delete_image', image_id) + except glance_exception.NotFound: + raise exception.ImageNotFound(image_id=image_id) + return result + + def delete_all(self): + """Clears out all images.""" + pass + + @classmethod + def _translate_to_glance(cls, image_meta): + image_meta = _convert_to_string(image_meta) + image_meta = _remove_read_only(image_meta) + return image_meta + + @classmethod + def _translate_from_glance(cls, image_meta): + image_meta = _limit_attributes(image_meta) + image_meta = _convert_timestamps_to_datetimes(image_meta) + image_meta = _convert_from_string(image_meta) + return image_meta + + @staticmethod + def _is_image_available(context, image_meta): + """Check image availability. + + Under Glance, images are always available if the context has + an auth_token. + + """ + if hasattr(context, 'auth_token') and context.auth_token: + return True + + if image_meta['is_public'] or context.is_admin: + return True + + properties = image_meta['properties'] + + if context.project_id and ('owner_id' in properties): + return str(properties['owner_id']) == str(context.project_id) + + if context.project_id and ('project_id' in properties): + return str(properties['project_id']) == str(context.project_id) + + try: + user_id = properties['user_id'] + except KeyError: + return False + + return str(user_id) == str(context.user_id) + + +# utility functions +def _convert_timestamps_to_datetimes(image_meta): + """Returns image with timestamp fields converted to datetime objects.""" + for attr in ['created_at', 'updated_at', 'deleted_at']: + if image_meta.get(attr): + image_meta[attr] = _parse_glance_iso8601_timestamp( + image_meta[attr]) + return image_meta + + +def _parse_glance_iso8601_timestamp(timestamp): + """Parse a subset of iso8601 timestamps into datetime objects.""" + iso_formats = ['%Y-%m-%dT%H:%M:%S.%f', '%Y-%m-%dT%H:%M:%S'] + + for iso_format in iso_formats: + try: + return timeutils.parse_strtime(timestamp, iso_format) + except ValueError: + pass + + raise ValueError(_('%(timestamp)s does not follow any of the ' + 'signatures: %(iso_formats)s') % locals()) + + +# NOTE(bcwaldon): used to store non-string data in glance metadata +def _json_loads(properties, attr): + prop = properties[attr] + if isinstance(prop, basestring): + properties[attr] = jsonutils.loads(prop) + + +def _json_dumps(properties, attr): + prop = properties[attr] + if not isinstance(prop, basestring): + properties[attr] = jsonutils.dumps(prop) + + +_CONVERT_PROPS = ('block_device_mapping', 'mappings') + + +def _convert(method, metadata): + metadata = copy.deepcopy(metadata) + properties = metadata.get('properties') + if properties: + for attr in _CONVERT_PROPS: + if attr in properties: + method(properties, attr) + + return metadata + + +def _convert_from_string(metadata): + return _convert(_json_loads, metadata) + + +def _convert_to_string(metadata): + return _convert(_json_dumps, metadata) + + +def _limit_attributes(image_meta): + IMAGE_ATTRIBUTES = ['size', 'disk_format', 'owner', + 'container_format', 'checksum', 'id', + 'name', 'created_at', 'updated_at', + 'deleted_at', 'deleted', 'status', + 'min_disk', 'min_ram', 'is_public'] + output = {} + for attr in IMAGE_ATTRIBUTES: + output[attr] = image_meta.get(attr) + + output['properties'] = image_meta.get('properties', {}) + + return output + + +def _remove_read_only(image_meta): + IMAGE_ATTRIBUTES = ['updated_at', 'created_at', 'deleted_at'] + output = copy.deepcopy(image_meta) + for attr in IMAGE_ATTRIBUTES: + if attr in output: + del output[attr] + return output + + +def _reraise_translated_image_exception(image_id): + """Transform the exception for the image but keep its traceback intact.""" + exc_type, exc_value, exc_trace = sys.exc_info() + new_exc = _translate_image_exception(image_id, exc_type, exc_value) + raise new_exc, None, exc_trace + + +def _reraise_translated_exception(): + """Transform the exception but keep its traceback intact.""" + exc_type, exc_value, exc_trace = sys.exc_info() + new_exc = _translate_plain_exception(exc_type, exc_value) + raise new_exc, None, exc_trace + + +def _translate_image_exception(image_id, exc_type, exc_value): + if exc_type in (glance_exception.Forbidden, + glance_exception.NotAuthenticated, + glance_exception.MissingCredentialError): + return exception.ImageNotAuthorized(image_id=image_id) + if exc_type is glance_exception.NotFound: + return exception.ImageNotFound(image_id=image_id) + if exc_type is glance_exception.Invalid: + return exception.Invalid(exc_value) + return exc_value + + +def _translate_plain_exception(exc_type, exc_value): + if exc_type in (glance_exception.Forbidden, + glance_exception.NotAuthenticated, + glance_exception.MissingCredentialError): + return exception.NotAuthorized(exc_value) + if exc_type is glance_exception.NotFound: + return exception.NotFound(exc_value) + if exc_type is glance_exception.Invalid: + return exception.Invalid(exc_value) + return exc_value + + +def get_remote_image_service(context, image_href): + """Create an image_service and parse the id from the given image_href. + + The image_href param can be an href of the form + 'http://example.com:9292/v1/images/b8b2c6f7-7345-4e2f-afa2-eedaba9cbbe3', + or just an id such as 'b8b2c6f7-7345-4e2f-afa2-eedaba9cbbe3'. If the + image_href is a standalone id, then the default image service is returned. + + :param image_href: href that describes the location of an image + :returns: a tuple of the form (image_service, image_id) + + """ + #NOTE(bcwaldon): If image_href doesn't look like a URI, assume its a + # standalone image ID + if '/' not in str(image_href): + image_service = get_default_image_service() + return image_service, image_href + + try: + (image_id, glance_host, glance_port) = _parse_image_ref(image_href) + glance_client = GlanceClientWrapper(context=context, + host=glance_host, port=glance_port) + except ValueError: + raise exception.InvalidImageRef(image_href=image_href) + + image_service = GlanceImageService(client=glance_client) + return image_service, image_id + + +def get_default_image_service(): + return GlanceImageService() diff --git a/cinder/tests/api/openstack/fakes.py b/cinder/tests/api/openstack/fakes.py index b6d2e34e8..509c7211a 100644 --- a/cinder/tests/api/openstack/fakes.py +++ b/cinder/tests/api/openstack/fakes.py @@ -30,7 +30,6 @@ from cinder.api.openstack import volume from cinder.api.openstack.volume import versions from cinder.api.openstack import wsgi as os_wsgi from cinder import context -from cinder.db.sqlalchemy import models from cinder import exception as exc from cinder import utils from cinder import wsgi @@ -215,6 +214,18 @@ def stub_volume_create(self, context, size, name, description, snapshot, return vol +def stub_volume_create_from_image(self, context, size, name, description, + snapshot, volume_type, metadata, + availability_zone): + vol = stub_volume('1') + vol['status'] = 'creating' + vol['size'] = size + vol['display_name'] = name + vol['display_description'] = description + vol['availability_zone'] = 'cinder' + return vol + + def stub_volume_update(self, context, *args, **param): pass diff --git a/cinder/tests/api/openstack/volume/contrib/test_volume_actions.py b/cinder/tests/api/openstack/volume/contrib/test_volume_actions.py index 8edf37d8e..2448c9f37 100644 --- a/cinder/tests/api/openstack/volume/contrib/test_volume_actions.py +++ b/cinder/tests/api/openstack/volume/contrib/test_volume_actions.py @@ -12,14 +12,19 @@ # License for the specific language governing permissions and limitations # under the License. +import datetime import webob -from cinder import volume +from cinder import exception from cinder import flags from cinder import test +from cinder import utils +from cinder import volume +from cinder.api.openstack.volume.contrib import volume_actions from cinder.openstack.common import jsonutils +from cinder.openstack.common.rpc import common as rpc_common from cinder.tests.api.openstack import fakes -from cinder import utils +from cinder.volume import api as volume_api FLAGS = flags.FLAGS @@ -72,7 +77,6 @@ class VolumeActionsTest(test.TestCase): req.headers["content-type"] = "application/json" res = req.get_response(fakes.wsgi_app()) - output = jsonutils.loads(res.body) self.assertEqual(res.status_int, 200) def test_terminate_connection(self): @@ -100,3 +104,142 @@ class VolumeActionsTest(test.TestCase): res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 202) + + +def stub_volume_get(self, context, volume_id): + volume = fakes.stub_volume(volume_id) + if volume_id == 5: + volume['status'] = 'in-use' + else: + volume['status'] = 'available' + return volume + + +def stub_upload_volume_to_image_service(self, context, volume, metadata, + force): + ret = {"id": volume['id'], + "updated_at": datetime.datetime(1, 1, 1, 1, 1, 1), + "status": 'uploading', + "display_description": volume['display_description'], + "size": volume['size'], + "volume_type": volume['volume_type'], + "image_id": 1, + "container_format": 'bare', + "disk_format": 'raw', + "image_name": 'image_name'} + return ret + + +class VolumeImageActionsTest(test.TestCase): + def setUp(self): + super(VolumeImageActionsTest, self).setUp() + self.controller = volume_actions.VolumeActionsController() + + self.stubs.Set(volume_api.API, 'get', stub_volume_get) + + def test_copy_volume_to_image(self): + self.stubs.Set(volume_api.API, + "copy_volume_to_image", + stub_upload_volume_to_image_service) + + id = 1 + vol = {"container_format": 'bare', + "disk_format": 'raw', + "image_name": 'image_name', + "force": True} + body = {"os-volume_upload_image": vol} + req = fakes.HTTPRequest.blank('/v1/tenant1/volumes/%s/action' % id) + res_dict = self.controller._volume_upload_image(req, id, body) + expected = {'os-volume_upload_image': {'id': id, + 'updated_at': datetime.datetime(1, 1, 1, 1, 1, 1), + 'status': 'uploading', + 'display_description': 'displaydesc', + 'size': 1, + 'volume_type': {'name': 'vol_type_name'}, + 'image_id': 1, + 'container_format': 'bare', + 'disk_format': 'raw', + 'image_name': 'image_name'}} + self.assertDictMatch(res_dict, expected) + + def test_copy_volume_to_image_volumenotfound(self): + def stub_volume_get_raise_exc(self, context, volume_id): + raise exception.VolumeNotFound(volume_id=volume_id) + + self.stubs.Set(volume_api.API, 'get', stub_volume_get_raise_exc) + + id = 1 + vol = {"container_format": 'bare', + "disk_format": 'raw', + "image_name": 'image_name', + "force": True} + body = {"os-volume_upload_image": vol} + req = fakes.HTTPRequest.blank('/v1/tenant1/volumes/%s/action' % id) + self.assertRaises(webob.exc.HTTPNotFound, + self.controller._volume_upload_image, + req, + id, + body) + + def test_copy_volume_to_image_invalidvolume(self): + def stub_upload_volume_to_image_service_raise(self, context, volume, + metadata, force): + raise exception.InvalidVolume + self.stubs.Set(volume_api.API, + "copy_volume_to_image", + stub_upload_volume_to_image_service_raise) + + id = 1 + vol = {"container_format": 'bare', + "disk_format": 'raw', + "image_name": 'image_name', + "force": True} + body = {"os-volume_upload_image": vol} + req = fakes.HTTPRequest.blank('/v1/tenant1/volumes/%s/action' % id) + self.assertRaises(webob.exc.HTTPBadRequest, + self.controller._volume_upload_image, + req, + id, + body) + + def test_copy_volume_to_image_valueerror(self): + def stub_upload_volume_to_image_service_raise(self, context, volume, + metadata, force): + raise ValueError + self.stubs.Set(volume_api.API, + "copy_volume_to_image", + stub_upload_volume_to_image_service_raise) + + id = 1 + vol = {"container_format": 'bare', + "disk_format": 'raw', + "image_name": 'image_name', + "force": True} + body = {"os-volume_upload_image": vol} + req = fakes.HTTPRequest.blank('/v1/tenant1/volumes/%s/action' % id) + self.assertRaises(webob.exc.HTTPBadRequest, + self.controller._volume_upload_image, + req, + id, + body) + + def test_copy_volume_to_image_remoteerror(self): + def stub_upload_volume_to_image_service_raise(self, context, volume, + metadata, force): + raise rpc_common.RemoteError + self.stubs.Set(volume_api.API, + "copy_volume_to_image", + stub_upload_volume_to_image_service_raise) + + id = 1 + vol = {"container_format": 'bare', + "disk_format": 'raw', + "image_name": 'image_name', + "force": True} + body = {"os-volume_upload_image": vol} + req = fakes.HTTPRequest.blank('/v1/tenant1/volumes/%s/action' % id) + self.assertRaises(webob.exc.HTTPBadRequest, + self.controller._volume_upload_image, + req, + id, + body) diff --git a/cinder/tests/api/openstack/volume/test_router.py b/cinder/tests/api/openstack/volume/test_router.py index 8ba9a0fe7..f6a1c12ae 100644 --- a/cinder/tests/api/openstack/volume/test_router.py +++ b/cinder/tests/api/openstack/volume/test_router.py @@ -30,6 +30,9 @@ LOG = logging.getLogger(__name__) class FakeController(object): + def __init__(self, ext_mgr=None): + self.ext_mgr = ext_mgr + def index(self, req): return {} @@ -41,12 +44,16 @@ def create_resource(): return wsgi.Resource(FakeController()) +def create_volume_resource(ext_mgr): + return wsgi.Resource(FakeController(ext_mgr)) + + class VolumeRouterTestCase(test.TestCase): def setUp(self): super(VolumeRouterTestCase, self).setUp() # NOTE(vish): versions is just returning text so, no need to stub. self.stubs.Set(snapshots, 'create_resource', create_resource) - self.stubs.Set(volumes, 'create_resource', create_resource) + self.stubs.Set(volumes, 'create_resource', create_volume_resource) self.app = volume.APIRouter() def test_versions(self): diff --git a/cinder/tests/api/openstack/volume/test_volumes.py b/cinder/tests/api/openstack/volume/test_volumes.py index 50c3f662a..2a63b561e 100644 --- a/cinder/tests/api/openstack/volume/test_volumes.py +++ b/cinder/tests/api/openstack/volume/test_volumes.py @@ -20,21 +20,43 @@ import webob from cinder.api.openstack.volume import volumes from cinder import db +from cinder.api.openstack.volume import extensions from cinder import exception from cinder import flags from cinder import test from cinder.tests.api.openstack import fakes +from cinder.tests.image import fake as fake_image from cinder.volume import api as volume_api FLAGS = flags.FLAGS NS = '{http://docs.openstack.org/volume/api/v1}' +TEST_SNAPSHOT_UUID = '00000000-0000-0000-0000-000000000001' + + +def stub_snapshot_get(self, context, snapshot_id): + if snapshot_id != TEST_SNAPSHOT_UUID: + raise exception.NotFound + + return { + 'id': snapshot_id, + 'volume_id': 12, + 'status': 'available', + 'volume_size': 100, + 'created_at': None, + 'display_name': 'Default name', + 'display_description': 'Default description', + } + class VolumeApiTest(test.TestCase): def setUp(self): super(VolumeApiTest, self).setUp() - self.controller = volumes.VolumeController() + self.ext_mgr = extensions.ExtensionManager() + self.ext_mgr.extensions = {} + fake_image.stub_out_image_service(self.stubs) + self.controller = volumes.VolumeController(self.ext_mgr) self.stubs.Set(db, 'volume_get_all', fakes.stub_volume_get_all) self.stubs.Set(db, 'volume_get_all_by_project', @@ -95,6 +117,82 @@ class VolumeApiTest(test.TestCase): req, body) + def test_volume_create_with_image_id(self): + self.stubs.Set(volume_api.API, "create", fakes.stub_volume_create) + self.ext_mgr.extensions = {'os-image-create': 'fake'} + vol = {"size": '1', + "display_name": "Volume Test Name", + "display_description": "Volume Test Desc", + "availability_zone": "cinder", + "imageRef": 'c905cedb-7281-47e4-8a62-f26bc5fc4c77'} + expected = {'volume': {'status': 'fakestatus', + 'display_description': 'Volume Test Desc', + 'availability_zone': 'cinder', + 'display_name': 'Volume Test Name', + 'attachments': [{'device': '/', + 'server_id': 'fakeuuid', + 'id': '1', + 'volume_id': '1'}], + 'volume_type': 'vol_type_name', + 'image_id': 'c905cedb-7281-47e4-8a62-f26bc5fc4c77', + 'snapshot_id': None, + 'metadata': {}, + 'id': '1', + 'created_at': datetime.datetime(1, 1, 1, 1, 1, 1), + 'size': 1} + } + body = {"volume": vol} + req = fakes.HTTPRequest.blank('/v1/volumes') + res_dict = self.controller.create(req, body) + self.assertEqual(res_dict, expected) + + def test_volume_create_with_image_id_and_snapshot_id(self): + self.stubs.Set(volume_api.API, "create", fakes.stub_volume_create) + self.stubs.Set(volume_api.API, "get_snapshot", stub_snapshot_get) + self.ext_mgr.extensions = {'os-image-create': 'fake'} + vol = {"size": '1', + "display_name": "Volume Test Name", + "display_description": "Volume Test Desc", + "availability_zone": "cinder", + "imageRef": 'c905cedb-7281-47e4-8a62-f26bc5fc4c77', + "snapshot_id": TEST_SNAPSHOT_UUID} + body = {"volume": vol} + req = fakes.HTTPRequest.blank('/v1/volumes') + self.assertRaises(webob.exc.HTTPBadRequest, + self.controller.create, + req, + body) + + def test_volume_create_with_image_id_is_integer(self): + self.stubs.Set(volume_api.API, "create", fakes.stub_volume_create) + self.ext_mgr.extensions = {'os-image-create': 'fake'} + vol = {"size": '1', + "display_name": "Volume Test Name", + "display_description": "Volume Test Desc", + "availability_zone": "cinder", + "imageRef": 1234} + body = {"volume": vol} + req = fakes.HTTPRequest.blank('/v1/volumes') + self.assertRaises(webob.exc.HTTPBadRequest, + self.controller.create, + req, + body) + + def test_volume_create_with_image_id_not_uuid_format(self): + self.stubs.Set(volume_api.API, "create", fakes.stub_volume_create) + self.ext_mgr.extensions = {'os-image-create': 'fake'} + vol = {"size": '1', + "display_name": "Volume Test Name", + "display_description": "Volume Test Desc", + "availability_zone": "cinder", + "imageRef": '12345'} + body = {"volume": vol} + req = fakes.HTTPRequest.blank('/v1/volumes') + self.assertRaises(webob.exc.HTTPBadRequest, + self.controller.create, + req, + body) + def test_volume_list(self): self.stubs.Set(volume_api.API, 'get_all', fakes.stub_volume_get_all_by_project) diff --git a/cinder/tests/image/__init__.py b/cinder/tests/image/__init__.py new file mode 100644 index 000000000..387872b21 --- /dev/null +++ b/cinder/tests/image/__init__.py @@ -0,0 +1,20 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# NOTE(vish): this forces the fixtures from tests/__init.py:setup() to work + +from cinder.tests import * diff --git a/cinder/tests/image/fake.py b/cinder/tests/image/fake.py new file mode 100644 index 000000000..c4b8d2c5d --- /dev/null +++ b/cinder/tests/image/fake.py @@ -0,0 +1,248 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 Justin Santa Barbara +# Copyright 2012 OpenStack LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Implementation of a fake image service""" + +import copy +import datetime + +from cinder import exception +from cinder import flags +import cinder.image.glance +from cinder.openstack.common import log as logging +from cinder import utils + + +LOG = logging.getLogger(__name__) + + +FLAGS = flags.FLAGS + + +class _FakeImageService(object): + """Mock (fake) image service for unit testing.""" + + def __init__(self): + self.images = {} + # NOTE(justinsb): The OpenStack API can't upload an image? + # So, make sure we've got one.. + timestamp = datetime.datetime(2011, 01, 01, 01, 02, 03) + + image1 = {'id': '155d900f-4e14-4e4c-a73d-069cbf4541e6', + 'name': 'fakeimage123456', + 'created_at': timestamp, + 'updated_at': timestamp, + 'deleted_at': None, + 'deleted': False, + 'status': 'active', + 'is_public': False, + 'container_format': 'raw', + 'disk_format': 'raw', + 'properties': {'kernel_id': FLAGS.null_kernel, + 'ramdisk_id': FLAGS.null_kernel, + 'architecture': 'x86_64'}} + + image2 = {'id': 'a2459075-d96c-40d5-893e-577ff92e721c', + 'name': 'fakeimage123456', + 'size': 1048576 * 1024 * 2, # Image of size 2GB + 'created_at': timestamp, + 'updated_at': timestamp, + 'deleted_at': None, + 'deleted': False, + 'status': 'active', + 'is_public': True, + 'container_format': 'ami', + 'disk_format': 'ami', + 'properties': {'kernel_id': FLAGS.null_kernel, + 'ramdisk_id': FLAGS.null_kernel}} + + image3 = {'id': '76fa36fc-c930-4bf3-8c8a-ea2a2420deb6', + 'name': 'fakeimage123456', + 'created_at': timestamp, + 'updated_at': timestamp, + 'deleted_at': None, + 'deleted': False, + 'status': 'active', + 'is_public': True, + 'container_format': None, + 'disk_format': None, + 'properties': {'kernel_id': FLAGS.null_kernel, + 'ramdisk_id': FLAGS.null_kernel}} + + image4 = {'id': 'cedef40a-ed67-4d10-800e-17455edce175', + 'name': 'fakeimage123456', + 'created_at': timestamp, + 'updated_at': timestamp, + 'deleted_at': None, + 'deleted': False, + 'status': 'active', + 'is_public': True, + 'container_format': 'ami', + 'disk_format': 'ami', + 'properties': {'kernel_id': FLAGS.null_kernel, + 'ramdisk_id': FLAGS.null_kernel}} + + image5 = {'id': 'c905cedb-7281-47e4-8a62-f26bc5fc4c77', + 'name': 'fakeimage123456', + 'size': 1048576, # Image of size 1MB + 'created_at': timestamp, + 'updated_at': timestamp, + 'deleted_at': None, + 'deleted': False, + 'status': 'active', + 'is_public': True, + 'container_format': 'ami', + 'disk_format': 'ami', + 'properties': {'kernel_id': + '155d900f-4e14-4e4c-a73d-069cbf4541e6', + 'ramdisk_id': None}} + + image6 = {'id': 'a440c04b-79fa-479c-bed1-0b816eaec379', + 'name': 'fakeimage6', + 'created_at': timestamp, + 'updated_at': timestamp, + 'deleted_at': None, + 'deleted': False, + 'status': 'active', + 'is_public': False, + 'container_format': 'ova', + 'disk_format': 'vhd', + 'properties': {'kernel_id': FLAGS.null_kernel, + 'ramdisk_id': FLAGS.null_kernel, + 'architecture': 'x86_64', + 'auto_disk_config': 'False'}} + + image7 = {'id': '70a599e0-31e7-49b7-b260-868f441e862b', + 'name': 'fakeimage7', + 'created_at': timestamp, + 'updated_at': timestamp, + 'deleted_at': None, + 'deleted': False, + 'status': 'active', + 'is_public': False, + 'container_format': 'ova', + 'disk_format': 'vhd', + 'properties': {'kernel_id': FLAGS.null_kernel, + 'ramdisk_id': FLAGS.null_kernel, + 'architecture': 'x86_64', + 'auto_disk_config': 'True'}} + + self.create(None, image1) + self.create(None, image2) + self.create(None, image3) + self.create(None, image4) + self.create(None, image5) + self.create(None, image6) + self.create(None, image7) + self._imagedata = {} + super(_FakeImageService, self).__init__() + + #TODO(bcwaldon): implement optional kwargs such as limit, sort_dir + def detail(self, context, **kwargs): + """Return list of detailed image information.""" + return copy.deepcopy(self.images.values()) + + def download(self, context, image_id, data): + self.show(context, image_id) + data.write(self._imagedata.get(image_id, '')) + + def show(self, context, image_id): + """Get data about specified image. + + Returns a dict containing image data for the given opaque image id. + + """ + image = self.images.get(str(image_id)) + if image: + return copy.deepcopy(image) + LOG.warn('Unable to find image id %s. Have images: %s', + image_id, self.images) + raise exception.ImageNotFound(image_id=image_id) + + def create(self, context, metadata, data=None): + """Store the image data and return the new image id. + + :raises: Duplicate if the image already exist. + + """ + image_id = str(metadata.get('id', utils.gen_uuid())) + metadata['id'] = image_id + if image_id in self.images: + raise exception.Duplicate() + self.images[image_id] = copy.deepcopy(metadata) + if data: + self._imagedata[image_id] = data.read() + return self.images[image_id] + + def update(self, context, image_id, metadata, data=None, + headers=None): + """Replace the contents of the given image with the new data. + + :raises: ImageNotFound if the image does not exist. + + """ + if not self.images.get(image_id): + raise exception.ImageNotFound(image_id=image_id) + try: + purge = headers['x-glance-registry-purge-props'] + except Exception: + purge = True + if purge: + self.images[image_id] = copy.deepcopy(metadata) + else: + image = self.images[image_id] + try: + image['properties'].update(metadata.pop('properties')) + except Exception: + pass + image.update(metadata) + return self.images[image_id] + + def delete(self, context, image_id): + """Delete the given image. + + :raises: ImageNotFound if the image does not exist. + + """ + removed = self.images.pop(image_id, None) + if not removed: + raise exception.ImageNotFound(image_id=image_id) + + def delete_all(self): + """Clears out all images.""" + self.images.clear() + +_fakeImageService = _FakeImageService() + + +def FakeImageService(): + return _fakeImageService + + +def FakeImageService_reset(): + global _fakeImageService + _fakeImageService = _FakeImageService() + + +def stub_out_image_service(stubs): + def fake_get_remote_image_service(context, image_href): + return (FakeImageService(), image_href) + stubs.Set(cinder.image.glance, 'get_remote_image_service', + lambda x, y: (FakeImageService(), y)) + stubs.Set(cinder.image.glance, 'get_default_image_service', + lambda: FakeImageService()) diff --git a/cinder/tests/policy.json b/cinder/tests/policy.json index 1d4eff39e..93162d445 100644 --- a/cinder/tests/policy.json +++ b/cinder/tests/policy.json @@ -20,6 +20,7 @@ "volume:get_snapshot": [], "volume:get_all_snapshots": [], + "volume_extension:volume_actions:upload_image": [], "volume_extension:types_manage": [], "volume_extension:types_extra_specs": [], "volume_extension:extended_snapshot_attributes": [] diff --git a/cinder/tests/test_volume.py b/cinder/tests/test_volume.py index fde6a397a..0cdd3596f 100644 --- a/cinder/tests/test_volume.py +++ b/cinder/tests/test_volume.py @@ -20,6 +20,8 @@ Tests for Volume Code. """ +import os +import datetime import cStringIO import logging @@ -31,6 +33,7 @@ from cinder import context from cinder import exception from cinder import db from cinder import flags +from cinder.tests.image import fake as fake_image from cinder.openstack.common import log as os_logging from cinder.openstack.common import importutils from cinder.openstack.common import rpc @@ -52,20 +55,23 @@ class VolumeTestCase(test.TestCase): volumes_dir=vol_tmpdir) self.volume = importutils.import_object(FLAGS.volume_manager) self.context = context.get_admin_context() + fake_image.stub_out_image_service(self.stubs) def tearDown(self): try: shutil.rmtree(FLAGS.volumes_dir) - except OSError, e: + except OSError: pass super(VolumeTestCase, self).tearDown() @staticmethod - def _create_volume(size='0', snapshot_id=None, metadata=None): + def _create_volume(size='0', snapshot_id=None, image_id=None, + metadata=None): """Create a volume object.""" vol = {} vol['size'] = size vol['snapshot_id'] = snapshot_id + vol['image_id'] = image_id vol['user_id'] = 'fake' vol['project_id'] = 'fake' vol['availability_zone'] = FLAGS.storage_availability_zone @@ -90,9 +96,9 @@ class VolumeTestCase(test.TestCase): volume_id) def test_create_delete_volume_with_metadata(self): - """Test volume can be created and deleted.""" + """Test volume can be created with metadata and deleted.""" test_meta = {'fake_key': 'fake_value'} - volume = self._create_volume('0', None, test_meta) + volume = self._create_volume('0', None, metadata=test_meta) volume_id = volume['id'] self.volume.create_volume(self.context, volume_id) result_meta = { @@ -361,6 +367,188 @@ class VolumeTestCase(test.TestCase): self.volume.delete_snapshot(self.context, snapshot_id) self.volume.delete_volume(self.context, volume_id) + def _create_volume_from_image(self, expected_status, + fakeout_copy_image_to_volume=False): + """Call copy image to volume, Test the status of volume after calling + copying image to volume.""" + def fake_local_path(volume): + return dst_path + + def fake_copy_image_to_volume(context, volume, image_id): + pass + + dst_fd, dst_path = tempfile.mkstemp() + os.close(dst_fd) + self.stubs.Set(self.volume.driver, 'local_path', fake_local_path) + if fakeout_copy_image_to_volume: + self.stubs.Set(self.volume, '_copy_image_to_volume', + fake_copy_image_to_volume) + + image_id = 'c905cedb-7281-47e4-8a62-f26bc5fc4c77' + volume_id = 1 + # creating volume testdata + db.volume_create(self.context, {'id': volume_id, + 'updated_at': datetime.datetime(1, 1, 1, 1, 1, 1), + 'display_description': 'Test Desc', + 'size': 20, + 'status': 'creating', + 'instance_uuid': None, + 'host': 'dummy'}) + try: + self.volume.create_volume(self.context, + volume_id, + image_id=image_id) + + volume = db.volume_get(self.context, volume_id) + self.assertEqual(volume['status'], expected_status) + finally: + # cleanup + db.volume_destroy(self.context, volume_id) + os.unlink(dst_path) + + def test_create_volume_from_image_status_downloading(self): + """Verify that before copying image to volume, it is in downloading + state.""" + self._create_volume_from_image('downloading', True) + + def test_create_volume_from_image_status_available(self): + """Verify that before copying image to volume, it is in available + state.""" + self._create_volume_from_image('available') + + def test_create_volume_from_image_exception(self): + """Verify that create volume from image, the volume status is + 'downloading'.""" + dst_fd, dst_path = tempfile.mkstemp() + os.close(dst_fd) + + self.stubs.Set(self.volume.driver, 'local_path', lambda x: dst_path) + + image_id = 'aaaaaaaa-0000-0000-0000-000000000000' + # creating volume testdata + volume_id = 1 + db.volume_create(self.context, {'id': volume_id, + 'updated_at': datetime.datetime(1, 1, 1, 1, 1, 1), + 'display_description': 'Test Desc', + 'size': 20, + 'status': 'creating', + 'host': 'dummy'}) + + self.assertRaises(exception.ImageNotFound, + self.volume.create_volume, + self.context, + volume_id, + None, + image_id) + volume = db.volume_get(self.context, volume_id) + self.assertEqual(volume['status'], "error") + # cleanup + db.volume_destroy(self.context, volume_id) + os.unlink(dst_path) + + def test_copy_volume_to_image_status_available(self): + dst_fd, dst_path = tempfile.mkstemp() + os.close(dst_fd) + + def fake_local_path(volume): + return dst_path + + self.stubs.Set(self.volume.driver, 'local_path', fake_local_path) + + image_id = '70a599e0-31e7-49b7-b260-868f441e862b' + # creating volume testdata + volume_id = 1 + db.volume_create(self.context, {'id': volume_id, + 'updated_at': datetime.datetime(1, 1, 1, 1, 1, 1), + 'display_description': 'Test Desc', + 'size': 20, + 'status': 'uploading', + 'instance_uuid': None, + 'host': 'dummy'}) + + try: + # start test + self.volume.copy_volume_to_image(self.context, + volume_id, + image_id) + + volume = db.volume_get(self.context, volume_id) + self.assertEqual(volume['status'], 'available') + finally: + # cleanup + db.volume_destroy(self.context, volume_id) + os.unlink(dst_path) + + def test_copy_volume_to_image_status_use(self): + dst_fd, dst_path = tempfile.mkstemp() + os.close(dst_fd) + + def fake_local_path(volume): + return dst_path + + self.stubs.Set(self.volume.driver, 'local_path', fake_local_path) + + #image_id = '70a599e0-31e7-49b7-b260-868f441e862b' + image_id = 'a440c04b-79fa-479c-bed1-0b816eaec379' + # creating volume testdata + volume_id = 1 + db.volume_create(self.context, + {'id': volume_id, + 'updated_at': datetime.datetime(1, 1, 1, 1, 1, 1), + 'display_description': 'Test Desc', + 'size': 20, + 'status': 'uploading', + 'instance_uuid': + 'b21f957d-a72f-4b93-b5a5-45b1161abb02', + 'host': 'dummy'}) + + try: + # start test + self.volume.copy_volume_to_image(self.context, + volume_id, + image_id) + + volume = db.volume_get(self.context, volume_id) + self.assertEqual(volume['status'], 'in-use') + finally: + # cleanup + db.volume_destroy(self.context, volume_id) + os.unlink(dst_path) + + def test_copy_volume_to_image_exception(self): + dst_fd, dst_path = tempfile.mkstemp() + os.close(dst_fd) + + def fake_local_path(volume): + return dst_path + + self.stubs.Set(self.volume.driver, 'local_path', fake_local_path) + + image_id = 'aaaaaaaa-0000-0000-0000-000000000000' + # creating volume testdata + volume_id = 1 + db.volume_create(self.context, {'id': volume_id, + 'updated_at': datetime.datetime(1, 1, 1, 1, 1, 1), + 'display_description': 'Test Desc', + 'size': 20, + 'status': 'in-use', + 'host': 'dummy'}) + + try: + # start test + self.assertRaises(exception.ImageNotFound, + self.volume.copy_volume_to_image, + self.context, + volume_id, + image_id) + + volume = db.volume_get(self.context, volume_id) + self.assertEqual(volume['status'], 'available') + finally: + # cleanup + db.volume_destroy(self.context, volume_id) + os.unlink(dst_path) + class DriverTestCase(test.TestCase): """Base Test class for Drivers.""" @@ -388,7 +576,7 @@ class DriverTestCase(test.TestCase): def tearDown(self): try: shutil.rmtree(FLAGS.volumes_dir) - except OSError, e: + except OSError: pass super(DriverTestCase, self).tearDown() diff --git a/cinder/utils.py b/cinder/utils.py index 480cd98fc..3819047c5 100644 --- a/cinder/utils.py +++ b/cinder/utils.py @@ -43,15 +43,12 @@ from xml.sax import saxutils from eventlet import event from eventlet import greenthread -from eventlet import semaphore from eventlet.green import subprocess -import iso8601 import netaddr from cinder import exception from cinder import flags from cinder.openstack.common import log as logging -from cinder.openstack.common import cfg from cinder.openstack.common import excutils from cinder.openstack.common import importutils from cinder.openstack.common import timeutils @@ -961,6 +958,18 @@ def read_cached_file(filename, cache_info, reload_func=None): return cache_info['data'] +def file_open(*args, **kwargs): + """Open file + + see built-in file() documentation for more details + + Note: The reason this is kept in a separate module is to easily + be able to provide a stub module that doesn't alter system + state at all (for unit tests) + """ + return file(*args, **kwargs) + + def hash_file(file_like_object): """Generate a hash for the contents of a file.""" checksum = hashlib.sha1() diff --git a/cinder/volume/api.py b/cinder/volume/api.py index ffe3dc09b..003b3cf2b 100644 --- a/cinder/volume/api.py +++ b/cinder/volume/api.py @@ -27,12 +27,12 @@ from eventlet import greenthread from cinder import exception from cinder import flags from cinder.openstack.common import cfg +from cinder.image import glance from cinder.openstack.common import log as logging from cinder.openstack.common import rpc import cinder.policy from cinder.openstack.common import timeutils from cinder import quota -from cinder import utils from cinder.db import base volume_host_opt = cfg.BoolOpt('snapshot_same_host', @@ -41,8 +41,10 @@ volume_host_opt = cfg.BoolOpt('snapshot_same_host', FLAGS = flags.FLAGS FLAGS.register_opt(volume_host_opt) +flags.DECLARE('storage_availability_zone', 'cinder.volume.manager') LOG = logging.getLogger(__name__) +GB = 1048576 * 1024 def wrap_check_policy(func): @@ -72,8 +74,14 @@ def check_policy(context, action, target_obj=None): class API(base.Base): """API for interacting with the volume manager.""" + def __init__(self, db_driver=None, image_service=None): + self.image_service = (image_service or + glance.get_default_image_service()) + super(API, self).__init__(db_driver) + def create(self, context, size, name, description, snapshot=None, - volume_type=None, metadata=None, availability_zone=None): + image_id=None, volume_type=None, metadata=None, + availability_zone=None): check_policy(context, 'create') if snapshot is not None: if snapshot['status'] != "available": @@ -85,7 +93,6 @@ class API(base.Base): snapshot_id = snapshot['id'] else: snapshot_id = None - if not isinstance(size, int) or size <= 0: msg = _('Volume size must be an integer and greater than 0') raise exception.InvalidInput(reason=msg) @@ -95,6 +102,15 @@ class API(base.Base): " %(size)sG volume") % locals()) raise exception.QuotaError(code="VolumeSizeTooLarge") + if image_id: + # check image existence + image_meta = self.image_service.show(context, image_id) + image_size_in_gb = image_meta['size'] / GB + #check image size is not larger than volume size. + if image_size_in_gb > size: + msg = _('Size of specified image is larger than volume size.') + raise exception.InvalidInput(reason=msg) + if availability_zone is None: availability_zone = FLAGS.storage_availability_zone @@ -116,9 +132,14 @@ class API(base.Base): 'volume_type_id': volume_type_id, 'metadata': metadata, } - volume = self.db.volume_create(context, options) - self._cast_create_volume(context, volume['id'], snapshot_id) + rpc.cast(context, + FLAGS.scheduler_topic, + {"method": "create_volume", + "args": {"topic": FLAGS.volume_topic, + "volume_id": volume['id'], + "snapshot_id": volume['snapshot_id'], + "image_id": image_id}}) return volume def _cast_create_volume(self, context, volume_id, snapshot_id): @@ -412,3 +433,40 @@ class API(base.Base): if i['key'] == key: return i['value'] return None + + def _check_volume_availability(self, context, volume, force): + """Check if the volume can be used.""" + if volume['status'] not in ['available', 'in-use']: + msg = _('Volume status must be available/in-use.') + raise exception.InvalidVolume(reason=msg) + if not force and 'in-use' == volume['status']: + msg = _('Volume status is in-use.') + raise exception.InvalidVolume(reason=msg) + + @wrap_check_policy + def copy_volume_to_image(self, context, volume, metadata, force): + """Create a new image from the specified volume.""" + self._check_volume_availability(context, volume, force) + + recv_metadata = self.image_service.create(context, metadata) + self.update(context, volume, {'status': 'uploading'}) + rpc.cast(context, + rpc.queue_get_for(context, + FLAGS.volume_topic, + volume['host']), + {"method": "copy_volume_to_image", + "args": {"volume_id": volume['id'], + "image_id": recv_metadata['id']}}) + + response = {"id": volume['id'], + "updated_at": volume['updated_at'], + "status": 'uploading', + "display_description": volume['display_description'], + "size": volume['size'], + "volume_type": volume['volume_type'], + "image_id": recv_metadata['id'], + "container_format": recv_metadata['container_format'], + "disk_format": recv_metadata['disk_format'], + "image_name": recv_metadata.get('name', None) + } + return response diff --git a/cinder/volume/driver.py b/cinder/volume/driver.py index 4ec73db75..846158017 100644 --- a/cinder/volume/driver.py +++ b/cinder/volume/driver.py @@ -237,6 +237,14 @@ class VolumeDriver(object): """Any initialization the volume driver does while starting""" pass + def copy_image_to_volume(self, context, volume, image_service, image_id): + """Fetch the image from image_service and write it to the volume.""" + raise NotImplementedError() + + def copy_volume_to_image(self, context, volume, image_service, image_id): + """Copy the volume to the specified image.""" + raise NotImplementedError() + class ISCSIDriver(VolumeDriver): """Executes commands relating to ISCSI volumes. @@ -466,6 +474,20 @@ class ISCSIDriver(VolumeDriver): "id:%(volume_id)s.") % locals()) raise + def copy_image_to_volume(self, context, volume, image_service, image_id): + """Fetch the image from image_service and write it to the volume.""" + volume_path = self.local_path(volume) + with utils.temporary_chown(volume_path): + with utils.file_open(volume_path, "wb") as image_file: + image_service.download(context, image_id, image_file) + + def copy_volume_to_image(self, context, volume, image_service, image_id): + """Copy the volume to the specified image.""" + volume_path = self.local_path(volume) + with utils.temporary_chown(volume_path): + with utils.file_open(volume_path) as volume_file: + image_service.update(context, image_id, {}, volume_file) + class FakeISCSIDriver(ISCSIDriver): """Logs calls instead of executing.""" diff --git a/cinder/volume/manager.py b/cinder/volume/manager.py index 7346a7559..1a7570dda 100644 --- a/cinder/volume/manager.py +++ b/cinder/volume/manager.py @@ -40,15 +40,14 @@ intact. from cinder import context from cinder import exception from cinder import flags +from cinder.image import glance from cinder.openstack.common import log as logging from cinder import manager from cinder.openstack.common import cfg from cinder.openstack.common import excutils from cinder.openstack.common import importutils -from cinder.openstack.common import rpc from cinder.openstack.common import timeutils from cinder import utils -from cinder.volume import volume_types LOG = logging.getLogger(__name__) @@ -99,7 +98,8 @@ class VolumeManager(manager.SchedulerDependentManager): else: LOG.info(_("volume %s: skipping export"), volume['name']) - def create_volume(self, context, volume_id, snapshot_id=None): + def create_volume(self, context, volume_id, snapshot_id=None, + image_id=None): """Creates and exports the volume.""" context = context.elevated() volume_ref = self.db.volume_get(context, volume_id) @@ -112,6 +112,11 @@ class VolumeManager(manager.SchedulerDependentManager): # before passing it to the driver. volume_ref['host'] = self.host + if image_id: + status = 'downloading' + else: + status = 'available' + try: vol_name = volume_ref['name'] vol_size = volume_ref['size'] @@ -138,11 +143,15 @@ class VolumeManager(manager.SchedulerDependentManager): now = timeutils.utcnow() self.db.volume_update(context, - volume_ref['id'], {'status': 'available', + volume_ref['id'], {'status': status, 'launched_at': now}) LOG.debug(_("volume %s: created successfully"), volume_ref['name']) self._reset_stats() - return volume_id + + if image_id: + #copy the image onto the volume. + self._copy_image_to_volume(context, volume_ref, image_id) + return volume_ref['id'] def delete_volume(self, context, volume_id): """Deletes and unexports volume.""" @@ -153,7 +162,7 @@ class VolumeManager(manager.SchedulerDependentManager): raise exception.VolumeAttached(volume_id=volume_id) if volume_ref['host'] != self.host: raise exception.InvalidVolume( - reason=_("Volume is not local to this node")) + reason=_("Volume is not local to this node")) self._reset_stats() try: @@ -161,7 +170,7 @@ class VolumeManager(manager.SchedulerDependentManager): self.driver.remove_export(context, volume_ref) LOG.debug(_("volume %s: deleting"), volume_ref['name']) self.driver.delete_volume(volume_ref) - except exception.VolumeIsBusy, e: + except exception.VolumeIsBusy: LOG.debug(_("volume %s: volume is busy"), volume_ref['name']) self.driver.ensure_export(context, volume_ref) self.db.volume_update(context, volume_ref['id'], @@ -245,6 +254,48 @@ class VolumeManager(manager.SchedulerDependentManager): # TODO(sleepsonthefloor): Is this 'elevated' appropriate? self.db.volume_detached(context.elevated(), volume_id) + def _copy_image_to_volume(self, context, volume, image_id): + """Downloads Glance image to the specified volume. """ + volume_id = volume['id'] + payload = {'volume_id': volume_id, 'image_id': image_id} + try: + self.driver.ensure_export(context.elevated(), volume) + image_service, image_id = glance.get_remote_image_service(context, + image_id) + self.driver.copy_image_to_volume(context, volume, image_service, + image_id) + LOG.debug(_("Downloaded image %(image_id)s to %(volume_id)s " + "successfully") % locals()) + self.db.volume_update(context, volume_id, + {'status': 'available'}) + except Exception, error: + with excutils.save_and_reraise_exception(): + payload['message'] = unicode(error) + self.db.volume_update(context, volume_id, {'status': 'error'}) + + def copy_volume_to_image(self, context, volume_id, image_id): + """Uploads the specified volume to Glance.""" + payload = {'volume_id': volume_id, 'image_id': image_id} + try: + volume = self.db.volume_get(context, volume_id) + self.driver.ensure_export(context.elevated(), volume) + image_service, image_id = glance.get_remote_image_service(context, + image_id) + self.driver.copy_volume_to_image(context, volume, image_service, + image_id) + LOG.debug(_("Uploaded volume %(volume_id)s to " + "image (%(image_id)s) successfully") % locals()) + except Exception, error: + with excutils.save_and_reraise_exception(): + payload['message'] = unicode(error) + finally: + if volume['instance_uuid'] is None: + self.db.volume_update(context, volume_id, + {'status': 'available'}) + else: + self.db.volume_update(context, volume_id, + {'status': 'in-use'}) + def initialize_connection(self, context, volume_id, connector): """Prepare volume for connection from host represented by connector. diff --git a/etc/cinder/rootwrap.d/volume.filters b/etc/cinder/rootwrap.d/volume.filters index 9451e5115..23fddbca8 100644 --- a/etc/cinder/rootwrap.d/volume.filters +++ b/etc/cinder/rootwrap.d/volume.filters @@ -27,3 +27,6 @@ lvdisplay: CommandFilter, /sbin/lvdisplay, root # nova/volume/driver.py: 'iscsiadm', '-m', 'node', '-T', ... iscsiadm: CommandFilter, /sbin/iscsiadm, root iscsiadm_usr: CommandFilter, /usr/bin/iscsiadm, root + +#nova/volume/.py: utils.temporary_chown(path, 0), ... +chown: CommandFilter, /bin/chown, root -- 2.45.2