From 1a431edff622ad62707cce085d2af6683a995e3f Mon Sep 17 00:00:00 2001 From: Ollie Leahy Date: Thu, 15 Nov 2012 11:48:27 +0000 Subject: [PATCH] Add db table for Glance Metadata MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit This commit implements the blueprint https://blueprints.launchpad.net/cinder/+spec/retain-glance-metadata-for-billing It creates the new table volume_glance_metadata in the cinder database, provides the CRUD methods for it, and populates the table when a volume or snapshot is created from a Glance image. Patch set 2: remove superflous line Patch set 3: Fix incorrect column types in sqlalchemy/models.py Patch set 4: Define exception class GlanceMetadataExists Change-Id: I8f98f6eaae005a33bfd49cea783774407b7aa120 --- cinder/db/api.py | 49 ++++++ cinder/db/sqlalchemy/api.py | 142 ++++++++++++++++++ .../versions/003_glance_metadata.py | 74 +++++++++ cinder/db/sqlalchemy/models.py | 13 +- cinder/exception.py | 5 + cinder/tests/test_volume_glance_metadata.py | 114 ++++++++++++++ cinder/volume/manager.py | 28 ++++ 7 files changed, 424 insertions(+), 1 deletion(-) create mode 100644 cinder/db/sqlalchemy/migrate_repo/versions/003_glance_metadata.py create mode 100644 cinder/tests/test_volume_glance_metadata.py diff --git a/cinder/db/api.py b/cinder/db/api.py index 7ac9acffa..bdd629db5 100644 --- a/cinder/db/api.py +++ b/cinder/db/api.py @@ -375,6 +375,55 @@ def volume_type_extra_specs_update_or_create(context, volume_type_id, ################### +def volume_glance_metadata_create(context, volume_id, key, value): + """Update the Glance metadata for the specified volume.""" + return IMPL.volume_glance_metadata_create(context, volume_id, + key, value) + + +def volume_glance_metadata_get(context, volume_id): + """Return the glance metadata for a volume.""" + return IMPL.volume_glance_metadata_get(context, volume_id) + + +def volume_snapshot_glance_metadata_get(context, snapshot_id): + """Return the Glance metadata for the specified snapshot.""" + return IMPL.volume_snapshot_glance_metadata_get(context, snapshot_id) + + +def volume_glance_metadata_copy_to_snapshot(context, snapshot_id, volume_id): + """ + Update the Glance metadata for a snapshot by copying all of the key:value + pairs from the originating volume. This is so that a volume created from + the snapshot will retain the original metadata. + """ + return IMPL.volume_glance_metadata_copy_to_snapshot(context, snapshot_id, + volume_id) + + +def volume_glance_metadata_copy_to_volume(context, volume_id, snapshot_id): + """ + Update the Glance metadata from a volume (created from a snapshot) by + copying all of the key:value pairs from the originating snapshot. This is + so that the Glance metadata from the original volume is retained. + """ + return IMPL.volume_glance_metadata_copy_to_volume(context, volume_id, + snapshot_id) + + +def volume_glance_metadata_delete_by_volume(context, volume_id): + """Delete the glance metadata for a volume.""" + return IMPL.volume_glance_metadata_delete_by_volume(context, volume_id) + + +def volume_glance_metadata_delete_by_snapshot(context, snapshot_id): + """Delete the glance metadata for a snapshot.""" + return IMPL.volume_glance_metadata_delete_by_snapshot(context, snapshot_id) + + +################### + + def sm_backend_conf_create(context, values): """Create a new SM Backend Config entry.""" return IMPL.sm_backend_conf_create(context, values) diff --git a/cinder/db/sqlalchemy/api.py b/cinder/db/sqlalchemy/api.py index bfc9303a8..5ef459031 100644 --- a/cinder/db/sqlalchemy/api.py +++ b/cinder/db/sqlalchemy/api.py @@ -142,6 +142,20 @@ def require_volume_exists(f): return wrapper +def require_snapshot_exists(f): + """Decorator to require the specified snapshot to exist. + + Requires the wrapped function to use context and snapshot_id as + their first two arguments. + """ + + def wrapper(context, snapshot_id, *args, **kwargs): + db.api.snapshot_get(context, snapshot_id) + return f(context, snapshot_id, *args, **kwargs) + wrapper.__name__ = f.__name__ + return wrapper + + def model_query(context, *args, **kwargs): """Query helper that accounts for context's `read_deleted` field. @@ -1439,6 +1453,134 @@ def volume_type_extra_specs_update_or_create(context, volume_type_id, #################### +@require_context +@require_volume_exists +def volume_glance_metadata_get(context, volume_id, session=None): + """Return the Glance metadata for the specified volume.""" + if not session: + session = get_session() + + return session.query(models.VolumeGlanceMetadata).\ + filter_by(volume_id=volume_id).\ + filter_by(deleted=False).all() + + +@require_context +@require_snapshot_exists +def volume_snapshot_glance_metadata_get(context, snapshot_id, session=None): + """Return the Glance metadata for the specified snapshot.""" + if not session: + session = get_session() + + return session.query(models.VolumeGlanceMetadata).\ + filter_by(snapshot_id=snapshot_id).\ + filter_by(deleted=False).all() + + +@require_context +@require_volume_exists +def volume_glance_metadata_create(context, volume_id, key, value, + session=None): + """ + Update the Glance metadata for a volume by adding a new key:value pair. + This API does not support changing the value of a key once it has been + created. + """ + if session is None: + session = get_session() + + with session.begin(): + rows = session.query(models.VolumeGlanceMetadata).\ + filter_by(volume_id=volume_id).\ + filter_by(key=key).\ + filter_by(deleted=False).all() + + if len(rows) > 0: + raise exception.GlanceMetadataExists(key=key, + volume_id=volume_id) + + vol_glance_metadata = models.VolumeGlanceMetadata() + vol_glance_metadata.volume_id = volume_id + vol_glance_metadata.key = key + vol_glance_metadata.value = value + + vol_glance_metadata.save(session=session) + + return + + +@require_context +@require_snapshot_exists +def volume_glance_metadata_copy_to_snapshot(context, snapshot_id, volume_id, + session=None): + """ + Update the Glance metadata for a snapshot by copying all of the key:value + pairs from the originating volume. This is so that a volume created from + the snapshot will retain the original metadata. + """ + if session is None: + session = get_session() + + metadata = volume_glance_metadata_get(context, volume_id, session=session) + with session.begin(): + for meta in metadata: + vol_glance_metadata = models.VolumeGlanceMetadata() + vol_glance_metadata.snapshot_id = snapshot_id + vol_glance_metadata.key = meta['key'] + vol_glance_metadata.value = meta['value'] + + vol_glance_metadata.save(session=session) + + +@require_context +@require_volume_exists +def volume_glance_metadata_copy_to_volume(context, volume_id, snapshot_id, + session=None): + """ + Update the Glance metadata from a volume (created from a snapshot) by + copying all of the key:value pairs from the originating snapshot. This is + so that the Glance metadata from the original volume is retained. + """ + if session is None: + session = get_session() + + metadata = volume_snapshot_glance_metadata_get(context, snapshot_id, + session=session) + with session.begin(): + for meta in metadata: + vol_glance_metadata = models.VolumeGlanceMetadata() + vol_glance_metadata.volume_id = volume_id + vol_glance_metadata.key = meta['key'] + vol_glance_metadata.value = meta['value'] + + vol_glance_metadata.save(session=session) + + +@require_context +def volume_glance_metadata_delete_by_volume(context, volume_id): + session = get_session() + session.query(models.VolumeGlanceMetadata).\ + filter_by(volume_id=volume_id).\ + filter_by(deleted=False).\ + update({'deleted': True, + 'deleted_at': timeutils.utcnow(), + 'updated_at': literal_column('updated_at')}) + + +@require_context +def volume_glance_metadata_delete_by_snapshot(context, snapshot_id): + session = get_session() + session.query(models.VolumeGlanceMetadata).\ + filter_by(snapshot_id=snapshot_id).\ + filter_by(deleted=False).\ + update({'deleted': True, + 'deleted_at': timeutils.utcnow(), + 'updated_at': literal_column('updated_at')}) + + +#################### + + @require_admin_context def sm_backend_conf_create(context, values): backend_conf = models.SMBackendConf() diff --git a/cinder/db/sqlalchemy/migrate_repo/versions/003_glance_metadata.py b/cinder/db/sqlalchemy/migrate_repo/versions/003_glance_metadata.py new file mode 100644 index 000000000..ff990a9e5 --- /dev/null +++ b/cinder/db/sqlalchemy/migrate_repo/versions/003_glance_metadata.py @@ -0,0 +1,74 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import Column, DateTime, Text, Boolean +from sqlalchemy import MetaData, Integer, String, Table, ForeignKey + +from cinder.openstack.common import log as logging + +LOG = logging.getLogger(__name__) + + +def upgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + # Just for the ForeignKey and column creation to succeed, these are not the + # actual definitions of tables . + # + volumes = Table('volumes', meta, + Column('id', Integer(), primary_key=True, nullable=False), + mysql_engine='InnoDB' + ) + snapshots = Table('snapshots', meta, + Column('id', Integer(), primary_key=True, nullable=False), + mysql_engine='InnoDB' + ) + # Create new table + volume_glance_metadata = Table('volume_glance_metadata', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('volume_id', String(length=36), ForeignKey('volumes.id')), + Column('snapshot_id', String(length=36), + ForeignKey('snapshots.id')), + Column('key', String(255)), + Column('value', Text), + mysql_engine='InnoDB' + ) + + try: + volume_glance_metadata.create() + except Exception: + LOG.exception("Exception while creating table " + "'volume_glance_metedata'") + meta.drop_all(tables=[volume_glance_metadata]) + raise + + +def downgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + volume_glance_metadata = Table('volume_glance_metadata', + meta, autoload=True) + try: + volume_glance_metadata.drop() + except Exception: + LOG.error(_("volume_glance_metadata table not dropped")) + raise diff --git a/cinder/db/sqlalchemy/models.py b/cinder/db/sqlalchemy/models.py index b053e284d..664230dfa 100644 --- a/cinder/db/sqlalchemy/models.py +++ b/cinder/db/sqlalchemy/models.py @@ -21,7 +21,7 @@ SQLAlchemy models for cinder data. """ -from sqlalchemy import Column, Integer, String, schema +from sqlalchemy import Column, Integer, String, Text, schema from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.declarative import declarative_base from sqlalchemy import ForeignKey, DateTime, Boolean @@ -205,6 +205,16 @@ class VolumeTypeExtraSpecs(BASE, CinderBase): ) +class VolumeGlanceMetadata(BASE, CinderBase): + """Glance metadata for a bootable volume""" + __tablename__ = 'volume_glance_metadata' + id = Column(Integer, primary_key=True, nullable=False) + volume_id = Column(String(36), ForeignKey('volumes.id')) + snapshot_id = Column(String(36), ForeignKey('snapshots.id')) + key = Column(String(255)) + value = Column(Text) + + class Quota(BASE, CinderBase): """Represents a single quota override for a project. @@ -378,6 +388,7 @@ def register_models(): VolumeMetadata, VolumeTypeExtraSpecs, VolumeTypes, + VolumeGlanceMetadata, ) engine = create_engine(FLAGS.sql_connection, echo=False) for model in models: diff --git a/cinder/exception.py b/cinder/exception.py index 045c8a6c2..c9dc09a19 100644 --- a/cinder/exception.py +++ b/cinder/exception.py @@ -490,3 +490,8 @@ class NfsNoSharesMounted(NotFound): class NfsNoSuitableShareFound(NotFound): message = _("There is no share which can host %(volume_size)sG") + + +class GlanceMetadataExists(Invalid): + message = _("Glance metadata cannot be updated, key %(key)s" + " exists for volume id %(volume_id)s") diff --git a/cinder/tests/test_volume_glance_metadata.py b/cinder/tests/test_volume_glance_metadata.py new file mode 100644 index 000000000..773a90f18 --- /dev/null +++ b/cinder/tests/test_volume_glance_metadata.py @@ -0,0 +1,114 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2011 Zadara Storage Inc. +# Copyright (c) 2011 OpenStack LLC. +# Copyright 2011 University of Southern California +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Unit Tests for volume types extra specs code +""" + +from cinder import context +from cinder import db +from cinder import exception +from cinder import test + + +class VolumeGlanceMetadataTestCase(test.TestCase): + + def setUp(self): + super(VolumeGlanceMetadataTestCase, self).setUp() + self.context = context.get_admin_context() + + def tearDown(self): + super(VolumeGlanceMetadataTestCase, self).tearDown() + + def test_vol_glance_metadata_bad_vol_id(self): + ctxt = context.get_admin_context() + self.assertRaises(exception.VolumeNotFound, + db.volume_glance_metadata_create, + ctxt, 1, 'key1', 'value1') + self.assertRaises(exception.VolumeNotFound, + db.volume_glance_metadata_get, ctxt, 1) + db.volume_glance_metadata_delete_by_volume(ctxt, 10) + + def test_vol_update_glance_metadata(self): + ctxt = context.get_admin_context() + db.volume_create(ctxt, {'id': 1}) + db.volume_create(ctxt, {'id': 2}) + vol_metadata = db.volume_glance_metadata_create(ctxt, 1, 'key1', + 'value1') + vol_metadata = db.volume_glance_metadata_create(ctxt, 2, 'key1', + 'value1') + vol_metadata = db.volume_glance_metadata_create(ctxt, 2, 'key2', + 'value2') + + expected_metadata_1 = {'volume_id': '1', + 'key': 'key1', + 'value': 'value1'} + + metadata = db.volume_glance_metadata_get(ctxt, 1) + self.assertEqual(len(metadata), 1) + for key, value in expected_metadata_1.items(): + self.assertEqual(metadata[0][key], value) + + expected_metadata_2 = ({'volume_id': '2', + 'key': 'key1', + 'value': 'value1'}, + {'volume_id': '2', + 'key': 'key2', + 'value': 'value2'}) + + metadata = db.volume_glance_metadata_get(ctxt, 2) + self.assertEqual(len(metadata), 2) + for expected, meta in zip(expected_metadata_2, metadata): + for key, value in expected.iteritems(): + self.assertEqual(meta[key], value) + + self.assertRaises(exception.GlanceMetadataExists, + db.volume_glance_metadata_create, + ctxt, 1, 'key1', 'value1a') + + metadata = db.volume_glance_metadata_get(ctxt, 1) + self.assertEqual(len(metadata), 1) + for key, value in expected_metadata_1.items(): + self.assertEqual(metadata[0][key], value) + + def test_vol_delete_glance_metadata(self): + ctxt = context.get_admin_context() + db.volume_create(ctxt, {'id': 1}) + db.volume_glance_metadata_delete_by_volume(ctxt, 1) + vol_metadata = db.volume_glance_metadata_create(ctxt, 1, 'key1', + 'value1') + db.volume_glance_metadata_delete_by_volume(ctxt, 1) + metadata = db.volume_glance_metadata_get(ctxt, 1) + self.assertEqual(len(metadata), 0) + db.volume_glance_metadata_delete_by_volume(ctxt, 1) + metadata = db.volume_glance_metadata_get(ctxt, 1) + self.assertEqual(len(metadata), 0) + + def test_vol_glance_metadata_copy_to_snapshot(self): + ctxt = context.get_admin_context() + db.volume_create(ctxt, {'id': 1}) + db.snapshot_create(ctxt, {'id': 100, 'volume_id': 1}) + vol_meta = db.volume_glance_metadata_create(ctxt, 1, 'key1', + 'value1') + db.volume_glance_metadata_copy_to_snapshot(ctxt, 100, 1) + + expected_meta = {'snapshot_id': '100', + 'key': 'key1', + 'value': 'value1'} + + for meta in db.volume_snapshot_glance_metadata_get(ctxt, 100): + for (key, value) in expected_meta.items(): + self.assertEquals(meta[key], value) diff --git a/cinder/volume/manager.py b/cinder/volume/manager.py index 7e5c2bb56..d07ff81b5 100644 --- a/cinder/volume/manager.py +++ b/cinder/volume/manager.py @@ -134,6 +134,7 @@ class VolumeManager(manager.SchedulerDependentManager): status = 'available' model_update = False + image_meta = None try: vol_name = volume_ref['name'] @@ -153,6 +154,7 @@ class VolumeManager(manager.SchedulerDependentManager): glance.get_remote_image_service(context, image_id) image_location = image_service.get_location(context, image_id) + image_meta = image_service.show(context, image_id) cloned = self.driver.clone_image(volume_ref, image_location) if not cloned: model_update = self.driver.create_volume(volume_ref) @@ -171,6 +173,11 @@ class VolumeManager(manager.SchedulerDependentManager): self.db.volume_update(context, volume_ref['id'], {'status': 'error'}) + if snapshot_id: + # Copy any Glance metadata from the original volume + self.db.volume_glance_metadata_copy_to_volume(context, + volume_ref['id'], snapshot_id) + now = timeutils.utcnow() self.db.volume_update(context, volume_ref['id'], {'status': status, @@ -179,6 +186,23 @@ class VolumeManager(manager.SchedulerDependentManager): self._reset_stats() if image_id and not cloned: + if image_meta: + # Copy all of the Glance image properties to the + # volume_glance_metadata table for future reference. + self.db.volume_glance_metadata_create(context, + volume_ref['id'], + 'image_id', image_id) + name = image_meta.get('name', None) + if name: + self.db.volume_glance_metadata_create(context, + volume_ref['id'], + 'image_name', name) + image_properties = image_meta.get('properties', {}) + for key, value in image_properties.items(): + self.db.volume_glance_metadata_create(context, + volume_ref['id'], + key, value) + #copy the image onto the volume. self._copy_image_to_volume(context, volume_ref, image_id) self._notify_about_volume_usage(context, volume_ref, "create.end") @@ -222,6 +246,7 @@ class VolumeManager(manager.SchedulerDependentManager): reservations = None LOG.exception(_("Failed to update usages deleting volume")) + self.db.volume_glance_metadata_delete_by_volume(context, volume_id) self.db.volume_destroy(context, volume_id) LOG.debug(_("volume %s: deleted successfully"), volume_ref['name']) self._notify_about_volume_usage(context, volume_ref, "delete.end") @@ -255,6 +280,8 @@ class VolumeManager(manager.SchedulerDependentManager): self.db.snapshot_update(context, snapshot_ref['id'], {'status': 'available', 'progress': '100%'}) + self.db.volume_glance_metadata_copy_to_snapshot(context, + snapshot_ref['id'], volume_id) LOG.debug(_("snapshot %s: created successfully"), snapshot_ref['name']) return snapshot_id @@ -278,6 +305,7 @@ class VolumeManager(manager.SchedulerDependentManager): snapshot_ref['id'], {'status': 'error_deleting'}) + self.db.volume_glance_metadata_delete_by_snapshot(context, snapshot_id) self.db.snapshot_destroy(context, snapshot_id) LOG.debug(_("snapshot %s: deleted successfully"), snapshot_ref['name']) return True -- 2.45.2