From a02cc07bd823e9795b1563ce058772795772bbd9 Mon Sep 17 00:00:00 2001 From: Brianna Poulos Date: Fri, 29 Aug 2014 17:19:29 -0400 Subject: [PATCH] Add support for backup encryption metadata This modification allows the encryption key UUID field, which has been added to the volume table, to remain valid when encrypted volumes are backed up and then restored, which enables the restored volume to be accessible and encrypted. This is related to patch https://review.openstack.org/#/c/39573/, except that it uses the backup metadata support recently added in patch https://review.openstack.org/#/c/51900/ rather than modifying the backup api. Change-Id: Ib91f8275271e6bd4f2d9e17499d16ec13bca2b84 Implements: blueprint backup-support-for-encrypted-volumes DocImpact --- cinder/backup/driver.py | 85 ++++++++++++++- cinder/exception.py | 4 + cinder/tests/test_backup_ceph.py | 8 +- cinder/tests/test_backup_driver_base.py | 131 ++++++++++++++++++++++- cinder/volume/api.py | 3 +- cinder/volume/flows/api/create_volume.py | 17 +-- 6 files changed, 226 insertions(+), 22 deletions(-) diff --git a/cinder/backup/driver.py b/cinder/backup/driver.py index c235c52b8..7e2cbd4e8 100644 --- a/cinder/backup/driver.py +++ b/cinder/backup/driver.py @@ -23,11 +23,12 @@ import six from cinder.db import base from cinder import exception -from cinder.i18n import _, _LI +from cinder.i18n import _, _LI, _LE, _LW +from cinder import keymgr from cinder.openstack.common import log as logging service_opts = [ - cfg.IntOpt('backup_metadata_version', default=1, + cfg.IntOpt('backup_metadata_version', default=2, help='Backup metadata version to be used when backing up ' 'volume metadata. If this number is bumped, make sure the ' 'service doing the restore supports the new version.') @@ -78,6 +79,10 @@ class BackupMetadataAPI(base.Base): LOG.info(_LI("Unable to serialize field '%s' - excluding " "from backup") % (key)) continue + # Copy the encryption key uuid for backup + if key is 'encryption_key_id' and value is not None: + value = keymgr.API().copy_key(self.context, value) + LOG.debug("Copying encryption key uuid for backup.") container[type_tag][key] = value LOG.debug("Completed fetching metadata type '%s'" % type_tag) @@ -149,6 +154,62 @@ class BackupMetadataAPI(base.Base): return subset + def _restore_vol_base_meta(self, metadata, volume_id, fields): + """Restore values to Volume object for provided fields.""" + LOG.debug("Restoring volume base metadata") + + # Ignore unencrypted backups. + key = 'encryption_key_id' + if key in fields and key in metadata and metadata[key] is not None: + self._restore_vol_encryption_meta(volume_id, + metadata['volume_type_id']) + + metadata = self._filter(metadata, fields) + self.db.volume_update(self.context, volume_id, metadata) + + def _restore_vol_encryption_meta(self, volume_id, src_volume_type_id): + """Restores the volume_type_id for encryption if needed. + + Only allow restoration of an encrypted backup if the destination + volume has the same volume type as the source volume. Otherwise + encryption will not work. If volume types are already the same, + no action is needed. + """ + dest_vol = self.db.volume_get(self.context, volume_id) + if dest_vol['volume_type_id'] != src_volume_type_id: + LOG.debug("Volume type id's do not match.") + # If the volume types do not match, and the destination volume + # does not have a volume type, force the destination volume + # to have the encrypted volume type, provided it still exists. + if dest_vol['volume_type_id'] is None: + try: + self.db.volume_type_get( + self.context, src_volume_type_id) + except exception.VolumeTypeNotFound: + LOG.debug("Volume type of source volume has been " + "deleted. Encrypted backup restore has " + "failed.") + msg = _LE("The source volume type '%s' is not " + "available.") % (src_volume_type_id) + raise exception.EncryptedBackupOperationFailed(msg) + # Update dest volume with src volume's volume_type_id. + LOG.debug("The volume type of the destination volume " + "will become the volume type of the source " + "volume.") + self.db.volume_update(self.context, volume_id, + {'volume_type_id': src_volume_type_id}) + else: + # Volume type id's do not match, and destination volume + # has a volume type. Throw exception. + LOG.warn(_LW("Destination volume type is different from " + "source volume type for an encrypted volume. " + "Encrypted backup restore has failed.")) + msg = _LE("The source volume type '%(src)s' is different " + "than the destination volume type '%(dest)s'.") % \ + {'src': src_volume_type_id, + 'dest': dest_vol['volume_type_id']} + raise exception.EncryptedBackupOperationFailed(msg) + def _restore_vol_meta(self, metadata, volume_id, fields): """Restore values to VolumeMetadata object for provided fields.""" LOG.debug("Restoring volume metadata") @@ -188,6 +249,24 @@ class BackupMetadataAPI(base.Base): self.TYPE_TAG_VOL_GLANCE_META: (self._restore_vol_glance_meta, [])} + def _v2_restore_factory(self): + """All metadata is backed up but we selectively restore. + + Returns a dictionary of the form: + + {: (, )} + + Empty field list indicates that all backed up fields should be + restored. + """ + return {self.TYPE_TAG_VOL_BASE_META: + (self._restore_vol_base_meta, + ['encryption_key_id']), + self.TYPE_TAG_VOL_META: + (self._restore_vol_meta, []), + self.TYPE_TAG_VOL_GLANCE_META: + (self._restore_vol_glance_meta, [])} + def get(self, volume_id): """Get volume metadata. @@ -214,6 +293,8 @@ class BackupMetadataAPI(base.Base): version = meta_container['version'] if version == 1: factory = self._v1_restore_factory() + elif version == 2: + factory = self._v2_restore_factory() else: msg = (_("Unsupported backup metadata version (%s)") % (version)) raise exception.BackupMetadataUnsupportedVersion(msg) diff --git a/cinder/exception.py b/cinder/exception.py index 5edad3ff8..30bfc4adf 100755 --- a/cinder/exception.py +++ b/cinder/exception.py @@ -528,6 +528,10 @@ class BackupRBDOperationFailed(BackupDriverException): message = _("Backup RBD operation failed") +class EncryptedBackupOperationFailed(BackupDriverException): + message = _("Backup operation of an encrypted volume failed.") + + class BackupNotFound(NotFound): message = _("Backup %(backup_id)s could not be found.") diff --git a/cinder/tests/test_backup_ceph.py b/cinder/tests/test_backup_ceph.py index 7d13ade87..d8b8895d1 100644 --- a/cinder/tests/test_backup_ceph.py +++ b/cinder/tests/test_backup_ceph.py @@ -911,7 +911,7 @@ class BackupCephTestCase(test.TestCase): @common_mocks def test_restore_metdata(self): - version = 1 + version = 2 def mock_read(*args): base_tag = driver.BackupMetadataAPI.TYPE_TAG_VOL_BASE_META @@ -927,7 +927,7 @@ class BackupCephTestCase(test.TestCase): self.assertTrue(self.mock_rados.Object.return_value.stat.called) self.assertTrue(self.mock_rados.Object.return_value.read.called) - version = 2 + version = 3 try: self.service._restore_metadata(self.backup, self.volume_id) except exception.BackupOperationError as exc: @@ -939,7 +939,7 @@ class BackupCephTestCase(test.TestCase): @common_mocks @mock.patch('cinder.backup.drivers.ceph.VolumeMetadataBackup', spec=True) - def test_backup_metata_already_exists(self, mock_meta_backup): + def test_backup_metadata_already_exists(self, mock_meta_backup): def mock_set(json_meta): msg = (_("Metadata backup object '%s' already exists") % @@ -994,7 +994,7 @@ class BackupCephTestCase(test.TestCase): glance_tag = driver.BackupMetadataAPI.TYPE_TAG_VOL_GLANCE_META return jsonutils.dumps({base_tag: {'image_name': 'image.base'}, glance_tag: {'image_name': 'image.glance'}, - 'version': 2}) + 'version': 3}) self.mock_rados.Object.return_value.read.side_effect = mock_read with mock.patch.object(ceph.VolumeMetadataBackup, '_exists') as \ diff --git a/cinder/tests/test_backup_driver_base.py b/cinder/tests/test_backup_driver_base.py index 74ef1aeae..fd91e0738 100644 --- a/cinder/tests/test_backup_driver_base.py +++ b/cinder/tests/test_backup_driver_base.py @@ -59,7 +59,7 @@ class BackupBaseDriverTestCase(test.TestCase): def test_get_metadata(self): json_metadata = self.driver.get_metadata(self.volume_id) metadata = jsonutils.loads(json_metadata) - self.assertEqual(metadata['version'], 1) + self.assertEqual(metadata['version'], 2) def test_put_metadata(self): metadata = {'version': 1} @@ -158,7 +158,7 @@ class BackupMetadataAPITestCase(test.TestCase): self.bak_meta_api.put(self.volume_id, meta) def test_put_invalid_version(self): - container = jsonutils.dumps({'version': 2}) + container = jsonutils.dumps({'version': 3}) self.assertRaises(exception.BackupMetadataUnsupportedVersion, self.bak_meta_api.put, self.volume_id, container) @@ -186,6 +186,21 @@ class BackupMetadataAPITestCase(test.TestCase): self.assertEqual(self.volume_display_description, vol['display_description']) + def test_v2_restore_factory(self): + fact = self.bak_meta_api._v2_restore_factory() + + keys = [self.bak_meta_api.TYPE_TAG_VOL_BASE_META, + self.bak_meta_api.TYPE_TAG_VOL_META, + self.bak_meta_api.TYPE_TAG_VOL_GLANCE_META] + + self.assertEqual(set(keys).symmetric_difference(set(fact.keys())), + set([])) + + for f in fact: + func = fact[f][0] + fields = fact[f][1] + func({}, self.volume_id, fields) + def test_restore_vol_glance_meta(self): fields = {} container = {} @@ -206,6 +221,118 @@ class BackupMetadataAPITestCase(test.TestCase): self.bak_meta_api._save_vol_meta(container, self.volume_id) self.bak_meta_api._restore_vol_meta(container, self.volume_id, fields) + def test_restore_vol_base_meta(self): + fields = {} + container = {} + self.bak_meta_api._save_vol_base_meta(container, self.volume_id) + self.bak_meta_api._restore_vol_base_meta(container, self.volume_id, + fields) + + def _create_encrypted_volume_db_entry(self, id, type_id, encrypted): + if encrypted: + vol = {'id': id, 'size': 1, 'status': 'available', + 'volume_type_id': type_id, 'encryption_key_id': 'fake_id'} + else: + vol = {'id': id, 'size': 1, 'status': 'available', + 'volume_type_id': type_id, 'encryption_key_id': None} + return db.volume_create(self.ctxt, vol)['id'] + + def test_restore_encrypted_vol_to_different_volume_type(self): + fields = ['encryption_key_id'] + container = {} + + # Create an encrypted volume + enc_vol1_id = self._create_encrypted_volume_db_entry(str(uuid.uuid4()), + 'enc_vol_type', + True) + + # Create a second encrypted volume, of a different volume type + enc_vol2_id = self._create_encrypted_volume_db_entry(str(uuid.uuid4()), + 'enc_vol_type2', + True) + + # Backup the first volume and attempt to restore to the second + self.bak_meta_api._save_vol_base_meta(container, enc_vol1_id) + self.assertRaises(exception.EncryptedBackupOperationFailed, + self.bak_meta_api._restore_vol_base_meta, + container[self.bak_meta_api.TYPE_TAG_VOL_BASE_META], + enc_vol2_id, fields) + + def test_restore_unencrypted_vol_to_different_volume_type(self): + fields = ['encryption_key_id'] + container = {} + + # Create an unencrypted volume + vol1_id = self._create_encrypted_volume_db_entry(str(uuid.uuid4()), + 'vol_type1', + False) + + # Create a second unencrypted volume, of a different volume type + vol2_id = self._create_encrypted_volume_db_entry(str(uuid.uuid4()), + 'vol_type2', + False) + + # Backup the first volume and restore to the second + self.bak_meta_api._save_vol_base_meta(container, vol1_id) + self.bak_meta_api._restore_vol_base_meta( + container[self.bak_meta_api.TYPE_TAG_VOL_BASE_META], vol2_id, + fields) + self.assertNotEqual( + db.volume_get(self.ctxt, vol1_id)['volume_type_id'], + db.volume_get(self.ctxt, vol2_id)['volume_type_id']) + + def test_restore_encrypted_vol_to_same_volume_type(self): + fields = ['encryption_key_id'] + container = {} + + # Create an encrypted volume + enc_vol1_id = self._create_encrypted_volume_db_entry(str(uuid.uuid4()), + 'enc_vol_type', + True) + + # Create an encrypted volume of the same type + enc_vol2_id = self._create_encrypted_volume_db_entry(str(uuid.uuid4()), + 'enc_vol_type', + True) + + # Backup the first volume and restore to the second + self.bak_meta_api._save_vol_base_meta(container, enc_vol1_id) + self.bak_meta_api._restore_vol_base_meta( + container[self.bak_meta_api.TYPE_TAG_VOL_BASE_META], enc_vol2_id, + fields) + + def test_restore_encrypted_vol_to_none_type_source_type_unavailable(self): + fields = ['encryption_key_id'] + container = {} + enc_vol_id = self._create_encrypted_volume_db_entry(str(uuid.uuid4()), + 'enc_vol_type', + True) + undef_vol_id = self._create_encrypted_volume_db_entry( + str(uuid.uuid4()), None, False) + self.bak_meta_api._save_vol_base_meta(container, enc_vol_id) + self.assertRaises(exception.EncryptedBackupOperationFailed, + self.bak_meta_api._restore_vol_base_meta, + container[self.bak_meta_api.TYPE_TAG_VOL_BASE_META], + undef_vol_id, fields) + + def test_restore_encrypted_vol_to_none_type_source_type_available(self): + fields = ['encryption_key_id'] + container = {} + db.volume_type_create(self.ctxt, {'id': 'enc_vol_type_id', + 'name': 'enc_vol_type'}) + enc_vol_id = self._create_encrypted_volume_db_entry(str(uuid.uuid4()), + 'enc_vol_type_id', + True) + undef_vol_id = self._create_encrypted_volume_db_entry( + str(uuid.uuid4()), None, False) + self.bak_meta_api._save_vol_base_meta(container, enc_vol_id) + self.bak_meta_api._restore_vol_base_meta( + container[self.bak_meta_api.TYPE_TAG_VOL_BASE_META], undef_vol_id, + fields) + self.assertEqual( + db.volume_get(self.ctxt, undef_vol_id)['volume_type_id'], + db.volume_get(self.ctxt, enc_vol_id)['volume_type_id']) + def test_filter(self): metadata = {'a': 1, 'b': 2, 'c': 3} self.assertEqual(metadata, self.bak_meta_api._filter(metadata, [])) diff --git a/cinder/volume/api.py b/cinder/volume/api.py index f1929957f..79173840f 100644 --- a/cinder/volume/api.py +++ b/cinder/volume/api.py @@ -153,7 +153,7 @@ class API(base.Base): def create(self, context, size, name, description, snapshot=None, image_id=None, volume_type=None, metadata=None, availability_zone=None, source_volume=None, - scheduler_hints=None, backup_source_volume=None, + scheduler_hints=None, source_replica=None, consistencygroup=None): # NOTE(jdg): we can have a create without size if we're @@ -224,7 +224,6 @@ class API(base.Base): 'source_volume': source_volume, 'scheduler_hints': scheduler_hints, 'key_manager': self.key_manager, - 'backup_source_volume': backup_source_volume, 'source_replica': source_replica, 'optional_args': {'is_quota_committed': False}, 'consistencygroup': consistencygroup diff --git a/cinder/volume/flows/api/create_volume.py b/cinder/volume/flows/api/create_volume.py index 5e054001c..73d5d5132 100644 --- a/cinder/volume/flows/api/create_volume.py +++ b/cinder/volume/flows/api/create_volume.py @@ -338,15 +338,13 @@ class ExtractVolumeRequestTask(flow_utils.CinderTask): return availability_zone def _get_encryption_key_id(self, key_manager, context, volume_type_id, - snapshot, source_volume, backup_source_volume): + snapshot, source_volume): encryption_key_id = None if volume_types.is_encrypted(context, volume_type_id): if snapshot is not None: # creating from snapshot encryption_key_id = snapshot['encryption_key_id'] elif source_volume is not None: # cloning volume encryption_key_id = source_volume['encryption_key_id'] - elif backup_source_volume is not None: # creating from backup - encryption_key_id = backup_source_volume['encryption_key_id'] # NOTE(joel-coffman): References to the encryption key should *not* # be copied because the key is deleted when the volume is deleted. @@ -360,8 +358,7 @@ class ExtractVolumeRequestTask(flow_utils.CinderTask): return encryption_key_id - def _get_volume_type_id(self, volume_type, source_volume, snapshot, - backup_source_volume): + def _get_volume_type_id(self, volume_type, source_volume, snapshot): volume_type_id = None if not volume_type and source_volume: volume_type_id = source_volume['volume_type_id'] @@ -374,8 +371,6 @@ class ExtractVolumeRequestTask(flow_utils.CinderTask): "be the same as the source volume.") LOG.warn(msg) volume_type_id = snapshot['volume_type_id'] - elif backup_source_volume is not None: - volume_type_id = backup_source_volume['volume_type_id'] else: volume_type_id = volume_type.get('id') @@ -383,7 +378,7 @@ class ExtractVolumeRequestTask(flow_utils.CinderTask): def execute(self, context, size, snapshot, image_id, source_volume, availability_zone, volume_type, metadata, - key_manager, backup_source_volume, source_replica, + key_manager, source_replica, consistencygroup): utils.check_exclusive_options(snapshot=snapshot, @@ -421,15 +416,13 @@ class ExtractVolumeRequestTask(flow_utils.CinderTask): volume_type = def_vol_type volume_type_id = self._get_volume_type_id(volume_type, - source_volume, snapshot, - backup_source_volume) + source_volume, snapshot) encryption_key_id = self._get_encryption_key_id(key_manager, context, volume_type_id, snapshot, - source_volume, - backup_source_volume) + source_volume) specs = {} if volume_type_id: -- 2.45.2