From: Xing Yang Date: Mon, 20 Jul 2015 03:18:58 +0000 (-0400) Subject: Clone CG X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=360518ca0134a6f1fd7c8b39e51370ea76b1fe90;p=openstack-build%2Fcinder-build.git Clone CG This patch modifies the existing "create CG from source" API to take an existing CG as a source, in addition to a CG snapshot. APIImpact DocImpact implements blueprint clone-cg Change-Id: Ieabc190a5d9a08e2c84e42140192e6ee3dac9433 --- diff --git a/cinder/api/contrib/consistencygroups.py b/cinder/api/contrib/consistencygroups.py index 9874c66d1..284e6234c 100644 --- a/cinder/api/contrib/consistencygroups.py +++ b/cinder/api/contrib/consistencygroups.py @@ -49,6 +49,7 @@ def make_consistencygroup_from_src(elem): elem.set('name') elem.set('description') elem.set('cgsnapshot_id') + elem.set('source_cgid') class ConsistencyGroupTemplate(xmlutil.TemplateBuilder): @@ -116,7 +117,7 @@ class CreateFromSrcDeserializer(wsgi.MetadataXMLDeserializer): consistencygroup_node = self.find_first_child_named( node, 'consistencygroup-from-src') - attributes = ['cgsnapshot', 'name', 'description'] + attributes = ['cgsnapshot', 'source_cgid', 'name', 'description'] for attr in attributes: if consistencygroup_node.getAttribute(attr): @@ -250,8 +251,7 @@ class ConsistencyGroupsController(wsgi.Controller): def create_from_src(self, req, body): """Create a new consistency group from a source. - The source can be a snapshot. It could be extended - in the future to support other sources. Note that + The source can be a CG snapshot or a CG. Note that this does not require volume_types as the "create" API above. """ @@ -263,23 +263,37 @@ class ConsistencyGroupsController(wsgi.Controller): name = consistencygroup.get('name', None) description = consistencygroup.get('description', None) cgsnapshot_id = consistencygroup.get('cgsnapshot_id', None) - if not cgsnapshot_id: - msg = _("Cgsnapshot id must be provided to create " - "consistency group %(name)s from source.") % {'name': name} + source_cgid = consistencygroup.get('source_cgid', None) + if not cgsnapshot_id and not source_cgid: + msg = _("Either 'cgsnapshot_id' or 'source_cgid' must be " + "provided to create consistency group %(name)s " + "from source.") % {'name': name} raise exc.HTTPBadRequest(explanation=msg) - LOG.info(_LI("Creating consistency group %(name)s from cgsnapshot " - "%(snap)s."), - {'name': name, 'snap': cgsnapshot_id}, - context=context) + if cgsnapshot_id and source_cgid: + msg = _("Cannot provide both 'cgsnapshot_id' and 'source_cgid' " + "to create consistency group %(name)s from " + "source.") % {'name': name} + raise exc.HTTPBadRequest(explanation=msg) + + if cgsnapshot_id: + LOG.info(_LI("Creating consistency group %(name)s from " + "cgsnapshot %(snap)s."), + {'name': name, 'snap': cgsnapshot_id}, + context=context) + elif source_cgid: + LOG.info(_LI("Creating consistency group %(name)s from " + "source consistency group %(source_cgid)s."), + {'name': name, 'source_cgid': source_cgid}, + context=context) try: new_consistencygroup = self.consistencygroup_api.create_from_src( - context, name, description, cgsnapshot_id) + context, name, description, cgsnapshot_id, source_cgid) except exception.InvalidConsistencyGroup as error: raise exc.HTTPBadRequest(explanation=error.msg) except exception.CgSnapshotNotFound as error: - raise exc.HTTPBadRequest(explanation=error.msg) + raise exc.HTTPNotFound(explanation=error.msg) except exception.ConsistencyGroupNotFound as error: raise exc.HTTPNotFound(explanation=error.msg) except exception.CinderException as error: diff --git a/cinder/consistencygroup/api.py b/cinder/consistencygroup/api.py index 9b5600157..f09eef7b5 100644 --- a/cinder/consistencygroup/api.py +++ b/cinder/consistencygroup/api.py @@ -161,30 +161,56 @@ class API(base.Base): return group - def create_from_src(self, context, name, description, cgsnapshot_id): + def create_from_src(self, context, name, description=None, + cgsnapshot_id=None, + source_cgid=None): check_policy(context, 'create') cgsnapshot = None orig_cg = None if cgsnapshot_id: - cgsnapshot = self.db.cgsnapshot_get(context, cgsnapshot_id) - if cgsnapshot: - orig_cg = self.db.consistencygroup_get( - context, - cgsnapshot['consistencygroup_id']) + try: + cgsnapshot = self.db.cgsnapshot_get(context, cgsnapshot_id) + except exception.CgSnapshotNotFound: + with excutils.save_and_reraise_exception(): + LOG.error(_LE("CG snapshot %(cgsnap) not found when " + "creating consistency group %(cg)s from " + "source."), + {'cg': name, 'cgsnap': cgsnapshot_id}) + orig_cg = self.db.consistencygroup_get( + context, + cgsnapshot['consistencygroup_id']) + + source_cg = None + if source_cgid: + try: + source_cg = self.db.consistencygroup_get( + context, source_cgid) + except exception.ConsistencyGroupNotFound: + with excutils.save_and_reraise_exception(): + LOG.error(_LE("Source CG %(source_cg) not found when " + "creating consistency group %(cg)s from " + "source."), + {'cg': name, 'source_cg': source_cgid}) options = {'user_id': context.user_id, 'project_id': context.project_id, 'status': "creating", 'name': name, 'description': description, - 'cgsnapshot_id': cgsnapshot_id} + 'cgsnapshot_id': cgsnapshot_id, + 'source_cgid': source_cgid} if orig_cg: options['volume_type_id'] = orig_cg.get('volume_type_id') options['availability_zone'] = orig_cg.get('availability_zone') options['host'] = orig_cg.get('host') + if source_cg: + options['volume_type_id'] = source_cg.get('volume_type_id') + options['availability_zone'] = source_cg.get('availability_zone') + options['host'] = source_cg.get('host') + group = None try: group = self.db.consistencygroup_create(context, options) @@ -202,7 +228,10 @@ class API(base.Base): LOG.error(msg) raise exception.InvalidConsistencyGroup(reason=msg) - self._create_cg_from_cgsnapshot(context, group, cgsnapshot) + if cgsnapshot: + self._create_cg_from_cgsnapshot(context, group, cgsnapshot) + elif source_cg: + self._create_cg_from_source_cg(context, group, source_cg) return group @@ -268,6 +297,68 @@ class API(base.Base): self.volume_rpcapi.create_consistencygroup_from_src( context, group, group['host'], cgsnapshot) + def _create_cg_from_source_cg(self, context, group, source_cg): + try: + source_vols = self.db.volume_get_all_by_group(context, + source_cg['id']) + + if not source_vols: + msg = _("Source CG is empty. No consistency group " + "will be created.") + raise exception.InvalidConsistencyGroup(reason=msg) + + for source_vol in source_vols: + kwargs = {} + kwargs['availability_zone'] = group.get('availability_zone') + kwargs['source_cg'] = source_cg + kwargs['consistencygroup'] = group + kwargs['source_volume'] = source_vol + volume_type_id = source_vol.get('volume_type_id') + if volume_type_id: + kwargs['volume_type'] = volume_types.get_volume_type( + context, volume_type_id) + + # Since source_cg is passed in, the following call will + # create a db entry for the volume, but will not call the + # volume manager to create a real volume in the backend yet. + # If error happens, taskflow will handle rollback of quota + # and removal of volume entry in the db. + try: + self.volume_api.create(context, + source_vol['size'], + None, + None, + **kwargs) + except exception.CinderException: + with excutils.save_and_reraise_exception(): + LOG.error(_LE("Error occurred when creating cloned " + "volume in the process of creating " + "consistency group %(group)s from " + "source CG %(source_cg)s."), + {'group': group['id'], + 'source_cg': source_cg['id']}) + except Exception: + with excutils.save_and_reraise_exception(): + try: + self.db.consistencygroup_destroy(context.elevated(), + group['id']) + finally: + LOG.error(_LE("Error occurred when creating consistency " + "group %(group)s from source CG " + "%(source_cg)s."), + {'group': group['id'], + 'source_cg': source_cg['id']}) + + volumes = self.db.volume_get_all_by_group(context, + group['id']) + for vol in volumes: + # Update the host field for the volume. + self.db.volume_update(context, vol['id'], + {'host': group.get('host')}) + + self.volume_rpcapi.create_consistencygroup_from_src( + context, group, group['host'], None, source_cg) + def _cast_create_consistencygroup(self, context, group_id, request_spec_list, filter_properties_list): diff --git a/cinder/db/sqlalchemy/migrate_repo/versions/051_add_source_cgid_column_to_consistencygroups.py b/cinder/db/sqlalchemy/migrate_repo/versions/051_add_source_cgid_column_to_consistencygroups.py new file mode 100644 index 000000000..044e3cc63 --- /dev/null +++ b/cinder/db/sqlalchemy/migrate_repo/versions/051_add_source_cgid_column_to_consistencygroups.py @@ -0,0 +1,37 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import Column +from sqlalchemy import MetaData, String, Table + + +def upgrade(migrate_engine): + """Add source_cgid column to consistencygroups.""" + meta = MetaData() + meta.bind = migrate_engine + + consistencygroups = Table('consistencygroups', meta, autoload=True) + source_cgid = Column('source_cgid', String(36)) + + consistencygroups.create_column(source_cgid) + consistencygroups.update().values(source_cgid=None).execute() + + +def downgrade(migrate_engine): + """Remove source_cgid column from consistencygroups.""" + meta = MetaData() + meta.bind = migrate_engine + + consistencygroups = Table('consistencygroups', meta, autoload=True) + source_cgid = consistencygroups.columns.source_cgid + + consistencygroups.drop_column(source_cgid) diff --git a/cinder/db/sqlalchemy/models.py b/cinder/db/sqlalchemy/models.py index 0863b8348..52639711a 100644 --- a/cinder/db/sqlalchemy/models.py +++ b/cinder/db/sqlalchemy/models.py @@ -85,6 +85,7 @@ class ConsistencyGroup(BASE, CinderBase): volume_type_id = Column(String(255)) status = Column(String(255)) cgsnapshot_id = Column(String(36)) + source_cgid = Column(String(36)) class Cgsnapshot(BASE, CinderBase): diff --git a/cinder/tests/unit/api/contrib/test_consistencygroups.py b/cinder/tests/unit/api/contrib/test_consistencygroups.py index 4982b65fa..4abba1a88 100644 --- a/cinder/tests/unit/api/contrib/test_consistencygroups.py +++ b/cinder/tests/unit/api/contrib/test_consistencygroups.py @@ -713,7 +713,7 @@ class ConsistencyGroupsAPITestCase(test.TestCase): db.consistencygroup_destroy(ctxt.elevated(), consistencygroup_id) - def test_create_consistencygroup_from_src(self): + def test_create_consistencygroup_from_src_cgsnapshot(self): self.stubs.Set(volume_api.API, "create", stubs.stub_volume_create) ctxt = context.RequestContext('fake', 'fake', auth_token=True) @@ -753,6 +753,76 @@ class ConsistencyGroupsAPITestCase(test.TestCase): db.volume_destroy(ctxt.elevated(), volume_id) db.consistencygroup_destroy(ctxt.elevated(), consistencygroup_id) + def test_create_consistencygroup_from_src_cg(self): + self.mock_object(volume_api.API, "create", stubs.stub_volume_create) + + ctxt = context.RequestContext('fake', 'fake', auth_token=True) + source_cgid = utils.create_consistencygroup(ctxt)['id'] + volume_id = utils.create_volume( + ctxt, + consistencygroup_id=source_cgid)['id'] + + test_cg_name = 'test cg' + body = {"consistencygroup-from-src": {"name": test_cg_name, + "description": + "Consistency Group 1", + "source_cgid": source_cgid}} + req = webob.Request.blank('/v2/fake/consistencygroups/create_from_src') + req.method = 'POST' + req.headers['Content-Type'] = 'application/json' + req.body = json.dumps(body) + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(202, res.status_int) + self.assertIn('id', res_dict['consistencygroup']) + self.assertEqual(test_cg_name, res_dict['consistencygroup']['name']) + + db.consistencygroup_destroy(ctxt.elevated(), + res_dict['consistencygroup']['id']) + db.volume_destroy(ctxt.elevated(), volume_id) + db.consistencygroup_destroy(ctxt.elevated(), source_cgid) + + def test_create_consistencygroup_from_src_both_snap_cg(self): + self.stubs.Set(volume_api.API, "create", stubs.stub_volume_create) + + ctxt = context.RequestContext('fake', 'fake', auth_token=True) + consistencygroup_id = utils.create_consistencygroup(ctxt)['id'] + volume_id = utils.create_volume( + ctxt, + consistencygroup_id=consistencygroup_id)['id'] + cgsnapshot_id = utils.create_cgsnapshot( + ctxt, + consistencygroup_id=consistencygroup_id)['id'] + snapshot_id = utils.create_snapshot( + ctxt, + volume_id, + cgsnapshot_id=cgsnapshot_id, + status='available')['id'] + + test_cg_name = 'test cg' + body = {"consistencygroup-from-src": {"name": test_cg_name, + "description": + "Consistency Group 1", + "cgsnapshot_id": cgsnapshot_id, + "source_cgid": + consistencygroup_id}} + req = webob.Request.blank('/v2/fake/consistencygroups/create_from_src') + req.method = 'POST' + req.headers['Content-Type'] = 'application/json' + req.body = json.dumps(body) + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(400, res.status_int) + self.assertEqual(400, res_dict['badRequest']['code']) + self.assertIsNotNone(res_dict['badRequest']['message']) + + db.snapshot_destroy(ctxt.elevated(), snapshot_id) + db.cgsnapshot_destroy(ctxt.elevated(), cgsnapshot_id) + db.volume_destroy(ctxt.elevated(), volume_id) + db.consistencygroup_destroy(ctxt.elevated(), consistencygroup_id) + def test_create_consistencygroup_from_src_invalid_body(self): name = 'cg1' body = {"invalid": {"name": name, @@ -767,11 +837,10 @@ class ConsistencyGroupsAPITestCase(test.TestCase): self.assertEqual(400, res.status_int) self.assertEqual(400, res_dict['badRequest']['code']) - msg = _("Missing required element 'consistencygroup-from-src' in " - "request body.") - self.assertEqual(msg, res_dict['badRequest']['message']) + # Missing 'consistencygroup-from-src' in the body. + self.assertIsNotNone(res_dict['badRequest']['message']) - def test_create_consistencygroup_from_src_no_cgsnapshot_id(self): + def test_create_consistencygroup_from_src_no_source_id(self): name = 'cg1' body = {"consistencygroup-from-src": {"name": name, "description": @@ -785,9 +854,7 @@ class ConsistencyGroupsAPITestCase(test.TestCase): self.assertEqual(400, res.status_int) self.assertEqual(400, res_dict['badRequest']['code']) - msg = (_('Cgsnapshot id must be provided to create ' - 'consistency group %s from source.') % name) - self.assertEqual(msg, res_dict['badRequest']['message']) + self.assertIsNotNone(res_dict['badRequest']['message']) def test_create_consistencygroup_from_src_no_host(self): ctxt = context.RequestContext('fake', 'fake', auth_token=True) @@ -854,18 +921,83 @@ class ConsistencyGroupsAPITestCase(test.TestCase): self.assertEqual(400, res.status_int) self.assertEqual(400, res_dict['badRequest']['code']) - msg = _("Invalid ConsistencyGroup: Cgsnahost is empty. No " - "consistency group will be created.") - self.assertIn(msg, res_dict['badRequest']['message']) + self.assertIsNotNone(res_dict['badRequest']['message']) db.cgsnapshot_destroy(ctxt.elevated(), cgsnapshot_id) db.volume_destroy(ctxt.elevated(), volume_id) db.consistencygroup_destroy(ctxt.elevated(), consistencygroup_id) + def test_create_consistencygroup_from_src_source_cg_empty(self): + ctxt = context.RequestContext('fake', 'fake', auth_token=True) + source_cgid = utils.create_consistencygroup( + ctxt)['id'] + + test_cg_name = 'test cg' + body = {"consistencygroup-from-src": {"name": test_cg_name, + "description": + "Consistency Group 1", + "source_cgid": source_cgid}} + req = webob.Request.blank('/v2/fake/consistencygroups/create_from_src') + req.method = 'POST' + req.headers['Content-Type'] = 'application/json' + req.body = json.dumps(body) + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(400, res.status_int) + self.assertEqual(400, res_dict['badRequest']['code']) + self.assertIsNotNone(res_dict['badRequest']['message']) + + db.consistencygroup_destroy(ctxt.elevated(), source_cgid) + + def test_create_consistencygroup_from_src_cgsnapshot_notfound(self): + ctxt = context.RequestContext('fake', 'fake', auth_token=True) + consistencygroup_id = utils.create_consistencygroup( + ctxt)['id'] + volume_id = utils.create_volume( + ctxt, + consistencygroup_id=consistencygroup_id)['id'] + + test_cg_name = 'test cg' + body = {"consistencygroup-from-src": {"name": test_cg_name, + "description": + "Consistency Group 1", + "cgsnapshot_id": "fake_cgsnap"}} + req = webob.Request.blank('/v2/fake/consistencygroups/create_from_src') + req.method = 'POST' + req.headers['Content-Type'] = 'application/json' + req.body = json.dumps(body) + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(404, res.status_int) + self.assertEqual(404, res_dict['itemNotFound']['code']) + self.assertIsNotNone(res_dict['itemNotFound']['message']) + + db.volume_destroy(ctxt.elevated(), volume_id) + db.consistencygroup_destroy(ctxt.elevated(), consistencygroup_id) + + def test_create_consistencygroup_from_src_source_cg_notfound(self): + test_cg_name = 'test cg' + body = {"consistencygroup-from-src": {"name": test_cg_name, + "description": + "Consistency Group 1", + "source_cgid": "fake_source_cg"}} + req = webob.Request.blank('/v2/fake/consistencygroups/create_from_src') + req.method = 'POST' + req.headers['Content-Type'] = 'application/json' + req.body = json.dumps(body) + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(404, res.status_int) + self.assertEqual(404, res_dict['itemNotFound']['code']) + self.assertIsNotNone(res_dict['itemNotFound']['message']) + @mock.patch.object(volume_api.API, 'create', side_effect=exception.CinderException( 'Create volume failed.')) - def test_create_consistencygroup_from_src_create_volume_failed( + def test_create_consistencygroup_from_src_cgsnapshot_create_volume_failed( self, mock_create): ctxt = context.RequestContext('fake', 'fake', auth_token=True) consistencygroup_id = utils.create_consistencygroup(ctxt)['id'] @@ -902,3 +1034,33 @@ class ConsistencyGroupsAPITestCase(test.TestCase): db.cgsnapshot_destroy(ctxt.elevated(), cgsnapshot_id) db.volume_destroy(ctxt.elevated(), volume_id) db.consistencygroup_destroy(ctxt.elevated(), consistencygroup_id) + + @mock.patch.object(volume_api.API, 'create', + side_effect=exception.CinderException( + 'Create volume failed.')) + def test_create_consistencygroup_from_src_cg_create_volume_failed( + self, mock_create): + ctxt = context.RequestContext('fake', 'fake', auth_token=True) + source_cgid = utils.create_consistencygroup(ctxt)['id'] + volume_id = utils.create_volume( + ctxt, + consistencygroup_id=source_cgid)['id'] + + test_cg_name = 'test cg' + body = {"consistencygroup-from-src": {"name": test_cg_name, + "description": + "Consistency Group 1", + "source_cgid": source_cgid}} + req = webob.Request.blank('/v2/fake/consistencygroups/create_from_src') + req.method = 'POST' + req.headers['Content-Type'] = 'application/json' + req.body = json.dumps(body) + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(400, res.status_int) + self.assertEqual(400, res_dict['badRequest']['code']) + self.assertIsNotNone(res_dict['badRequest']['message']) + + db.volume_destroy(ctxt.elevated(), volume_id) + db.consistencygroup_destroy(ctxt.elevated(), source_cgid) diff --git a/cinder/tests/unit/api/v2/stubs.py b/cinder/tests/unit/api/v2/stubs.py index 41763430e..9875294e6 100644 --- a/cinder/tests/unit/api/v2/stubs.py +++ b/cinder/tests/unit/api/v2/stubs.py @@ -64,7 +64,7 @@ def stub_volume(id, **kwargs): return volume -def stub_volume_create(self, context, size, name, description, snapshot, +def stub_volume_create(self, context, size, name, description, snapshot=None, **param): vol = stub_volume('1') vol['size'] = size diff --git a/cinder/tests/unit/test_migrations.py b/cinder/tests/unit/test_migrations.py index 90f664942..589c68052 100644 --- a/cinder/tests/unit/test_migrations.py +++ b/cinder/tests/unit/test_migrations.py @@ -847,6 +847,15 @@ class MigrationsMixin(test_migrations.WalkVersionsMixin): volumes = db_utils.get_table(engine, 'volumes') self.assertNotIn('previous_status', volumes.c) + def _check_051(self, engine, data): + consistencygroups = db_utils.get_table(engine, 'consistencygroups') + self.assertIsInstance(consistencygroups.c.source_cgid.type, + sqlalchemy.types.VARCHAR) + + def _post_downgrade_051(self, engine): + consistencygroups = db_utils.get_table(engine, 'consistencygroups') + self.assertNotIn('source_cgid', consistencygroups.c) + def test_walk_versions(self): self.walk_versions(True, False) diff --git a/cinder/tests/unit/test_volume.py b/cinder/tests/unit/test_volume.py index 7ac782716..daf3a53b5 100644 --- a/cinder/tests/unit/test_volume.py +++ b/cinder/tests/unit/test_volume.py @@ -4797,26 +4797,35 @@ class VolumeTestCase(BaseVolumeTestCase): return_value=(None, None)) @mock.patch('cinder.volume.drivers.lvm.LVMVolumeDriver.' 'create_volume_from_snapshot') - def test_create_consistencygroup_from_src(self, mock_create_from_src, + @mock.patch('cinder.volume.drivers.lvm.LVMVolumeDriver.' + 'create_cloned_volume') + def test_create_consistencygroup_from_src(self, + mock_create_cloned_vol, + mock_create_vol_from_snap, + mock_create_from_src, mock_delete_cgsnap, mock_create_cgsnap, - mock_delete_cg, mock_create_cg, - mock_create_volume): + mock_delete_cg, + mock_create_cg): """Test consistencygroup can be created and deleted.""" group = tests_utils.create_consistencygroup( self.context, availability_zone=CONF.storage_availability_zone, - volume_type='type1,type2') + volume_type='type1,type2', + status='available') group_id = group['id'] volume = tests_utils.create_volume( self.context, consistencygroup_id=group_id, - **self.volume_params) + status='available', + host=CONF.host, + size=1) volume_id = volume['id'] cgsnapshot_returns = self._create_cgsnapshot(group_id, volume_id) cgsnapshot_id = cgsnapshot_returns[0]['id'] snapshot_id = cgsnapshot_returns[1]['id'] + # Create CG from source CG snapshot. group2 = tests_utils.create_consistencygroup( self.context, availability_zone=CONF.storage_availability_zone, @@ -4846,6 +4855,9 @@ class VolumeTestCase(BaseVolumeTestCase): 'consistencygroup_id': group2_id } self.assertEqual('available', cg2['status']) + self.assertEqual(group2_id, cg2['id']) + self.assertEqual(cgsnapshot_id, cg2['cgsnapshot_id']) + self.assertIsNone(cg2['source_cgid']) msg = self.notifier.notifications[2] self.assertEqual('consistencygroup.create.start', msg['event_type']) @@ -4883,8 +4895,34 @@ class VolumeTestCase(BaseVolumeTestCase): self.context, group2_id) + # Create CG from source CG. + group3 = tests_utils.create_consistencygroup( + self.context, + availability_zone=CONF.storage_availability_zone, + volume_type='type1,type2', + source_cgid=group_id) + group3_id = group3['id'] + volume3 = tests_utils.create_volume( + self.context, + consistencygroup_id=group3_id, + source_volid=volume_id, + **self.volume_params) + volume3_id = volume3['id'] + self.volume.create_volume(self.context, volume3_id) + self.volume.create_consistencygroup_from_src( + self.context, group3_id, source_cgid=group_id) + + cg3 = db.consistencygroup_get( + self.context, + group3_id) + self.assertEqual('available', cg3['status']) + self.assertEqual(group3_id, cg3['id']) + self.assertEqual(group_id, cg3['source_cgid']) + self.assertIsNone(cg3['cgsnapshot_id']) + self.volume.delete_cgsnapshot(self.context, cgsnapshot_id) self.volume.delete_consistencygroup(self.context, group_id) + self.volume.delete_consistencygroup(self.context, group3_id) def test_sort_snapshots(self): vol1 = {'id': '1', 'name': 'volume 1', @@ -4931,6 +4969,51 @@ class VolumeTestCase(BaseVolumeTestCase): self.volume._sort_snapshots, volumes, []) + def test_sort_source_vols(self): + vol1 = {'id': '1', 'name': 'volume 1', + 'source_volid': '1', + 'consistencygroup_id': '2'} + vol2 = {'id': '2', 'name': 'volume 2', + 'source_volid': '2', + 'consistencygroup_id': '2'} + vol3 = {'id': '3', 'name': 'volume 3', + 'source_volid': '3', + 'consistencygroup_id': '2'} + src_vol1 = {'id': '1', 'name': 'source vol 1', + 'consistencygroup_id': '1'} + src_vol2 = {'id': '2', 'name': 'source vol 2', + 'consistencygroup_id': '1'} + src_vol3 = {'id': '3', 'name': 'source vol 3', + 'consistencygroup_id': '1'} + volumes = [] + src_vols = [] + volumes.append(vol1) + volumes.append(vol2) + volumes.append(vol3) + src_vols.append(src_vol2) + src_vols.append(src_vol3) + src_vols.append(src_vol1) + i = 0 + for vol in volumes: + src_vol = src_vols[i] + i += 1 + self.assertNotEqual(vol['source_volid'], src_vol['id']) + sorted_src_vols = self.volume._sort_source_vols(volumes, src_vols) + i = 0 + for vol in volumes: + src_vol = sorted_src_vols[i] + i += 1 + self.assertEqual(vol['source_volid'], src_vol['id']) + + src_vols[2]['id'] = '9999' + self.assertRaises(exception.VolumeNotFound, + self.volume._sort_source_vols, + volumes, src_vols) + + self.assertRaises(exception.InvalidInput, + self.volume._sort_source_vols, + volumes, []) + @staticmethod def _create_cgsnapshot(group_id, volume_id, size='0'): """Create a cgsnapshot object.""" diff --git a/cinder/tests/unit/test_volume_rpcapi.py b/cinder/tests/unit/test_volume_rpcapi.py index 8d3d3fc93..90ef502c2 100644 --- a/cinder/tests/unit/test_volume_rpcapi.py +++ b/cinder/tests/unit/test_volume_rpcapi.py @@ -25,6 +25,7 @@ from cinder import db from cinder import objects from cinder import test from cinder.tests.unit import fake_snapshot +from cinder.tests.unit import utils as tests_utils from cinder.volume import rpcapi as volume_rpcapi @@ -53,12 +54,41 @@ class VolumeRpcAPITestCase(test.TestCase): 'display_name': 'fake_name', 'display_description': 'fake_description'} snapshot = db.snapshot_create(self.context, snpshot) + + source_group = tests_utils.create_consistencygroup( + self.context, + availability_zone=CONF.storage_availability_zone, + volume_type='type1,type2', + host='fakehost@fakedrv#fakepool') + + cgsnapshot = tests_utils.create_cgsnapshot( + self.context, + consistencygroup_id=source_group['id']) + + group = tests_utils.create_consistencygroup( + self.context, + availability_zone=CONF.storage_availability_zone, + volume_type='type1,type2', + host='fakehost@fakedrv#fakepool', + cgsnapshot_id=cgsnapshot['id']) + + group2 = tests_utils.create_consistencygroup( + self.context, + availability_zone=CONF.storage_availability_zone, + volume_type='type1,type2', + host='fakehost@fakedrv#fakepool', + source_cgid=source_group['id']) + self.fake_volume = jsonutils.to_primitive(volume) self.fake_volume_metadata = volume["volume_metadata"] self.fake_snapshot = jsonutils.to_primitive(snapshot) self.fake_snapshot_obj = fake_snapshot.fake_snapshot_obj(self.context, **snpshot) self.fake_reservations = ["RESERVATION"] + self.fake_cg = jsonutils.to_primitive(group) + self.fake_cg2 = jsonutils.to_primitive(group2) + self.fake_src_cg = jsonutils.to_primitive(source_group) + self.fake_cgsnap = jsonutils.to_primitive(cgsnapshot) def test_serialized_volume_has_id(self): self.assertIn('id', self.fake_volume) @@ -105,6 +135,27 @@ class VolumeRpcAPITestCase(test.TestCase): del expected_msg['new_volume'] expected_msg['new_volume_id'] = volume['id'] + if 'group' in expected_msg: + group = expected_msg['group'] + del expected_msg['group'] + expected_msg['group_id'] = group['id'] + + if 'cgsnapshot' in expected_msg: + cgsnapshot = expected_msg['cgsnapshot'] + if cgsnapshot: + del expected_msg['cgsnapshot'] + expected_msg['cgsnapshot_id'] = cgsnapshot['id'] + else: + expected_msg['cgsnapshot_id'] = None + + if 'source_cg' in expected_msg: + source_cg = expected_msg['source_cg'] + if source_cg: + del expected_msg['source_cg'] + expected_msg['source_cgid'] = source_cg['id'] + else: + expected_msg['source_cgid'] = None + if 'host' in kwargs: host = kwargs['host'] else: @@ -307,3 +358,21 @@ class VolumeRpcAPITestCase(test.TestCase): rpc_method='cast', volume=self.fake_volume, version='1.17') + + def test_create_consistencygroup_from_src_cgsnapshot(self): + self._test_volume_api('create_consistencygroup_from_src', + rpc_method='cast', + group=self.fake_cg, + host='fakehost', + cgsnapshot=self.fake_cgsnap, + source_cg=None, + version='1.25') + + def test_create_consistencygroup_from_src_cg(self): + self._test_volume_api('create_consistencygroup_from_src', + rpc_method='cast', + group=self.fake_cg2, + host='fakehost', + cgsnapshot=None, + source_cg=self.fake_src_cg, + version='1.25') diff --git a/cinder/tests/unit/utils.py b/cinder/tests/unit/utils.py index 9333fef02..d244d150d 100644 --- a/cinder/tests/unit/utils.py +++ b/cinder/tests/unit/utils.py @@ -107,6 +107,7 @@ def create_consistencygroup(ctxt, availability_zone='fake_az', volume_type_id=None, cgsnapshot_id=None, + source_cgid=None, **kwargs): """Create a consistencygroup object in the DB.""" cg = {} @@ -119,6 +120,8 @@ def create_consistencygroup(ctxt, cg['availability_zone'] = availability_zone if volume_type_id: cg['volume_type_id'] = volume_type_id + cg['cgsnapshot_id'] = cgsnapshot_id + cg['source_cgid'] = source_cgid for key in kwargs: cg[key] = kwargs[key] return db.consistencygroup_create(ctxt, cg) diff --git a/cinder/volume/api.py b/cinder/volume/api.py index d8043cce7..c825a65a9 100644 --- a/cinder/volume/api.py +++ b/cinder/volume/api.py @@ -191,7 +191,7 @@ class API(base.Base): availability_zone=None, source_volume=None, scheduler_hints=None, source_replica=None, consistencygroup=None, - cgsnapshot=None, multiattach=False): + cgsnapshot=None, multiattach=False, source_cg=None): # NOTE(jdg): we can have a create without size if we're # doing a create from snap or volume. Currently @@ -209,7 +209,7 @@ class API(base.Base): 'than zero).') % size raise exception.InvalidInput(reason=msg) - if consistencygroup and not cgsnapshot: + if consistencygroup and (not cgsnapshot and not source_cg): if not volume_type: msg = _("volume_type must be provided when creating " "a volume in a consistency group.") @@ -278,8 +278,10 @@ class API(base.Base): 'multiattach': multiattach, } try: - sched_rpcapi = self.scheduler_rpcapi if not cgsnapshot else None - volume_rpcapi = self.volume_rpcapi if not cgsnapshot else None + sched_rpcapi = (self.scheduler_rpcapi if (not cgsnapshot and + not source_cg) else None) + volume_rpcapi = (self.volume_rpcapi if (not cgsnapshot and + not source_cg) else None) flow_engine = create_volume.get_flow(self.db, self.image_service, availability_zones, diff --git a/cinder/volume/driver.py b/cinder/volume/driver.py index 2922d0870..f9e553098 100644 --- a/cinder/volume/driver.py +++ b/cinder/volume/driver.py @@ -1359,7 +1359,8 @@ class VolumeDriver(ConsistencyGroupVD, TransferVD, ManageableVD, ExtendVD, raise NotImplementedError() def create_consistencygroup_from_src(self, context, group, volumes, - cgsnapshot=None, snapshots=None): + cgsnapshot=None, snapshots=None, + source_cg=None, source_vols=None): """Creates a consistencygroup from source. :param context: the context of the caller. @@ -1367,9 +1368,11 @@ class VolumeDriver(ConsistencyGroupVD, TransferVD, ManageableVD, ExtendVD, :param volumes: a list of volume dictionaries in the group. :param cgsnapshot: the dictionary of the cgsnapshot as source. :param snapshots: a list of snapshot dictionaries in the cgsnapshot. + :param source_cg: the dictionary of a consistency group as source. + :param source_vols: a list of volume dictionaries in the source_cg. :return model_update, volumes_model_update - Currently the source can only be cgsnapshot. + The source can be cgsnapshot or a source cg. param volumes is retrieved directly from the db. It is a list of cinder.db.sqlalchemy.models.Volume to be precise. It cannot be diff --git a/cinder/volume/manager.py b/cinder/volume/manager.py index a231e2f63..3b937939f 100644 --- a/cinder/volume/manager.py +++ b/cinder/volume/manager.py @@ -77,6 +77,7 @@ QUOTAS = quota.QUOTAS CGQUOTAS = quota.CGQUOTAS VALID_REMOVE_VOL_FROM_CG_STATUS = ('available', 'in-use',) VALID_CREATE_CG_SRC_SNAP_STATUS = ('available',) +VALID_CREATE_CG_SRC_CG_STATUS = ('available',) volume_manager_opts = [ cfg.StrOpt('volume_driver', @@ -188,7 +189,7 @@ def locked_snapshot_operation(f): class VolumeManager(manager.SchedulerDependentManager): """Manages attachable block storage devices.""" - RPC_API_VERSION = '1.24' + RPC_API_VERSION = '1.25' target = messaging.Target(version=RPC_API_VERSION) @@ -1956,10 +1957,11 @@ class VolumeManager(manager.SchedulerDependentManager): return group_ref['id'] def create_consistencygroup_from_src(self, context, group_id, - cgsnapshot_id=None): + cgsnapshot_id=None, + source_cgid=None): """Creates the consistency group from source. - Currently the source can only be a cgsnapshot. + The source can be a CG snapshot or a source CG. """ group_ref = self.db.consistencygroup_get(context, group_id) @@ -1995,9 +1997,49 @@ class VolumeManager(manager.SchedulerDependentManager): 'valid': VALID_CREATE_CG_SRC_SNAP_STATUS}) raise exception.InvalidConsistencyGroup(reason=msg) + source_cg = None + source_vols = None + if source_cgid: + try: + source_cg = self.db.consistencygroup_get( + context, source_cgid) + except exception.ConsistencyGroupNotFound: + LOG.error(_LE("Create consistency group " + "from source cg-%(cg)s failed: " + "ConsistencyGroupNotFound."), + {'cg': source_cgid}, + resource={'type': 'consistency_group', + 'id': group_ref['id']}) + raise + if source_cg: + source_vols = self.db.volume_get_all_by_group( + context, source_cgid) + for source_vol in source_vols: + if (source_vol['status'] not in + VALID_CREATE_CG_SRC_CG_STATUS): + msg = (_("Cannot create consistency group " + "%(group)s because source volume " + "%(source_vol)s is not in a valid " + "state. Valid states are: " + "%(valid)s.") % + {'group': group_id, + 'source_vol': source_vol['id'], + 'valid': VALID_CREATE_CG_SRC_CG_STATUS}) + raise exception.InvalidConsistencyGroup(reason=msg) + # Sort source snapshots so that they are in the same order as their # corresponding target volumes. - sorted_snapshots = self._sort_snapshots(volumes, snapshots) + sorted_snapshots = None + if cgsnapshot and snapshots: + sorted_snapshots = self._sort_snapshots(volumes, snapshots) + + # Sort source volumes so that they are in the same order as their + # corresponding target volumes. + sorted_source_vols = None + if source_cg and source_vols: + sorted_source_vols = self._sort_source_vols(volumes, + source_vols) + self._notify_about_consistencygroup_usage( context, group_ref, "create.start") @@ -2006,7 +2048,7 @@ class VolumeManager(manager.SchedulerDependentManager): model_update, volumes_model_update = ( self.driver.create_consistencygroup_from_src( context, group_ref, volumes, cgsnapshot, - sorted_snapshots)) + sorted_snapshots, source_cg, sorted_source_vols)) if volumes_model_update: for update in volumes_model_update: @@ -2022,9 +2064,15 @@ class VolumeManager(manager.SchedulerDependentManager): context, group_id, {'status': 'error'}) + if cgsnapshot_id: + source = _("snapshot-%s") % cgsnapshot_id + elif source_cgid: + source = _("cg-%s") % source_cgid + else: + source = None LOG.error(_LE("Create consistency group " - "from snapshot-%(snap)s failed."), - {'snap': cgsnapshot_id}, + "from source %(source)s failed."), + {'source': source}, resource={'type': 'consistency_group', 'id': group_ref['id']}) # Update volume status to 'error' as well. @@ -2078,15 +2126,41 @@ class VolumeManager(manager.SchedulerDependentManager): return sorted_snapshots + def _sort_source_vols(self, volumes, source_vols): + # Sort source volumes so that they are in the same order as their + # corresponding target volumes. Each source volume in the source_vols + # list should have a corresponding target volume in the volumes list. + if not volumes or not source_vols or len(volumes) != len(source_vols): + msg = _("Input volumes or source volumes are invalid.") + LOG.error(msg) + raise exception.InvalidInput(reason=msg) + + sorted_source_vols = [] + for vol in volumes: + found_source_vols = filter( + lambda source_vol: source_vol['id'] == vol['source_volid'], + source_vols) + if not found_source_vols: + LOG.error(_LE("Source volumes cannot be found for target " + "volume %(volume_id)s."), + {'volume_id': vol['id']}) + raise exception.VolumeNotFound( + volume_id=vol['source_volid']) + sorted_source_vols.extend(found_source_vols) + + return sorted_source_vols + def _update_volume_from_src(self, context, vol, update, group_id=None): try: - snapshot = objects.Snapshot.get_by_id(context, vol['snapshot_id']) - orig_vref = self.db.volume_get(context, - snapshot.volume_id) - if orig_vref.bootable: - update['bootable'] = True - self.db.volume_glance_metadata_copy_to_volume( - context, vol['id'], vol['snapshot_id']) + snapshot_id = vol.get('snapshot_id') + if snapshot_id: + snapshot = objects.Snapshot.get_by_id(context, snapshot_id) + orig_vref = self.db.volume_get(context, + snapshot.volume_id) + if orig_vref.bootable: + update['bootable'] = True + self.db.volume_glance_metadata_copy_to_volume( + context, vol['id'], snapshot_id) except exception.SnapshotNotFound: LOG.error(_LE("Source snapshot %(snapshot_id)s cannot be found."), {'snapshot_id': vol['snapshot_id']}) diff --git a/cinder/volume/rpcapi.py b/cinder/volume/rpcapi.py index 8a027b81b..4952ecceb 100644 --- a/cinder/volume/rpcapi.py +++ b/cinder/volume/rpcapi.py @@ -68,6 +68,7 @@ class VolumeAPI(object): source_volid, source_replicaid, consistencygroup_id and cgsnapshot_id from create_volume. All off them are already passed either in request_spec or available in the DB. + 1.25 - Add source_cg to create_consistencygroup_from_src. """ BASE_RPC_API_VERSION = '1.0' @@ -77,7 +78,7 @@ class VolumeAPI(object): target = messaging.Target(topic=CONF.volume_topic, version=self.BASE_RPC_API_VERSION) serializer = objects_base.CinderObjectSerializer() - self.client = rpc.get_client(target, '1.24', serializer=serializer) + self.client = rpc.get_client(target, '1.25', serializer=serializer) def create_consistencygroup(self, ctxt, group, host): new_host = utils.extract_host(host) @@ -101,12 +102,14 @@ class VolumeAPI(object): remove_volumes=remove_volumes) def create_consistencygroup_from_src(self, ctxt, group, host, - cgsnapshot=None): + cgsnapshot=None, + source_cg=None): new_host = utils.extract_host(host) - cctxt = self.client.prepare(server=new_host, version='1.22') + cctxt = self.client.prepare(server=new_host, version='1.25') cctxt.cast(ctxt, 'create_consistencygroup_from_src', group_id=group['id'], - cgsnapshot_id=cgsnapshot['id']) + cgsnapshot_id=cgsnapshot['id'] if cgsnapshot else None, + source_cgid=source_cg['id'] if source_cg else None) def create_cgsnapshot(self, ctxt, group, cgsnapshot):