From: Xing Yang Date: Wed, 31 Dec 2014 02:27:56 +0000 (-0500) Subject: Modify Consistency Group API X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=1a62a6e60fda73bf31256fbf684dc03bb6cf0038;p=openstack-build%2Fcinder-build.git Modify Consistency Group API This patch addressed the following: * Modify Consistency Group * Added an API that supports adding existing volumes to CG and removing volumes from CG after it is created. It also allows the name and the description to be modified. * Added a volume driver API accordingly. Change-Id: I473cff65191e6e16dc22110f23efd376bfd3178a Implements: blueprint consistency-groups-kilo-update --- diff --git a/cinder/api/contrib/consistencygroups.py b/cinder/api/contrib/consistencygroups.py index fd15531aa..2e183eef4 100644 --- a/cinder/api/contrib/consistencygroups.py +++ b/cinder/api/contrib/consistencygroups.py @@ -201,6 +201,65 @@ class ConsistencyGroupsController(wsgi.Controller): dict(new_consistencygroup.iteritems())) return retval + @wsgi.serializers(xml=ConsistencyGroupTemplate) + def update(self, req, id, body): + """Update the consistency group. + + Expected format of the input parameter 'body': + { + "consistencygroup": + { + "name": "my_cg", + "description": "My consistency group", + "add_volumes": "volume-uuid-1,volume-uuid-2,..." + "remove_volumes": "volume-uuid-8,volume-uuid-9,..." + } + } + """ + LOG.debug('Update called for consistency group %s.', id) + + if not body: + msg = _("Missing request body.") + raise exc.HTTPBadRequest(explanation=msg) + if not self.is_valid_body(body, 'consistencygroup'): + msg = _("Incorrect request body format.") + raise exc.HTTPBadRequest(explanation=msg) + context = req.environ['cinder.context'] + + consistencygroup = body.get('consistencygroup', None) + name = consistencygroup.get('name', None) + description = consistencygroup.get('description', None) + add_volumes = consistencygroup.get('add_volumes', None) + remove_volumes = consistencygroup.get('remove_volumes', None) + + if (not name and not description and not add_volumes + and not remove_volumes): + msg = _("Name, description, add_volumes, and remove_volumes " + "can not be all empty in the request body.") + raise exc.HTTPBadRequest(explanation=msg) + + LOG.info(_LI("Updating consistency group %(id)s with name %(name)s " + "description: %(description)s add_volumes: " + "%(add_volumes)s remove_volumes: %(remove_volumes)s."), + {'id': id, 'name': name, + 'description': description, + 'add_volumes': add_volumes, + 'remove_volumes': remove_volumes}, + context=context) + + try: + group = self.consistencygroup_api.get(context, id) + self.consistencygroup_api.update( + context, group, name, description, + add_volumes, remove_volumes) + except exception.ConsistencyGroupNotFound: + msg = _("Consistency group %s could not be found.") % id + raise exc.HTTPNotFound(explanation=msg) + except exception.InvalidConsistencyGroup as error: + raise exc.HTTPBadRequest(explanation=error.msg) + + return webob.Response(status_int=202) + class Consistencygroups(extensions.ExtensionDescriptor): """consistency groups support.""" @@ -215,6 +274,6 @@ class Consistencygroups(extensions.ExtensionDescriptor): res = extensions.ResourceExtension( Consistencygroups.alias, ConsistencyGroupsController(), collection_actions={'detail': 'GET'}, - member_actions={'delete': 'POST'}) + member_actions={'delete': 'POST', 'update': 'PUT'}) resources.append(res) return resources diff --git a/cinder/consistencygroup/api.py b/cinder/consistencygroup/api.py index 9c90c0a38..7da4181da 100644 --- a/cinder/consistencygroup/api.py +++ b/cinder/consistencygroup/api.py @@ -33,6 +33,7 @@ from cinder import quota from cinder.scheduler import rpcapi as scheduler_rpcapi from cinder.volume import api as volume_api from cinder.volume import rpcapi as volume_rpcapi +from cinder.volume import utils as vol_utils from cinder.volume import volume_types @@ -41,6 +42,7 @@ CONF.import_opt('storage_availability_zone', 'cinder.volume.manager') LOG = logging.getLogger(__name__) CGQUOTAS = quota.CGQUOTAS +VALID_REMOVE_VOL_FROM_CG_STATUS = ('available', 'in-use',) def wrap_check_policy(func): @@ -286,10 +288,188 @@ class API(base.Base): self.volume_rpcapi.delete_consistencygroup(context, group) - @wrap_check_policy - def update(self, context, group, fields): + def update(self, context, group, name, description, + add_volumes, remove_volumes): + """Update consistency group.""" + if group['status'] not in ["available"]: + msg = _("Consistency group status must be available, " + "but current status is: %s.") % group['status'] + raise exception.InvalidConsistencyGroup(reason=msg) + + add_volumes_list = [] + remove_volumes_list = [] + if add_volumes: + add_volumes = add_volumes.strip(',') + add_volumes_list = add_volumes.split(',') + if remove_volumes: + remove_volumes = remove_volumes.strip(',') + remove_volumes_list = remove_volumes.split(',') + + invalid_uuids = [] + for uuid in add_volumes_list: + if uuid in remove_volumes_list: + invalid_uuids.append(uuid) + if invalid_uuids: + msg = _("UUIDs %s are in both add and remove volume " + "list.") % invalid_uuids + raise exception.InvalidVolume(reason=msg) + + volumes = self.db.volume_get_all_by_group(context, group['id']) + + # Validate name. + if not name or name == group['name']: + name = None + + # Validate description. + if not description or description == group['description']: + description = None + + # Validate volumes in add_volumes and remove_volumes. + add_volumes_new = "" + remove_volumes_new = "" + if add_volumes_list: + add_volumes_new = self._validate_add_volumes( + context, volumes, add_volumes_list, group) + if remove_volumes_list: + remove_volumes_new = self._validate_remove_volumes( + volumes, remove_volumes_list, group) + + if (not name and not description and not add_volumes_new and + not remove_volumes_new): + msg = (_("Cannot update consistency group %(group_id)s " + "because no valid name, description, add_volumes, " + "or remove_volumes were provided.") % + {'group_id': group['id']}) + raise exception.InvalidConsistencyGroup(reason=msg) + + now = timeutils.utcnow() + fields = {'updated_at': now} + + # Update name and description in db now. No need to + # to send them over thru an RPC call. + if name: + fields['name'] = name + if description: + fields['description'] = description + if not add_volumes_new and not remove_volumes_new: + # Only update name or description. Set status to available. + fields['status'] = 'available' + else: + fields['status'] = 'updating' + self.db.consistencygroup_update(context, group['id'], fields) + # Do an RPC call only if the update request includes + # adding/removing volumes. add_volumes_new and remove_volumes_new + # are strings of volume UUIDs separated by commas with no spaces + # in between. + if add_volumes_new or remove_volumes_new: + self.volume_rpcapi.update_consistencygroup( + context, group, + add_volumes=add_volumes_new, + remove_volumes=remove_volumes_new) + + def _validate_remove_volumes(self, volumes, remove_volumes_list, group): + # Validate volumes in remove_volumes. + remove_volumes_new = "" + for volume in volumes: + if volume['id'] in remove_volumes_list: + if volume['status'] not in VALID_REMOVE_VOL_FROM_CG_STATUS: + msg = (_("Cannot remove volume %(volume_id)s from " + "consistency group %(group_id)s because volume " + "is in an invalid state: %(status)s. Valid " + "states are: %(valid)s.") % + {'volume_id': volume['id'], + 'group_id': group['id'], + 'status': volume['status'], + 'valid': VALID_REMOVE_VOL_FROM_CG_STATUS}) + raise exception.InvalidVolume(reason=msg) + # Volume currently in CG. It will be removed from CG. + if remove_volumes_new: + remove_volumes_new += "," + remove_volumes_new += volume['id'] + + for rem_vol in remove_volumes_list: + if rem_vol not in remove_volumes_new: + msg = (_("Cannot remove volume %(volume_id)s from " + "consistency group %(group_id)s because it " + "is not in the group.") % + {'volume_id': rem_vol, + 'group_id': group['id']}) + raise exception.InvalidVolume(reason=msg) + + return remove_volumes_new + + def _validate_add_volumes(self, context, volumes, add_volumes_list, group): + add_volumes_new = "" + for volume in volumes: + if volume['id'] in add_volumes_list: + # Volume already in CG. Remove from add_volumes. + add_volumes_list.remove(volume['id']) + + for add_vol in add_volumes_list: + try: + add_vol_ref = self.db.volume_get(context, add_vol) + except exception.VolumeNotFound: + msg = (_("Cannot add volume %(volume_id)s to consistency " + "group %(group_id)s because volume cannot be " + "found.") % + {'volume_id': add_vol, + 'group_id': group['id']}) + raise exception.InvalidVolume(reason=msg) + if add_vol_ref: + add_vol_type_id = add_vol_ref.get('volume_type_id', None) + if not add_vol_type_id: + msg = (_("Cannot add volume %(volume_id)s to consistency " + "group %(group_id)s because it has no volume " + "type.") % + {'volume_id': add_vol_ref['id'], + 'group_id': group['id']}) + raise exception.InvalidVolume(reason=msg) + if add_vol_type_id not in group['volume_type_id']: + msg = (_("Cannot add volume %(volume_id)s to consistency " + "group %(group_id)s because volume type " + "%(volume_type)s is not supported by the " + "group.") % + {'volume_id': add_vol_ref['id'], + 'group_id': group['id'], + 'volume_type': add_vol_type_id}) + raise exception.InvalidVolume(reason=msg) + if (add_vol_ref['status'] not in + VALID_REMOVE_VOL_FROM_CG_STATUS): + msg = (_("Cannot add volume %(volume_id)s to consistency " + "group %(group_id)s because volume is in an " + "invalid state: %(status)s. Valid states are: " + "%(valid)s.") % + {'volume_id': add_vol_ref['id'], + 'group_id': group['id'], + 'status': add_vol_ref['status'], + 'valid': VALID_REMOVE_VOL_FROM_CG_STATUS}) + raise exception.InvalidVolume(reason=msg) + + # group['host'] and add_vol_ref['host'] are in this format: + # 'host@backend#pool'. Extract host (host@backend) before + # doing comparison. + vol_host = vol_utils.extract_host(add_vol_ref['host']) + group_host = vol_utils.extract_host(group['host']) + if group_host != vol_host: + raise exception.InvalidVolume( + reason=_("Volume is not local to this node.")) + + # Volume exists. It will be added to CG. + if add_volumes_new: + add_volumes_new += "," + add_volumes_new += add_vol_ref['id'] + + else: + msg = (_("Cannot add volume %(volume_id)s to consistency " + "group %(group_id)s because volume does not exist.") % + {'volume_id': add_vol_ref['id'], + 'group_id': group['id']}) + raise exception.InvalidVolume(reason=msg) + + return add_volumes_new + def get(self, context, group_id): rv = self.db.consistencygroup_get(context, group_id) group = dict(rv.iteritems()) @@ -326,11 +506,6 @@ class API(base.Base): return groups - def get_group(self, context, group_id): - check_policy(context, 'get_group') - rv = self.db.consistencygroup_get(context, group_id) - return dict(rv.iteritems()) - def create_cgsnapshot(self, context, group, name, description): diff --git a/cinder/db/sqlalchemy/api.py b/cinder/db/sqlalchemy/api.py index 7b5cc1e63..6e3d85050 100644 --- a/cinder/db/sqlalchemy/api.py +++ b/cinder/db/sqlalchemy/api.py @@ -1251,7 +1251,7 @@ def volume_get_all_by_host(context, host, filters=None): return [] -@require_admin_context +@require_context def volume_get_all_by_group(context, group_id, filters=None): """Retrieves all volumes associated with the group_id. diff --git a/cinder/tests/api/contrib/test_consistencygroups.py b/cinder/tests/api/contrib/test_consistencygroups.py index 11889e25d..3d98e8525 100644 --- a/cinder/tests/api/contrib/test_consistencygroups.py +++ b/cinder/tests/api/contrib/test_consistencygroups.py @@ -29,6 +29,7 @@ from cinder import db from cinder.i18n import _ from cinder import test from cinder.tests.api import fakes +from cinder.tests import utils class ConsistencyGroupsAPITestCase(test.TestCase): @@ -456,3 +457,219 @@ class ConsistencyGroupsAPITestCase(test.TestCase): msg = (_('volume_types must be provided to create ' 'consistency group %s.') % name) self.assertEqual(msg, res_dict['badRequest']['message']) + + def test_update_consistencygroup_success(self): + volume_type_id = '123456' + ctxt = context.RequestContext('fake', 'fake') + consistencygroup_id = self._create_consistencygroup(status='available', + host='test_host') + remove_volume_id = utils.create_volume( + ctxt, + volume_type_id=volume_type_id, + consistencygroup_id=consistencygroup_id)['id'] + remove_volume_id2 = utils.create_volume( + ctxt, + volume_type_id=volume_type_id, + consistencygroup_id=consistencygroup_id)['id'] + + self.assertEqual('available', + self._get_consistencygroup_attrib(consistencygroup_id, + 'status')) + + cg_volumes = db.volume_get_all_by_group(ctxt.elevated(), + consistencygroup_id) + cg_vol_ids = [cg_vol['id'] for cg_vol in cg_volumes] + self.assertIn(remove_volume_id, cg_vol_ids) + self.assertIn(remove_volume_id2, cg_vol_ids) + + add_volume_id = utils.create_volume( + ctxt, + volume_type_id=volume_type_id)['id'] + add_volume_id2 = utils.create_volume( + ctxt, + volume_type_id=volume_type_id)['id'] + req = webob.Request.blank('/v2/fake/consistencygroups/%s/update' % + consistencygroup_id) + req.method = 'PUT' + req.headers['Content-Type'] = 'application/json' + name = 'newcg' + description = 'New Consistency Group Description' + add_volumes = add_volume_id + "," + add_volume_id2 + remove_volumes = remove_volume_id + "," + remove_volume_id2 + body = {"consistencygroup": {"name": name, + "description": description, + "add_volumes": add_volumes, + "remove_volumes": remove_volumes, }} + req.body = json.dumps(body) + res = req.get_response(fakes.wsgi_app()) + + self.assertEqual(202, res.status_int) + self.assertEqual('updating', + self._get_consistencygroup_attrib(consistencygroup_id, + 'status')) + + db.consistencygroup_destroy(ctxt.elevated(), consistencygroup_id) + + def test_update_consistencygroup_add_volume_not_found(self): + ctxt = context.RequestContext('fake', 'fake') + consistencygroup_id = self._create_consistencygroup(status='available') + req = webob.Request.blank('/v2/fake/consistencygroups/%s/update' % + consistencygroup_id) + req.method = 'PUT' + req.headers['Content-Type'] = 'application/json' + body = {"consistencygroup": {"name": None, + "description": None, + "add_volumes": "fake-volume-uuid", + "remove_volumes": None, }} + req.body = json.dumps(body) + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(400, res.status_int) + self.assertEqual(400, res_dict['badRequest']['code']) + msg = (_("Invalid volume: Cannot add volume fake-volume-uuid " + "to consistency group %(group_id)s because volume cannot " + "be found.") % + {'group_id': consistencygroup_id}) + self.assertEqual(msg, res_dict['badRequest']['message']) + + db.consistencygroup_destroy(ctxt.elevated(), consistencygroup_id) + + def test_update_consistencygroup_remove_volume_not_found(self): + ctxt = context.RequestContext('fake', 'fake') + consistencygroup_id = self._create_consistencygroup(status='available') + req = webob.Request.blank('/v2/fake/consistencygroups/%s/update' % + consistencygroup_id) + req.method = 'PUT' + req.headers['Content-Type'] = 'application/json' + body = {"consistencygroup": {"name": None, + "description": "new description", + "add_volumes": None, + "remove_volumes": "fake-volume-uuid", }} + req.body = json.dumps(body) + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(400, res.status_int) + self.assertEqual(400, res_dict['badRequest']['code']) + msg = (_("Invalid volume: Cannot remove volume fake-volume-uuid " + "from consistency group %(group_id)s because it is not " + "in the group.") % + {'group_id': consistencygroup_id}) + self.assertEqual(msg, res_dict['badRequest']['message']) + + db.consistencygroup_destroy(ctxt.elevated(), consistencygroup_id) + + def test_update_consistencygroup_empty_parameters(self): + ctxt = context.RequestContext('fake', 'fake') + consistencygroup_id = self._create_consistencygroup(status='available') + req = webob.Request.blank('/v2/fake/consistencygroups/%s/update' % + consistencygroup_id) + req.method = 'PUT' + req.headers['Content-Type'] = 'application/json' + body = {"consistencygroup": {"name": "", + "description": "", + "add_volumes": None, + "remove_volumes": None, }} + req.body = json.dumps(body) + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(400, res.status_int) + self.assertEqual(400, res_dict['badRequest']['code']) + self.assertEqual('Name, description, add_volumes, and remove_volumes ' + 'can not be all empty in the request body.', + res_dict['badRequest']['message']) + + db.consistencygroup_destroy(ctxt.elevated(), consistencygroup_id) + + def test_update_consistencygroup_add_volume_invalid_state(self): + volume_type_id = '123456' + ctxt = context.RequestContext('fake', 'fake') + consistencygroup_id = self._create_consistencygroup(status='available') + add_volume_id = utils.create_volume( + ctxt, + volume_type_id=volume_type_id, + status='wrong_status')['id'] + req = webob.Request.blank('/v2/fake/consistencygroups/%s/update' % + consistencygroup_id) + req.method = 'PUT' + req.headers['Content-Type'] = 'application/json' + add_volumes = add_volume_id + body = {"consistencygroup": {"name": "", + "description": "", + "add_volumes": add_volumes, + "remove_volumes": None, }} + req.body = json.dumps(body) + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(400, res.status_int) + self.assertEqual(400, res_dict['badRequest']['code']) + msg = (_("Invalid volume: Cannot add volume %(volume_id)s " + "to consistency group %(group_id)s because volume is in an " + "invalid state: %(status)s. Valid states are: ('available', " + "'in-use').") % + {'volume_id': add_volume_id, + 'group_id': consistencygroup_id, + 'status': 'wrong_status'}) + self.assertEqual(msg, res_dict['badRequest']['message']) + + db.consistencygroup_destroy(ctxt.elevated(), consistencygroup_id) + + def test_update_consistencygroup_add_volume_invalid_volume_type(self): + ctxt = context.RequestContext('fake', 'fake') + consistencygroup_id = self._create_consistencygroup(status='available') + wrong_type = 'wrong-volume-type-id' + add_volume_id = utils.create_volume( + ctxt, + volume_type_id=wrong_type)['id'] + req = webob.Request.blank('/v2/fake/consistencygroups/%s/update' % + consistencygroup_id) + req.method = 'PUT' + req.headers['Content-Type'] = 'application/json' + add_volumes = add_volume_id + body = {"consistencygroup": {"name": "", + "description": "", + "add_volumes": add_volumes, + "remove_volumes": None, }} + req.body = json.dumps(body) + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(400, res.status_int) + self.assertEqual(400, res_dict['badRequest']['code']) + msg = (_("Invalid volume: Cannot add volume %(volume_id)s " + "to consistency group %(group_id)s because volume type " + "%(volume_type)s is not supported by the group.") % + {'volume_id': add_volume_id, + 'group_id': consistencygroup_id, + 'volume_type': wrong_type}) + self.assertEqual(msg, res_dict['badRequest']['message']) + + db.consistencygroup_destroy(ctxt.elevated(), consistencygroup_id) + + def test_update_consistencygroup_invalid_state(self): + ctxt = context.RequestContext('fake', 'fake') + wrong_status = 'wrong_status' + consistencygroup_id = self._create_consistencygroup( + status=wrong_status) + req = webob.Request.blank('/v2/fake/consistencygroups/%s/update' % + consistencygroup_id) + req.method = 'PUT' + req.headers['Content-Type'] = 'application/json' + body = {"consistencygroup": {"name": "new name", + "description": None, + "add_volumes": None, + "remove_volumes": None, }} + req.body = json.dumps(body) + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(400, res.status_int) + self.assertEqual(400, res_dict['badRequest']['code']) + msg = _("Invalid ConsistencyGroup: Consistency group status must be " + "available, but current status is: %s.") % wrong_status + self.assertEqual(msg, res_dict['badRequest']['message']) + + db.consistencygroup_destroy(ctxt.elevated(), consistencygroup_id) diff --git a/cinder/tests/policy.json b/cinder/tests/policy.json index 75cc24d75..4d54a74d1 100644 --- a/cinder/tests/policy.json +++ b/cinder/tests/policy.json @@ -86,6 +86,7 @@ "consistencygroup:create" : "", "consistencygroup:delete": "", + "consistencygroup:update": "", "consistencygroup:get": "", "consistencygroup:get_all": "", diff --git a/cinder/tests/test_volume.py b/cinder/tests/test_volume.py index bc3c8e100..42d5fcb3c 100644 --- a/cinder/tests/test_volume.py +++ b/cinder/tests/test_volume.py @@ -3528,30 +3528,20 @@ class VolumeTestCase(BaseVolumeTestCase): # clean up self.volume.delete_volume(self.context, volume['id']) - def test_create_delete_consistencygroup(self): + @mock.patch.object(CGQUOTAS, "reserve", + return_value=["RESERVATION"]) + @mock.patch.object(CGQUOTAS, "commit") + @mock.patch.object(CGQUOTAS, "rollback") + @mock.patch.object(driver.VolumeDriver, + "create_consistencygroup", + return_value={'status': 'available'}) + @mock.patch.object(driver.VolumeDriver, + "delete_consistencygroup", + return_value=({'status': 'deleted'}, [])) + def test_create_delete_consistencygroup(self, fake_delete_cg, + fake_create_cg, fake_rollback, + fake_commit, fake_reserve): """Test consistencygroup can be created and deleted.""" - # Need to stub out reserve, commit, and rollback - def fake_reserve(context, expire=None, project_id=None, **deltas): - return ["RESERVATION"] - - def fake_commit(context, reservations, project_id=None): - pass - - def fake_rollback(context, reservations, project_id=None): - pass - - self.stubs.Set(CGQUOTAS, "reserve", fake_reserve) - self.stubs.Set(CGQUOTAS, "commit", fake_commit) - self.stubs.Set(CGQUOTAS, "rollback", fake_rollback) - - rval = {'status': 'available'} - driver.VolumeDriver.create_consistencygroup = \ - mock.Mock(return_value=rval) - - rval = {'status': 'deleted'}, [] - driver.VolumeDriver.delete_consistencygroup = \ - mock.Mock(return_value=rval) - group = tests_utils.create_consistencygroup( self.context, availability_zone=CONF.storage_availability_zone, @@ -3598,6 +3588,96 @@ class VolumeTestCase(BaseVolumeTestCase): self.context, group_id) + @mock.patch.object(CGQUOTAS, "reserve", + return_value=["RESERVATION"]) + @mock.patch.object(CGQUOTAS, "commit") + @mock.patch.object(CGQUOTAS, "rollback") + @mock.patch.object(driver.VolumeDriver, + "create_consistencygroup", + return_value={'status': 'available'}) + @mock.patch.object(driver.VolumeDriver, + "update_consistencygroup") + def test_update_consistencygroup(self, fake_update_cg, + fake_create_cg, fake_rollback, + fake_commit, fake_reserve): + """Test consistencygroup can be updated.""" + group = tests_utils.create_consistencygroup( + self.context, + availability_zone=CONF.storage_availability_zone, + volume_type='type1,type2') + group_id = group['id'] + self.volume.create_consistencygroup(self.context, group_id) + + volume = tests_utils.create_volume( + self.context, + consistencygroup_id=group_id, + **self.volume_params) + volume_id = volume['id'] + self.volume.create_volume(self.context, volume_id) + + volume2 = tests_utils.create_volume( + self.context, + consistencygroup_id=None, + **self.volume_params) + volume_id2 = volume2['id'] + self.volume.create_volume(self.context, volume_id2) + + fake_update_cg.return_value = ( + {'status': 'available'}, + [{'id': volume_id2, 'status': 'available'}], + [{'id': volume_id, 'status': 'available'}]) + + self.volume.update_consistencygroup(self.context, group_id, + add_volumes=volume_id2, + remove_volumes=volume_id) + cg = db.consistencygroup_get( + self.context, + group_id) + expected = { + 'status': 'available', + 'name': 'test_cg', + 'availability_zone': 'nova', + 'tenant_id': 'fake', + 'created_at': 'DONTCARE', + 'user_id': 'fake', + 'consistencygroup_id': group_id + } + self.assertEqual('available', cg['status']) + self.assertEqual(10, len(fake_notifier.NOTIFICATIONS)) + msg = fake_notifier.NOTIFICATIONS[6] + self.assertEqual('consistencygroup.update.start', msg['event_type']) + self.assertDictMatch(expected, msg['payload']) + msg = fake_notifier.NOTIFICATIONS[8] + self.assertEqual('consistencygroup.update.end', msg['event_type']) + self.assertDictMatch(expected, msg['payload']) + cgvolumes = db.volume_get_all_by_group(self.context, group_id) + cgvol_ids = [cgvol['id'] for cgvol in cgvolumes] + # Verify volume is removed. + self.assertNotIn(volume_id, cgvol_ids) + # Verify volume is added. + self.assertIn(volume_id2, cgvol_ids) + + self.volume_params['status'] = 'wrong-status' + volume3 = tests_utils.create_volume( + self.context, + consistencygroup_id=None, + **self.volume_params) + volume_id3 = volume3['id'] + + volume_get_orig = self.volume.db.volume_get + self.volume.db.volume_get = mock.Mock( + return_value={'status': 'wrong_status', + 'id': volume_id3}) + # Try to add a volume in wrong status + self.assertRaises(exception.InvalidVolume, + self.volume.update_consistencygroup, + self.context, + group_id, + add_volumes=volume_id3, + remove_volumes=None) + self.volume.db.volume_get.reset_mock() + self.volume.db.volume_get = volume_get_orig + @staticmethod def _create_cgsnapshot(group_id, volume_id, size='0'): """Create a cgsnapshot object.""" diff --git a/cinder/volume/driver.py b/cinder/volume/driver.py index 17cee2407..0dcbe5e24 100644 --- a/cinder/volume/driver.py +++ b/cinder/volume/driver.py @@ -1115,6 +1115,34 @@ class VolumeDriver(ConsistencyGroupVD, TransferVD, ManageableVD, ExtendVD, """Deletes a consistency group.""" raise NotImplementedError() + def update_consistencygroup(self, context, group, + add_volumes=None, remove_volumes=None): + """Updates a consistency group. + + :param context: the context of the caller. + :param group: the dictionary of the consistency group to be updated. + :param add_volumes: a list of volume dictionaries to be added. + :param remove_volumes: a list of volume dictionaries to be removed. + :return model_update, add_volumes_update, remove_volumes_update + + model_update is a dictionary that the driver wants the manager + to update upon a successful return. If None is returned, the manager + will set the status to 'available'. + + add_volumes_update and remove_volumes_update are lists of dictionaries + that the driver wants the manager to update upon a successful return. + Note that each entry requires a {'id': xxx} so that the correct + volume entry can be updated. If None is returned, the volume will + remain its original status. Also note that you cannot directly + assign add_volumes to add_volumes_update as add_volumes is a list of + cinder.db.sqlalchemy.models.Volume objects and cannot be used for + db update directly. Same with remove_volumes. + + If the driver throws an exception, the status of the group as well as + those of the volumes to be added/removed will be set to 'error'. + """ + raise NotImplementedError() + def create_cgsnapshot(self, context, cgsnapshot): """Creates a cgsnapshot.""" raise NotImplementedError() diff --git a/cinder/volume/manager.py b/cinder/volume/manager.py index 08b6bdf4a..496d8de6e 100644 --- a/cinder/volume/manager.py +++ b/cinder/volume/manager.py @@ -72,6 +72,7 @@ LOG = logging.getLogger(__name__) QUOTAS = quota.QUOTAS CGQUOTAS = quota.CGQUOTAS +VALID_REMOVE_VOL_FROM_CG_STATUS = ('available', 'in-use',) volume_manager_opts = [ cfg.StrOpt('volume_driver', @@ -160,7 +161,7 @@ def locked_snapshot_operation(f): class VolumeManager(manager.SchedulerDependentManager): """Manages attachable block storage devices.""" - RPC_API_VERSION = '1.19' + RPC_API_VERSION = '1.21' target = messaging.Target(version=RPC_API_VERSION) @@ -1362,12 +1363,14 @@ class VolumeManager(manager.SchedulerDependentManager): context, group, event_suffix, + volumes=None, extra_usage_info=None): vol_utils.notify_about_consistencygroup_usage( context, group, event_suffix, extra_usage_info=extra_usage_info, host=self.host) - volumes = self.db.volume_get_all_by_group(context, group['id']) + if not volumes: + volumes = self.db.volume_get_all_by_group(context, group['id']) if volumes: for volume in volumes: vol_utils.notify_about_volume_usage( @@ -1378,13 +1381,15 @@ class VolumeManager(manager.SchedulerDependentManager): context, cgsnapshot, event_suffix, + snapshots=None, extra_usage_info=None): vol_utils.notify_about_cgsnapshot_usage( context, cgsnapshot, event_suffix, extra_usage_info=extra_usage_info, host=self.host) - snapshots = self.db.snapshot_get_all_for_cgsnapshot(context, - cgsnapshot['id']) + if not snapshots: + snapshots = self.db.snapshot_get_all_for_cgsnapshot( + context, cgsnapshot['id']) if snapshots: for snapshot in snapshots: vol_utils.notify_about_snapshot_usage( @@ -1857,11 +1862,148 @@ class VolumeManager(manager.SchedulerDependentManager): LOG.info(_LI("Consistency group %s: deleted successfully."), group_id) self._notify_about_consistencygroup_usage( - context, group_ref, "delete.end") + context, group_ref, "delete.end", volumes) self.publish_service_capabilities(context) return True + def update_consistencygroup(self, context, group_id, + add_volumes=None, remove_volumes=None): + """Updates consistency group. + + Update consistency group by adding volumes to the group, + or removing volumes from the group. + """ + LOG.info(_LI("Consistency group %s: updating"), group_id) + group = self.db.consistencygroup_get(context, group_id) + + add_volumes_ref = [] + remove_volumes_ref = [] + add_volumes_list = [] + remove_volumes_list = [] + if add_volumes: + add_volumes_list = add_volumes.split(',') + if remove_volumes: + remove_volumes_list = remove_volumes.split(',') + for add_vol in add_volumes_list: + try: + add_vol_ref = self.db.volume_get(context, add_vol) + except exception.VolumeNotFound: + LOG.error(_LE("Cannot add volume %(volume_id)s to consistency " + "group %(group_id)s because volume cannot be " + "found."), + {'volume_id': add_vol_ref['id'], + 'group_id': group_id}) + raise + if add_vol_ref['status'] not in ['in-use', 'available']: + msg = (_("Cannot add volume %(volume_id)s to consistency " + "group %(group_id)s because volume is in an invalid " + "state: %(status)s. Valid states are: %(valid)s.") % + {'volume_id': add_vol_ref['id'], + 'group_id': group_id, + 'status': add_vol_ref['status'], + 'valid': VALID_REMOVE_VOL_FROM_CG_STATUS}) + raise exception.InvalidVolume(reason=msg) + # self.host is 'host@backend' + # volume_ref['host'] is 'host@backend#pool' + # Extract host before doing comparison + new_host = vol_utils.extract_host(add_vol_ref['host']) + if new_host != self.host: + raise exception.InvalidVolume( + reason=_("Volume is not local to this node.")) + add_volumes_ref.append(add_vol_ref) + + for remove_vol in remove_volumes_list: + try: + remove_vol_ref = self.db.volume_get(context, remove_vol) + except exception.VolumeNotFound: + LOG.error(_LE("Cannot remove volume %(volume_id)s from " + "consistency group %(group_id)s because volume " + "cannot be found."), + {'volume_id': remove_vol_ref['id'], + 'group_id': group_id}) + raise + remove_volumes_ref.append(remove_vol_ref) + + self._notify_about_consistencygroup_usage( + context, group, "update.start") + + try: + utils.require_driver_initialized(self.driver) + + LOG.debug("Consistency group %(group_id)s: updating", + {'group_id': group['id']}) + + model_update, add_volumes_update, remove_volumes_update = ( + self.driver.update_consistencygroup( + context, group, + add_volumes=add_volumes_ref, + remove_volumes=remove_volumes_ref)) + + if add_volumes_update: + for update in add_volumes_update: + self.db.volume_update(context, update['id'], update) + + if remove_volumes_update: + for update in remove_volumes_update: + self.db.volume_update(context, update['id'], update) + + if model_update: + if model_update['status'] in ['error']: + msg = (_('Error occurred when updating consistency group ' + '%s.') % group_id) + LOG.exception(msg) + raise exception.VolumeDriverException(message=msg) + self.db.consistencygroup_update(context, group_id, + model_update) + + except exception.VolumeDriverException: + with excutils.save_and_reraise_exception(): + LOG.error(_LE("Error occurred in the volume driver when " + "updating consistency group %(group_id)s."), + {'group_id': group_id}) + self.db.consistencygroup_update(context, group_id, + {'status': 'error'}) + for add_vol in add_volumes_ref: + self.db.volume_update(context, add_vol['id'], + {'status': 'error'}) + for rem_vol in remove_volumes_ref: + self.db.volume_update(context, rem_vol['id'], + {'status': 'error'}) + except Exception: + with excutils.save_and_reraise_exception(): + LOG.error(_LE("Error occurred when updating consistency " + "group %(group_id)s."), + {'group_id': group['id']}) + self.db.consistencygroup_update(context, group_id, + {'status': 'error'}) + for add_vol in add_volumes_ref: + self.db.volume_update(context, add_vol['id'], + {'status': 'error'}) + for rem_vol in remove_volumes_ref: + self.db.volume_update(context, rem_vol['id'], + {'status': 'error'}) + + now = timeutils.utcnow() + self.db.consistencygroup_update(context, group_id, + {'status': 'available', + 'updated_at': now}) + for add_vol in add_volumes_ref: + self.db.volume_update(context, add_vol['id'], + {'consistencygroup_id': group_id, + 'updated_at': now}) + for rem_vol in remove_volumes_ref: + self.db.volume_update(context, rem_vol['id'], + {'consistencygroup_id': None, + 'updated_at': now}) + + LOG.info(_LI("Consistency group %s: updated successfully."), + group_id) + self._notify_about_consistencygroup_usage( + context, group, "update.end") + + return True + def create_cgsnapshot(self, context, group_id, cgsnapshot_id): """Creates the cgsnapshot.""" caller_context = context @@ -2038,7 +2180,7 @@ class VolumeManager(manager.SchedulerDependentManager): LOG.info(_LI("cgsnapshot %s: deleted successfully"), cgsnapshot_ref['id']) self._notify_about_cgsnapshot_usage( - context, cgsnapshot_ref, "delete.end") + context, cgsnapshot_ref, "delete.end", snapshots) return True diff --git a/cinder/volume/rpcapi.py b/cinder/volume/rpcapi.py index 93cd177e7..18a8ec710 100644 --- a/cinder/volume/rpcapi.py +++ b/cinder/volume/rpcapi.py @@ -61,6 +61,7 @@ class VolumeAPI(object): 1.19 - Adds update_migrated_volume 1.20 - Adds support for sending objects over RPC in create_snapshot() and delete_snapshot() + 1.21 - Adds update_consistencygroup. ''' BASE_RPC_API_VERSION = '1.0' @@ -70,7 +71,7 @@ class VolumeAPI(object): target = messaging.Target(topic=CONF.volume_topic, version=self.BASE_RPC_API_VERSION) serializer = objects_base.CinderObjectSerializer() - self.client = rpc.get_client(target, '1.20', serializer=serializer) + self.client = rpc.get_client(target, '1.21', serializer=serializer) def create_consistencygroup(self, ctxt, group, host): new_host = utils.extract_host(host) @@ -84,6 +85,15 @@ class VolumeAPI(object): cctxt.cast(ctxt, 'delete_consistencygroup', group_id=group['id']) + def update_consistencygroup(self, ctxt, group, add_volumes=None, + remove_volumes=None): + host = utils.extract_host(group['host']) + cctxt = self.client.prepare(server=host, version='1.21') + cctxt.cast(ctxt, 'update_consistencygroup', + group_id=group['id'], + add_volumes=add_volumes, + remove_volumes=remove_volumes) + def create_cgsnapshot(self, ctxt, group, cgsnapshot): host = utils.extract_host(group['host']) diff --git a/etc/cinder/policy.json b/etc/cinder/policy.json index 36816060f..a552c0122 100644 --- a/etc/cinder/policy.json +++ b/etc/cinder/policy.json @@ -73,6 +73,7 @@ "consistencygroup:create" : "group:nobody", "consistencygroup:delete": "group:nobody", + "consistencygroup:update": "group:nobody", "consistencygroup:get": "group:nobody", "consistencygroup:get_all": "group:nobody",