From 39c1a8e08dc854e22ada315a46c20c99df2facf8 Mon Sep 17 00:00:00 2001 From: Patrick East Date: Thu, 4 Dec 2014 18:51:06 -0800 Subject: [PATCH] Add support to PureISCSIDriver for Consistency Groups This change adds implementations for the required driver methods for Consistency Groups to be supported with the PureISCSIDriver. There is a nice direct mapping between Consistency Groups and Purity Protection Groups which makes the implementation pretty straightforward. Implements: blueprint pure-iscsi-consistency-group Change-Id: I8b97947d4788a7a7ba2dfa1d2b07645eafba76ec --- cinder/tests/test_pure.py | 267 ++++++++++++++++++++++++++++++++++ cinder/volume/drivers/pure.py | 146 ++++++++++++++++++- 2 files changed, 410 insertions(+), 3 deletions(-) diff --git a/cinder/tests/test_pure.py b/cinder/tests/test_pure.py index e5fc1d032..46d26dd31 100644 --- a/cinder/tests/test_pure.py +++ b/cinder/tests/test_pure.py @@ -49,7 +49,11 @@ VOLUME = {"name": "volume-" + VOLUME_ID, "host": "irrelevant", "volume_type": None, "volume_type_id": None, + "consistencygroup_id": None } +VOLUME_WITH_CGROUP = VOLUME.copy() +VOLUME_WITH_CGROUP['consistencygroup_id'] = \ + "4a2f7e3a-312a-40c5-96a8-536b8a0fe074" SRC_VOL_ID = "dc7a294d-5964-4379-a15f-ce5554734efc" SRC_VOL = {"name": "volume-" + SRC_VOL_ID, "id": SRC_VOL_ID, @@ -58,6 +62,7 @@ SRC_VOL = {"name": "volume-" + SRC_VOL_ID, "host": "irrelevant", "volume_type": None, "volume_type_id": None, + "consistencygroup_id": None } SNAPSHOT_ID = "04fe2f9a-d0c4-4564-a30d-693cc3657b47" SNAPSHOT = {"name": "snapshot-" + SNAPSHOT_ID, @@ -66,7 +71,11 @@ SNAPSHOT = {"name": "snapshot-" + SNAPSHOT_ID, "volume_name": "volume-" + SRC_VOL_ID, "volume_size": 2, "display_name": "fake_snapshot", + "cgsnapshot_id": None } +SNAPSHOT_WITH_CGROUP = SNAPSHOT.copy() +SNAPSHOT_WITH_CGROUP['cgsnapshot_id'] = \ + "4a2f7e3a-312a-40c5-96a8-536b8a0fe075" INITIATOR_IQN = "iqn.1993-08.org.debian:01:222" CONNECTOR = {"initiator": INITIATOR_IQN, "host": HOSTNAME} TARGET_IQN = "iqn.2010-06.com.purestorage:flasharray.12345abc" @@ -155,9 +164,22 @@ class PureISCSIDriverTestCase(test.TestCase): self.assert_error_propagates([self.array.create_volume], self.driver.create_volume, VOLUME) + @mock.patch(DRIVER_OBJ + "._add_volume_to_consistency_group", + autospec=True) + def test_create_volume_with_cgroup(self, mock_add_to_cgroup): + vol_name = VOLUME_WITH_CGROUP["name"] + "-cinder" + + self.driver.create_volume(VOLUME_WITH_CGROUP) + + mock_add_to_cgroup\ + .assert_called_with(self.driver, + VOLUME_WITH_CGROUP['consistencygroup_id'], + vol_name) + def test_create_volume_from_snapshot(self): vol_name = VOLUME["name"] + "-cinder" snap_name = SNAPSHOT["volume_name"] + "-cinder." + SNAPSHOT["name"] + # Branch where extend unneeded self.driver.create_volume_from_snapshot(VOLUME, SNAPSHOT) self.array.copy_volume.assert_called_with(snap_name, vol_name) @@ -166,6 +188,7 @@ class PureISCSIDriverTestCase(test.TestCase): [self.array.copy_volume], self.driver.create_volume_from_snapshot, VOLUME, SNAPSHOT) self.assertFalse(self.array.extend_volume.called) + # Branch where extend needed SNAPSHOT["volume_size"] = 1 # resize so smaller than VOLUME self.driver.create_volume_from_snapshot(VOLUME, SNAPSHOT) @@ -177,6 +200,33 @@ class PureISCSIDriverTestCase(test.TestCase): self.driver.create_volume_from_snapshot, VOLUME, SNAPSHOT) SNAPSHOT["volume_size"] = 2 # reset size + @mock.patch(DRIVER_OBJ + "._add_volume_to_consistency_group", + autospec=True) + @mock.patch(DRIVER_OBJ + "._extend_if_needed", autospec=True) + @mock.patch(DRIVER_PATH + "._get_pgroup_vol_snap_name", autospec=True) + def test_create_volume_from_cgsnapshot(self, mock_get_snap_name, + mock_extend_if_needed, + mock_add_to_cgroup): + vol_name = VOLUME_WITH_CGROUP["name"] + "-cinder" + snap_name = "consisgroup-4a2f7e3a-312a-40c5-96a8-536b8a0f" \ + "e074-cinder.4a2f7e3a-312a-40c5-96a8-536b8a0fe075."\ + + vol_name + mock_get_snap_name.return_value = snap_name + + self.driver.create_volume_from_snapshot(VOLUME_WITH_CGROUP, + SNAPSHOT_WITH_CGROUP) + + self.array.copy_volume.assert_called_with(snap_name, vol_name) + self.assertTrue(mock_get_snap_name.called) + self.assertTrue(mock_extend_if_needed.called) + + self.driver.create_volume_from_snapshot(VOLUME_WITH_CGROUP, + SNAPSHOT_WITH_CGROUP) + mock_add_to_cgroup\ + .assert_called_with(self.driver, + VOLUME_WITH_CGROUP['consistencygroup_id'], + vol_name) + def test_create_cloned_volume(self): vol_name = VOLUME["name"] + "-cinder" src_name = SRC_VOL["name"] + "-cinder" @@ -199,6 +249,18 @@ class PureISCSIDriverTestCase(test.TestCase): self.driver.create_cloned_volume, VOLUME, SRC_VOL) SRC_VOL["size"] = 2 # reset size + @mock.patch(DRIVER_OBJ + "._add_volume_to_consistency_group", + autospec=True) + def test_create_cloned_volume_with_cgroup(self, mock_add_to_cgroup): + vol_name = VOLUME_WITH_CGROUP["name"] + "-cinder" + + self.driver.create_cloned_volume(VOLUME_WITH_CGROUP, SRC_VOL) + + mock_add_to_cgroup\ + .assert_called_with(self.driver, + VOLUME_WITH_CGROUP['consistencygroup_id'], + vol_name) + def test_delete_volume_already_deleted(self): self.array.list_volume_hosts.side_effect = exception.PureAPIException( code=400, reason="Volume does not exist") @@ -468,6 +530,7 @@ class PureISCSIDriverTestCase(test.TestCase): "total_capacity_gb": TOTAL_SPACE, "free_capacity_gb": FREE_SPACE, "reserved_percentage": 0, + "consistencygroup_support": True } real_result = self.driver.get_volume_stats(refresh=True) self.assertDictMatch(result, real_result) @@ -480,6 +543,210 @@ class PureISCSIDriverTestCase(test.TestCase): self.assert_error_propagates([self.array.extend_volume], self.driver.extend_volume, VOLUME, 3) + def test_get_pgroup_name_from_id(self): + id = "4a2f7e3a-312a-40c5-96a8-536b8a0fe074" + expected_name = "consisgroup-%s-cinder" % id + actual_name = pure._get_pgroup_name_from_id(id) + self.assertEqual(expected_name, actual_name) + + def test_get_pgroup_snap_suffix(self): + cgsnap = mock.Mock() + cgsnap.id = "4a2f7e3a-312a-40c5-96a8-536b8a0fe074" + expected_suffix = "cgsnapshot-%s-cinder" % cgsnap.id + actual_suffix = pure._get_pgroup_snap_suffix(cgsnap) + self.assertEqual(expected_suffix, actual_suffix) + + def test_get_pgroup_snap_name(self): + cg_id = "4a2f7e3a-312a-40c5-96a8-536b8a0fe074" + cgsnap_id = "4a2f7e3a-312a-40c5-96a8-536b8a0fe075" + + mock_cgsnap = mock.Mock() + mock_cgsnap.consistencygroup_id = cg_id + mock_cgsnap.id = cgsnap_id + expected_name = "consisgroup-%(cg)s-cinder.cgsnapshot-%(snap)s-cinder"\ + % {"cg": cg_id, "snap": cgsnap_id} + + actual_name = pure._get_pgroup_snap_name(mock_cgsnap) + + self.assertEqual(expected_name, actual_name) + + def test_get_pgroup_vol_snap_name(self): + cg_id = "4a2f7e3a-312a-40c5-96a8-536b8a0fe074" + cgsnap_id = "4a2f7e3a-312a-40c5-96a8-536b8a0fe075" + volume_name = "volume-4a2f7e3a-312a-40c5-96a8-536b8a0fe075" + + mock_snap = mock.Mock() + mock_snap.cgsnapshot = mock.Mock() + mock_snap.cgsnapshot.consistencygroup_id = cg_id + mock_snap.cgsnapshot.id = cgsnap_id + mock_snap.volume_name = volume_name + + expected_name = "consisgroup-%(cg)s-cinder.cgsnapshot-%(snap)s-cinder"\ + ".%(vol)s-cinder" % {"cg": cg_id, + "snap": cgsnap_id, + "vol": volume_name} + + actual_name = pure._get_pgroup_vol_snap_name(mock_snap) + + self.assertEqual(expected_name, actual_name) + + def test_create_consistencygroup(self): + mock_cgroup = mock.Mock() + mock_cgroup.id = "4a2f7e3a-312a-40c5-96a8-536b8a0fe074" + + model_update = self.driver.create_consistencygroup(None, mock_cgroup) + + expected_name = pure._get_pgroup_name_from_id(mock_cgroup.id) + self.array.create_pgroup.assert_called_with(expected_name) + self.assertEqual({'status': 'available'}, model_update) + + self.assert_error_propagates( + [self.array.create_pgroup], + self.driver.create_consistencygroup, None, mock_cgroup) + + @mock.patch(DRIVER_OBJ + ".delete_volume", autospec=True) + def test_delete_consistencygroup(self, mock_delete_volume): + mock_cgroup = mock.MagicMock() + mock_cgroup.id = "4a2f7e3a-312a-40c5-96a8-536b8a0fe074" + mock_cgroup['status'] = "deleted" + mock_context = mock.Mock() + self.driver.db = mock.Mock() + mock_volume = mock.MagicMock() + expected_volumes = [mock_volume] + self.driver.db.volume_get_all_by_group.return_value = expected_volumes + + model_update, volumes = \ + self.driver.delete_consistencygroup(mock_context, mock_cgroup) + + expected_name = pure._get_pgroup_name_from_id(mock_cgroup.id) + self.array.delete_pgroup.assert_called_with(expected_name) + self.assertEqual(expected_volumes, volumes) + self.assertEqual(mock_cgroup['status'], model_update['status']) + mock_delete_volume.assert_called_with(self.driver, mock_volume) + + self.array.delete_pgroup.side_effect = exception.PureAPIException( + code=400, reason="Protection group has been destroyed.") + self.driver.delete_consistencygroup(mock_context, mock_cgroup) + self.array.delete_pgroup.assert_called_with(expected_name) + mock_delete_volume.assert_called_with(self.driver, mock_volume) + + self.array.delete_pgroup.side_effect = exception.PureAPIException( + code=400, reason="Protection group does not exist") + self.driver.delete_consistencygroup(mock_context, mock_cgroup) + self.array.delete_pgroup.assert_called_with(expected_name) + mock_delete_volume.assert_called_with(self.driver, mock_volume) + + self.array.delete_pgroup.side_effect = exception.PureAPIException( + code=400, reason="Some other error") + self.assertRaises(exception.PureAPIException, + self.driver.delete_consistencygroup, + mock_context, + mock_volume) + + self.array.delete_pgroup.side_effect = exception.PureAPIException( + code=500, reason="Another different error") + self.assertRaises(exception.PureAPIException, + self.driver.delete_consistencygroup, + mock_context, + mock_volume) + + self.array.delete_pgroup.side_effect = None + self.assert_error_propagates( + [self.array.delete_pgroup], + self.driver.delete_consistencygroup, mock_context, mock_cgroup) + + def test_create_cgsnapshot(self): + mock_cgsnap = mock.Mock() + mock_cgsnap.id = "4a2f7e3a-312a-40c5-96a8-536b8a0fe074" + mock_cgsnap.consistencygroup_id = \ + "4a2f7e3a-312a-40c5-96a8-536b8a0fe075" + mock_context = mock.Mock() + self.driver.db = mock.Mock() + mock_snap = mock.MagicMock() + expected_snaps = [mock_snap] + self.driver.db.snapshot_get_all_for_cgsnapshot.return_value = \ + expected_snaps + + model_update, snapshots = \ + self.driver.create_cgsnapshot(mock_context, mock_cgsnap) + + expected_pgroup_name = \ + pure._get_pgroup_name_from_id(mock_cgsnap.consistencygroup_id) + expected_snap_suffix = pure._get_pgroup_snap_suffix(mock_cgsnap) + self.array.create_pgroup_snapshot\ + .assert_called_with(expected_pgroup_name, expected_snap_suffix) + self.assertEqual({'status': 'available'}, model_update) + self.assertEqual(expected_snaps, snapshots) + self.assertEqual('available', mock_snap.status) + + self.assert_error_propagates( + [self.array.create_pgroup_snapshot], + self.driver.create_cgsnapshot, mock_context, mock_cgsnap) + + @mock.patch(DRIVER_PATH + "._get_pgroup_snap_name", autospec=True) + def test_delete_cgsnapshot(self, mock_get_snap_name): + snap_name = "consisgroup-4a2f7e3a-312a-40c5-96a8-536b8a0f" \ + "e074-cinder.4a2f7e3a-312a-40c5-96a8-536b8a0fe075" + mock_get_snap_name.return_value = snap_name + mock_cgsnap = mock.Mock() + mock_cgsnap.status = 'deleted' + mock_context = mock.Mock() + mock_snap = mock.MagicMock() + expected_snaps = [mock_snap] + self.driver.db = mock.Mock() + self.driver.db.snapshot_get_all_for_cgsnapshot.return_value = \ + expected_snaps + + model_update, snapshots = \ + self.driver.delete_cgsnapshot(mock_context, mock_cgsnap) + + self.array.delete_pgroup_snapshot.assert_called_with(snap_name) + self.assertEqual({'status': mock_cgsnap.status}, model_update) + self.assertEqual(expected_snaps, snapshots) + self.assertEqual('deleted', mock_snap.status) + + self.array.delete_pgroup_snapshot.side_effect = \ + exception.PureAPIException( + code=400, + reason="Protection group snapshot has been destroyed." + ) + self.driver.delete_cgsnapshot(mock_context, mock_cgsnap) + self.array.delete_pgroup_snapshot.assert_called_with(snap_name) + + self.array.delete_pgroup_snapshot.side_effect = \ + exception.PureAPIException( + code=400, + reason="Protection group snapshot does not exist" + ) + self.driver.delete_cgsnapshot(mock_context, mock_cgsnap) + self.array.delete_pgroup_snapshot.assert_called_with(snap_name) + + self.array.delete_pgroup_snapshot.side_effect = \ + exception.PureAPIException( + code=400, + reason="Some other error" + ) + self.assertRaises(exception.PureAPIException, + self.driver.delete_cgsnapshot, + mock_context, + mock_cgsnap) + + self.array.delete_pgroup_snapshot.side_effect = \ + exception.PureAPIException( + code=500, + reason="Another different error" + ) + self.assertRaises(exception.PureAPIException, + self.driver.delete_cgsnapshot, + mock_context, + mock_cgsnap) + + self.array.delete_pgroup_snapshot.side_effect = None + + self.assert_error_propagates( + [self.array.delete_pgroup_snapshot], + self.driver.delete_cgsnapshot, mock_context, mock_cgsnap) + class FlashArrayBaseTestCase(test.TestCase): diff --git a/cinder/volume/drivers/pure.py b/cinder/volume/drivers/pure.py index 31dfe760e..d8235d06c 100644 --- a/cinder/volume/drivers/pure.py +++ b/cinder/volume/drivers/pure.py @@ -48,6 +48,9 @@ CONF.register_opts(PURE_OPTS) INVALID_CHARACTERS = re.compile(r"[^-a-zA-Z0-9]") GENERATED_NAME = re.compile(r".*-[a-f0-9]{32}-cinder$") +ERR_MSG_NOT_EXIST = "does not exist" +ERR_MSG_PENDING_ERADICATION = "has been destroyed" + def _get_vol_name(volume): """Return the name of the volume Purity will use.""" @@ -59,6 +62,28 @@ def _get_snap_name(snapshot): return "%s-cinder.%s" % (snapshot["volume_name"], snapshot["name"]) +def _get_pgroup_name_from_id(id): + return "consisgroup-%s-cinder" % id + + +def _get_pgroup_snap_suffix(cgsnapshot): + return "cgsnapshot-%s-cinder" % cgsnapshot.id + + +def _get_pgroup_snap_name(cgsnapshot): + """Return the name of the pgroup snapshot that Purity will use""" + return "%s.%s" % (_get_pgroup_name_from_id(cgsnapshot.consistencygroup_id), + _get_pgroup_snap_suffix(cgsnapshot)) + + +def _get_pgroup_vol_snap_name(snapshot): + """Return the name of the snapshot that Purity will use for a volume.""" + cg_name = _get_pgroup_name_from_id(snapshot.cgsnapshot.consistencygroup_id) + cgsnapshot_id = _get_pgroup_snap_suffix(snapshot.cgsnapshot) + volume_name = snapshot.volume_name + return "%s.%s.%s-cinder" % (cg_name, cgsnapshot_id, volume_name) + + def _generate_purity_host_name(name): """Return a valid Purity host name based on the name passed in.""" if len(name) > 23: @@ -71,7 +96,7 @@ def _generate_purity_host_name(name): class PureISCSIDriver(san.SanISCSIDriver): """Performs volume management on Pure Storage FlashArray.""" - VERSION = "2.0.0" + VERSION = "2.0.1" def __init__(self, *args, **kwargs): execute = kwargs.pop("execute", utils.execute) @@ -102,16 +127,31 @@ class PureISCSIDriver(san.SanISCSIDriver): vol_name = _get_vol_name(volume) vol_size = volume["size"] * units.Gi self._array.create_volume(vol_name, vol_size) + + if volume['consistencygroup_id']: + self._add_volume_to_consistency_group( + volume['consistencygroup_id'], + vol_name + ) LOG.debug("Leave PureISCSIDriver.create_volume.") def create_volume_from_snapshot(self, volume, snapshot): """Creates a volume from a snapshot.""" LOG.debug("Enter PureISCSIDriver.create_volume_from_snapshot.") vol_name = _get_vol_name(volume) - snap_name = _get_snap_name(snapshot) + if snapshot['cgsnapshot_id']: + snap_name = _get_pgroup_vol_snap_name(snapshot) + else: + snap_name = _get_snap_name(snapshot) + self._array.copy_volume(snap_name, vol_name) self._extend_if_needed(vol_name, snapshot["volume_size"], volume["size"]) + if volume['consistencygroup_id']: + self._add_volume_to_consistency_group( + volume['consistencygroup_id'], + vol_name + ) LOG.debug("Leave PureISCSIDriver.create_volume_from_snapshot.") def create_cloned_volume(self, volume, src_vref): @@ -121,6 +161,13 @@ class PureISCSIDriver(san.SanISCSIDriver): src_name = _get_vol_name(src_vref) self._array.copy_volume(src_name, vol_name) self._extend_if_needed(vol_name, src_vref["size"], volume["size"]) + + if volume['consistencygroup_id']: + self._add_volume_to_consistency_group( + volume['consistencygroup_id'], + vol_name + ) + LOG.debug("Leave PureISCSIDriver.create_cloned_volume.") def _extend_if_needed(self, vol_name, src_size, vol_size): @@ -142,7 +189,7 @@ class PureISCSIDriver(san.SanISCSIDriver): except exception.PureAPIException as err: with excutils.save_and_reraise_exception() as ctxt: if err.kwargs["code"] == 400 and \ - "Volume does not exist" in err.msg: + ERR_MSG_NOT_EXIST in err.msg: # Happens if the volume does not exist. ctxt.reraise = False LOG.warn(_LW("Volume deletion failed with message: %s") @@ -329,6 +376,7 @@ class PureISCSIDriver(san.SanISCSIDriver): "total_capacity_gb": total, "free_capacity_gb": free, "reserved_percentage": 0, + "consistencygroup_support": True } self._stats = data @@ -340,6 +388,98 @@ class PureISCSIDriver(san.SanISCSIDriver): self._array.extend_volume(vol_name, new_size) LOG.debug("Leave PureISCSIDriver.extend_volume.") + def _add_volume_to_consistency_group(self, consistencygroup_id, vol_name): + pgroup_name = _get_pgroup_name_from_id(consistencygroup_id) + self._array.add_volume_to_pgroup(pgroup_name, vol_name) + + def create_consistencygroup(self, context, group): + """Creates a consistencygroup.""" + LOG.debug("Enter PureISCSIDriver.create_consistencygroup") + + self._array.create_pgroup(_get_pgroup_name_from_id(group.id)) + + model_update = {'status': 'available'} + + LOG.debug("Leave PureISCSIDriver.create_consistencygroup") + return model_update + + def delete_consistencygroup(self, context, group): + """Deletes a consistency group.""" + LOG.debug("Enter PureISCSIDriver.delete_consistencygroup") + + try: + self._array.delete_pgroup(_get_pgroup_name_from_id(group.id)) + except exception.PureAPIException as err: + with excutils.save_and_reraise_exception() as ctxt: + if (err.kwargs["code"] == 400 and + (ERR_MSG_PENDING_ERADICATION in err.msg or + ERR_MSG_NOT_EXIST in err.msg)): + # Treat these as a "success" case since we are trying + # to delete them anyway. + ctxt.reraise = False + LOG.warning(_LW("Unable to delete Protection Group: %s"), + err.msg) + + volumes = self.db.volume_get_all_by_group(context, group.id) + + for volume in volumes: + self.delete_volume(volume) + volume.status = 'deleted' + + model_update = {'status': group['status']} + + LOG.debug("Leave PureISCSIDriver.delete_consistencygroup") + return model_update, volumes + + def create_cgsnapshot(self, context, cgsnapshot): + """Creates a cgsnapshot.""" + LOG.debug("Enter PureISCSIDriver.create_cgsnapshot") + + pgroup_name = _get_pgroup_name_from_id(cgsnapshot.consistencygroup_id) + pgsnap_suffix = _get_pgroup_snap_suffix(cgsnapshot) + self._array.create_pgroup_snapshot(pgroup_name, pgsnap_suffix) + + snapshots = self.db.snapshot_get_all_for_cgsnapshot( + context, cgsnapshot.id) + + for snapshot in snapshots: + snapshot.status = 'available' + + model_update = {'status': 'available'} + + LOG.debug("Leave PureISCSIDriver.create_cgsnapshot") + return model_update, snapshots + + def delete_cgsnapshot(self, context, cgsnapshot): + """Deletes a cgsnapshot.""" + LOG.debug("Enter PureISCSIDriver.delete_cgsnapshot") + + pgsnap_name = _get_pgroup_snap_name(cgsnapshot) + + try: + self._array.delete_pgroup_snapshot(pgsnap_name) + except exception.PureAPIException as err: + with excutils.save_and_reraise_exception() as ctxt: + if (err.kwargs["code"] == 400 and + (ERR_MSG_PENDING_ERADICATION in err.msg or + ERR_MSG_NOT_EXIST in err.msg)): + # Treat these as a "success" case since we are trying + # to delete them anyway. + ctxt.reraise = False + LOG.warning(_LW("Unable to delete Protection Group " + "Snapshot: %s"), err.msg) + + snapshots = self.db.snapshot_get_all_for_cgsnapshot( + context, cgsnapshot.id) + + for snapshot in snapshots: + snapshot.status = 'deleted' + + model_update = {'status': cgsnapshot.status} + + LOG.debug("Leave PureISCSIDriver.delete_cgsnapshot") + return model_update, snapshots + class FlashArray(object): """Wrapper for Pure Storage REST API.""" -- 2.45.2