VNX: Fix issue in deleting cg/cgsnapshot

If user deletes a cg/cgsnapshot quickly after its creation,
driver would report 'not found' warning, and return success
for the deletion, but actually, the object underlying the VNX
is not deleted.

This fix tries to wait the cg and cg snapshot to be available
to make sure it's eligible for deletion.

liberty-backport-potential
Closes-Bug: 1499615
Change-Id: Ifae57b9e95e01b1789a37ac7c03e9aad65cd50f7

diff --git a/cinder/tests/unit/test_emc_vnxdirect.py b/cinder/tests/unit/test_emc_vnxdirect.py
index 8e23887d0592ee1c850bb9f0651e7d8504d09f4c..76b3407267bd00a3be4f0df3efd89a88026e92cb 100644 (file)
--- a/cinder/tests/unit/test_emc_vnxdirect.py
+++ b/cinder/tests/unit/test_emc_vnxdirect.py
@@ -477,6 +477,10 @@ class EMCVNXCLIDriverTestData(object):
             output += out
         return (output, 0)
 
+    def SNAP_NOT_EXIST(self):
+        return ("Could not retrieve the specified (Snapshot).\n "
+                "The (Snapshot) may not exist", 9)
+
     NDU_LIST_CMD = ('ndu', '-list')
     NDU_LIST_RESULT = ("Name of the software package:   -Compression " +
                        "Name of the software package:   -Deduplication " +
@@ -740,6 +744,9 @@ class EMCVNXCLIDriverTestData(object):
     def GET_CG_BY_NAME_CMD(self, cg_name):
         return ('snap', '-group', '-list', '-id', cg_name)
 
+    def GET_SNAP(self, snap_name):
+        return ('snap', '-list', '-id', snap_name)
+
     def REMOVE_LUNS_FROM_CG_CMD(self, cg_name, remove_ids):
         return ('snap', '-group', '-rmmember', '-id', cg_name, '-res',
                 ','.join(remove_ids))
@@ -781,6 +788,9 @@ Member LUN ID(s):  1, 3
 State:  Ready
 """ % {'cg_name': cg_name}, 0
 
+    def CG_NOT_FOUND(self):
+        return ("Cannot find the consistency group. \n\n", 13)
+
     def CG_REPL_ERROR(self):
         return """
         The specified LUN is already a member
@@ -3834,8 +3844,9 @@ Time Remaining:  0 second(s)
 
     def test_create_consistency_group(self):
         cg_name = self.testData.test_cg['id']
-        commands = [self.testData.CREATE_CONSISTENCYGROUP_CMD(cg_name)]
-        results = [SUCCEED]
+        commands = [self.testData.CREATE_CONSISTENCYGROUP_CMD(cg_name),
+                    self.testData.GET_CG_BY_NAME_CMD(cg_name)]
+        results = [SUCCEED, self.testData.CG_PROPERTY(cg_name)]
         fake_cli = self.driverSetup(commands, results)
 
         model_update = self.driver.create_consistencygroup(
@@ -3844,7 +3855,30 @@ Time Remaining:  0 second(s)
         expect_cmd = [
             mock.call(
                 *self.testData.CREATE_CONSISTENCYGROUP_CMD(
-                    cg_name), poll=False)]
+                    cg_name), poll=False),
+            mock.call(
+                *self.testData.GET_CG_BY_NAME_CMD(cg_name))]
+        fake_cli.assert_has_calls(expect_cmd)
+
+    def test_create_consistency_group_retry(self):
+        cg_name = self.testData.test_cg['id']
+        commands = [self.testData.CREATE_CONSISTENCYGROUP_CMD(cg_name),
+                    self.testData.GET_CG_BY_NAME_CMD(cg_name)]
+        results = [SUCCEED,
+                   [self.testData.CG_NOT_FOUND(),
+                    self.testData.CG_PROPERTY(cg_name)]]
+        fake_cli = self.driverSetup(commands, results)
+        model_update = self.driver.create_consistencygroup(
+            None, self.testData.test_cg)
+        self.assertDictMatch({'status': 'available'}, model_update)
+        expect_cmd = [
+            mock.call(
+                *self.testData.CREATE_CONSISTENCYGROUP_CMD(
+                    cg_name), poll=False),
+            mock.call(
+                *self.testData.GET_CG_BY_NAME_CMD(cg_name)),
+            mock.call(
+                *self.testData.GET_CG_BY_NAME_CMD(cg_name))]
         fake_cli.assert_has_calls(expect_cmd)
 
     @mock.patch(
@@ -3882,8 +3916,10 @@ Time Remaining:  0 second(s)
     def test_create_cgsnapshot(self):
         cgsnapshot = self.testData.test_cgsnapshot['id']
         cg_name = self.testData.test_cgsnapshot['consistencygroup_id']
-        commands = [self.testData.CREATE_CG_SNAPSHOT(cg_name, cgsnapshot)]
-        results = [SUCCEED]
+        commands = [self.testData.CREATE_CG_SNAPSHOT(cg_name, cgsnapshot),
+                    self.testData.GET_SNAP(cgsnapshot)]
+        results = [SUCCEED,
+                   SUCCEED]
         fake_cli = self.driverSetup(commands, results)
         snapshot_obj = fake_snapshot.fake_snapshot_obj(
             self.testData.SNAPS_IN_SNAP_GROUP())
@@ -3893,7 +3929,32 @@ Time Remaining:  0 second(s)
         expect_cmd = [
             mock.call(
                 *self.testData.CREATE_CG_SNAPSHOT(
-                    cg_name, cgsnapshot))]
+                    cg_name, cgsnapshot)),
+            mock.call(
+                *self.testData.GET_SNAP(cgsnapshot))]
+        fake_cli.assert_has_calls(expect_cmd)
+
+    def test_create_cgsnapshot_retry(self):
+        cgsnapshot = self.testData.test_cgsnapshot['id']
+        cg_name = self.testData.test_cgsnapshot['consistencygroup_id']
+        commands = [self.testData.CREATE_CG_SNAPSHOT(cg_name, cgsnapshot),
+                    self.testData.GET_SNAP(cgsnapshot)]
+        results = [SUCCEED,
+                   [self.testData.SNAP_NOT_EXIST(), SUCCEED]]
+        fake_cli = self.driverSetup(commands, results)
+        snapshot_obj = fake_snapshot.fake_snapshot_obj(
+            self.testData.SNAPS_IN_SNAP_GROUP())
+        snapshot_obj.consistencygroup_id = cg_name
+        self.driver.create_cgsnapshot(None, self.testData.test_cgsnapshot,
+                                      [snapshot_obj])
+        expect_cmd = [
+            mock.call(
+                *self.testData.CREATE_CG_SNAPSHOT(
+                    cg_name, cgsnapshot)),
+            mock.call(
+                *self.testData.GET_SNAP(cgsnapshot)),
+            mock.call(
+                *self.testData.GET_SNAP(cgsnapshot))]
         fake_cli.assert_has_calls(expect_cmd)
 
     def test_delete_cgsnapshot(self):
@@ -3961,6 +4022,8 @@ Time Remaining:  0 second(s)
         expect_cmd = [
             mock.call(
                 *self.testData.CREATE_CG_SNAPSHOT(cg_name, tmp_cgsnapshot)),
+            mock.call(
+                *self.testData.GET_SNAP(tmp_cgsnapshot)),
             mock.call(*self.testData.SNAP_MP_CREATE_CMD(name='vol1',
                                                         source='clone1'),
                       poll=False),
@@ -4219,6 +4282,7 @@ Time Remaining:  0 second(s)
             mock.call(*td.MIGRATION_VERIFY_CMD(6232), poll=True),
             mock.call(*td.CREATE_CONSISTENCYGROUP_CMD(
                       new_cg['id'], [6231, 6232]), poll=True),
+            mock.call(*td.GET_CG_BY_NAME_CMD(new_cg.id)),
             mock.call(*td.DELETE_CG_SNAPSHOT(copied_snap_name))]
         self.assertEqual(expect_cmd, fake_cli.call_args_list)
 

diff --git a/cinder/volume/drivers/emc/emc_vnx_cli.py b/cinder/volume/drivers/emc/emc_vnx_cli.py
index 8614921c107362f866c757d124c758a7088e74e6..e49cea9fa1f4ef75c4910276abbff28e4635d3f9 100644 (file)
--- a/cinder/volume/drivers/emc/emc_vnx_cli.py
+++ b/cinder/volume/drivers/emc/emc_vnx_cli.py
@@ -835,9 +835,8 @@ class CommandLineHelper(object):
             return _lun_state_validation(data)
 
         self._wait_for_a_condition(lun_is_ready,
-                                   None,
-                                   INTERVAL_5_SEC,
-                                   lambda ex:
+                                   interval=INTERVAL_5_SEC,
+                                   ignorable_exception_arbiter=lambda ex:
                                    isinstance(ex, exception.EMCVnxCLICmdError))
         lun = self.get_lun_by_name(name, VNXLunProperties.lun_all, False)
         return lun
@@ -884,14 +883,15 @@ class CommandLineHelper(object):
 
     def _wait_for_a_condition(self, testmethod, timeout=None,
                               interval=INTERVAL_5_SEC,
-                              ignorable_exception_arbiter=lambda ex: True):
+                              ignorable_exception_arbiter=lambda ex: True,
+                              *args, **kwargs):
         start_time = time.time()
         if timeout is None:
             timeout = self.timeout
 
         def _inner():
             try:
-                test_value = testmethod()
+                test_value = testmethod(*args, **kwargs)
             except Exception as ex:
                 test_value = False
                 with excutils.save_and_reraise_exception(
@@ -988,6 +988,12 @@ class CommandLineHelper(object):
             else:
                 self._raise_cli_error(command_create_cg, rc, out)
 
+        self._wait_for_a_condition(self.get_consistency_group_by_name,
+                                   cg_name=cg_name,
+                                   interval=INTERVAL_5_SEC,
+                                   ignorable_exception_arbiter=lambda ex:
+                                   isinstance(ex, exception.EMCVnxCLICmdError))
+
     def get_consistency_group_by_name(self, cg_name):
         cmd = ('snap', '-group', '-list', '-id', cg_name)
         data = {
@@ -1011,6 +1017,8 @@ class CommandLineHelper(object):
                                 if luns_of_cg else [])
                 LOG.debug("Found consistent group %s.", data['Name'])
 
+        else:
+            self._raise_cli_error(cmd, rc, out)
         return data
 
     def add_lun_to_consistency_group(self, cg_name, lun_id, poll=False):
@@ -1089,6 +1097,20 @@ class CommandLineHelper(object):
                             {'name': snap_name, 'msg': out})
             else:
                 self._raise_cli_error(create_cg_snap_cmd, rc, out)
+        self._wait_for_a_condition(self.check_snapshot,
+                                   snap_name=snap_name,
+                                   interval=INTERVAL_30_SEC,
+                                   ignorable_exception_arbiter=lambda ex:
+                                   isinstance(ex, exception.EMCVnxCLICmdError))
+
+    def check_snapshot(self, snap_name, poll=True):
+        """check if a snapshot/cgsnapshot is existed."""
+        cmd_get = ('snap', '-list', '-id', snap_name)
+        out, rc = self.command_execute(*cmd_get)
+        if rc == 0:
+            return True
+        else:
+            self._raise_cli_error(cmd_get, rc, out)
 
     def delete_cgsnapshot(self, snap_name):
         """Delete a cgsnapshot (snap group)."""