From: peter_wang Date: Tue, 8 Dec 2015 08:18:31 +0000 (-0500) Subject: VNX: Fix issue in deleting cg/cgsnapshot X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=f7b5655d188bfde96333760bbe634f45d4a16395;p=openstack-build%2Fcinder-build.git VNX: Fix issue in deleting cg/cgsnapshot If user deletes a cg/cgsnapshot quickly after its creation, driver would report 'not found' warning, and return success for the deletion, but actually, the object underlying the VNX is not deleted. This fix tries to wait the cg and cg snapshot to be available to make sure it's eligible for deletion. liberty-backport-potential Closes-Bug: 1499615 Change-Id: Ifae57b9e95e01b1789a37ac7c03e9aad65cd50f7 --- diff --git a/cinder/tests/unit/test_emc_vnxdirect.py b/cinder/tests/unit/test_emc_vnxdirect.py index 8e23887d0..76b340726 100644 --- a/cinder/tests/unit/test_emc_vnxdirect.py +++ b/cinder/tests/unit/test_emc_vnxdirect.py @@ -477,6 +477,10 @@ class EMCVNXCLIDriverTestData(object): output += out return (output, 0) + def SNAP_NOT_EXIST(self): + return ("Could not retrieve the specified (Snapshot).\n " + "The (Snapshot) may not exist", 9) + NDU_LIST_CMD = ('ndu', '-list') NDU_LIST_RESULT = ("Name of the software package: -Compression " + "Name of the software package: -Deduplication " + @@ -740,6 +744,9 @@ class EMCVNXCLIDriverTestData(object): def GET_CG_BY_NAME_CMD(self, cg_name): return ('snap', '-group', '-list', '-id', cg_name) + def GET_SNAP(self, snap_name): + return ('snap', '-list', '-id', snap_name) + def REMOVE_LUNS_FROM_CG_CMD(self, cg_name, remove_ids): return ('snap', '-group', '-rmmember', '-id', cg_name, '-res', ','.join(remove_ids)) @@ -781,6 +788,9 @@ Member LUN ID(s): 1, 3 State: Ready """ % {'cg_name': cg_name}, 0 + def CG_NOT_FOUND(self): + return ("Cannot find the consistency group. \n\n", 13) + def CG_REPL_ERROR(self): return """ The specified LUN is already a member @@ -3834,8 +3844,9 @@ Time Remaining: 0 second(s) def test_create_consistency_group(self): cg_name = self.testData.test_cg['id'] - commands = [self.testData.CREATE_CONSISTENCYGROUP_CMD(cg_name)] - results = [SUCCEED] + commands = [self.testData.CREATE_CONSISTENCYGROUP_CMD(cg_name), + self.testData.GET_CG_BY_NAME_CMD(cg_name)] + results = [SUCCEED, self.testData.CG_PROPERTY(cg_name)] fake_cli = self.driverSetup(commands, results) model_update = self.driver.create_consistencygroup( @@ -3844,7 +3855,30 @@ Time Remaining: 0 second(s) expect_cmd = [ mock.call( *self.testData.CREATE_CONSISTENCYGROUP_CMD( - cg_name), poll=False)] + cg_name), poll=False), + mock.call( + *self.testData.GET_CG_BY_NAME_CMD(cg_name))] + fake_cli.assert_has_calls(expect_cmd) + + def test_create_consistency_group_retry(self): + cg_name = self.testData.test_cg['id'] + commands = [self.testData.CREATE_CONSISTENCYGROUP_CMD(cg_name), + self.testData.GET_CG_BY_NAME_CMD(cg_name)] + results = [SUCCEED, + [self.testData.CG_NOT_FOUND(), + self.testData.CG_PROPERTY(cg_name)]] + fake_cli = self.driverSetup(commands, results) + model_update = self.driver.create_consistencygroup( + None, self.testData.test_cg) + self.assertDictMatch({'status': 'available'}, model_update) + expect_cmd = [ + mock.call( + *self.testData.CREATE_CONSISTENCYGROUP_CMD( + cg_name), poll=False), + mock.call( + *self.testData.GET_CG_BY_NAME_CMD(cg_name)), + mock.call( + *self.testData.GET_CG_BY_NAME_CMD(cg_name))] fake_cli.assert_has_calls(expect_cmd) @mock.patch( @@ -3882,8 +3916,10 @@ Time Remaining: 0 second(s) def test_create_cgsnapshot(self): cgsnapshot = self.testData.test_cgsnapshot['id'] cg_name = self.testData.test_cgsnapshot['consistencygroup_id'] - commands = [self.testData.CREATE_CG_SNAPSHOT(cg_name, cgsnapshot)] - results = [SUCCEED] + commands = [self.testData.CREATE_CG_SNAPSHOT(cg_name, cgsnapshot), + self.testData.GET_SNAP(cgsnapshot)] + results = [SUCCEED, + SUCCEED] fake_cli = self.driverSetup(commands, results) snapshot_obj = fake_snapshot.fake_snapshot_obj( self.testData.SNAPS_IN_SNAP_GROUP()) @@ -3893,7 +3929,32 @@ Time Remaining: 0 second(s) expect_cmd = [ mock.call( *self.testData.CREATE_CG_SNAPSHOT( - cg_name, cgsnapshot))] + cg_name, cgsnapshot)), + mock.call( + *self.testData.GET_SNAP(cgsnapshot))] + fake_cli.assert_has_calls(expect_cmd) + + def test_create_cgsnapshot_retry(self): + cgsnapshot = self.testData.test_cgsnapshot['id'] + cg_name = self.testData.test_cgsnapshot['consistencygroup_id'] + commands = [self.testData.CREATE_CG_SNAPSHOT(cg_name, cgsnapshot), + self.testData.GET_SNAP(cgsnapshot)] + results = [SUCCEED, + [self.testData.SNAP_NOT_EXIST(), SUCCEED]] + fake_cli = self.driverSetup(commands, results) + snapshot_obj = fake_snapshot.fake_snapshot_obj( + self.testData.SNAPS_IN_SNAP_GROUP()) + snapshot_obj.consistencygroup_id = cg_name + self.driver.create_cgsnapshot(None, self.testData.test_cgsnapshot, + [snapshot_obj]) + expect_cmd = [ + mock.call( + *self.testData.CREATE_CG_SNAPSHOT( + cg_name, cgsnapshot)), + mock.call( + *self.testData.GET_SNAP(cgsnapshot)), + mock.call( + *self.testData.GET_SNAP(cgsnapshot))] fake_cli.assert_has_calls(expect_cmd) def test_delete_cgsnapshot(self): @@ -3961,6 +4022,8 @@ Time Remaining: 0 second(s) expect_cmd = [ mock.call( *self.testData.CREATE_CG_SNAPSHOT(cg_name, tmp_cgsnapshot)), + mock.call( + *self.testData.GET_SNAP(tmp_cgsnapshot)), mock.call(*self.testData.SNAP_MP_CREATE_CMD(name='vol1', source='clone1'), poll=False), @@ -4219,6 +4282,7 @@ Time Remaining: 0 second(s) mock.call(*td.MIGRATION_VERIFY_CMD(6232), poll=True), mock.call(*td.CREATE_CONSISTENCYGROUP_CMD( new_cg['id'], [6231, 6232]), poll=True), + mock.call(*td.GET_CG_BY_NAME_CMD(new_cg.id)), mock.call(*td.DELETE_CG_SNAPSHOT(copied_snap_name))] self.assertEqual(expect_cmd, fake_cli.call_args_list) diff --git a/cinder/volume/drivers/emc/emc_vnx_cli.py b/cinder/volume/drivers/emc/emc_vnx_cli.py index 8614921c1..e49cea9fa 100644 --- a/cinder/volume/drivers/emc/emc_vnx_cli.py +++ b/cinder/volume/drivers/emc/emc_vnx_cli.py @@ -835,9 +835,8 @@ class CommandLineHelper(object): return _lun_state_validation(data) self._wait_for_a_condition(lun_is_ready, - None, - INTERVAL_5_SEC, - lambda ex: + interval=INTERVAL_5_SEC, + ignorable_exception_arbiter=lambda ex: isinstance(ex, exception.EMCVnxCLICmdError)) lun = self.get_lun_by_name(name, VNXLunProperties.lun_all, False) return lun @@ -884,14 +883,15 @@ class CommandLineHelper(object): def _wait_for_a_condition(self, testmethod, timeout=None, interval=INTERVAL_5_SEC, - ignorable_exception_arbiter=lambda ex: True): + ignorable_exception_arbiter=lambda ex: True, + *args, **kwargs): start_time = time.time() if timeout is None: timeout = self.timeout def _inner(): try: - test_value = testmethod() + test_value = testmethod(*args, **kwargs) except Exception as ex: test_value = False with excutils.save_and_reraise_exception( @@ -988,6 +988,12 @@ class CommandLineHelper(object): else: self._raise_cli_error(command_create_cg, rc, out) + self._wait_for_a_condition(self.get_consistency_group_by_name, + cg_name=cg_name, + interval=INTERVAL_5_SEC, + ignorable_exception_arbiter=lambda ex: + isinstance(ex, exception.EMCVnxCLICmdError)) + def get_consistency_group_by_name(self, cg_name): cmd = ('snap', '-group', '-list', '-id', cg_name) data = { @@ -1011,6 +1017,8 @@ class CommandLineHelper(object): if luns_of_cg else []) LOG.debug("Found consistent group %s.", data['Name']) + else: + self._raise_cli_error(cmd, rc, out) return data def add_lun_to_consistency_group(self, cg_name, lun_id, poll=False): @@ -1089,6 +1097,20 @@ class CommandLineHelper(object): {'name': snap_name, 'msg': out}) else: self._raise_cli_error(create_cg_snap_cmd, rc, out) + self._wait_for_a_condition(self.check_snapshot, + snap_name=snap_name, + interval=INTERVAL_30_SEC, + ignorable_exception_arbiter=lambda ex: + isinstance(ex, exception.EMCVnxCLICmdError)) + + def check_snapshot(self, snap_name, poll=True): + """check if a snapshot/cgsnapshot is existed.""" + cmd_get = ('snap', '-list', '-id', snap_name) + out, rc = self.command_execute(*cmd_get) + if rc == 0: + return True + else: + self._raise_cli_error(cmd_get, rc, out) def delete_cgsnapshot(self, snap_name): """Delete a cgsnapshot (snap group)."""