From a76eae7ffedc9dda5bf61dc3d713e59844094c2f Mon Sep 17 00:00:00 2001 From: Mike Perez Date: Wed, 18 Mar 2015 02:19:45 -0700 Subject: [PATCH] Add retry to create resource in Datera driver If the volume is still in a creating state, wait, and retry a few times. Otherwise, raise an exception for the manager to set the volume into an error state. Closes-Bug: #1433543 Change-Id: If49fd9229f08301cedbd63399fe46475e73a83ef --- cinder/tests/volume/drivers/datera.py | 63 +++++++++++++++++++++++---- cinder/volume/drivers/datera.py | 40 ++++++++++++----- 2 files changed, 83 insertions(+), 20 deletions(-) diff --git a/cinder/tests/volume/drivers/datera.py b/cinder/tests/volume/drivers/datera.py index 8071817ba..8f03caf15 100644 --- a/cinder/tests/volume/drivers/datera.py +++ b/cinder/tests/volume/drivers/datera.py @@ -55,12 +55,15 @@ class DateraVolumeTestCase(test.TestCase): def test_volume_create_success(self): self.mock_api.return_value = { - 'uuid': 'c20aba21-6ef6-446b-b374-45733b4883ba', - 'size': '1073741824', - 'name': 'volume-00000001', - 'parent': '00000000-0000-0000-0000-000000000000', - 'numReplicas': '2', - 'subType': 'IS_ORIGINAL' + u'status': u'available', + u'name': u'volume-00000001', + u'parent': u'00000000-0000-0000-0000-000000000000', + u'uuid': u'c20aba21-6ef6-446b-b374-45733b4883ba', + u'snapshots': {}, + u'targets': {}, + u'num_replicas': u'2', + u'sub_type': u'IS_ORIGINAL', + u'size': u'1073741824' } self.assertIsNone(self.driver.create_volume(self.volume)) @@ -69,12 +72,50 @@ class DateraVolumeTestCase(test.TestCase): self.assertRaises(exception.DateraAPIException, self.driver.create_volume, self.volume) + def test_volume_create_delay(self): + """Verify after 1st retry volume becoming available is a success.""" + + def _progress_api_return(mock_api): + if mock_api.retry_count == 1: + return { + u'status': u'unavailable', + u'name': u'test', + u'parent': u'00000000-0000-0000-0000-000000000000', + u'uuid': u'9c1666fe-4f1a-4891-b33d-e710549527fe', + u'snapshots': {}, + u'targets': {}, + u'num_replicas': u'2', + u'sub_type': u'IS_ORIGINAL', + u'size': u'1073741824' + } + else: + self.mock_api.retry_count += 1 + return { + u'status': u'available', + u'name': u'test', + u'parent': u'00000000-0000-0000-0000-000000000000', + u'uuid': u'9c1666fe-4f1a-4891-b33d-e710549527fe', + u'snapshots': {}, + u'targets': {}, + u'num_replicas': u'2', + u'sub_type': u'IS_ORIGINAL', + u'size': u'1073741824' + } + + self.mock_api.retry_count = 0 + self.mock_api.return_value = _progress_api_return(self.mock_api) + self.assertEqual(1, self.mock_api.retry_count) + self.assertIsNone(self.driver.create_volume(self.volume)) + def test_create_cloned_volume_success(self): self.mock_api.return_value = { + 'status': 'available', 'uuid': 'c20aba21-6ef6-446b-b374-45733b4883ba', 'size': '1073741824', 'name': 'volume-00000001', 'parent': '7f91abfa-7964-41ed-88fc-207c3a290b4f', + 'snapshots': {}, + 'targets': {}, 'numReplicas': '2', 'subType': 'IS_CLONE' } @@ -121,7 +162,7 @@ class DateraVolumeTestCase(test.TestCase): ctxt = context.get_admin_context() expected = { 'provider_location': u'172.28.121.10:3260 iqn.2013-05.com.daterain' - 'c::01:sn:fc372bc0490b2dbe 1' + 'c::01:sn:fc372bc0490b2dbe 0' } self.assertEqual(expected, self.driver.ensure_export(ctxt, self.volume)) @@ -137,7 +178,7 @@ class DateraVolumeTestCase(test.TestCase): ctxt = context.get_admin_context() expected = { 'provider_location': u'172.28.121.10:3260 iqn.2013-05.com.daterain' - 'c::01:sn:fc372bc0490b2dbe 1' + 'c::01:sn:fc372bc0490b2dbe 0' } self.assertEqual(expected, self.driver.create_export(ctxt, self.volume)) @@ -169,9 +210,12 @@ class DateraVolumeTestCase(test.TestCase): def test_create_snapshot_success(self): self.mock_api.return_value = { + u'status': u'available', u'uuid': u'0bb34f0c-fea4-48e0-bf96-591120ac7e3c', u'parent': u'c20aba21-6ef6-446b-b374-45733b4883ba', u'subType': u'IS_SNAPSHOT', + u'snapshots': {}, + u'targets': {}, u'numReplicas': 2, u'size': u'1073741824', u'name': u'snapshot-00000001' @@ -210,8 +254,11 @@ class DateraVolumeTestCase(test.TestCase): def test_create_volume_from_snapshot_success(self): self.mock_api.return_value = { + u'status': u'available', u'uuid': u'c20aba21-6ef6-446b-b374-45733b4883ba', u'parent': u'0bb34f0c-fea4-48e0-bf96-591120ac7e3c', + u'snapshots': {}, + u'targets': {}, u'subType': u'IS_ORIGINAL', u'numReplicas': 2, u'size': u'1073741824', diff --git a/cinder/volume/drivers/datera.py b/cinder/volume/drivers/datera.py index 46fedd9fa..1432baeea 100644 --- a/cinder/volume/drivers/datera.py +++ b/cinder/volume/drivers/datera.py @@ -24,6 +24,7 @@ import requests from cinder import exception from cinder.i18n import _, _LE, _LW from cinder.openstack.common import versionutils +from cinder import utils from cinder.volume.drivers.san import san LOG = logging.getLogger(__name__) @@ -122,24 +123,39 @@ class DateraDriver(san.SanISCSIDriver): self._login() + @utils.retry(exception.VolumeDriverException, retries=3) + def _wait_for_resource(self, id, resource_type): + result = self._issue_api_request(resource_type, 'get', id) + if result['status'] == 'available': + return + else: + raise exception.VolumeDriverException(msg=_('Resource not ready.')) + + def _create_resource(self, resource, resource_type, body): + result = self._issue_api_request(resource_type, 'post', body=body) + + if result['status'] == 'available': + return + self._wait_for_resource(resource['id'], resource_type) + def create_volume(self, volume): """Create a logical volume.""" - params = { + body = { 'name': volume['display_name'] or volume['id'], 'size': str(volume['size'] * units.Gi), 'uuid': volume['id'], 'numReplicas': self.num_replicas } - self._issue_api_request('volumes', 'post', body=params) + self._create_resource(volume, 'volumes', body) def create_cloned_volume(self, volume, src_vref): - data = { + body = { 'name': volume['display_name'] or volume['id'], 'uuid': volume['id'], 'clone_uuid': src_vref['id'], 'numReplicas': self.num_replicas } - self._issue_api_request('volumes', 'post', body=data) + self._create_resource(volume, 'volumes', body) def delete_volume(self, volume): try: @@ -192,20 +208,20 @@ class DateraDriver(san.SanISCSIDriver): LOG.info(msg, snapshot['id']) def create_snapshot(self, snapshot): - data = { + body = { 'uuid': snapshot['id'], 'parentUUID': snapshot['volume_id'] } - self._issue_api_request('snapshots', 'post', body=data) + self._create_resource(snapshot, 'snapshots', body) def create_volume_from_snapshot(self, volume, snapshot): - data = { + body = { 'name': volume['display_name'] or volume['id'], 'uuid': volume['id'], 'snapshot_uuid': snapshot['id'], 'numReplicas': self.num_replicas } - self._issue_api_request('volumes', 'post', body=data) + self._create_resource(volume, 'volumes', body) def get_volume_stats(self, refresh=False): """Get volume stats. @@ -225,10 +241,10 @@ class DateraDriver(san.SanISCSIDriver): return self.cluster_stats def extend_volume(self, volume, new_size): - data = { + body = { 'size': str(new_size * units.Gi) } - self._issue_api_request('volumes', 'put', body=data, + self._issue_api_request('volumes', 'put', body=body, resource=volume['id']) def _update_cluster_stats(self): @@ -254,7 +270,7 @@ class DateraDriver(san.SanISCSIDriver): def _login(self): """Use the san_login and san_password to set self.auth_token.""" - data = { + body = { 'name': self.username, 'password': self.password } @@ -265,7 +281,7 @@ class DateraDriver(san.SanISCSIDriver): try: LOG.debug('Getting Datera auth token.') - results = self._issue_api_request('login', 'post', body=data, + results = self._issue_api_request('login', 'post', body=body, sensitive=True) self.auth_token = results['key'] except exception.NotAuthorized: -- 2.45.2