Add retry to create resource in Datera driver

If the volume is still in a creating state, wait, and retry a few times.
Otherwise, raise an exception for the manager to set the volume into an
error state.

Closes-Bug: #1433543
Change-Id: If49fd9229f08301cedbd63399fe46475e73a83ef

diff --git a/cinder/tests/volume/drivers/datera.py b/cinder/tests/volume/drivers/datera.py
index 8071817ba6f5e59900ce9f25fde08f3d662a3494..8f03caf15f0e1fa7c8f4defbd2a8c72f8273a1d4 100644 (file)
--- a/cinder/tests/volume/drivers/datera.py
+++ b/cinder/tests/volume/drivers/datera.py
@@ -55,12 +55,15 @@ class DateraVolumeTestCase(test.TestCase):
 
     def test_volume_create_success(self):
         self.mock_api.return_value = {
-            'uuid': 'c20aba21-6ef6-446b-b374-45733b4883ba',
-            'size': '1073741824',
-            'name': 'volume-00000001',
-            'parent': '00000000-0000-0000-0000-000000000000',
-            'numReplicas': '2',
-            'subType': 'IS_ORIGINAL'
+            u'status': u'available',
+            u'name': u'volume-00000001',
+            u'parent': u'00000000-0000-0000-0000-000000000000',
+            u'uuid': u'c20aba21-6ef6-446b-b374-45733b4883ba',
+            u'snapshots': {},
+            u'targets': {},
+            u'num_replicas': u'2',
+            u'sub_type': u'IS_ORIGINAL',
+            u'size': u'1073741824'
         }
         self.assertIsNone(self.driver.create_volume(self.volume))
 
@@ -69,12 +72,50 @@ class DateraVolumeTestCase(test.TestCase):
         self.assertRaises(exception.DateraAPIException,
                           self.driver.create_volume, self.volume)
 
+    def test_volume_create_delay(self):
+        """Verify after 1st retry volume becoming available is a success."""
+
+        def _progress_api_return(mock_api):
+            if mock_api.retry_count == 1:
+                return {
+                    u'status': u'unavailable',
+                    u'name': u'test',
+                    u'parent': u'00000000-0000-0000-0000-000000000000',
+                    u'uuid': u'9c1666fe-4f1a-4891-b33d-e710549527fe',
+                    u'snapshots': {},
+                    u'targets': {},
+                    u'num_replicas': u'2',
+                    u'sub_type': u'IS_ORIGINAL',
+                    u'size': u'1073741824'
+                }
+            else:
+                self.mock_api.retry_count += 1
+                return {
+                    u'status': u'available',
+                    u'name': u'test',
+                    u'parent': u'00000000-0000-0000-0000-000000000000',
+                    u'uuid': u'9c1666fe-4f1a-4891-b33d-e710549527fe',
+                    u'snapshots': {},
+                    u'targets': {},
+                    u'num_replicas': u'2',
+                    u'sub_type': u'IS_ORIGINAL',
+                    u'size': u'1073741824'
+                }
+
+        self.mock_api.retry_count = 0
+        self.mock_api.return_value = _progress_api_return(self.mock_api)
+        self.assertEqual(1, self.mock_api.retry_count)
+        self.assertIsNone(self.driver.create_volume(self.volume))
+
     def test_create_cloned_volume_success(self):
         self.mock_api.return_value = {
+            'status': 'available',
             'uuid': 'c20aba21-6ef6-446b-b374-45733b4883ba',
             'size': '1073741824',
             'name': 'volume-00000001',
             'parent': '7f91abfa-7964-41ed-88fc-207c3a290b4f',
+            'snapshots': {},
+            'targets': {},
             'numReplicas': '2',
             'subType': 'IS_CLONE'
         }
@@ -121,7 +162,7 @@ class DateraVolumeTestCase(test.TestCase):
         ctxt = context.get_admin_context()
         expected = {
             'provider_location': u'172.28.121.10:3260 iqn.2013-05.com.daterain'
-                                 'c::01:sn:fc372bc0490b2dbe 1'
+                                 'c::01:sn:fc372bc0490b2dbe 0'
         }
         self.assertEqual(expected, self.driver.ensure_export(ctxt,
                                                              self.volume))
@@ -137,7 +178,7 @@ class DateraVolumeTestCase(test.TestCase):
         ctxt = context.get_admin_context()
         expected = {
             'provider_location': u'172.28.121.10:3260 iqn.2013-05.com.daterain'
-                                 'c::01:sn:fc372bc0490b2dbe 1'
+                                 'c::01:sn:fc372bc0490b2dbe 0'
         }
         self.assertEqual(expected, self.driver.create_export(ctxt,
                                                              self.volume))
@@ -169,9 +210,12 @@ class DateraVolumeTestCase(test.TestCase):
 
     def test_create_snapshot_success(self):
         self.mock_api.return_value = {
+            u'status': u'available',
             u'uuid': u'0bb34f0c-fea4-48e0-bf96-591120ac7e3c',
             u'parent': u'c20aba21-6ef6-446b-b374-45733b4883ba',
             u'subType': u'IS_SNAPSHOT',
+            u'snapshots': {},
+            u'targets': {},
             u'numReplicas': 2,
             u'size': u'1073741824',
             u'name': u'snapshot-00000001'
@@ -210,8 +254,11 @@ class DateraVolumeTestCase(test.TestCase):
 
     def test_create_volume_from_snapshot_success(self):
         self.mock_api.return_value = {
+            u'status': u'available',
             u'uuid': u'c20aba21-6ef6-446b-b374-45733b4883ba',
             u'parent': u'0bb34f0c-fea4-48e0-bf96-591120ac7e3c',
+            u'snapshots': {},
+            u'targets': {},
             u'subType': u'IS_ORIGINAL',
             u'numReplicas': 2,
             u'size': u'1073741824',

diff --git a/cinder/volume/drivers/datera.py b/cinder/volume/drivers/datera.py
index 46fedd9fa626ee3f877ee37c904f645bc380dde6..1432baeeaee037316254d4cbaece8d5728a175b8 100644 (file)
--- a/cinder/volume/drivers/datera.py
+++ b/cinder/volume/drivers/datera.py
@@ -24,6 +24,7 @@ import requests
 from cinder import exception
 from cinder.i18n import _, _LE, _LW
 from cinder.openstack.common import versionutils
+from cinder import utils
 from cinder.volume.drivers.san import san
 
 LOG = logging.getLogger(__name__)
@@ -122,24 +123,39 @@ class DateraDriver(san.SanISCSIDriver):
 
         self._login()
 
+    @utils.retry(exception.VolumeDriverException, retries=3)
+    def _wait_for_resource(self, id, resource_type):
+        result = self._issue_api_request(resource_type, 'get', id)
+        if result['status'] == 'available':
+            return
+        else:
+            raise exception.VolumeDriverException(msg=_('Resource not ready.'))
+
+    def _create_resource(self, resource, resource_type, body):
+        result = self._issue_api_request(resource_type, 'post', body=body)
+
+        if result['status'] == 'available':
+            return
+        self._wait_for_resource(resource['id'], resource_type)
+
     def create_volume(self, volume):
         """Create a logical volume."""
-        params = {
+        body = {
             'name': volume['display_name'] or volume['id'],
             'size': str(volume['size'] * units.Gi),
             'uuid': volume['id'],
             'numReplicas': self.num_replicas
         }
-        self._issue_api_request('volumes', 'post', body=params)
+        self._create_resource(volume, 'volumes', body)
 
     def create_cloned_volume(self, volume, src_vref):
-        data = {
+        body = {
             'name': volume['display_name'] or volume['id'],
             'uuid': volume['id'],
             'clone_uuid': src_vref['id'],
             'numReplicas': self.num_replicas
         }
-        self._issue_api_request('volumes', 'post', body=data)
+        self._create_resource(volume, 'volumes', body)
 
     def delete_volume(self, volume):
         try:
@@ -192,20 +208,20 @@ class DateraDriver(san.SanISCSIDriver):
             LOG.info(msg, snapshot['id'])
 
     def create_snapshot(self, snapshot):
-        data = {
+        body = {
             'uuid': snapshot['id'],
             'parentUUID': snapshot['volume_id']
         }
-        self._issue_api_request('snapshots', 'post', body=data)
+        self._create_resource(snapshot, 'snapshots', body)
 
     def create_volume_from_snapshot(self, volume, snapshot):
-        data = {
+        body = {
             'name': volume['display_name'] or volume['id'],
             'uuid': volume['id'],
             'snapshot_uuid': snapshot['id'],
             'numReplicas': self.num_replicas
         }
-        self._issue_api_request('volumes', 'post', body=data)
+        self._create_resource(volume, 'volumes', body)
 
     def get_volume_stats(self, refresh=False):
         """Get volume stats.
@@ -225,10 +241,10 @@ class DateraDriver(san.SanISCSIDriver):
         return self.cluster_stats
 
     def extend_volume(self, volume, new_size):
-        data = {
+        body = {
             'size': str(new_size * units.Gi)
         }
-        self._issue_api_request('volumes', 'put', body=data,
+        self._issue_api_request('volumes', 'put', body=body,
                                 resource=volume['id'])
 
     def _update_cluster_stats(self):
@@ -254,7 +270,7 @@ class DateraDriver(san.SanISCSIDriver):
 
     def _login(self):
         """Use the san_login and san_password to set self.auth_token."""
-        data = {
+        body = {
             'name': self.username,
             'password': self.password
         }
@@ -265,7 +281,7 @@ class DateraDriver(san.SanISCSIDriver):
 
         try:
             LOG.debug('Getting Datera auth token.')
-            results = self._issue_api_request('login', 'post', body=data,
+            results = self._issue_api_request('login', 'post', body=body,
                                               sensitive=True)
             self.auth_token = results['key']
         except exception.NotAuthorized: