Make X-IO volume driver wait for delete volume to complete

Add a check to make sure the volume has been deleted from
the backend storage before returning from delete_volume.

Change-Id: Icc662de476fcb21ff2f7ca6aa69e46c434214d7e
Closes-Bug: #1482044

diff --git a/cinder/tests/unit/test_xio.py b/cinder/tests/unit/test_xio.py
index 955c6f8fc15e8e131d25846ffa3c19f6dddbd542..1cdfc7c591d159ac3172a3132ff241ad71f0dbc5 100644 (file)
--- a/cinder/tests/unit/test_xio.py
+++ b/cinder/tests/unit/test_xio.py
@@ -865,7 +865,7 @@ class XIOISEDriverTestCase(object):
             protocol = 'fibre_channel'
         exp_result = {}
         exp_result = {'vendor_name': "X-IO",
-                      'driver_version': "1.1.2",
+                      'driver_version': "1.1.3",
                       'volume_backend_name': backend_name,
                       'reserved_percentage': 0,
                       'total_capacity_gb': 100,
@@ -987,7 +987,34 @@ class XIOISEDriverTestCase(object):
                                      ISE_GET_ALLOC_WITH_EP_RESP,
                                      ISE_DELETE_ALLOC_RESP,
                                      ISE_GET_VOL1_STATUS_RESP,
-                                     ISE_DELETE_VOLUME_RESP])
+                                     ISE_DELETE_VOLUME_RESP,
+                                     ISE_GET_VOL_STATUS_404_RESP])
+        self.setup_driver()
+        self.driver.delete_volume(VOLUME1)
+
+    def test_delete_volume_delayed(self, mock_req):
+        mock_req.side_effect = iter([ISE_GET_QUERY_RESP,
+                                     ISE_GET_ALLOC_WITH_EP_RESP,
+                                     ISE_DELETE_ALLOC_RESP,
+                                     ISE_GET_VOL1_STATUS_RESP,
+                                     ISE_DELETE_VOLUME_RESP,
+                                     ISE_GET_VOL1_STATUS_RESP,
+                                     ISE_GET_VOL1_STATUS_RESP,
+                                     ISE_GET_VOL_STATUS_404_RESP])
+        self.setup_driver()
+        self.driver.delete_volume(VOLUME1)
+
+    def test_delete_volume_timeout(self, mock_req):
+        mock_req.side_effect = iter([ISE_GET_QUERY_RESP,
+                                     ISE_GET_ALLOC_WITH_EP_RESP,
+                                     ISE_DELETE_ALLOC_RESP,
+                                     ISE_GET_VOL1_STATUS_RESP,
+                                     ISE_DELETE_VOLUME_RESP,
+                                     ISE_GET_VOL1_STATUS_RESP,
+                                     ISE_GET_VOL1_STATUS_RESP,
+                                     ISE_GET_VOL1_STATUS_RESP])
+
+        self.configuration.ise_completion_retries = 3
         self.setup_driver()
         self.driver.delete_volume(VOLUME1)
 

diff --git a/cinder/volume/drivers/xio.py b/cinder/volume/drivers/xio.py
index 92aad19ba999128479de28613586f8087a6d56c9..13ea346f0a1b0f275a2136b7ba9f76e6131aec8a 100644 (file)
--- a/cinder/volume/drivers/xio.py
+++ b/cinder/volume/drivers/xio.py
@@ -55,6 +55,7 @@ LOG = logging.getLogger(__name__)
 OPERATIONAL_STATUS = 'OPERATIONAL'
 PREPARED_STATUS = 'PREPARED'
 INVALID_STATUS = 'VALID'
+NOTFOUND_STATUS = 'NOT FOUND'
 
 
 # Raise exception for X-IO driver
@@ -64,13 +65,14 @@ def RaiseXIODriverException():
 
 class XIOISEDriver(object):
 
-    VERSION = '1.1.2'
+    VERSION = '1.1.3'
 
     # Version   Changes
     # 1.0.0     Base driver
     # 1.1.0     QoS, affinity, retype and thin support
     # 1.1.1     Fix retry loop (Bug 1429283)
     # 1.1.2     Fix host object deletion (Bug 1433450).
+    # 1.1.3     Wait for volume/snapshot to be deleted.
 
     def __init__(self, *args, **kwargs):
         super(XIOISEDriver, self).__init__()
@@ -573,7 +575,7 @@ class XIOISEDriver(object):
         """Return status of ISE volume"""
         vol_info = {}
         vol_info['value'] = ''
-        vol_info['string'] = ''
+        vol_info['string'] = NOTFOUND_STATUS
         vol_info['details'] = ''
         vol_info['location'] = ''
         vol_info['size'] = ''
@@ -1186,14 +1188,13 @@ class XIOISEDriver(object):
 
     def _delete_volume(self, volume):
         """Delete specified volume"""
-        LOG.debug("X-IO delete_volume called.")
         # First unpresent volume from all hosts.
         self._alloc_location(volume, '', 1)
         # Get volume status. Location string for volume comes back
         # in response. Used for DELETE call below.
         vol_info = self._get_volume_info(volume['name'])
         if vol_info['location'] == '':
-            LOG.warning(_LW("Delete volume: %s not found!"), volume['name'])
+            LOG.warning(_LW("%s not found!"), volume['name'])
             return
         # Make DELETE call.
         args = {}
@@ -1203,8 +1204,25 @@ class XIOISEDriver(object):
         args['status'] = 204
         retries = self.configuration.ise_completion_retries
         resp = self._wait_for_completion(self._help_call_method, args, retries)
-        if resp['status'] == 204:
-            LOG.info(_LI("Volume %s deleted."), volume['name'])
+        if resp['status'] != 204:
+            LOG.warning(_LW("DELETE call failed for %s!"), volume['name'])
+            return
+        # DELETE call successful, now wait for completion.
+        # We do that by waiting for the REST call to return Volume Not Found.
+        args['method'] = ''
+        args['url'] = ''
+        args['name'] = volume['name']
+        args['status_string'] = NOTFOUND_STATUS
+        retries = self.configuration.ise_completion_retries
+        vol_info = self._wait_for_completion(self._help_wait_for_status,
+                                             args, retries)
+        if NOTFOUND_STATUS in vol_info['string']:
+            # Volume no longer present on the backend.
+            LOG.info(_LI("Successfully deleted %s."), volume['name'])
+            return
+        # If we come here it means the volume is still present
+        # on the backend.
+        LOG.error(_LE("Timed out deleting %s!"), volume['name'])
         return
 
     def delete_volume(self, volume):