Add retries to delete a volume in the RBD driver

This patch adds retries to delete a volume. After N tries of a
volume deletion failed we raise VolumeIsBusy exception.

Change-Id: I9499be0c5985f9e8a3e55d1c9add01ad5cd11789

diff --git a/cinder/tests/unit/test_rbd.py b/cinder/tests/unit/test_rbd.py
index 434ca109e67aa0b2ad53fc9aa955ecd85b49f3c5..101d05b3c050b88a249c257048a9a6eb78b320fd 100644 (file)
--- a/cinder/tests/unit/test_rbd.py
+++ b/cinder/tests/unit/test_rbd.py
@@ -313,10 +313,10 @@ class RBDTestCase(test.TestCase):
                     self.assertFalse(
                         self.mock_rbd.Image.return_value.unprotect_snap.called)
                     self.assertEqual(
-                        1, self.mock_rbd.RBD.return_value.remove.call_count)
+                        3, self.mock_rbd.RBD.return_value.remove.call_count)
+                    self.assertEqual(3, len(RAISED_EXCEPTIONS))
                     # Make sure the exception was raised
-                    self.assertEqual(RAISED_EXCEPTIONS,
-                                     [self.mock_rbd.ImageBusy])
+                    self.assertIn(self.mock_rbd.ImageBusy, RAISED_EXCEPTIONS)
 
     @common_mocks
     def test_delete_volume_not_found(self):

diff --git a/cinder/volume/drivers/rbd.py b/cinder/volume/drivers/rbd.py
index 72a914228ea677ed25f0b7df3830259c6b270059..71e766d0048713b309f6ab1bea7e28304259dea6 100644 (file)
--- a/cinder/volume/drivers/rbd.py
+++ b/cinder/volume/drivers/rbd.py
@@ -699,10 +699,14 @@ class RBDDriver(driver.TransferVD, driver.ExtendVD,
             finally:
                 rbd_image.close()
 
+            @utils.retry(self.rbd.ImageBusy, retries=3)
+            def _try_remove_volume(client, volume_name):
+                self.RBDProxy().remove(client.ioctx, volume_name)
+
             if clone_snap is None:
                 LOG.debug("deleting rbd volume %s", volume_name)
                 try:
-                    self.RBDProxy().remove(client.ioctx, volume_name)
+                    _try_remove_volume(client, volume_name)
                 except self.rbd.ImageBusy:
                     msg = (_("ImageBusy error raised while deleting rbd "
                              "volume. This may have been caused by a "