Fix failure with rbd on slow ceph clusters

Make rados connection interval and retries configurable
for _try_remove_volume() function

Otherwise on slow ceph clusters, we can get following problem:

"ImageBusy error raised while deleting rbd volume. This may have been
caused by a connection from a client that has crashed and, if so,
may be resolved by retrying the delete after 30 seconds has elapsed."

Change-Id: I1230715663ea00c3eb4241154e6f194dee0e23d4
Co-Authored-By: Dirk Mueller <dirk@dmllr.de>
Closes-Bug: #1554045

diff --git a/cinder/tests/unit/test_rbd.py b/cinder/tests/unit/test_rbd.py
index c1f4a49041f8db7e7cd3df61f4246eccb333b195..bdfb38e7b565b400592e53ce5326df02cec5457f 100644 (file)
--- a/cinder/tests/unit/test_rbd.py
+++ b/cinder/tests/unit/test_rbd.py
@@ -144,6 +144,8 @@ class RBDTestCase(test.TestCase):
         self.cfg.rbd_user = None
         self.cfg.volume_dd_blocksize = '1M'
         self.cfg.rbd_store_chunk_size = 4
+        self.cfg.rados_connection_retries = 3
+        self.cfg.rados_connection_interval = 5
 
         mock_exec = mock.Mock()
         mock_exec.return_value = ('', '')

diff --git a/cinder/volume/drivers/rbd.py b/cinder/volume/drivers/rbd.py
index 91a02133e58d42c067a118348c79f6687ee2cd9f..5f4f1bcac5df9f1fca289554b1d3f1af0e080bce 100644 (file)
--- a/cinder/volume/drivers/rbd.py
+++ b/cinder/volume/drivers/rbd.py
@@ -702,7 +702,9 @@ class RBDDriver(driver.TransferVD, driver.ExtendVD,
             finally:
                 rbd_image.close()
 
-            @utils.retry(self.rbd.ImageBusy, retries=3)
+            @utils.retry(self.rbd.ImageBusy,
+                         self.configuration.rados_connection_interval,
+                         self.configuration.rados_connection_retries)
             def _try_remove_volume(client, volume_name):
                 self.RBDProxy().remove(client.ioctx, volume_name)