Add a generic retry decorator to cinder/utils

Retries are something that we use in a number of
places, and could probably use in a few more.  Rather
than continue writing custom retry code or silly loops
let's add a generic retry decorator class that can be
initialized and used as needed.

This patch leverages the retrying library that's already
in OpenStack Requirements and used in other places.  We
just add a wrapper around it for some logging and add
a bit of error handling incase somebody sets retries to 0

Co-Authored-By: Josh Harlow <harlowja@yahoo-inc.com>
Change-Id: I629373d3c75be591ccde5608c8c85f2debd47447

diff --git a/cinder/tests/test_utils.py b/cinder/tests/test_utils.py
index 1eeec4854e996c6c155755165cb7ed307fdd656c..7bfccc72b9741ef3acec9efb654e734d7541f780 100644 (file)
--- a/cinder/tests/test_utils.py
+++ b/cinder/tests/test_utils.py
@@ -16,6 +16,7 @@
 import datetime
 import hashlib
 import os
+import time
 import uuid
 
 import mock
@@ -785,7 +786,6 @@ class AuditPeriodTest(test.TestCase):
 
     def setUp(self):
         super(AuditPeriodTest, self).setUp()
-        #a fairly random time to test with
         test_time = datetime.datetime(second=23,
                                       minute=12,
                                       hour=8,
@@ -1380,3 +1380,91 @@ class IsBlkDeviceTestCase(test.TestCase):
     def test_fail_is_blk_device(self, mock_os_stat, mock_S_ISBLK):
         dev = 'device_exception'
         self.assertFalse(utils.is_blk_device(dev))
+
+
+class WrongException(Exception):
+        pass
+
+
+class TestRetryDecorator(test.TestCase):
+    def setUp(self):
+        super(TestRetryDecorator, self).setUp()
+
+    def test_no_retry_required(self):
+        self.counter = 0
+
+        with mock.patch.object(time, 'sleep') as mock_sleep:
+            @utils.retry(exception.VolumeBackendAPIException,
+                         interval=2,
+                         retries=3,
+                         backoff_rate=2)
+            def succeeds():
+                self.counter += 1
+                return 'success'
+
+            ret = succeeds()
+            self.assertFalse(mock_sleep.called)
+            self.assertEqual(ret, 'success')
+            self.assertEqual(self.counter, 1)
+
+    def test_retries_once(self):
+        self.counter = 0
+        interval = 2
+        backoff_rate = 2
+        retries = 3
+
+        with mock.patch.object(time, 'sleep') as mock_sleep:
+            @utils.retry(exception.VolumeBackendAPIException,
+                         interval,
+                         retries,
+                         backoff_rate)
+            def fails_once():
+                self.counter += 1
+                if self.counter < 2:
+                    raise exception.VolumeBackendAPIException(data='fake')
+                else:
+                    return 'success'
+
+            ret = fails_once()
+            self.assertEqual(ret, 'success')
+            self.assertEqual(self.counter, 2)
+            self.assertEqual(mock_sleep.call_count, 1)
+            mock_sleep.assert_called_with(interval * backoff_rate)
+
+    def test_limit_is_reached(self):
+        self.counter = 0
+        retries = 3
+        interval = 2
+        backoff_rate = 4
+
+        with mock.patch.object(time, 'sleep') as mock_sleep:
+            @utils.retry(exception.VolumeBackendAPIException,
+                         interval,
+                         retries,
+                         backoff_rate)
+            def always_fails():
+                self.counter += 1
+                raise exception.VolumeBackendAPIException(data='fake')
+
+            self.assertRaises(exception.VolumeBackendAPIException,
+                              always_fails)
+            self.assertEqual(retries, self.counter)
+
+            expected_sleep_arg = []
+
+            for i in xrange(retries):
+                if i > 0:
+                    interval *= backoff_rate
+                    expected_sleep_arg.append(float(interval))
+
+            mock_sleep.assert_has_calls(map(mock.call, expected_sleep_arg))
+
+    def test_wrong_exception_no_retry(self):
+
+        with mock.patch.object(time, 'sleep') as mock_sleep:
+            @utils.retry(exception.VolumeBackendAPIException)
+            def raise_unexpected_error():
+                raise WrongException("wrong exception")
+
+            self.assertRaises(WrongException, raise_unexpected_error)
+            self.assertFalse(mock_sleep.called)

diff --git a/cinder/utils.py b/cinder/utils.py
index 674ca5b952b2291652aefdd4f491d63938c7c208..b1f1a44d85a8db2dc002550f3b7297a54282d1ec 100644 (file)
--- a/cinder/utils.py
+++ b/cinder/utils.py
@@ -40,6 +40,7 @@ from oslo_concurrency import processutils
 from oslo_config import cfg
 from oslo_utils import importutils
 from oslo_utils import timeutils
+import retrying
 import six
 
 from cinder.brick.initiator import connector
@@ -766,3 +767,39 @@ def is_blk_device(dev):
     except Exception:
         LOG.debug('Path %s not found in is_blk_device check' % dev)
         return False
+
+
+def retry(exceptions, interval=1, retries=3, backoff_rate=2):
+
+    def _retry_on_exception(e):
+        return isinstance(e, exceptions)
+
+    def _backoff_sleep(previous_attempt_number, delay_since_first_attempt_ms):
+        exp = backoff_rate ** previous_attempt_number
+        wait_for = max(0, interval * exp)
+        LOG.debug("Sleeping for %s seconds", wait_for)
+        return wait_for * 1000.0
+
+    def _print_stop(previous_attempt_number, delay_since_first_attempt_ms):
+        delay_since_first_attempt = delay_since_first_attempt_ms / 1000.0
+        LOG.debug("Failed attempt %s", previous_attempt_number)
+        LOG.debug("Have been at this for %s seconds",
+                  delay_since_first_attempt)
+        return previous_attempt_number == retries
+
+    if retries < 1:
+        raise ValueError('Retries must be greater than or '
+                         'equal to 1 (received: %s). ' % retries)
+
+    def _decorator(f):
+
+        @six.wraps(f)
+        def _wrapper(*args, **kwargs):
+            r = retrying.Retrying(retry_on_exception=_retry_on_exception,
+                                  wait_func=_backoff_sleep,
+                                  stop_func=_print_stop)
+            return r.call(f, *args, **kwargs)
+
+        return _wrapper
+
+    return _decorator