From 49dd3785229200d649d1291e556ae30566763bce Mon Sep 17 00:00:00 2001 From: Sean McGinnis Date: Thu, 12 Feb 2015 14:52:56 -0600 Subject: [PATCH] Dell Storage Center: Add retries to API calls In heavily loaded networks we have seen some cases of temporary ConnectionErrors when making REST API calls. There are usually successful calls just prior and immediately after these failures, so it appears to be a transient condition. This patch utilizes the recently merged retry decorator to add some retry handling to the REST API calls when this condition is encountered. With a test script looping through repeatedly running CI against this first patch I was able to validate that it addresses the issue. Out of 20 runs, a full 2/3 of the tests passed showing that the retry was used. These test runs would have failed without the retry. The output from these test runs can be viewed here: http://oslogs.compellent.com/?C=N;O=D Pertinent results are dell-sc-iscsi-1554792015-12* Once merged we should be able to enable full third party CI testing with some expectation of reliable results. Closes-Bug: 1422742 Change-Id: I1279a7696068aee534498bb18f4f6b4fbb7a33d5 --- cinder/volume/drivers/dell/dell_storagecenter_api.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cinder/volume/drivers/dell/dell_storagecenter_api.py b/cinder/volume/drivers/dell/dell_storagecenter_api.py index f9df554d2..6c4e918e3 100644 --- a/cinder/volume/drivers/dell/dell_storagecenter_api.py +++ b/cinder/volume/drivers/dell/dell_storagecenter_api.py @@ -21,6 +21,7 @@ import requests from cinder import exception from cinder.i18n import _, _LE, _LI, _LW from cinder.openstack.common import log as logging +from cinder import utils LOG = logging.getLogger(__name__) @@ -77,12 +78,14 @@ class HttpClient(object): def __formatUrl(self, url): return '%s%s' % (self.baseUrl, url if url[0] != '/' else url[1:]) + @utils.retry(exceptions=(requests.ConnectionError, )) def get(self, url): return self.session.get( self.__formatUrl(url), headers=self.header, verify=self.verify) + @utils.retry(exceptions=(requests.ConnectionError, )) def post(self, url, payload): return self.session.post( self.__formatUrl(url), @@ -91,6 +94,7 @@ class HttpClient(object): headers=self.header, verify=self.verify) + @utils.retry(exceptions=(requests.ConnectionError, )) def put(self, url, payload): return self.session.put( self.__formatUrl(url), @@ -99,6 +103,7 @@ class HttpClient(object): headers=self.header, verify=self.verify) + @utils.retry(exceptions=(requests.ConnectionError, )) def delete(self, url): return self.session.delete( self.__formatUrl(url), @@ -181,7 +186,7 @@ class StorageCenterApi(object): LOG.debug('Unable to find result where %(attr)s is %(val)s', {'attr': attribute, 'val': value}) - LOG.debug('Blob was %(blob)s', {'blob': blob}) + LOG.debug('Blob was %(blob)s', {'blob': blob.text}) return rsp def _get_json(self, blob): -- 2.45.2