From: John Griffith <john.griffith@solidfire.com>
Date: Wed, 20 Feb 2013 04:34:01 +0000 (-0700)
Subject: Handle maxclonepervolume/node limits in SF driver.
X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=737468e5c80a35b859dcda29706c88e2976339a3;p=openstack-build%2Fcinder-build.git

Handle maxclonepervolume/node limits in SF driver.

The SolidFire cluster has a max simultaneous clones per volume and node
limt that we weren't accounting for.  Add some logic to detect this situation
and loop until enough simultaneous clones are completed to allow processing
of the next one in line.

Fixes bug: 1129810

Change-Id: Id63c1b142a81df5d9d2c4e39fb821b03ebac63a2
---

diff --git a/cinder/volume/drivers/solidfire.py b/cinder/volume/drivers/solidfire.py
index fa4dce238..664bd18d8 100644
--- a/cinder/volume/drivers/solidfire.py
+++ b/cinder/volume/drivers/solidfire.py
@@ -89,6 +89,10 @@ class SolidFire(SanISCSIDriver):
         and returns results in a dict as well.
 
         """
+        max_simultaneous_clones = ['xMaxSnapshotsPerVolumeExceeded',
+                                   'xMaxClonesPerVolumeExceeded',
+                                   'xMaxSnapshotsPerNodeExceeded',
+                                   'xMaxClonesPerNodeExceeded']
         host = FLAGS.san_ip
         # For now 443 is the only port our server accepts requests on
         port = 443
@@ -135,7 +139,6 @@ class SolidFire(SanISCSIDriver):
                 data = response.read()
                 try:
                     data = json.loads(data)
-
                 except (TypeError, ValueError), exc:
                     connection.close()
                     msg = _("Call to json.loads() raised "
@@ -145,12 +148,24 @@ class SolidFire(SanISCSIDriver):
                 connection.close()
 
             LOG.debug(_("Results of SolidFire API call: %s"), data)
-            if ('error' in data and
-                    'xDBVersionMismatch' in data['error']['name']):
-                LOG.debug(_('Detected xDBVersionMismatch, '
-                            'retry %s of 5') % (5 - retry_count))
-                time.sleep(1)
-                retry_count -= 1
+
+            if 'error' in data:
+                if data['error']['name'] in max_simultaneous_clones:
+                    LOG.warning(_('Clone operation '
+                                  'encountered: %s') % data['error']['name'])
+                    LOG.warning(_(
+                        'Waiting for outstanding operation '
+                        'before retrying snapshot: %s') % params['name'])
+                    time.sleep(5)
+                    # Don't decrement the retry count for this one
+                elif 'xDBVersionMismatch' in data['error']['name']:
+                    LOG.debug(_('Detected xDBVersionMismatch, '
+                                'retry %s of 5') % (5 - retry_count))
+                    time.sleep(1)
+                    retry_count -= 1
+                else:
+                    msg = _("API response: %s") % data
+                    raise exception.SolidFireAPIException(msg)
             else:
                 retry_count = 0
 
@@ -299,6 +314,9 @@ class SolidFire(SanISCSIDriver):
 
         sf_volume_id = data['result']['volumeID']
         model_update = self._get_model_info(sfaccount, sf_volume_id)
+        if model_update is None:
+            mesg = _('Failed to get model update from clone')
+            raise exception.SolidFireAPIDataException(mesg)
 
         return (data, sfaccount, model_update)