Handle maxclonepervolume/node limits in SF driver.

The SolidFire cluster has a max simultaneous clones per volume and node
limt that we weren't accounting for.  Add some logic to detect this situation
and loop until enough simultaneous clones are completed to allow processing
of the next one in line.

Fixes bug: 1129810

Change-Id: Id63c1b142a81df5d9d2c4e39fb821b03ebac63a2

diff --git a/cinder/volume/drivers/solidfire.py b/cinder/volume/drivers/solidfire.py
index fa4dce2383fc4d557fc321ae9103351283088bd8..664bd18d85328ea50f86ef9a1ae4a5385c766aa1 100644 (file)
--- a/cinder/volume/drivers/solidfire.py
+++ b/cinder/volume/drivers/solidfire.py
@@ -89,6 +89,10 @@ class SolidFire(SanISCSIDriver):
         and returns results in a dict as well.
 
         """
+        max_simultaneous_clones = ['xMaxSnapshotsPerVolumeExceeded',
+                                   'xMaxClonesPerVolumeExceeded',
+                                   'xMaxSnapshotsPerNodeExceeded',
+                                   'xMaxClonesPerNodeExceeded']
         host = FLAGS.san_ip
         # For now 443 is the only port our server accepts requests on
         port = 443
@@ -135,7 +139,6 @@ class SolidFire(SanISCSIDriver):
                 data = response.read()
                 try:
                     data = json.loads(data)
-
                 except (TypeError, ValueError), exc:
                     connection.close()
                     msg = _("Call to json.loads() raised "
@@ -145,12 +148,24 @@ class SolidFire(SanISCSIDriver):
                 connection.close()
 
             LOG.debug(_("Results of SolidFire API call: %s"), data)
-            if ('error' in data and
-                    'xDBVersionMismatch' in data['error']['name']):
-                LOG.debug(_('Detected xDBVersionMismatch, '
-                            'retry %s of 5') % (5 - retry_count))
-                time.sleep(1)
-                retry_count -= 1
+
+            if 'error' in data:
+                if data['error']['name'] in max_simultaneous_clones:
+                    LOG.warning(_('Clone operation '
+                                  'encountered: %s') % data['error']['name'])
+                    LOG.warning(_(
+                        'Waiting for outstanding operation '
+                        'before retrying snapshot: %s') % params['name'])
+                    time.sleep(5)
+                    # Don't decrement the retry count for this one
+                elif 'xDBVersionMismatch' in data['error']['name']:
+                    LOG.debug(_('Detected xDBVersionMismatch, '
+                                'retry %s of 5') % (5 - retry_count))
+                    time.sleep(1)
+                    retry_count -= 1
+                else:
+                    msg = _("API response: %s") % data
+                    raise exception.SolidFireAPIException(msg)
             else:
                 retry_count = 0
 
@@ -299,6 +314,9 @@ class SolidFire(SanISCSIDriver):
 
         sf_volume_id = data['result']['volumeID']
         model_update = self._get_model_info(sfaccount, sf_volume_id)
+        if model_update is None:
+            mesg = _('Failed to get model update from clone')
+            raise exception.SolidFireAPIDataException(mesg)
 
         return (data, sfaccount, model_update)