Eqlx: Fixes the retries on Network Connection Error

Volume creations were failing due to ssh connection
errors throwing ProcessExecutionError
and retries were not happening. Made a fix
to retry commands on both network errors and
backend api exceptions.
The parameter eqlx_cli_max_retries determines
the number of retry attempts made.

Closes Bug: #1441719
Change-Id: I54b76afc6c9c5d51dd9c38831ee0a1c731c7b161

diff --git a/cinder/tests/unit/test_eqlx.py b/cinder/tests/unit/test_eqlx.py
index cb928c6d2b720bc1087150ad0534b7474b40c3a1..85082559ecb22bd64aac009aa2018730bf83b347 100644 (file)
--- a/cinder/tests/unit/test_eqlx.py
+++ b/cinder/tests/unit/test_eqlx.py
@@ -345,6 +345,33 @@ class DellEQLSanISCSIDriverTestCase(test.TestCase):
         self.assertEqual(num_attempts + 1,
                          self.driver._ssh_execute.call_count)
 
+    @mock.patch.object(greenthread, 'sleep')
+    def test_ensure_connection_retries(self, _gt_sleep):
+        num_attempts = 3
+        self.driver.configuration.eqlx_cli_max_retries = num_attempts
+        self.mock_object(self.driver, '_ssh_execute',
+                         mock.Mock(side_effect=
+                                   processutils.ProcessExecutionError
+                                   (stdout='% Error ... some error.\n')))
+        # mocks for calls in _run_ssh
+        self.mock_object(utils, 'check_ssh_injection')
+        self.mock_object(ssh_utils, 'SSHPool')
+
+        sshpool = ssh_utils.SSHPool("127.0.0.1", 22, 10,
+                                    "test",
+                                    password="test",
+                                    min_size=1,
+                                    max_size=1)
+        self.driver.sshpool = mock.Mock(return_value=sshpool)
+        ssh = mock.Mock(paramiko.SSHClient)
+        self.driver.sshpool.item().__enter__ = mock.Mock(return_value=ssh)
+        self.driver.sshpool.item().__exit__ = mock.Mock(return_value=False)
+        # now call the execute
+        self.assertRaises(exception.VolumeBackendAPIException,
+                          self.driver._eql_execute, "fake command")
+        self.assertEqual(num_attempts + 1,
+                         self.driver._ssh_execute.call_count)
+
     @mock.patch.object(greenthread, 'sleep')
     def test_ensure_retries_on_channel_timeout(self, _gt_sleep):
         num_attempts = 3

diff --git a/cinder/volume/drivers/eqlx.py b/cinder/volume/drivers/eqlx.py
index f04be1659d383f97186db851c16f72f370c8a8a1..187d27df4acb2de77a780da68e8a5029ca699197 100644 (file)
--- a/cinder/volume/drivers/eqlx.py
+++ b/cinder/volume/drivers/eqlx.py
@@ -245,8 +245,6 @@ class DellEQLSanISCSIDriver(san.SanISCSIDriver):
                         return self._ssh_execute(
                             ssh, command,
                             timeout=self.configuration.eqlx_cli_timeout)
-                    except processutils.ProcessExecutionError:
-                        raise
                     except Exception as e:
                         LOG.exception(e)
                         greenthread.sleep(random.randint(20, 500) / 100.0)