Storwize SVC multiple management IPs

Right now Storwize SVC does not support multiple management IPs.
This patch adds this feature so that if the primary IP fails, it
will switch to the secondary IP that the user sets.

DocImpact
Adds config option 'storwize_san_secondary_ip'

Implements: blueprint storwize-add-support-for-multiple-management-ips
Change-Id: Ib82ba5b43e92027bfe39873a556baec796bb457e

diff --git a/cinder/tests/unit/test_storwize_svc.py b/cinder/tests/unit/test_storwize_svc.py
index f510da8cf4f77903b165773dce989445353ad868..16feb926cdd1fc4b959d4ac9af9e8ffc379e88b6 100644 (file)
--- a/cinder/tests/unit/test_storwize_svc.py
+++ b/cinder/tests/unit/test_storwize_svc.py
@@ -18,6 +18,7 @@
 Tests for the IBM Storwize family and SVC volume driver.
 """
 
+import paramiko
 import random
 import re
 import time
@@ -33,6 +34,7 @@ from cinder import context
 from cinder import exception
 from cinder.i18n import _
 from cinder.objects import fields
+from cinder import ssh_utils
 from cinder import test
 from cinder.tests.unit import utils as testutils
 from cinder import utils
@@ -2507,8 +2509,11 @@ class StorwizeSVCCommonDriverTestCase(test.TestCase):
         if self.USESIM:
             self.driver = StorwizeSVCISCSIFakeDriver(
                 configuration=conf.Configuration(None))
+            self._driver = storwize_svc_iscsi.StorwizeSVCISCSIDriver(
+                configuration=conf.Configuration(None))
 
             self._def_flags = {'san_ip': 'hostname',
+                               'storwize_san_secondary_ip': 'secondaryname',
                                'san_login': 'user',
                                'san_password': 'pass',
                                'storwize_svc_volpool_name': 'openstack',
@@ -2621,6 +2626,59 @@ class StorwizeSVCCommonDriverTestCase(test.TestCase):
         # Finally, check with good parameters
         self.driver.do_setup(None)
 
+    @mock.patch.object(ssh_utils, 'SSHPool')
+    @mock.patch.object(processutils, 'ssh_execute')
+    def test_run_ssh_set_up_with_san_ip(self, mock_ssh_execute, mock_ssh_pool):
+        ssh_cmd = ['svcinfo']
+        self._driver._run_ssh(ssh_cmd)
+
+        mock_ssh_pool.assert_called_once_with(
+            self._driver.configuration.san_ip,
+            self._driver.configuration.san_ssh_port,
+            self._driver.configuration.ssh_conn_timeout,
+            self._driver.configuration.san_login,
+            password=self._driver.configuration.san_password,
+            privatekey=self._driver.configuration.san_private_key,
+            min_size=self._driver.configuration.ssh_min_pool_conn,
+            max_size=self._driver.configuration.ssh_max_pool_conn)
+
+    @mock.patch.object(ssh_utils, 'SSHPool')
+    @mock.patch.object(processutils, 'ssh_execute')
+    def test_run_ssh_set_up_with_secondary_ip(self, mock_ssh_execute,
+                                              mock_ssh_pool):
+        mock_ssh_pool.side_effect = [paramiko.SSHException, mock.MagicMock()]
+        ssh_cmd = ['svcinfo']
+        self._driver._run_ssh(ssh_cmd)
+
+        mock_ssh_pool.assert_called_with(
+            self._driver.configuration.storwize_san_secondary_ip,
+            self._driver.configuration.san_ssh_port,
+            self._driver.configuration.ssh_conn_timeout,
+            self._driver.configuration.san_login,
+            password=self._driver.configuration.san_password,
+            privatekey=self._driver.configuration.san_private_key,
+            min_size=self._driver.configuration.ssh_min_pool_conn,
+            max_size=self._driver.configuration.ssh_max_pool_conn)
+
+    @mock.patch.object(ssh_utils, 'SSHPool')
+    @mock.patch.object(processutils, 'ssh_execute')
+    def test_run_ssh_fail_to_secondary_ip(self, mock_ssh_execute,
+                                          mock_ssh_pool):
+        mock_ssh_execute.side_effect = [processutils.ProcessExecutionError,
+                                        mock.MagicMock()]
+        ssh_cmd = ['svcinfo']
+        self._driver._run_ssh(ssh_cmd)
+
+        mock_ssh_pool.assert_called_with(
+            self._driver.configuration.storwize_san_secondary_ip,
+            self._driver.configuration.san_ssh_port,
+            self._driver.configuration.ssh_conn_timeout,
+            self._driver.configuration.san_login,
+            password=self._driver.configuration.san_password,
+            privatekey=self._driver.configuration.san_private_key,
+            min_size=self._driver.configuration.ssh_min_pool_conn,
+            max_size=self._driver.configuration.ssh_max_pool_conn)
+
     def _generate_vol_info(self, vol_name, vol_id):
         rand_id = six.text_type(random.randint(10000, 99999))
         if vol_name:

diff --git a/cinder/volume/drivers/ibm/storwize_svc/storwize_svc_common.py b/cinder/volume/drivers/ibm/storwize_svc/storwize_svc_common.py
index 98fc51e828bf1bd43889cf674a9d46edf2913269..d8a2e9316a0d4eaeccfabcc754bfe0f12a7695fd 100644 (file)
--- a/cinder/volume/drivers/ibm/storwize_svc/storwize_svc_common.py
+++ b/cinder/volume/drivers/ibm/storwize_svc/storwize_svc_common.py
@@ -15,6 +15,7 @@
 #
 
 import math
+import paramiko
 import random
 import re
 import string
@@ -33,6 +34,8 @@ import six
 
 from cinder import context
 from cinder import exception
+from cinder import ssh_utils
+from cinder import utils as cinder_utils
 from cinder.i18n import _, _LE, _LI, _LW
 from cinder.objects import fields
 from cinder.volume import driver
@@ -97,6 +100,10 @@ storwize_svc_opts = [
                help='If operating in stretched cluster mode, specify the '
                     'name of the pool in which mirrored copies are stored.'
                     'Example: "pool2"'),
+    cfg.StrOpt('storwize_san_secondary_ip',
+               default=None,
+               help='Specifies secondary management IP or hostname to be '
+                    'used if san_ip is invalid or becomes inaccessible.'),
     cfg.BoolOpt('storwize_svc_vol_nofmtdisk',
                 default=False,
                 help='Specifies that the volume not be formatted during '
@@ -1965,6 +1972,100 @@ class StorwizeSVCCommonDriver(san.SanDriver,
 
         LOG.debug('leave: check_for_setup_error')
 
+    def _run_ssh(self, cmd_list, check_exit_code=True, attempts=1):
+        cinder_utils.check_ssh_injection(cmd_list)
+        command = ' '.join(cmd_list)
+        if not self.sshpool:
+            try:
+                self.sshpool = self._set_up_sshpool(self.configuration.san_ip)
+            except paramiko.SSHException:
+                LOG.warning(_LW('Unable to use san_ip to create SSHPool. Now '
+                                'attempting to use storwize_san_secondary_ip '
+                                'to create SSHPool.'))
+                if self.configuration.storwize_san_secondary_ip is not None:
+                    self.sshpool = self._set_up_sshpool(
+                        self.configuration.storwize_san_secondary_ip)
+                else:
+                    LOG.warning(_LW('Unable to create SSHPool using san_ip '
+                                    'and not able to use '
+                                    'storwize_san_secondary_ip since it is '
+                                    'not configured.'))
+                    raise
+        try:
+            self._ssh_execute(self.sshpool.command,
+                              check_exit_code, attempts)
+
+        except Exception:
+            # Need to check if creating an SSHPool storwize_san_secondary_ip
+            # before raising an error.
+
+            if self.configuration.storwize_san_secondary_ip is not None:
+
+                LOG.warning(_LW("Unable to execute SSH command. "
+                                "Attempting to switch IP to %s."),
+                            self.configuration.storwize_san_secondary_ip)
+                self.sshpool = self._set_up_sshpool(
+                    self.configuration.storwize_san_secondary_ip)
+                self._ssh_execute(self.sshpool.command,
+                                  check_exit_code, attempts)
+            else:
+                LOG.warning(_LW('Unable to execute SSH command. '
+                                'Not able to use '
+                                'storwize_san_secondary_ip since it is '
+                                'not configured.'))
+                with excutils.save_and_reraise_exception():
+                    LOG.error(_LE("Error running SSH command: %s"),
+                              command)
+
+    def _set_up_sshpool(self, ip):
+        password = self.configuration.san_password
+        privatekey = self.configuration.san_private_key
+        min_size = self.configuration.ssh_min_pool_conn
+        max_size = self.configuration.ssh_max_pool_conn
+        sshpool = ssh_utils.SSHPool(
+            ip,
+            self.configuration.san_ssh_port,
+            self.configuration.ssh_conn_timeout,
+            self.configuration.san_login,
+            password=password,
+            privatekey=privatekey,
+            min_size=min_size,
+            max_size=max_size)
+
+        return sshpool
+
+    def _ssh_execute(self, sshpool, command,
+                     check_exit_code = True, attempts=1):
+        try:
+            with sshpool.item() as ssh:
+                while attempts > 0:
+                    attempts -= 1
+                    try:
+                        return processutils.ssh_execute(
+                            ssh,
+                            command,
+                            check_exit_code=check_exit_code)
+                    except Exception as e:
+                            LOG.error(_LE('Error has occurred: %s'), e)
+                            last_exception = e
+                            greenthread.sleep(random.randint(20, 500) / 100.0)
+                    try:
+                        raise processutils.ProcessExecutionError(
+                            exit_code=last_exception.exit_code,
+                            stdout=last_exception.stdout,
+                            stderr=last_exception.stderr,
+                            cmd=last_exception.cmd)
+                    except AttributeError:
+                        raise processutils.ProcessExecutionError(
+                            exit_code=-1,
+                            stdout="",
+                            stderr="Error running SSH command",
+                            cmd=command)
+
+        except Exception:
+            with excutils.save_and_reraise_exception():
+                LOG.error(_LE("Error running SSH command: %s"), command)
+
     def ensure_export(self, ctxt, volume):
         """Check that the volume exists on the storage.
 

diff --git a/releasenotes/notes/storwize-multiple-management-ip-1cd364d63879d9b8.yaml b/releasenotes/notes/storwize-multiple-management-ip-1cd364d63879d9b8.yaml
new file mode 100644 (file)
index 0000000..6390ca7
--- /dev/null
+++ b/releasenotes/notes/storwize-multiple-management-ip-1cd364d63879d9b8.yaml
@@ -0,0 +1,3 @@
+---
+features:
+  - Add multiple management IP support to Storwize SVC driver.