From: Stefan Amann Date: Mon, 26 Jan 2015 12:49:16 +0000 (+0100) Subject: Adjust Cinder to support FCP on System z systems X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=c67a8aaa9ccebb8307f08ba77001b58e2eac2922;p=openstack-build%2Fcinder-build.git Adjust Cinder to support FCP on System z systems There are some platform specific changes needed to allow Cinder to support FCP on System z: the System z specific format of the device file paths, adding and removing devices explicitely, and ignoring vHBAs which are offline. Partial-Implements blueprint linux-systemz Change-Id: Ia61c37246235bec618de5bb7f718e8b5520bc9ae --- diff --git a/cinder/brick/initiator/connector.py b/cinder/brick/initiator/connector.py index 84f24e853..c03ff39df 100644 --- a/cinder/brick/initiator/connector.py +++ b/cinder/brick/initiator/connector.py @@ -15,11 +15,13 @@ import copy import os +import platform import socket import time from oslo_concurrency import lockutils from oslo_concurrency import processutils as putils +import six from cinder.brick import exception from cinder.brick import executor @@ -31,6 +33,9 @@ from cinder.i18n import _, _LE, _LW from cinder.openstack.common import log as logging from cinder.openstack.common import loopingcall +S390X = "s390x" +S390 = "s390" + LOG = logging.getLogger(__name__) synchronized = lockutils.synchronized_with_prefix('brick-') @@ -105,9 +110,10 @@ class InitiatorConnector(executor.Executor): def factory(protocol, root_helper, driver=None, execute=putils.execute, use_multipath=False, device_scan_attempts=DEVICE_SCAN_ATTEMPTS_DEFAULT, + arch=platform.machine(), *args, **kwargs): - """Build a Connector object based upon protocol.""" - LOG.debug("Factory for %s" % protocol) + """Build a Connector object based upon protocol and architecture.""" + LOG.debug("Factory for %s on %s" % (protocol, arch)) protocol = protocol.upper() if protocol == "ISCSI": return ISCSIConnector(root_helper=root_helper, @@ -124,13 +130,22 @@ class InitiatorConnector(executor.Executor): device_scan_attempts=device_scan_attempts, *args, **kwargs) elif protocol == "FIBRE_CHANNEL": - return FibreChannelConnector(root_helper=root_helper, - driver=driver, - execute=execute, - use_multipath=use_multipath, - device_scan_attempts= - device_scan_attempts, - *args, **kwargs) + if arch in (S390, S390X): + return FibreChannelConnectorS390X(root_helper=root_helper, + driver=driver, + execute=execute, + use_multipath=use_multipath, + device_scan_attempts= + device_scan_attempts, + *args, **kwargs) + else: + return FibreChannelConnector(root_helper=root_helper, + driver=driver, + execute=execute, + use_multipath=use_multipath, + device_scan_attempts= + device_scan_attempts, + *args, **kwargs) elif protocol == "AOE": return AoEConnector(root_helper=root_helper, driver=driver, @@ -673,31 +688,12 @@ class FibreChannelConnector(InitiatorConnector): LOG.debug("execute = %s" % self._execute) device_info = {'type': 'block'} + hbas = self._linuxfc.get_fc_hbas_info() ports = connection_properties['target_wwn'] - wwns = [] - # we support a list of wwns or a single wwn - if isinstance(ports, list): - for wwn in ports: - wwns.append(str(wwn)) - elif isinstance(ports, basestring): - wwns.append(str(ports)) + possible_devs = self._get_possible_devices(hbas, ports) - # We need to look for wwns on every hba - # because we don't know ahead of time - # where they will show up. - hbas = self._linuxfc.get_fc_hbas_info() - host_devices = [] - for hba in hbas: - pci_num = self._get_pci_num(hba) - if pci_num is not None: - for wwn in wwns: - target_wwn = "0x%s" % wwn.lower() - host_device = ("/dev/disk/by-path/pci-%s-fc-%s-lun-%s" % - (pci_num, - target_wwn, - connection_properties.get( - 'target_lun', 0))) - host_devices.append(host_device) + lun = connection_properties.get('target_lun', 0) + host_devices = self._get_host_devices(possible_devs, lun) if len(host_devices) == 0: # this is empty because we don't have any FC HBAs @@ -770,6 +766,50 @@ class FibreChannelConnector(InitiatorConnector): device_info['devices'] = devices return device_info + def _get_host_devices(self, possible_devs, lun): + host_devices = [] + for pci_num, target_wwn in possible_devs: + host_device = "/dev/disk/by-path/pci-%s-fc-%s-lun-%s" % ( + pci_num, + target_wwn, + lun) + host_devices.append(host_device) + return host_devices + + def _get_possible_devices(self, hbas, wwnports): + """Compute the possible valid fibre channel device options. + + :param hbas: available hba devices. + :param wwnports: possible wwn addresses. Can either be string + or list of strings. + + :returns: list of (pci_id, wwn) tuples + + Given one or more wwn (mac addresses for fibre channel) ports + do the matrix math to figure out a set of pci device, wwn + tuples that are potentially valid (they won't all be). This + provides a search space for the device connection. + + """ + # the wwn (think mac addresses for fiber channel devices) can + # either be a single value or a list. Normalize it to a list + # for further operations. + wwns = [] + if isinstance(wwnports, list): + for wwn in wwnports: + wwns.append(str(wwn)) + elif isinstance(wwnports, six.string_types): + wwns.append(str(wwnports)) + + raw_devices = [] + for hba in hbas: + pci_num = self._get_pci_num(hba) + if pci_num is not None: + for wwn in wwns: + target_wwn = "0x%s" % wwn.lower() + raw_devices.append((pci_num, target_wwn)) + return raw_devices + @synchronized('connect_volume') def disconnect_volume(self, connection_properties, device_info): """Detach the volume from instance_name. @@ -790,6 +830,9 @@ class FibreChannelConnector(InitiatorConnector): LOG.debug("devices to remove = %s" % devices) self._linuxscsi.flush_multipath_device(multipath_id) + self._remove_devices(connection_properties, devices) + + def _remove_devices(self, connection_properties, devices): # There may have been more than 1 device mounted # by the kernel for this volume. We have to remove # all of them @@ -818,6 +861,69 @@ class FibreChannelConnector(InitiatorConnector): return pci_num +class FibreChannelConnectorS390X(FibreChannelConnector): + """Connector class to attach/detach Fibre Channel volumes on S390X arch.""" + + def __init__(self, root_helper, driver=None, + execute=putils.execute, use_multipath=False, + device_scan_attempts=DEVICE_SCAN_ATTEMPTS_DEFAULT, + *args, **kwargs): + super(FibreChannelConnectorS390X, self).__init__(root_helper, + driver=driver, + execute=execute, + device_scan_attempts= + device_scan_attempts, + *args, **kwargs) + LOG.debug("Initializing Fibre Channel connector for S390") + self._linuxscsi = linuxscsi.LinuxSCSI(root_helper, execute) + self._linuxfc = linuxfc.LinuxFibreChannelS390X(root_helper, execute) + self.use_multipath = use_multipath + + def set_execute(self, execute): + super(FibreChannelConnectorS390X, self).set_execute(execute) + self._linuxscsi.set_execute(execute) + self._linuxfc.set_execute(execute) + + def _get_host_devices(self, possible_devs, lun): + host_devices = [] + for pci_num, target_wwn in possible_devs: + target_lun = self._get_lun_string(lun) + host_device = self._get_device_file_path( + pci_num, + target_wwn, + target_lun) + self._linuxfc.configure_scsi_device(pci_num, target_wwn, + target_lun) + host_devices.append(host_device) + return host_devices + + def _get_lun_string(self, lun): + target_lun = 0 + if lun < 256: + target_lun = "0x00%02x000000000000" % lun + elif lun <= 0xffffffff: + target_lun = "0x%08x00000000" % lun + return target_lun + + def _get_device_file_path(self, pci_num, target_wwn, target_lun): + host_device = "/dev/disk/by-path/ccw-%s-zfcp-%s:%s" % ( + pci_num, + target_wwn, + target_lun) + return host_device + + def _remove_devices(self, connection_properties, devices): + hbas = self._linuxfc.get_fc_hbas_info() + ports = connection_properties['target_wwn'] + possible_devs = self._get_possible_devices(hbas, ports) + lun = connection_properties.get('target_lun', 0) + target_lun = self._get_lun_string(lun) + for pci_num, target_wwn in possible_devs: + self._linuxfc.deconfigure_scsi_device(pci_num, + target_wwn, + target_lun) + + class AoEConnector(InitiatorConnector): """Connector class to attach/detach AoE volumes.""" def __init__(self, root_helper, driver=None, diff --git a/cinder/brick/initiator/linuxfc.py b/cinder/brick/initiator/linuxfc.py index 7600dc2e7..c6818b587 100644 --- a/cinder/brick/initiator/linuxfc.py +++ b/cinder/brick/initiator/linuxfc.py @@ -138,3 +138,75 @@ class LinuxFibreChannel(linuxscsi.LinuxSCSI): wwnns.append(wwnn) return wwnns + + +class LinuxFibreChannelS390X(LinuxFibreChannel): + def __init__(self, root_helper, execute=putils.execute, + *args, **kwargs): + super(LinuxFibreChannelS390X, self).__init__(root_helper, execute, + *args, **kwargs) + + def get_fc_hbas_info(self): + """Get Fibre Channel WWNs and device paths from the system, if any.""" + + hbas = self.get_fc_hbas() + if not hbas: + return [] + + hbas_info = [] + for hba in hbas: + if hba['port_state'] == 'Online': + wwpn = hba['port_name'].replace('0x', '') + wwnn = hba['node_name'].replace('0x', '') + device_path = hba['ClassDevicepath'] + device = hba['ClassDevice'] + hbas_info.append({'port_name': wwpn, + 'node_name': wwnn, + 'host_device': device, + 'device_path': device_path}) + return hbas_info + + def configure_scsi_device(self, device_number, target_wwn, lun): + """Write the LUN to the port's unit_add attribute. + + If auto-discovery of LUNs is disabled on s390 platforms + luns need to be added to the configuration through the + unit_add interface + """ + LOG.debug("Configure lun for s390: device_number=(%(device_num)s) " + "target_wwn=(%(target_wwn)s) target_lun=(%(target_lun)s)", + {'device_num': device_number, + 'target_wwn': target_wwn, + 'target_lun': lun}) + zfcp_device_command = ("/sys/bus/ccw/drivers/zfcp/%s/%s/unit_add" % + (device_number, target_wwn)) + LOG.debug("unit_add call for s390 execute: %s", zfcp_device_command) + try: + self.echo_scsi_command(zfcp_device_command, lun) + except putils.ProcessExecutionError as exc: + msg = _LW("unit_add call for s390 failed exit (%(code)s), " + "stderr (%(stderr)s)") + LOG.warn(msg, {'code': exc.exit_code, 'stderr': exc.stderr}) + + def deconfigure_scsi_device(self, device_number, target_wwn, lun): + """Write the LUN to the port's unit_remove attribute. + + If auto-discovery of LUNs is disabled on s390 platforms + luns need to be removed from the configuration through the + unit_remove interface + """ + LOG.debug("Deconfigure lun for s390: " + "device_number=(%(device_num)s) " + "target_wwn=(%(target_wwn)s) target_lun=(%(target_lun)s)", + {'device_num': device_number, + 'target_wwn': target_wwn, + 'target_lun': lun}) + zfcp_device_command = ("/sys/bus/ccw/drivers/zfcp/%s/%s/unit_remove" % + (device_number, target_wwn)) + LOG.debug("unit_remove call for s390 execute: %s", zfcp_device_command) + try: + self.echo_scsi_command(zfcp_device_command, lun) + except putils.ProcessExecutionError as exc: + msg = _LW("unit_remove call for s390 failed exit (%(code)s), " + "stderr (%(stderr)s)") + LOG.warn(msg, {'code': exc.exit_code, 'stderr': exc.stderr}) diff --git a/cinder/tests/brick/test_brick_connector.py b/cinder/tests/brick/test_brick_connector.py index 7aee1daf7..7daee4904 100644 --- a/cinder/tests/brick/test_brick_connector.py +++ b/cinder/tests/brick/test_brick_connector.py @@ -112,6 +112,10 @@ class ConnectorTestCase(test.TestCase): obj = connector.InitiatorConnector.factory('fibre_channel', None) self.assertEqual(obj.__class__.__name__, "FibreChannelConnector") + obj = connector.InitiatorConnector.factory('fibre_channel', None, + arch='s390x') + self.assertEqual(obj.__class__.__name__, "FibreChannelConnectorS390X") + obj = connector.InitiatorConnector.factory('aoe', None) self.assertEqual(obj.__class__.__name__, "AoEConnector") @@ -724,6 +728,45 @@ class FibreChannelConnectorTestCase(ConnectorTestCase): connection_info['data']) +class FibreChannelConnectorS390XTestCase(ConnectorTestCase): + + def setUp(self): + super(FibreChannelConnectorS390XTestCase, self).setUp() + self.connector = connector.FibreChannelConnectorS390X( + None, execute=self.fake_execute, use_multipath=False) + self.assertIsNotNone(self.connector) + self.assertIsNotNone(self.connector._linuxfc) + self.assertEqual(self.connector._linuxfc.__class__.__name__, + "LinuxFibreChannelS390X") + self.assertIsNotNone(self.connector._linuxscsi) + + @mock.patch.object(linuxfc.LinuxFibreChannelS390X, 'configure_scsi_device') + def test_get_host_devices(self, mock_configure_scsi_device): + lun = 2 + possible_devs = [(3, 5), ] + devices = self.connector._get_host_devices(possible_devs, lun) + mock_configure_scsi_device.assert_called_with(3, 5, + "0x0002000000000000") + self.assertEqual(1, len(devices)) + device_path = "/dev/disk/by-path/ccw-3-zfcp-5:0x0002000000000000" + self.assertEqual(devices[0], device_path) + + @mock.patch.object(connector.FibreChannelConnectorS390X, + '_get_possible_devices', return_value=[(3, 5), ]) + @mock.patch.object(linuxfc.LinuxFibreChannelS390X, 'get_fc_hbas_info', + return_value=[]) + @mock.patch.object(linuxfc.LinuxFibreChannelS390X, + 'deconfigure_scsi_device') + def test_remove_devices(self, mock_deconfigure_scsi_device, + mock_get_fc_hbas_info, mock_get_possible_devices): + connection_properties = {'target_wwn': 5, 'target_lun': 2} + self.connector._remove_devices(connection_properties, devices=None) + mock_deconfigure_scsi_device.assert_called_with(3, 5, + "0x0002000000000000") + mock_get_fc_hbas_info.assert_called_once_with() + mock_get_possible_devices.assert_called_once_with([], 5) + + class FakeFixedIntervalLoopingCall(object): def __init__(self, f=None, *args, **kw): self.args = args diff --git a/cinder/tests/brick/test_brick_linuxfc.py b/cinder/tests/brick/test_brick_linuxfc.py index 8e93ef71a..55de4fdfc 100644 --- a/cinder/tests/brick/test_brick_linuxfc.py +++ b/cinder/tests/brick/test_brick_linuxfc.py @@ -168,3 +168,77 @@ Class = "fc_host" """ + + +class LinuxFCS390XTestCase(LinuxFCTestCase): + + def setUp(self): + super(LinuxFCS390XTestCase, self).setUp() + self.cmds = [] + self.stubs.Set(os.path, 'exists', lambda x: True) + self.lfc = linuxfc.LinuxFibreChannelS390X(None, + execute=self.fake_execute) + + def test_get_fc_hbas_info(self): + def fake_exec(a, b, c, d, run_as_root=True, root_helper='sudo'): + return SYSTOOL_FC_S390X, None + self.stubs.Set(self.lfc, "_execute", fake_exec) + hbas_info = self.lfc.get_fc_hbas_info() + expected = [{'device_path': '/sys/devices/css0/0.0.02ea/' + '0.0.3080/host0/fc_host/host0', + 'host_device': 'host0', + 'node_name': '1234567898765432', + 'port_name': 'c05076ffe680a960'}] + self.assertEqual(expected, hbas_info) + + def test_configure_scsi_device(self): + device_number = "0.0.2319" + target_wwn = "0x50014380242b9751" + lun = 1 + self.lfc.configure_scsi_device(device_number, target_wwn, lun) + expected_commands = [('tee -a /sys/bus/ccw/drivers/zfcp/' + '0.0.2319/0x50014380242b9751/unit_add')] + self.assertEqual(expected_commands, self.cmds) + + def test_deconfigure_scsi_device(self): + device_number = "0.0.2319" + target_wwn = "0x50014380242b9751" + lun = 1 + self.lfc.deconfigure_scsi_device(device_number, target_wwn, lun) + expected_commands = [('tee -a /sys/bus/ccw/drivers/zfcp/' + '0.0.2319/0x50014380242b9751/unit_remove')] + self.assertEqual(expected_commands, self.cmds) + +SYSTOOL_FC_S390X = """ +Class = "fc_host" + + Class Device = "host0" + Class Device path = "/sys/devices/css0/0.0.02ea/0.0.3080/host0/fc_host/host0" + active_fc4s = "0x00 0x00 0x01 0x00 0x00 0x00 0x00 0x00 0x00 0x00 \ + 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 \ + 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 " + dev_loss_tmo = "60" + maxframe_size = "2112 bytes" + node_name = "0x1234567898765432" + permanent_port_name = "0xc05076ffe6803081" + port_id = "0x010014" + port_name = "0xc05076ffe680a960" + port_state = "Online" + port_type = "NPIV VPORT" + serial_number = "IBM00000000000P30" + speed = "8 Gbit" + supported_classes = "Class 2, Class 3" + supported_fc4s = "0x00 0x00 0x01 0x00 0x00 0x00 0x00 0x00 0x00 0x00 \ + 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 \ + 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 " + supported_speeds = "2 Gbit, 4 Gbit" + symbolic_name = "IBM 2827 00000000000P30 \ + PCHID: 0308 NPIV UlpId: 01EA0A00 DEVNO: 0.0.1234 NAME: dummy" + tgtid_bind_type = "wwpn (World Wide Port Name)" + uevent = + + Device = "host0" + Device path = "/sys/devices/css0/0.0.02ea/0.0.3080/host0" + uevent = "DEVTYPE=scsi_host" + +"""