From 0a155bea36aee44c5e7ecd026efe9537eee25eda Mon Sep 17 00:00:00 2001 From: "Walter A. Boring IV" Date: Tue, 11 Jun 2013 15:27:56 -0700 Subject: [PATCH] Add Brick iSCSI attach/detach. This patch adds the new brick initiator connector object which contains the code to do volume attach and detach to a host machine. It includes hooks to work in both cinder and nova. Nova has different exec wrapper and it also needs to talk to a hypervisor at certain points during detach. This patch also pulls the copy/pasted code from nova in the base ISCSIDriver's _attach_volume method to use this new brick library. This patch also includes a fix in the ISCSIDriver's copy_volume_to_image code that didn't actually detach a volume when it was done with the operation. Bug 1194962 This patch also includes a fix for iSCSI detaches where the iSCSI LUN wasn't being removed from the system until the very last detach issues an iscsiadm logout. Bug 1112483 This patch includes a fix for iSCSI multipath detaches where the multipath device and the iSCSI LUNs for the multipath device were never removed from the kernel. Bug 1112483 Blueprint cinder-refactor-attach Change-Id: Ieb181f896adb9230bbb6a2e5c42f261d61a0f140 --- cinder/brick/initiator/__init__.py | 16 ++ cinder/brick/initiator/connector.py | 375 ++++++++++++++++++++++++++ cinder/brick/initiator/executor.py | 36 +++ cinder/brick/initiator/host_driver.py | 30 +++ cinder/brick/initiator/linuxscsi.py | 157 +++++++++++ cinder/tests/test_brick_initiator.py | 282 +++++++++++++++++++ cinder/volume/driver.py | 94 ++----- etc/cinder/rootwrap.d/volume.filters | 5 + 8 files changed, 918 insertions(+), 77 deletions(-) create mode 100644 cinder/brick/initiator/__init__.py create mode 100644 cinder/brick/initiator/connector.py create mode 100644 cinder/brick/initiator/executor.py create mode 100644 cinder/brick/initiator/host_driver.py create mode 100644 cinder/brick/initiator/linuxscsi.py create mode 100644 cinder/tests/test_brick_initiator.py diff --git a/cinder/brick/initiator/__init__.py b/cinder/brick/initiator/__init__.py new file mode 100644 index 000000000..5e8da711f --- /dev/null +++ b/cinder/brick/initiator/__init__.py @@ -0,0 +1,16 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/cinder/brick/initiator/connector.py b/cinder/brick/initiator/connector.py new file mode 100644 index 000000000..ac67f0d71 --- /dev/null +++ b/cinder/brick/initiator/connector.py @@ -0,0 +1,375 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import executor +import host_driver +import linuxscsi +import os +import time + +from oslo.config import cfg + +from cinder import exception +from cinder.openstack.common.gettextutils import _ +from cinder.openstack.common import lockutils +from cinder.openstack.common import log as logging +from cinder.openstack.common import processutils as putils + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF +synchronized = lockutils.synchronized_with_prefix('brick-') + + +class InitiatorConnector(executor.Executor): + def __init__(self, driver=None, execute=putils.execute, + root_helper="sudo", *args, **kwargs): + super(InitiatorConnector, self).__init__(execute, root_helper, + *args, **kwargs) + if not driver: + driver = host_driver.HostDriver() + self.set_driver(driver) + + self._linuxscsi = linuxscsi.LinuxSCSI(execute, root_helper) + + def set_driver(self, driver): + """The driver used to find used LUNs.""" + + self.driver = driver + + def connect_volume(self, connection_properties): + raise NotImplementedError() + + def disconnect_volume(self, connection_properties): + raise NotImplementedError() + + +class ISCSIConnector(InitiatorConnector): + """"Connector class to attach/detach iSCSI volumes.""" + + def __init__(self, driver=None, execute=putils.execute, + root_helper="sudo", use_multipath=False, + *args, **kwargs): + super(ISCSIConnector, self).__init__(driver, execute, root_helper, + *args, **kwargs) + self.use_multipath = use_multipath + + @synchronized('connect_volume') + def connect_volume(self, connection_properties): + """Attach the volume to instance_name. + + connection_properties for iSCSI must include: + target_portal - ip and optional port + target_iqn - iSCSI Qualified Name + target_lun - LUN id of the volume + """ + + device_info = {'type': 'block'} + + if self.use_multipath: + #multipath installed, discovering other targets if available + target_portal = connection_properties['target_portal'] + out = self._run_iscsiadm_bare(['-m', + 'discovery', + '-t', + 'sendtargets', + '-p', + target_portal], + check_exit_code=[0, 255])[0] \ + or "" + + for ip in self._get_target_portals_from_iscsiadm_output(out): + props = connection_properties.copy() + props['target_portal'] = ip + self._connect_to_iscsi_portal(props) + + self._rescan_iscsi() + else: + self._connect_to_iscsi_portal(connection_properties) + + host_device = self._get_device_path(connection_properties) + + # The /dev/disk/by-path/... node is not always present immediately + # TODO(justinsb): This retry-with-delay is a pattern, move to utils? + tries = 0 + while not os.path.exists(host_device): + if tries >= CONF.num_iscsi_scan_tries: + raise exception.CinderException( + _("iSCSI device not found at %s") % (host_device)) + + LOG.warn(_("ISCSI volume not yet found at: %(host_device)s. " + "Will rescan & retry. Try number: %(tries)s"), + {'host_device': host_device, + 'tries': tries}) + + # The rescan isn't documented as being necessary(?), but it helps + self._run_iscsiadm(connection_properties, ("--rescan",)) + + tries = tries + 1 + if not os.path.exists(host_device): + time.sleep(tries ** 2) + + if tries != 0: + LOG.debug(_("Found iSCSI node %(host_device)s " + "(after %(tries)s rescans)"), + {'host_device': host_device, 'tries': tries}) + + if self.use_multipath: + #we use the multipath device instead of the single path device + self._rescan_multipath() + multipath_device = self._get_multipath_device_name(host_device) + if multipath_device is not None: + host_device = multipath_device + + device_info['path'] = host_device + return device_info + + @synchronized('connect_volume') + def disconnect_volume(self, connection_properties): + """Detach the volume from instance_name. + + connection_properties for iSCSI must include: + target_portal - ip and optional port + target_iqn - iSCSI Qualified Name + target_lun - LUN id of the volume + """ + host_device = self._get_device_path(connection_properties) + multipath_device = None + if self.use_multipath: + multipath_device = self._get_multipath_device_name(host_device) + if multipath_device: + self._linuxscsi.remove_multipath_device(multipath_device) + return self._disconnect_volume_multipath_iscsi( + connection_properties, multipath_device) + + # remove the device from the scsi subsystem + # this eliminates any stale entries until logout + dev_name = self._linuxscsi.get_name_from_path(host_device) + if dev_name: + self._linuxscsi.remove_scsi_device(dev_name) + + # NOTE(vish): Only disconnect from the target if no luns from the + # target are in use. + device_prefix = ("/dev/disk/by-path/ip-%(portal)s-iscsi-%(iqn)s-lun-" % + {'portal': connection_properties['target_portal'], + 'iqn': connection_properties['target_iqn']}) + devices = self.driver.get_all_block_devices() + devices = [dev for dev in devices if dev.startswith(device_prefix)] + + if not devices: + self._disconnect_from_iscsi_portal(connection_properties) + + def _get_device_path(self, connection_properties): + path = ("/dev/disk/by-path/ip-%(portal)s-iscsi-%(iqn)s-lun-%(lun)s" % + {'portal': connection_properties['target_portal'], + 'iqn': connection_properties['target_iqn'], + 'lun': connection_properties.get('target_lun', 0)}) + return path + + def _run_iscsiadm(self, connection_properties, iscsi_command, **kwargs): + check_exit_code = kwargs.pop('check_exit_code', 0) + (out, err) = self._execute('iscsiadm', '-m', 'node', '-T', + connection_properties['target_iqn'], + '-p', + connection_properties['target_portal'], + *iscsi_command, run_as_root=True, + root_helper=self._root_helper, + check_exit_code=check_exit_code) + LOG.debug("iscsiadm %s: stdout=%s stderr=%s" % + (iscsi_command, out, err)) + return (out, err) + + def _iscsiadm_update(self, connection_properties, property_key, + property_value, **kwargs): + iscsi_command = ('--op', 'update', '-n', property_key, + '-v', property_value) + return self._run_iscsiadm(connection_properties, iscsi_command, + **kwargs) + + def _get_target_portals_from_iscsiadm_output(self, output): + return [line.split()[0] for line in output.splitlines()] + + def _disconnect_volume_multipath_iscsi(self, connection_properties, + multipath_name): + """This removes a multipath device and it's LUNs.""" + LOG.debug("Disconnect multipath device %s" % multipath_name) + self._rescan_iscsi() + self._rescan_multipath() + block_devices = self.driver.get_all_block_devices() + devices = [] + for dev in block_devices: + if "/mapper/" in dev: + devices.append(dev) + else: + mpdev = self._get_multipath_device_name(dev) + if mpdev: + devices.append(mpdev) + + if not devices: + # disconnect if no other multipath devices + self._disconnect_mpath(connection_properties) + return + + other_iqns = [self._get_multipath_iqn(device) + for device in devices] + + if connection_properties['target_iqn'] not in other_iqns: + # disconnect if no other multipath devices with same iqn + self._disconnect_mpath(connection_properties) + return + + # else do not disconnect iscsi portals, + # as they are used for other luns + return + + def _connect_to_iscsi_portal(self, connection_properties): + # NOTE(vish): If we are on the same host as nova volume, the + # discovery makes the target so we don't need to + # run --op new. Therefore, we check to see if the + # target exists, and if we get 255 (Not Found), then + # we run --op new. This will also happen if another + # volume is using the same target. + try: + self._run_iscsiadm(connection_properties, ()) + except putils.ProcessExecutionError as exc: + # iscsiadm returns 21 for "No records found" after version 2.0-871 + if exc.exit_code in [21, 255]: + self._run_iscsiadm(connection_properties, ('--op', 'new')) + else: + raise + + if connection_properties.get('auth_method'): + self._iscsiadm_update(connection_properties, + "node.session.auth.authmethod", + connection_properties['auth_method']) + self._iscsiadm_update(connection_properties, + "node.session.auth.username", + connection_properties['auth_username']) + self._iscsiadm_update(connection_properties, + "node.session.auth.password", + connection_properties['auth_password']) + + #duplicate logins crash iscsiadm after load, + #so we scan active sessions to see if the node is logged in. + out = self._run_iscsiadm_bare(["-m", "session"], + run_as_root=True, + check_exit_code=[0, 1, 21])[0] or "" + + portals = [{'portal': p.split(" ")[2], 'iqn': p.split(" ")[3]} + for p in out.splitlines() if p.startswith("tcp:")] + + stripped_portal = connection_properties['target_portal'].split(",")[0] + if len(portals) == 0 or len([s for s in portals + if stripped_portal == + s['portal'].split(",")[0] + and + s['iqn'] == + connection_properties['target_iqn']] + ) == 0: + try: + self._run_iscsiadm(connection_properties, + ("--login",), + check_exit_code=[0, 255]) + except putils.ProcessExecutionError as err: + #as this might be one of many paths, + #only set successfull logins to startup automatically + if err.exit_code in [15]: + self._iscsiadm_update(connection_properties, + "node.startup", + "automatic") + return + + self._iscsiadm_update(connection_properties, + "node.startup", + "automatic") + + def _disconnect_from_iscsi_portal(self, connection_properties): + self._iscsiadm_update(connection_properties, "node.startup", "manual", + check_exit_code=[0, 21, 255]) + self._run_iscsiadm(connection_properties, ("--logout",), + check_exit_code=[0, 21, 255]) + self._run_iscsiadm(connection_properties, ('--op', 'delete'), + check_exit_code=[0, 21, 255]) + + def _get_multipath_device_name(self, single_path_device): + device = os.path.realpath(single_path_device) + out = self._run_multipath(['-ll', + device], + check_exit_code=[0, 1])[0] + mpath_line = [line for line in out.splitlines() + if "scsi_id" not in line] # ignore udev errors + if len(mpath_line) > 0 and len(mpath_line[0]) > 0: + return "/dev/mapper/%s" % mpath_line[0].split(" ")[0] + + return None + + def _get_iscsi_devices(self): + try: + devices = list(os.walk('/dev/disk/by-path'))[0][-1] + except IndexError: + return [] + return [entry for entry in devices if entry.startswith("ip-")] + + def _disconnect_mpath(self, connection_properties): + entries = self._get_iscsi_devices() + ips = [ip.split("-")[1] for ip in entries + if connection_properties['target_iqn'] in ip] + for ip in ips: + props = connection_properties.copy() + props['target_portal'] = ip + self._disconnect_from_iscsi_portal(props) + + self._rescan_multipath() + + def _get_multipath_iqn(self, multipath_device): + entries = self._get_iscsi_devices() + for entry in entries: + entry_real_path = os.path.realpath("/dev/disk/by-path/%s" % entry) + entry_multipath = self._get_multipath_device_name(entry_real_path) + if entry_multipath == multipath_device: + return entry.split("iscsi-")[1].split("-lun")[0] + return None + + def _run_iscsiadm_bare(self, iscsi_command, **kwargs): + check_exit_code = kwargs.pop('check_exit_code', 0) + (out, err) = self._execute('iscsiadm', + *iscsi_command, + run_as_root=True, + root_helper=self._root_helper, + check_exit_code=check_exit_code) + LOG.debug("iscsiadm %s: stdout=%s stderr=%s" % + (iscsi_command, out, err)) + return (out, err) + + def _run_multipath(self, multipath_command, **kwargs): + check_exit_code = kwargs.pop('check_exit_code', 0) + (out, err) = self._execute('multipath', + *multipath_command, + run_as_root=True, + root_helper=self._root_helper, + check_exit_code=check_exit_code) + LOG.debug("multipath %s: stdout=%s stderr=%s" % + (multipath_command, out, err)) + return (out, err) + + def _rescan_iscsi(self): + self._run_iscsiadm_bare(('-m', 'node', '--rescan'), + check_exit_code=[0, 1, 21, 255]) + self._run_iscsiadm_bare(('-m', 'session', '--rescan'), + check_exit_code=[0, 1, 21, 255]) + + def _rescan_multipath(self): + self._run_multipath('-r', check_exit_code=[0, 1, 21]) diff --git a/cinder/brick/initiator/executor.py b/cinder/brick/initiator/executor.py new file mode 100644 index 000000000..f70fea670 --- /dev/null +++ b/cinder/brick/initiator/executor.py @@ -0,0 +1,36 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# (c) Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Generic exec utility that allows us to set the + execute and root_helper attributes for putils. + Some projects need their own execute wrapper + and root_helper settings, so this provides that hook. +""" + +from cinder.openstack.common import processutils as putils + + +class Executor(object): + def __init__(self, execute=putils.execute, root_helper="sudo", + *args, **kwargs): + self.set_execute(execute) + self.set_root_helper(root_helper) + + def set_execute(self, execute): + self._execute = execute + + def set_root_helper(self, helper): + self._root_helper = helper diff --git a/cinder/brick/initiator/host_driver.py b/cinder/brick/initiator/host_driver.py new file mode 100644 index 000000000..e675b3705 --- /dev/null +++ b/cinder/brick/initiator/host_driver.py @@ -0,0 +1,30 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + + +class HostDriver(object): + + def get_all_block_devices(self): + """Get the list of all block devices seen in /dev/disk/by-path/""" + dir = "/dev/disk/by-path/" + files = os.listdir(dir) + devices = [] + for file in files: + devices.append(dir + file) + return devices diff --git a/cinder/brick/initiator/linuxscsi.py b/cinder/brick/initiator/linuxscsi.py new file mode 100644 index 000000000..d5000d2fa --- /dev/null +++ b/cinder/brick/initiator/linuxscsi.py @@ -0,0 +1,157 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# (c) Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Generic linux scsi subsystem and Multipath utilities. + + Note, this is not iSCSI. +""" + +import executor +import os + +from cinder.openstack.common.gettextutils import _ +from cinder.openstack.common import log as logging +from cinder.openstack.common import processutils as putils + +LOG = logging.getLogger(__name__) + + +class LinuxSCSI(executor.Executor): + def __init__(self, execute=putils.execute, root_helper="sudo", + *args, **kwargs): + super(LinuxSCSI, self).__init__(execute, root_helper, + *args, **kwargs) + + def echo_scsi_command(self, path, content): + """Used to echo strings to scsi subsystem.""" + + args = ["-a", path] + kwargs = dict(process_input=content, + run_as_root=True, + root_helper=self._root_helper) + self._execute('tee', *args, **kwargs) + + def get_name_from_path(self, path): + """Translates /dev/disk/by-path/ entry to /dev/sdX.""" + + name = os.path.realpath(path) + if name.startswith("/dev/"): + return name + else: + return None + + def remove_scsi_device(self, device): + """Removes a scsi device based upon /dev/sdX name.""" + + path = "/sys/block/%s/device/delete" % device.replace("/dev/", "") + if os.path.exists(path): + LOG.debug("Remove SCSI device(%s) with %s" % (device, path)) + self.echo_scsi_command(path, "1") + + def remove_multipath_device(self, multipath_name): + """This removes LUNs associated with a multipath device + and the multipath device itself. + """ + + LOG.debug("remove multipath device %s" % multipath_name) + mpath_dev = self.find_multipath_device(multipath_name) + if mpath_dev: + devices = mpath_dev['devices'] + LOG.debug("multipath LUNs to remove %s" % devices) + for device in devices: + self.remove_scsi_device(device['device']) + self.flush_multipath_device(mpath_dev['id']) + + def flush_multipath_device(self, device): + try: + self._execute('multipath', '-f', device, run_as_root=True, + root_helper=self._root_helper) + except putils.ProcessExecutionError as exc: + LOG.warn(_("multipath call failed exit (%(code)s)") + % {'code': exc.exit_code}) + + def flush_multipath_devices(self): + try: + self._execute('multipath', '-F', run_as_root=True, + root_helper=self._root_helper) + except putils.ProcessExecutionError as exc: + LOG.warn(_("multipath call failed exit (%(code)s)") + % {'code': exc.exit_code}) + + def find_multipath_device(self, device): + """Find a multipath device associated with a LUN device name. + + device can be either a /dev/sdX entry or a multipath id. + """ + + mdev = None + devices = [] + out = None + try: + (out, err) = self._execute('multipath', '-l', device, + run_as_root=True, + root_helper=self._root_helper) + LOG.error("PISS = %s" % out) + except putils.ProcessExecutionError as exc: + LOG.warn(_("multipath call failed exit (%(code)s)") + % {'code': exc.exit_code}) + return None + + if out: + lines = out.strip() + lines = lines.split("\n") + if lines: + line = lines[0] + info = line.split(" ") + # device line output is different depending + # on /etc/multipath.conf settings. + if info[1][:2] == "dm": + mdev = "/dev/%s" % info[1] + mdev_id = info[0] + elif info[2][:2] == "dm": + mdev = "/dev/%s" % info[2] + mdev_id = info[1].replace('(', '') + mdev_id = mdev_id.replace(')', '') + + if mdev is None: + LOG.warn(_("Couldn't find multipath device %(line)s") + % {'line': line}) + return None + + LOG.debug(_("Found multipath device = %(mdev)s") + % {'mdev': mdev}) + device_lines = lines[3:] + for dev_line in device_lines: + if dev_line.find("policy") != -1: + continue + + dev_line = dev_line.lstrip(' |-`') + dev_info = dev_line.split() + address = dev_info[0].split(":") + + dev = {'device': '/dev/%s' % dev_info[1], + 'host': address[0], 'channel': address[1], + 'id': address[2], 'lun': address[3] + } + + devices.append(dev) + + if mdev is not None: + info = {"device": mdev, + "id": mdev_id, + "devices": devices} + return info + return None diff --git a/cinder/tests/test_brick_initiator.py b/cinder/tests/test_brick_initiator.py new file mode 100644 index 000000000..0acbce3c8 --- /dev/null +++ b/cinder/tests/test_brick_initiator.py @@ -0,0 +1,282 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os.path +import string + +from cinder.brick.initiator import connector +from cinder.brick.initiator import host_driver +from cinder.brick.initiator import linuxscsi +from cinder.openstack.common import log as logging +from cinder import test + +LOG = logging.getLogger(__name__) + + +class ConnectorTestCase(test.TestCase): + + def setUp(self): + super(ConnectorTestCase, self).setUp() + self.cmds = [] + self.stubs.Set(os.path, 'exists', lambda x: True) + + def fake_init(obj): + return + + def fake_execute(self, *cmd, **kwargs): + self.cmds.append(string.join(cmd)) + return "", None + + def test_connect_volume(self): + self.connector = connector.InitiatorConnector() + self.assertRaises(NotImplementedError, + self.connector.connect_volume, None) + + def test_disconnect_volume(self): + self.connector = connector.InitiatorConnector() + self.assertRaises(NotImplementedError, + self.connector.connect_volume, None) + + +class HostDriverTestCase(test.TestCase): + + def setUp(self): + super(HostDriverTestCase, self).setUp() + self.devlist = ['device1', 'device2'] + self.stubs.Set(os, 'listdir', lambda x: self.devlist) + + def test_host_driver(self): + expected = ['/dev/disk/by-path/' + dev for dev in self.devlist] + driver = host_driver.HostDriver() + actual = driver.get_all_block_devices() + self.assertEquals(expected, actual) + + +class LinuxSCSITestCase(test.TestCase): + def setUp(self): + super(LinuxSCSITestCase, self).setUp() + self.cmds = [] + self.stubs.Set(os.path, 'realpath', lambda x: '/dev/sdc') + self.linuxscsi = linuxscsi.LinuxSCSI(execute=self.fake_execute) + + def fake_execute(self, *cmd, **kwargs): + self.cmds.append(string.join(cmd)) + return "", None + + def test_get_name_from_path(self): + device_name = "/dev/sdc" + self.stubs.Set(os.path, 'realpath', lambda x: device_name) + disk_path = ("/dev/disk/by-path/ip-10.10.220.253:3260-" + "iscsi-iqn.2000-05.com.3pardata:21810002ac00383d-lun-0") + name = self.linuxscsi.get_name_from_path(disk_path) + self.assertTrue(name, device_name) + self.stubs.Set(os.path, 'realpath', lambda x: "bogus") + name = self.linuxscsi.get_name_from_path(disk_path) + self.assertIsNone(name) + + def test_remove_scsi_device(self): + self.stubs.Set(os.path, "exists", lambda x: False) + self.linuxscsi.remove_scsi_device("sdc") + expected_commands = [] + self.assertEqual(expected_commands, self.cmds) + self.stubs.Set(os.path, "exists", lambda x: True) + self.linuxscsi.remove_scsi_device("sdc") + expected_commands = [('tee -a /sys/block/sdc/device/delete')] + self.assertEqual(expected_commands, self.cmds) + + def test_flush_multipath_device(self): + self.linuxscsi.flush_multipath_device('/dev/dm-9') + expected_commands = [('multipath -f /dev/dm-9')] + self.assertEqual(expected_commands, self.cmds) + + def test_flush_multipath_devices(self): + self.linuxscsi.flush_multipath_devices() + expected_commands = [('multipath -F')] + self.assertEqual(expected_commands, self.cmds) + + def test_remove_multipath_device(self): + def fake_find_multipath_device(device): + devices = [{'device': '/dev/sde', 'host': 0, + 'channel': 0, 'id': 0, 'lun': 1}, + {'device': '/dev/sdf', 'host': 2, + 'channel': 0, 'id': 0, 'lun': 1}, ] + + info = {"device": "dm-3", + "id": "350002ac20398383d", + "devices": devices} + return info + + self.stubs.Set(os.path, "exists", lambda x: True) + self.stubs.Set(self.linuxscsi, 'find_multipath_device', + fake_find_multipath_device) + + self.linuxscsi.remove_multipath_device('/dev/dm-3') + expected_commands = [('tee -a /sys/block/sde/device/delete'), + ('tee -a /sys/block/sdf/device/delete'), + ('multipath -f 350002ac20398383d'), ] + self.assertEqual(expected_commands, self.cmds) + + def test_find_multipath_device_3par(self): + def fake_execute(*cmd, **kwargs): + out = ("mpath6 (350002ac20398383d) dm-3 3PARdata,VV\n" + "size=2.0G features='0' hwhandler='0' wp=rw\n" + "`-+- policy='round-robin 0' prio=-1 status=active\n" + " |- 0:0:0:1 sde 8:64 active undef running\n" + " `- 2:0:0:1 sdf 8:80 active undef running\n" + ) + return out, None + + def fake_execute2(*cmd, **kwargs): + out = ("350002ac20398383d dm-3 3PARdata,VV\n" + "size=2.0G features='0' hwhandler='0' wp=rw\n" + "`-+- policy='round-robin 0' prio=-1 status=active\n" + " |- 0:0:0:1 sde 8:64 active undef running\n" + " `- 2:0:0:1 sdf 8:80 active undef running\n" + ) + return out, None + + self.stubs.Set(self.linuxscsi, '_execute', fake_execute) + + info = self.linuxscsi.find_multipath_device('/dev/sde') + LOG.error("info = %s" % info) + self.assertEqual("/dev/dm-3", info["device"]) + self.assertEqual("/dev/sde", info['devices'][0]['device']) + self.assertEqual("0", info['devices'][0]['host']) + self.assertEqual("0", info['devices'][0]['id']) + self.assertEqual("0", info['devices'][0]['channel']) + self.assertEqual("1", info['devices'][0]['lun']) + + self.assertEqual("/dev/sdf", info['devices'][1]['device']) + self.assertEqual("2", info['devices'][1]['host']) + self.assertEqual("0", info['devices'][1]['id']) + self.assertEqual("0", info['devices'][1]['channel']) + self.assertEqual("1", info['devices'][1]['lun']) + + def test_find_multipath_device_svc(self): + def fake_execute(*cmd, **kwargs): + out = ("36005076da00638089c000000000004d5 dm-2 IBM,2145\n" + "size=954M features='1 queue_if_no_path' hwhandler='0'" + " wp=rw\n" + "|-+- policy='round-robin 0' prio=-1 status=active\n" + "| |- 6:0:2:0 sde 8:64 active undef running\n" + "| `- 6:0:4:0 sdg 8:96 active undef running\n" + "`-+- policy='round-robin 0' prio=-1 status=enabled\n" + " |- 6:0:3:0 sdf 8:80 active undef running\n" + " `- 6:0:5:0 sdh 8:112 active undef running\n" + ) + return out, None + + self.stubs.Set(self.linuxscsi, '_execute', fake_execute) + + info = self.linuxscsi.find_multipath_device('/dev/sde') + LOG.error("info = %s" % info) + self.assertEqual("/dev/dm-2", info["device"]) + self.assertEqual("/dev/sde", info['devices'][0]['device']) + self.assertEqual("6", info['devices'][0]['host']) + self.assertEqual("0", info['devices'][0]['channel']) + self.assertEqual("2", info['devices'][0]['id']) + self.assertEqual("0", info['devices'][0]['lun']) + + self.assertEqual("/dev/sdf", info['devices'][2]['device']) + self.assertEqual("6", info['devices'][2]['host']) + self.assertEqual("0", info['devices'][2]['channel']) + self.assertEqual("3", info['devices'][2]['id']) + self.assertEqual("0", info['devices'][2]['lun']) + + def test_find_multipath_device_ds8000(self): + def fake_execute(*cmd, **kwargs): + out = ("36005076303ffc48e0000000000000101 dm-2 IBM,2107900\n" + "size=1.0G features='1 queue_if_no_path' hwhandler='0'" + " wp=rw\n" + "`-+- policy='round-robin 0' prio=-1 status=active\n" + " |- 6:0:2:0 sdd 8:64 active undef running\n" + " `- 6:1:0:3 sdc 8:32 active undef running\n" + ) + return out, None + + self.stubs.Set(self.linuxscsi, '_execute', fake_execute) + + info = self.linuxscsi.find_multipath_device('/dev/sdd') + LOG.error("info = %s" % info) + self.assertEqual("/dev/dm-2", info["device"]) + self.assertEqual("/dev/sdd", info['devices'][0]['device']) + self.assertEqual("6", info['devices'][0]['host']) + self.assertEqual("0", info['devices'][0]['channel']) + self.assertEqual("2", info['devices'][0]['id']) + self.assertEqual("0", info['devices'][0]['lun']) + + self.assertEqual("/dev/sdc", info['devices'][1]['device']) + self.assertEqual("6", info['devices'][1]['host']) + self.assertEqual("1", info['devices'][1]['channel']) + self.assertEqual("0", info['devices'][1]['id']) + self.assertEqual("3", info['devices'][1]['lun']) + + +class ISCSIConnectorTestCase(ConnectorTestCase): + + def setUp(self): + super(ISCSIConnectorTestCase, self).setUp() + self.connector = connector.ISCSIConnector(execute=self.fake_execute) + self.stubs.Set(self.connector._linuxscsi, + 'get_name_from_path', + lambda x: "/dev/sdb") + + def tearDown(self): + super(ISCSIConnectorTestCase, self).tearDown() + + def iscsi_connection(self, volume, location, iqn): + return { + 'driver_volume_type': 'iscsi', + 'data': { + 'volume_id': volume['id'], + 'target_portal': location, + 'target_iqn': iqn, + 'target_lun': 1, + } + } + + def test_connect_volume(self): + self.stubs.Set(os.path, 'exists', lambda x: True) + location = '10.0.2.15:3260' + name = 'volume-00000001' + iqn = 'iqn.2010-10.org.openstack:%s' % name + vol = {'id': 1, 'name': name} + connection_info = self.iscsi_connection(vol, location, iqn) + conf = self.connector.connect_volume(connection_info['data']) + dev_str = '/dev/disk/by-path/ip-%s-iscsi-%s-lun-1' % (location, iqn) + self.assertEquals(conf['type'], 'block') + self.assertEquals(conf['path'], dev_str) + + self.connector.disconnect_volume(connection_info['data']) + expected_commands = [('iscsiadm -m node -T %s -p %s' % + (iqn, location)), + ('iscsiadm -m session'), + ('iscsiadm -m node -T %s -p %s --login' % + (iqn, location)), + ('iscsiadm -m node -T %s -p %s --op update' + ' -n node.startup -v automatic' % (iqn, + location)), + ('tee -a /sys/block/sdb/device/delete'), + ('iscsiadm -m node -T %s -p %s --op update' + ' -n node.startup -v manual' % (iqn, location)), + ('iscsiadm -m node -T %s -p %s --logout' % + (iqn, location)), + ('iscsiadm -m node -T %s -p %s --op delete' % + (iqn, location)), ] + LOG.debug("self.cmds = %s" % self.cmds) + LOG.debug("expected = %s" % expected_commands) + + self.assertEqual(expected_commands, self.cmds) diff --git a/cinder/volume/driver.py b/cinder/volume/driver.py index 5033a56a7..0cec2a953 100644 --- a/cinder/volume/driver.py +++ b/cinder/volume/driver.py @@ -26,6 +26,7 @@ import time from oslo.config import cfg +from cinder.brick.initiator import connector as initiator from cinder import exception from cinder.image import image_utils from cinder.openstack.common import log as logging @@ -417,6 +418,7 @@ class ISCSIDriver(VolumeDriver): image_id, volume_path) finally: + self._detach_volume(iscsi_properties) self.terminate_connection(volume, connector) def copy_volume_to_image(self, context, volume, image_service, image_meta): @@ -434,6 +436,7 @@ class ISCSIDriver(VolumeDriver): image_meta, volume_path) finally: + self._detach_volume(iscsi_properties) self.terminate_connection(volume, connector) def _attach_volume(self, context, volume, connector): @@ -443,84 +446,11 @@ class ISCSIDriver(VolumeDriver): init_conn = self.initialize_connection(volume, connector) iscsi_properties = init_conn['data'] - # code "inspired by" nova/virt/libvirt/volume.py - try: - self._run_iscsiadm(iscsi_properties, ()) - except exception.ProcessExecutionError as exc: - # iscsiadm returns 21 for "No records found" after version 2.0-871 - if exc.exit_code in [21, 255]: - self._run_iscsiadm(iscsi_properties, ('--op', 'new')) - else: - raise - - if iscsi_properties.get('auth_method'): - self._iscsiadm_update(iscsi_properties, - "node.session.auth.authmethod", - iscsi_properties['auth_method']) - self._iscsiadm_update(iscsi_properties, - "node.session.auth.username", - iscsi_properties['auth_username']) - self._iscsiadm_update(iscsi_properties, - "node.session.auth.password", - iscsi_properties['auth_password']) - - host_device = ("/dev/disk/by-path/ip-%s-iscsi-%s-lun-%s" % - (iscsi_properties['target_portal'], - iscsi_properties['target_iqn'], - iscsi_properties.get('target_lun', 0))) - - out = self._run_iscsiadm_bare(["-m", "session"], - run_as_root=True, - check_exit_code=[0, 1, 21])[0] or "" - - portals = [{'portal': p.split(" ")[2], 'iqn': p.split(" ")[3]} - for p in out.splitlines() if p.startswith("tcp:")] - - stripped_portal = iscsi_properties['target_portal'].split(",")[0] - length_iqn = [s for s in portals - if stripped_portal == - s['portal'].split(",")[0] and - s['iqn'] == iscsi_properties['target_iqn']] - if len(portals) == 0 or len(length_iqn) == 0: - try: - self._run_iscsiadm(iscsi_properties, ("--login",), - check_exit_code=[0, 255]) - except exception.ProcessExecutionError as err: - if err.exit_code in [15]: - self._iscsiadm_update(iscsi_properties, - "node.startup", - "automatic") - return iscsi_properties, host_device - else: - raise - - self._iscsiadm_update(iscsi_properties, - "node.startup", "automatic") - - tries = 0 - while not os.path.exists(host_device): - if tries >= self.configuration.num_iscsi_scan_tries: - raise exception.CinderException(_("iSCSI device " - "not found " - "at %s") % (host_device)) + # Use Brick's code to do attach/detach + iscsi = initiator.ISCSIConnector() + conf = iscsi.connect_volume(iscsi_properties) - LOG.warn(_("ISCSI volume not yet found at: %(host_device)s. " - "Will rescan & retry. Try number: %(tries)s.") % - {'host_device': host_device, 'tries': tries}) - - # The rescan isn't documented as being necessary(?), - # but it helps - self._run_iscsiadm(iscsi_properties, ("--rescan",)) - - tries = tries + 1 - if not os.path.exists(host_device): - time.sleep(tries ** 2) - - if tries != 0: - LOG.debug(_("Found iSCSI node %(host_device)s " - "(after %(tries)s rescans).") % - {'host_device': host_device, - 'tries': tries}) + host_device = conf['path'] if not self._check_valid_device(host_device): raise exception.DeviceUnavailable(path=host_device, @@ -529,8 +459,18 @@ class ISCSIDriver(VolumeDriver): "via the path " "%(path)s.") % {'path': host_device})) + LOG.debug("Volume attached %s" % host_device) return iscsi_properties, host_device + def _detach_volume(self, iscsi_properties): + LOG.debug("Detach volume %s:%s:%s" % + (iscsi_properties["target_portal"], + iscsi_properties["target_iqn"], + iscsi_properties["target_lun"])) + # Use Brick's code to do attach/detach + iscsi = initiator.ISCSIConnector() + conf = iscsi.disconnect_volume(iscsi_properties) + def get_volume_stats(self, refresh=False): """Get volume status. diff --git a/etc/cinder/rootwrap.d/volume.filters b/etc/cinder/rootwrap.d/volume.filters index 11224ceda..09989d8ca 100644 --- a/etc/cinder/rootwrap.d/volume.filters +++ b/etc/cinder/rootwrap.d/volume.filters @@ -56,3 +56,8 @@ lvs: CommandFilter, lvs, root # cinder/volumes/drivers/hds/hds.py: hus_cmd: CommandFilter, hus_cmd, root + +# cinder/brick/initiator/connector.py: +ls: CommandFilter, ls, root +tee: CommandFilter, tee, root +multipath: CommandFilter, multipath, root -- 2.45.2