Ensure flushing of IO prior to removing FC device

This patch flushes IO for each SCSI device that
brick removes from the system.  The patch also
flushes a multipath device prior to removing
each underlying LUN associated with the
multipath device.   We were seeing kernel
IO errors in the syslog without calling flush
on the block device, during a detach for a
multipath FibreChannel copy image to volume
action.

Change-Id: I2fae3dccd4fcab22c8f59dcd297581ee9577b779
Closes-Bug: 1318950

diff --git a/cinder/brick/initiator/connector.py b/cinder/brick/initiator/connector.py
index 3ffc6b5901396bae0b6ed46ff2c9aa79961a437e..6abc09b330762536146183238a0cc298f78fcb8a 100644 (file)
--- a/cinder/brick/initiator/connector.py
+++ b/cinder/brick/initiator/connector.py
@@ -693,6 +693,7 @@ class FibreChannelConnector(InitiatorConnector):
             mdev_info = self._linuxscsi.find_multipath_device(multipath_id)
             devices = mdev_info['devices']
             LOG.debug("devices to remove = %s" % devices)
+            self._linuxscsi.flush_multipath_device(multipath_id)
 
         # There may have been more than 1 device mounted
         # by the kernel for this volume.  We have to remove

diff --git a/cinder/brick/initiator/linuxscsi.py b/cinder/brick/initiator/linuxscsi.py
index 83e41d99db9b39fd43e75c224f6033c5dc7eece5..dd18bc2d9c8f57dc2a77489d747be6cc16a94b01 100644 (file)
--- a/cinder/brick/initiator/linuxscsi.py
+++ b/cinder/brick/initiator/linuxscsi.py
@@ -55,6 +55,9 @@ class LinuxSCSI(executor.Executor):
 
         path = "/sys/block/%s/device/delete" % device.replace("/dev/", "")
         if os.path.exists(path):
+            # flush any outstanding IO first
+            self.flush_device_io(device)
+
             LOG.debug("Remove SCSI device(%s) with %s" % (device, path))
             self.echo_scsi_command(path, "1")
 
@@ -91,8 +94,20 @@ class LinuxSCSI(executor.Executor):
                 self.remove_scsi_device(device['device'])
             self.flush_multipath_device(mpath_dev['id'])
 
+    def flush_device_io(self, device):
+        """This is used to flush any remaining IO in the buffers."""
+        try:
+            LOG.debug("Flushing IO for device %s" % device)
+            self._execute('blockdev', '--flushbufs', device, run_as_root=True,
+                          root_helper=self._root_helper)
+        except putils.ProcessExecutionError as exc:
+            msg = _("Failed to flush IO buffers prior to removing"
+                    "device: (%(code)s)") % {'code': exc.exit_code}
+            LOG.warn(msg)
+
     def flush_multipath_device(self, device):
         try:
+            LOG.debug("Flush multipath device %s" % device)
             self._execute('multipath', '-f', device, run_as_root=True,
                           root_helper=self._root_helper)
         except putils.ProcessExecutionError as exc:

diff --git a/cinder/tests/brick/test_brick_connector.py b/cinder/tests/brick/test_brick_connector.py
index 775392e469b0f2ad803a8d8d08216c1e3df72fb8..3728a00b6f69cbe63c2d2ae247ecac3c7f98b7c3 100644 (file)
--- a/cinder/tests/brick/test_brick_connector.py
+++ b/cinder/tests/brick/test_brick_connector.py
@@ -177,6 +177,7 @@ class ISCSIConnectorTestCase(ConnectorTestCase):
                               % (iqn, location)),
                              ('iscsiadm -m node --rescan'),
                              ('iscsiadm -m session --rescan'),
+                             ('blockdev --flushbufs /dev/sdb'),
                              ('tee -a /sys/block/sdb/device/delete'),
                              ('iscsiadm -m node -T %s -p %s --op update'
                               ' -n node.startup -v manual' % (iqn, location)),

diff --git a/cinder/tests/brick/test_brick_linuxscsi.py b/cinder/tests/brick/test_brick_linuxscsi.py
index 47b73dcfb1cf09a9b4adb12e5d7d10e09f6bb188..9611093eaf1edf2379e4e762967ba8a7fafbe3fb 100644 (file)
--- a/cinder/tests/brick/test_brick_linuxscsi.py
+++ b/cinder/tests/brick/test_brick_linuxscsi.py
@@ -51,12 +51,14 @@ class LinuxSCSITestCase(test.TestCase):
 
     def test_remove_scsi_device(self):
         self.stubs.Set(os.path, "exists", lambda x: False)
-        self.linuxscsi.remove_scsi_device("sdc")
+        self.linuxscsi.remove_scsi_device("/dev/sdc")
         expected_commands = []
         self.assertEqual(expected_commands, self.cmds)
         self.stubs.Set(os.path, "exists", lambda x: True)
-        self.linuxscsi.remove_scsi_device("sdc")
-        expected_commands = [('tee -a /sys/block/sdc/device/delete')]
+        self.linuxscsi.remove_scsi_device("/dev/sdc")
+        expected_commands = [
+            ('blockdev --flushbufs /dev/sdc'),
+            ('tee -a /sys/block/sdc/device/delete')]
         self.assertEqual(expected_commands, self.cmds)
 
     def test_flush_multipath_device(self):
@@ -86,9 +88,12 @@ class LinuxSCSITestCase(test.TestCase):
                        fake_find_multipath_device)
 
         self.linuxscsi.remove_multipath_device('/dev/dm-3')
-        expected_commands = [('tee -a /sys/block/sde/device/delete'),
-                             ('tee -a /sys/block/sdf/device/delete'),
-                             ('multipath -f 350002ac20398383d'), ]
+        expected_commands = [
+            ('blockdev --flushbufs /dev/sde'),
+            ('tee -a /sys/block/sde/device/delete'),
+            ('blockdev --flushbufs /dev/sdf'),
+            ('tee -a /sys/block/sdf/device/delete'),
+            ('multipath -f 350002ac20398383d'), ]
         self.assertEqual(expected_commands, self.cmds)
 
     def test_find_multipath_device_3par(self):