Wrap ML2 delete_port with db retry decorator

ML2 delete_port operation currently involves locking ports
and bindings tables which may lead to DBDeadlock errors in certain
cases when several ports are deleted concurrently.
That may happen due to specifics of Galera working in active-active
mode: it may throw deadlock errors when it fails to validate
a change with other members of the cluster.
The fix adds retries to delete port operation to overcome such
deadlocks

Closes-Bug: #1422504
Change-Id: I684691d59c5ac370d74314c3c91857dc709b2d9b

diff --git a/neutron/plugins/ml2/plugin.py b/neutron/plugins/ml2/plugin.py
index 2f209db7723a58d27d69a9d8ce9882c69e05a5aa..535bd86c97ec9312df074977c374d3b4cc566e95 100644 (file)
--- a/neutron/plugins/ml2/plugin.py
+++ b/neutron/plugins/ml2/plugin.py
@@ -1235,6 +1235,8 @@ class Ml2Plugin(db_base_plugin_v2.NeutronDbPluginV2,
                 raise e.errors[0].error
             raise exc.ServicePortInUse(port_id=port_id, reason=e)
 
+    @oslo_db_api.wrap_db_retry(max_retries=db_api.MAX_RETRIES,
+                               retry_on_deadlock=True)
     def delete_port(self, context, id, l3_port_check=True):
         self._pre_delete_port(context, id, l3_port_check)
         # TODO(armax): get rid of the l3 dependency in the with block

diff --git a/neutron/tests/unit/plugins/ml2/test_plugin.py b/neutron/tests/unit/plugins/ml2/test_plugin.py
index 21b90976a321b0639067c050d85924754c30fd30..e972b0228966cab66f2d3940858226b9ea938f61 100644 (file)
--- a/neutron/tests/unit/plugins/ml2/test_plugin.py
+++ b/neutron/tests/unit/plugins/ml2/test_plugin.py
@@ -583,6 +583,21 @@ class TestMl2PortsV2(test_plugin.TestPortsV2, Ml2PluginV2TestCase):
             # by the called method
             self.assertIsNone(l3plugin.disassociate_floatingips(ctx, port_id))
 
+    def test_delete_port_tolerates_db_deadlock(self):
+        ctx = context.get_admin_context()
+        plugin = manager.NeutronManager.get_plugin()
+        with self.port() as port:
+            port_db, binding = ml2_db.get_locked_port_and_binding(
+                ctx.session, port['port']['id'])
+            with mock.patch('neutron.plugins.ml2.plugin.'
+                            'db.get_locked_port_and_binding') as lock:
+                lock.side_effect = [db_exc.DBDeadlock,
+                                    (port_db, binding)]
+                plugin.delete_port(ctx, port['port']['id'])
+                self.assertEqual(2, lock.call_count)
+                self.assertRaises(
+                    exc.PortNotFound, plugin.get_port, ctx, port['port']['id'])
+
 
 class TestMl2PluginOnly(Ml2PluginV2TestCase):
     """For testing methods that don't call drivers"""