Avoid race with udev during ovs agent startup

After taking down the veth link between the physical bridge and the integration
bridge call udevadm settle to wait for any udev events to be completely
processed by the operating system before recreating the veth pair.

Some distributions (e.g. openSUSE) have udev rules installed by default that
call e.g. ifdown <interface> during the remove event. If that is processed
after the ovs agent already brought up the veth pair again the veth pair's
link will be down after the agent completed startup and networking will be
broken for all VM instances.

Change-Id: I95520ea96a9804c5261a0c994bbca137535cc37c
Closes-Bug: #1218556

diff --git a/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py b/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py
index eefe384367fda0000ddc115b872292b30b452a64..b3fad63a7dade259eae5d868e5f890059e0a257e 100644 (file)
--- a/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py
+++ b/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py
@@ -32,6 +32,7 @@ from oslo.config import cfg
 from neutron.agent import l2population_rpc
 from neutron.agent.linux import ip_lib
 from neutron.agent.linux import ovs_lib
+from neutron.agent.linux import utils
 from neutron.agent import rpc as agent_rpc
 from neutron.agent import securitygroups_rpc as sg_rpc
 from neutron.common import config as logging_config
@@ -778,6 +779,10 @@ class OVSNeutronAgent(sg_rpc.SecurityGroupAgentRpcCallbackMixin,
             br.delete_port(phys_veth_name)
             if ip_lib.device_exists(int_veth_name, self.root_helper):
                 ip_lib.IPDevice(int_veth_name, self.root_helper).link.delete()
+                # Give udev a chance to process its rules here, to avoid
+                # race conditions between commands launched by udev rules
+                # and the subsequent call to ip_wrapper.add_veth
+                utils.execute(['/sbin/udevadm', 'settle', '--timeout=10'])
             int_veth, phys_veth = ip_wrapper.add_veth(int_veth_name,
                                                       phys_veth_name)
             self.int_ofports[physical_network] = self.int_br.add_port(int_veth)

diff --git a/neutron/tests/unit/openvswitch/test_ovs_neutron_agent.py b/neutron/tests/unit/openvswitch/test_ovs_neutron_agent.py
index e74fc3149589ee86f34d3ed4d865640aa7e448e3..8b415b2e033d6aa551af4ef1eedec4aaaf5b11e2 100644 (file)
--- a/neutron/tests/unit/openvswitch/test_ovs_neutron_agent.py
+++ b/neutron/tests/unit/openvswitch/test_ovs_neutron_agent.py
@@ -23,6 +23,7 @@ import testtools
 
 from neutron.agent.linux import ip_lib
 from neutron.agent.linux import ovs_lib
+from neutron.agent.linux import utils
 from neutron.common import constants as n_const
 from neutron.openstack.common.rpc import common as rpc_common
 from neutron.plugins.openvswitch.agent import ovs_neutron_agent
@@ -327,6 +328,7 @@ class TestOvsNeutronAgent(base.BaseTestCase):
         with contextlib.nested(
             mock.patch.object(ip_lib, "device_exists"),
             mock.patch.object(sys, "exit"),
+            mock.patch.object(utils, "execute"),
             mock.patch.object(ovs_lib.OVSBridge, "remove_all_flows"),
             mock.patch.object(ovs_lib.OVSBridge, "add_flow"),
             mock.patch.object(ovs_lib.OVSBridge, "add_port"),
@@ -337,15 +339,26 @@ class TestOvsNeutronAgent(base.BaseTestCase):
             mock.patch.object(ip_lib.IpLinkCommand, "delete"),
             mock.patch.object(ip_lib.IpLinkCommand, "set_up"),
             mock.patch.object(ip_lib.IpLinkCommand, "set_mtu")
-        ) as (devex_fn, sysexit_fn, remflows_fn, ovs_addfl_fn,
+        ) as (devex_fn, sysexit_fn, utilsexec_fn, remflows_fn, ovs_addfl_fn,
               ovs_addport_fn, ovs_delport_fn, br_addport_fn,
               br_delport_fn, addveth_fn, linkdel_fn, linkset_fn, linkmtu_fn):
             devex_fn.return_value = True
+            parent = mock.MagicMock()
+            parent.attach_mock(utilsexec_fn, 'utils_execute')
+            parent.attach_mock(linkdel_fn, 'link_delete')
+            parent.attach_mock(addveth_fn, 'add_veth')
             addveth_fn.return_value = (ip_lib.IPDevice("int-br-eth1"),
                                        ip_lib.IPDevice("phy-br-eth1"))
             ovs_addport_fn.return_value = "int_ofport"
             br_addport_fn.return_value = "phys_veth"
             self.agent.setup_physical_bridges({"physnet1": "br-eth"})
+            expected_calls = [mock.call.link_delete(),
+                              mock.call.utils_execute(['/sbin/udevadm',
+                                                       'settle',
+                                                       '--timeout=10']),
+                              mock.call.add_veth('int-br-eth',
+                                                 'phy-br-eth')]
+            parent.assert_has_calls(expected_calls, any_order=False)
             self.assertEqual(self.agent.int_ofports["physnet1"],
                              "phys_veth")
             self.assertEqual(self.agent.phys_ofports["physnet1"],