From: Ralf Haferkamp <rhafer@suse.de>
Date: Thu, 29 Aug 2013 18:50:55 +0000 (+0200)
Subject: Avoid race with udev during ovs agent startup
X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=8d88ee7411d43f148b45d0a145fe32a75765a3ac;p=openstack-build%2Fneutron-build.git

Avoid race with udev during ovs agent startup

After taking down the veth link between the physical bridge and the integration
bridge call udevadm settle to wait for any udev events to be completely
processed by the operating system before recreating the veth pair.

Some distributions (e.g. openSUSE) have udev rules installed by default that
call e.g. ifdown <interface> during the remove event. If that is processed
after the ovs agent already brought up the veth pair again the veth pair's
link will be down after the agent completed startup and networking will be
broken for all VM instances.

Change-Id: I95520ea96a9804c5261a0c994bbca137535cc37c
Closes-Bug: #1218556
---

diff --git a/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py b/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py
index eefe38436..b3fad63a7 100644
--- a/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py
+++ b/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py
@@ -32,6 +32,7 @@ from oslo.config import cfg
 from neutron.agent import l2population_rpc
 from neutron.agent.linux import ip_lib
 from neutron.agent.linux import ovs_lib
+from neutron.agent.linux import utils
 from neutron.agent import rpc as agent_rpc
 from neutron.agent import securitygroups_rpc as sg_rpc
 from neutron.common import config as logging_config
@@ -778,6 +779,10 @@ class OVSNeutronAgent(sg_rpc.SecurityGroupAgentRpcCallbackMixin,
             br.delete_port(phys_veth_name)
             if ip_lib.device_exists(int_veth_name, self.root_helper):
                 ip_lib.IPDevice(int_veth_name, self.root_helper).link.delete()
+                # Give udev a chance to process its rules here, to avoid
+                # race conditions between commands launched by udev rules
+                # and the subsequent call to ip_wrapper.add_veth
+                utils.execute(['/sbin/udevadm', 'settle', '--timeout=10'])
             int_veth, phys_veth = ip_wrapper.add_veth(int_veth_name,
                                                       phys_veth_name)
             self.int_ofports[physical_network] = self.int_br.add_port(int_veth)
diff --git a/neutron/tests/unit/openvswitch/test_ovs_neutron_agent.py b/neutron/tests/unit/openvswitch/test_ovs_neutron_agent.py
index e74fc3149..8b415b2e0 100644
--- a/neutron/tests/unit/openvswitch/test_ovs_neutron_agent.py
+++ b/neutron/tests/unit/openvswitch/test_ovs_neutron_agent.py
@@ -23,6 +23,7 @@ import testtools
 
 from neutron.agent.linux import ip_lib
 from neutron.agent.linux import ovs_lib
+from neutron.agent.linux import utils
 from neutron.common import constants as n_const
 from neutron.openstack.common.rpc import common as rpc_common
 from neutron.plugins.openvswitch.agent import ovs_neutron_agent
@@ -327,6 +328,7 @@ class TestOvsNeutronAgent(base.BaseTestCase):
         with contextlib.nested(
             mock.patch.object(ip_lib, "device_exists"),
             mock.patch.object(sys, "exit"),
+            mock.patch.object(utils, "execute"),
             mock.patch.object(ovs_lib.OVSBridge, "remove_all_flows"),
             mock.patch.object(ovs_lib.OVSBridge, "add_flow"),
             mock.patch.object(ovs_lib.OVSBridge, "add_port"),
@@ -337,15 +339,26 @@ class TestOvsNeutronAgent(base.BaseTestCase):
             mock.patch.object(ip_lib.IpLinkCommand, "delete"),
             mock.patch.object(ip_lib.IpLinkCommand, "set_up"),
             mock.patch.object(ip_lib.IpLinkCommand, "set_mtu")
-        ) as (devex_fn, sysexit_fn, remflows_fn, ovs_addfl_fn,
+        ) as (devex_fn, sysexit_fn, utilsexec_fn, remflows_fn, ovs_addfl_fn,
               ovs_addport_fn, ovs_delport_fn, br_addport_fn,
               br_delport_fn, addveth_fn, linkdel_fn, linkset_fn, linkmtu_fn):
             devex_fn.return_value = True
+            parent = mock.MagicMock()
+            parent.attach_mock(utilsexec_fn, 'utils_execute')
+            parent.attach_mock(linkdel_fn, 'link_delete')
+            parent.attach_mock(addveth_fn, 'add_veth')
             addveth_fn.return_value = (ip_lib.IPDevice("int-br-eth1"),
                                        ip_lib.IPDevice("phy-br-eth1"))
             ovs_addport_fn.return_value = "int_ofport"
             br_addport_fn.return_value = "phys_veth"
             self.agent.setup_physical_bridges({"physnet1": "br-eth"})
+            expected_calls = [mock.call.link_delete(),
+                              mock.call.utils_execute(['/sbin/udevadm',
+                                                       'settle',
+                                                       '--timeout=10']),
+                              mock.call.add_veth('int-br-eth',
+                                                 'phy-br-eth')]
+            parent.assert_has_calls(expected_calls, any_order=False)
             self.assertEqual(self.agent.int_ofports["physnet1"],
                              "phys_veth")
             self.assertEqual(self.agent.phys_ofports["physnet1"],