Make sure OVS restarts when Exception occurred

This fix let flows in br-tun automatically recover from an Exception,
which is an ideal situation.
Simplly improve a missed flag will make sure OVS restart properly
after we walked out of Exception loop.

Change-Id: Id0ac9399ec39fef19ce71566670ed245c681192e
Closes-Bug: #1439472

diff --git a/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py b/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py
index 9e510c70b369d44001153346448af66f1d0945db..bc946e718af20f2a4819d3825f6ef53e2553928e 100644 (file)
--- a/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py
+++ b/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py
@@ -1508,6 +1508,7 @@ class OVSNeutronAgent(sg_rpc.SecurityGroupAgentRpcCallbackMixin,
         updated_ports_copy = set()
         ancillary_ports = set()
         tunnel_sync = True
+        ovs_restarted = False
         while self.run_daemon_loop:
             start = time.time()
             port_stats = {'regular': {'added': 0,
@@ -1552,7 +1553,7 @@ class OVSNeutronAgent(sg_rpc.SecurityGroupAgentRpcCallbackMixin,
                 except Exception:
                     LOG.exception(_LE("Error while synchronizing tunnels"))
                     tunnel_sync = True
-            ovs_restarted = (ovs_status == constants.OVS_RESTARTED)
+            ovs_restarted |= (ovs_status == constants.OVS_RESTARTED)
             if self._agent_has_updates(polling_manager) or ovs_restarted:
                 try:
                     LOG.debug("Agent rpc_loop - iteration:%(iter_num)d - "
@@ -1620,6 +1621,10 @@ class OVSNeutronAgent(sg_rpc.SecurityGroupAgentRpcCallbackMixin,
                             sync = sync | rc
 
                     polling_manager.polling_completed()
+                    # Keep this flag in the last line of "try" block,
+                    # so we can sure that no other Exception occurred.
+                    if not sync:
+                        ovs_restarted = False
                 except Exception:
                     LOG.exception(_LE("Error while processing VIF ports"))
                     # Put the ports back in self.updated_port

diff --git a/neutron/tests/unit/plugins/openvswitch/agent/test_ovs_neutron_agent.py b/neutron/tests/unit/plugins/openvswitch/agent/test_ovs_neutron_agent.py
index ee14e967a1435e0fc7003ee55d8122622b6cc884..b013c53469511786bb921dca0ae0c0bdffc9d9b3 100644 (file)
--- a/neutron/tests/unit/plugins/openvswitch/agent/test_ovs_neutron_agent.py
+++ b/neutron/tests/unit/plugins/openvswitch/agent/test_ovs_neutron_agent.py
@@ -1032,7 +1032,7 @@ class TestOvsNeutronAgent(base.BaseTestCase):
             self.agent.tunnel_delete(context=None, **kwargs)
             self.assertTrue(clean_tun_fn.called)
 
-    def test_ovs_status(self):
+    def _test_ovs_status(self, *args):
         reply2 = {'current': set(['tap0']),
                   'added': set(['tap2']),
                   'removed': set([])}
@@ -1065,11 +1065,7 @@ class TestOvsNeutronAgent(base.BaseTestCase):
             scan_ports.side_effect = [reply2, reply3]
             process_network_ports.side_effect = [
                 False, Exception('Fake exception to get out of the loop')]
-            check_ovs_status.side_effect = [constants.OVS_NORMAL,
-                                            constants.OVS_DEAD,
-                                            constants.OVS_RESTARTED]
-
-            # This will exit after the third loop
+            check_ovs_status.side_effect = args
             try:
                 self.agent.daemon_loop()
             except Exception:
@@ -1080,19 +1076,23 @@ class TestOvsNeutronAgent(base.BaseTestCase):
             mock.call(set(), set())
         ])
         process_network_ports.assert_has_calls([
-            mock.call({'current': set(['tap0']),
-                       'removed': set([]),
-                       'added': set(['tap2'])}, False),
-            mock.call({'current': set(['tap2']),
-                       'removed': set(['tap0']),
-                       'added': set([])}, True)
+            mock.call(reply2, False),
+            mock.call(reply3, True)
         ])
         self.assertTrue(update_stale.called)
-        # Verify the second time through the loop we triggered an
-        # OVS restart and re-setup the bridges
+        # Verify the OVS restart we triggered in the loop
+        # re-setup the bridges
         setup_int_br.assert_has_calls([mock.call()])
         setup_phys_br.assert_has_calls([mock.call({})])
 
+    def test_ovs_status(self):
+        self._test_ovs_status(constants.OVS_NORMAL,
+                              constants.OVS_DEAD,
+                              constants.OVS_RESTARTED)
+        # OVS will not DEAD in some exception, like DBConnectionError.
+        self._test_ovs_status(constants.OVS_NORMAL,
+                              constants.OVS_RESTARTED)
+
     def test_set_rpc_timeout(self):
         self.agent._handle_sigterm(None, None)
         for rpc_client in (self.agent.plugin_rpc.client,