Make sure OVS restarts when Exception occurred

This fix let flows in br-tun automatically recover from an Exception,
which is an ideal situation.
Simplly improve a missed flag will make sure OVS restart properly
after we walked out of Exception loop.

Closes-Bug: #1439472
(cherry picked from commit d72572729152e709c5f7ebae2896d5f66748b59b)

Conflicts:
	neutron/plugins/openvswitch/agent/ovs_neutron_agent.py

Change-Id: Id0ac9399ec39fef19ce71566670ed245c681192e

diff --git a/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py b/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py
index 457f1399fb64bd1089888e2deea819b04849231e..c1ccd7ef8599ed9ffe84934398bf852006853c91 100644 (file)
--- a/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py
+++ b/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py
@@ -1509,6 +1509,7 @@ class OVSNeutronAgent(sg_rpc.SecurityGroupAgentRpcCallbackMixin,
         ancillary_ports = set()
         tunnel_sync = True
         ovs_status = constants.OVS_NORMAL
+        ovs_restarted = False
         while self.run_daemon_loop:
             start = time.time()
             port_stats = {'regular': {'added': 0,
@@ -1553,7 +1554,7 @@ class OVSNeutronAgent(sg_rpc.SecurityGroupAgentRpcCallbackMixin,
                 except Exception:
                     LOG.exception(_LE("Error while synchronizing tunnels"))
                     tunnel_sync = True
-            ovs_restarted = (ovs_status == constants.OVS_RESTARTED)
+            ovs_restarted |= (ovs_status == constants.OVS_RESTARTED)
             if self._agent_has_updates(polling_manager) or ovs_restarted:
                 try:
                     LOG.debug("Agent rpc_loop - iteration:%(iter_num)d - "
@@ -1621,6 +1622,10 @@ class OVSNeutronAgent(sg_rpc.SecurityGroupAgentRpcCallbackMixin,
                             sync = sync | rc
 
                     polling_manager.polling_completed()
+                    # Keep this flag in the last line of "try" block,
+                    # so we can sure that no other Exception occurred.
+                    if not sync:
+                        ovs_restarted = False
                 except Exception:
                     LOG.exception(_LE("Error while processing VIF ports"))
                     # Put the ports back in self.updated_port

diff --git a/neutron/tests/unit/plugins/openvswitch/agent/test_ovs_neutron_agent.py b/neutron/tests/unit/plugins/openvswitch/agent/test_ovs_neutron_agent.py
index 060fc4a631483f0f16831351bf0170f3a46e0312..48315595588c70bbeaa7df50b30981277c678889 100644 (file)
--- a/neutron/tests/unit/plugins/openvswitch/agent/test_ovs_neutron_agent.py
+++ b/neutron/tests/unit/plugins/openvswitch/agent/test_ovs_neutron_agent.py
@@ -1031,7 +1031,7 @@ class TestOvsNeutronAgent(base.BaseTestCase):
             self.agent.tunnel_delete(context=None, **kwargs)
             self.assertTrue(clean_tun_fn.called)
 
-    def test_ovs_status(self):
+    def _test_ovs_status(self, *args):
         reply2 = {'current': set(['tap0']),
                   'added': set(['tap2']),
                   'removed': set([])}
@@ -1064,11 +1064,7 @@ class TestOvsNeutronAgent(base.BaseTestCase):
             scan_ports.side_effect = [reply2, reply3]
             process_network_ports.side_effect = [
                 False, Exception('Fake exception to get out of the loop')]
-            check_ovs_status.side_effect = [constants.OVS_NORMAL,
-                                            constants.OVS_DEAD,
-                                            constants.OVS_RESTARTED]
-
-            # This will exit after the third loop
+            check_ovs_status.side_effect = args
             try:
                 self.agent.daemon_loop()
             except Exception:
@@ -1079,19 +1075,23 @@ class TestOvsNeutronAgent(base.BaseTestCase):
             mock.call(set(), set())
         ])
         process_network_ports.assert_has_calls([
-            mock.call({'current': set(['tap0']),
-                       'removed': set([]),
-                       'added': set(['tap2'])}, False),
-            mock.call({'current': set(['tap2']),
-                       'removed': set(['tap0']),
-                       'added': set([])}, True)
+            mock.call(reply2, False),
+            mock.call(reply3, True)
         ])
         self.assertTrue(update_stale.called)
-        # Verify the second time through the loop we triggered an
-        # OVS restart and re-setup the bridges
+        # Verify the OVS restart we triggered in the loop
+        # re-setup the bridges
         setup_int_br.assert_has_calls([mock.call()])
         setup_phys_br.assert_has_calls([mock.call({})])
 
+    def test_ovs_status(self):
+        self._test_ovs_status(constants.OVS_NORMAL,
+                              constants.OVS_DEAD,
+                              constants.OVS_RESTARTED)
+        # OVS will not DEAD in some exception, like DBConnectionError.
+        self._test_ovs_status(constants.OVS_NORMAL,
+                              constants.OVS_RESTARTED)
+
     def test_set_rpc_timeout(self):
         self.agent._handle_sigterm(None, None)
         for rpc_client in (self.agent.plugin_rpc.client,