Improve exception handling in _process_router_update()

in _process_router_update() where _process_router_if_compatible()
method is called only RouterNotCompatibleWithAgent exception is handled.
In case any other (intermittent) exception happens inside
_process_router_if_compatible() (e.g. MessagingTimeout on fetching
external net from server) it results in a situation where agent completely
forgets about the router and continues working as usual while server
shows router hosted by agent.

This patch adds handler for broader exception there and set
fullsync = True like it's done earlier in _process_router_update()
when getting routers from server.

Change-Id: I5bfc540a926349848702100744f97c597b852604
Closes-Bug: #1423571

diff --git a/neutron/agent/l3/agent.py b/neutron/agent/l3/agent.py
index 7faaf09db7a1705729c7473d542eb7bf0e56b60a..287a13b9fc8bb324510d9b7b2278cb0b75034a79 100644 (file)
--- a/neutron/agent/l3/agent.py
+++ b/neutron/agent/l3/agent.py
@@ -1127,6 +1127,12 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback,
                     LOG.error(_LE("Removing incompatible router '%s'"),
                               router['id'])
                     self._router_removed(router['id'])
+            except Exception:
+                msg = _LE("Failed to process compatible router '%s'")
+                LOG.exception(msg, update.id)
+                self.fullsync = True
+                continue
+
             LOG.debug("Finished a router update for %s", update.id)
             rp.fetched_and_processed(update.timestamp)
 

diff --git a/neutron/tests/unit/test_l3_agent.py b/neutron/tests/unit/test_l3_agent.py
index 118eab87d32568b27a97e09acb51c44ae2c55d59..83a2a20166ae80652b26dc4ca91ab5a0e2fe77d4 100644 (file)
--- a/neutron/tests/unit/test_l3_agent.py
+++ b/neutron/tests/unit/test_l3_agent.py
@@ -1641,6 +1641,35 @@ class TestBasicRouterOperations(BasicRouterOperationsFramework):
         self.assertEqual('1234', agent.conf.router_id)
         self.assertFalse(agent._clean_stale_namespaces)
 
+    def test_process_routers_update_rpc_timeout_on_get_routers(self):
+        agent = l3_agent.L3NATAgent(HOSTNAME, self.conf)
+        agent.fullsync = False
+        agent._process_router_if_compatible = mock.Mock()
+        self.plugin_api.get_routers.side_effect = (
+            oslo_messaging.MessagingTimeout)
+        agent._queue = mock.Mock()
+        update = mock.Mock()
+        update.router = None
+        agent._queue.each_update_to_next_router.side_effect = [
+            [(None, update)]]
+
+        agent._process_router_update()
+        self.assertTrue(agent.fullsync)
+        self.assertFalse(agent._process_router_if_compatible.called)
+
+    def test_process_routers_update_rpc_timeout_on_get_ext_net(self):
+        agent = l3_agent.L3NATAgent(HOSTNAME, self.conf)
+        agent.fullsync = False
+        agent._process_router_if_compatible = mock.Mock()
+        agent._process_router_if_compatible.side_effect = (
+            oslo_messaging.MessagingTimeout)
+        agent._queue = mock.Mock()
+        agent._queue.each_update_to_next_router.side_effect = [
+            [(None, mock.Mock())]]
+
+        agent._process_router_update()
+        self.assertTrue(agent.fullsync)
+
     def test_process_router_if_compatible_with_no_ext_net_in_conf(self):
         agent = l3_agent.L3NATAgent(HOSTNAME, self.conf)
         self.plugin_api.get_external_network_id.return_value = 'aaa'