From: Oleg Bondarev <obondarev@mirantis.com>
Date: Fri, 20 Feb 2015 14:11:44 +0000 (+0300)
Subject: Improve exception handling in _process_router_update()
X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=4f55c8f689861631a8060e9918066792f56dfcf5;p=openstack-build%2Fneutron-build.git

Improve exception handling in _process_router_update()

in _process_router_update() where _process_router_if_compatible()
method is called only RouterNotCompatibleWithAgent exception is handled.
In case any other (intermittent) exception happens inside
_process_router_if_compatible() (e.g. MessagingTimeout on fetching
external net from server) it results in a situation where agent completely
forgets about the router and continues working as usual while server
shows router hosted by agent.

This patch adds handler for broader exception there and set
fullsync = True like it's done earlier in _process_router_update()
when getting routers from server.

Change-Id: I5bfc540a926349848702100744f97c597b852604
Closes-Bug: #1423571
---

diff --git a/neutron/agent/l3/agent.py b/neutron/agent/l3/agent.py
index 7faaf09db..287a13b9f 100644
--- a/neutron/agent/l3/agent.py
+++ b/neutron/agent/l3/agent.py
@@ -1127,6 +1127,12 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback,
                     LOG.error(_LE("Removing incompatible router '%s'"),
                               router['id'])
                     self._router_removed(router['id'])
+            except Exception:
+                msg = _LE("Failed to process compatible router '%s'")
+                LOG.exception(msg, update.id)
+                self.fullsync = True
+                continue
+
             LOG.debug("Finished a router update for %s", update.id)
             rp.fetched_and_processed(update.timestamp)
 
diff --git a/neutron/tests/unit/test_l3_agent.py b/neutron/tests/unit/test_l3_agent.py
index 118eab87d..83a2a2016 100644
--- a/neutron/tests/unit/test_l3_agent.py
+++ b/neutron/tests/unit/test_l3_agent.py
@@ -1641,6 +1641,35 @@ class TestBasicRouterOperations(BasicRouterOperationsFramework):
         self.assertEqual('1234', agent.conf.router_id)
         self.assertFalse(agent._clean_stale_namespaces)
 
+    def test_process_routers_update_rpc_timeout_on_get_routers(self):
+        agent = l3_agent.L3NATAgent(HOSTNAME, self.conf)
+        agent.fullsync = False
+        agent._process_router_if_compatible = mock.Mock()
+        self.plugin_api.get_routers.side_effect = (
+            oslo_messaging.MessagingTimeout)
+        agent._queue = mock.Mock()
+        update = mock.Mock()
+        update.router = None
+        agent._queue.each_update_to_next_router.side_effect = [
+            [(None, update)]]
+
+        agent._process_router_update()
+        self.assertTrue(agent.fullsync)
+        self.assertFalse(agent._process_router_if_compatible.called)
+
+    def test_process_routers_update_rpc_timeout_on_get_ext_net(self):
+        agent = l3_agent.L3NATAgent(HOSTNAME, self.conf)
+        agent.fullsync = False
+        agent._process_router_if_compatible = mock.Mock()
+        agent._process_router_if_compatible.side_effect = (
+            oslo_messaging.MessagingTimeout)
+        agent._queue = mock.Mock()
+        agent._queue.each_update_to_next_router.side_effect = [
+            [(None, mock.Mock())]]
+
+        agent._process_router_update()
+        self.assertTrue(agent.fullsync)
+
     def test_process_router_if_compatible_with_no_ext_net_in_conf(self):
         agent = l3_agent.L3NATAgent(HOSTNAME, self.conf)
         self.plugin_api.get_external_network_id.return_value = 'aaa'