Fixes linuxbridge agent downs with tap device deletion timing issue

Fixes bug 1136264

Add try/except in deamon loop for preventing agent broken down by
unhandled exception.

Change-Id: I99a4f2b485f134b240630a895378d0f61d6382d8

diff --git a/quantum/plugins/linuxbridge/agent/linuxbridge_quantum_agent.py b/quantum/plugins/linuxbridge/agent/linuxbridge_quantum_agent.py
index ea28ee2c802577b686a3434add752b765c00d9bd..27e3e14bf106c92a37e003634a478b4ea7c67a54 100755 (executable)
--- a/quantum/plugins/linuxbridge/agent/linuxbridge_quantum_agent.py
+++ b/quantum/plugins/linuxbridge/agent/linuxbridge_quantum_agent.py
@@ -601,16 +601,24 @@ class LinuxBridgeQuantumAgentRPC(sg_rpc.SecurityGroupAgentRpcMixin):
                 LOG.info(_("Agent out of sync with plugin!"))
                 devices.clear()
                 sync = False
-
-            device_info = self.br_mgr.update_devices(devices)
-
-            # notify plugin about device deltas
-            if device_info:
-                LOG.debug(_("Agent loop has new devices!"))
-                # If treat devices fails - indicates must resync with plugin
-                sync = self.process_network_devices(device_info)
-                devices = device_info['current']
-
+            device_info = {}
+            try:
+                device_info = self.br_mgr.update_devices(devices)
+            except Exception:
+                LOG.exception(_("Update devices failed"))
+                sync = True
+            try:
+                # notify plugin about device deltas
+                if device_info:
+                    LOG.debug(_("Agent loop has new devices!"))
+                    # If treat devices fails - indicates must resync with
+                    # plugin
+                    sync = self.process_network_devices(device_info)
+                    devices = device_info['current']
+            except Exception:
+                LOG.exception(_("Error in agent loop. Devices info: %s"),
+                              device_info)
+                sync = True
             # sleep till end of polling interval
             elapsed = (time.time() - start)
             if (elapsed < self.polling_interval):

diff --git a/quantum/tests/unit/linuxbridge/test_lb_quantum_agent.py b/quantum/tests/unit/linuxbridge/test_lb_quantum_agent.py
index a03a29adf2931232f1207d61fcae416190d02c20..da290022774a990f9fc7439a1d8523614be2acae 100644 (file)
--- a/quantum/tests/unit/linuxbridge/test_lb_quantum_agent.py
+++ b/quantum/tests/unit/linuxbridge/test_lb_quantum_agent.py
@@ -14,6 +14,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import contextlib
+
 import mock
 from oslo.config import cfg
 import testtools
@@ -52,3 +54,59 @@ class TestLinuxBridge(testtools.TestCase):
             result = self.linux_bridge.ensure_physical_in_bridge(
                 'network_id', 'physnet1', 7)
         self.assertTrue(vlan_bridge_func.called)
+
+
+class TestLinuxBridgeAgent(testtools.TestCase):
+
+    def setUp(self):
+        super(TestLinuxBridgeAgent, self).setUp()
+        self.lbmgr_patcher = mock.patch('quantum.plugins.linuxbridge.agent.'
+                                        'linuxbridge_quantum_agent.'
+                                        'LinuxBridgeManager')
+        self.lbmgr_mock = self.lbmgr_patcher.start()
+        self.addCleanup(self.lbmgr_patcher.stop)
+
+    def test_update_devices_failed(self):
+        lbmgr_instance = self.lbmgr_mock.return_value
+        lbmgr_instance.update_devices.side_effect = RuntimeError
+        agent = linuxbridge_quantum_agent.LinuxBridgeQuantumAgentRPC({},
+                                                                     0,
+                                                                     None)
+        raise_exception = [0]
+
+        def info_mock(msg):
+            if raise_exception[0] < 2:
+                raise_exception[0] += 1
+            else:
+                raise RuntimeError()
+
+        with mock.patch.object(linuxbridge_quantum_agent.LOG, 'info') as log:
+            log.side_effect = info_mock
+            with testtools.ExpectedException(RuntimeError):
+                agent.daemon_loop()
+            self.assertEqual(3, log.call_count)
+
+    def test_process_network_devices_failed(self):
+        device_info = {'current': [1, 2, 3]}
+        lbmgr_instance = self.lbmgr_mock.return_value
+        lbmgr_instance.update_devices.return_value = device_info
+        agent = linuxbridge_quantum_agent.LinuxBridgeQuantumAgentRPC({},
+                                                                     0,
+                                                                     None)
+        raise_exception = [0]
+
+        def info_mock(msg):
+            if raise_exception[0] < 2:
+                raise_exception[0] += 1
+            else:
+                raise RuntimeError()
+
+        with contextlib.nested(
+            mock.patch.object(linuxbridge_quantum_agent.LOG, 'info'),
+            mock.patch.object(agent, 'process_network_devices')
+        ) as (log, process_network_devices):
+            log.side_effect = info_mock
+            process_network_devices.side_effect = RuntimeError
+            with testtools.ExpectedException(RuntimeError):
+                agent.daemon_loop()
+            self.assertEqual(3, log.call_count)