LBaaS: handle NotFound exceptions in update_status callback

LBaaS agent may send update_status requests to server on objects
which were already deleted from db: this is due to creating and
deleting objects with a high rate (like tempest API tests do).
As a result errors and stacktraces appear in server and agent logs.
The proposed solution is to catch NotFound exceptions and print a warning.

Change-Id: I4446b678893d0bda578ad7ccdf3f109cb1c91b4d
Closes-Bug: #1260682

diff --git a/neutron/services/loadbalancer/drivers/haproxy/plugin_driver.py b/neutron/services/loadbalancer/drivers/haproxy/plugin_driver.py
index ad42b0c987a4b2ec0ad23263c8ee9c52723a34b5..8aa06ed3aa4e08e7330eb9b577ca3fb88d859bf2 100644 (file)
--- a/neutron/services/loadbalancer/drivers/haproxy/plugin_driver.py
+++ b/neutron/services/loadbalancer/drivers/haproxy/plugin_driver.py
@@ -166,12 +166,20 @@ class LoadBalancerCallbacks(object):
         }
         if obj_type not in model_mapping:
             raise q_exc.Invalid(_('Unknown object type: %s') % obj_type)
-        elif obj_type == 'health_monitor':
-            self.plugin.update_pool_health_monitor(
-                context, obj_id['monitor_id'], obj_id['pool_id'], status)
-        else:
-            self.plugin.update_status(
-                context, model_mapping[obj_type], obj_id, status)
+        try:
+            if obj_type == 'health_monitor':
+                self.plugin.update_pool_health_monitor(
+                    context, obj_id['monitor_id'], obj_id['pool_id'], status)
+            else:
+                self.plugin.update_status(
+                    context, model_mapping[obj_type], obj_id, status)
+        except q_exc.NotFound:
+            # update_status may come from agent on an object which was
+            # already deleted from db with other request
+            LOG.warning(_('Cannot update status: %(obj_type)s %(obj_id)s '
+                          'not found in the DB, it was probably deleted '
+                          'concurrently'),
+                        {'obj_type': obj_type, 'obj_id': obj_id})
 
     def pool_destroyed(self, context, pool_id=None):
         """Agent confirmation hook that a pool has been destroyed.

diff --git a/neutron/tests/unit/services/loadbalancer/drivers/haproxy/test_plugin_driver.py b/neutron/tests/unit/services/loadbalancer/drivers/haproxy/test_plugin_driver.py
index 5c521fdaea77eb71ff4a8df8ee22f0ac7e01511d..894bf13fb8ac3c148e00e256d01de88b60a10b3a 100644 (file)
--- a/neutron/tests/unit/services/loadbalancer/drivers/haproxy/test_plugin_driver.py
+++ b/neutron/tests/unit/services/loadbalancer/drivers/haproxy/test_plugin_driver.py
@@ -25,6 +25,7 @@ from neutron.common import exceptions
 from neutron import context
 from neutron.db.loadbalancer import loadbalancer_db as ldb
 from neutron.db import servicetype_db as st_db
+from neutron.extensions import loadbalancer
 from neutron.extensions import portbindings
 from neutron import manager
 from neutron.openstack.common import uuidutils
@@ -350,6 +351,15 @@ class TestLoadBalancerCallbacks(TestLoadBalancerPluginBase):
             p = self.plugin_instance.get_pool(ctx, pool_id)
             self.assertEqual('ACTIVE', p['status'])
 
+    def test_update_status_pool_deleted_already(self):
+        with mock.patch.object(plugin_driver, 'LOG') as mock_log:
+            pool_id = 'deleted_pool'
+            ctx = context.get_admin_context()
+            self.assertRaises(loadbalancer.PoolNotFound,
+                              self.plugin_instance.get_pool, ctx, pool_id)
+            self.callbacks.update_status(ctx, 'pool', pool_id, 'ACTIVE')
+            self.assertTrue(mock_log.warning.called)
+
     def test_update_status_health_monitor(self):
         with contextlib.nested(
             self.pool(),