Reduce db calls count in get_devices_details_list

Each Neutron agent will impose db calls to Neutron Server
to query devices, port and networks.
Network caching is added to reduce the number
of db calls on get_devices_details_list.
Added unit tests for the check caching.

Change-Id: I933dfe9b020b15b39bc932e62d599c5d654347e1
Closes-Bug: #1370361

diff --git a/neutron/plugins/ml2/plugin.py b/neutron/plugins/ml2/plugin.py
index 1faacf5a256994d21719ee624fd3f401d44e556f..30675717f9e3f468ff785a0cfee7c4d3f08e0736 100644 (file)
--- a/neutron/plugins/ml2/plugin.py
+++ b/neutron/plugins/ml2/plugin.py
@@ -1192,7 +1192,8 @@ class Ml2Plugin(db_base_plugin_v2.NeutronDbPluginV2,
         self.notifier.port_delete(context, id)
         self.notify_security_groups_member_updated(context, port)
 
-    def get_bound_port_context(self, plugin_context, port_id, host=None):
+    def get_bound_port_context(self, plugin_context, port_id, host=None,
+                               cached_networks=None):
         session = plugin_context.session
         with session.begin(subtransactions=True):
             try:
@@ -1209,7 +1210,11 @@ class Ml2Plugin(db_base_plugin_v2.NeutronDbPluginV2,
                           port_id)
                 return
             port = self._make_port_dict(port_db)
-            network = self.get_network(plugin_context, port['network_id'])
+            network = (cached_networks or {}).get(port['network_id'])
+
+            if not network:
+                network = self.get_network(plugin_context, port['network_id'])
+
             if port['device_owner'] == const.DEVICE_OWNER_DVR_INTERFACE:
                 binding = db.get_dvr_port_binding_by_host(
                     session, port['id'], host)

diff --git a/neutron/plugins/ml2/rpc.py b/neutron/plugins/ml2/rpc.py
index 16a26704a9cab05f5189512ac78ca020316bb39f..80d7013e143128ddad22b2764d5867ad92944b20 100644 (file)
--- a/neutron/plugins/ml2/rpc.py
+++ b/neutron/plugins/ml2/rpc.py
@@ -57,6 +57,9 @@ class RpcCallbacks(type_tunnel.TunnelRpcCallbackMixin):
         agent_id = kwargs.get('agent_id')
         device = kwargs.get('device')
         host = kwargs.get('host')
+        # cached networks used for reducing number of network db calls
+        # for server internal usage only
+        cached_networks = kwargs.get('cached_networks')
         LOG.debug("Device %(device)s details requested by agent "
                   "%(agent_id)s with host %(host)s",
                   {'device': device, 'agent_id': agent_id, 'host': host})
@@ -65,7 +68,8 @@ class RpcCallbacks(type_tunnel.TunnelRpcCallbackMixin):
         port_id = plugin._device_to_port_id(device)
         port_context = plugin.get_bound_port_context(rpc_context,
                                                      port_id,
-                                                     host)
+                                                     host,
+                                                     cached_networks)
         if not port_context:
             LOG.warning(_LW("Device %(device)s requested by agent "
                             "%(agent_id)s not found in database"),
@@ -74,6 +78,11 @@ class RpcCallbacks(type_tunnel.TunnelRpcCallbackMixin):
 
         segment = port_context.bottom_bound_segment
         port = port_context.current
+        # caching information about networks for future use
+        if cached_networks is not None:
+            if port['network_id'] not in cached_networks:
+                cached_networks[port['network_id']] = (
+                    port_context.network.current)
 
         if not segment:
             LOG.warning(_LW("Device %(device)s requested by agent "
@@ -108,10 +117,13 @@ class RpcCallbacks(type_tunnel.TunnelRpcCallbackMixin):
         return entry
 
     def get_devices_details_list(self, rpc_context, **kwargs):
+        # cached networks used for reducing number of network db calls
+        cached_networks = {}
         return [
             self.get_device_details(
                 rpc_context,
                 device=device,
+                cached_networks=cached_networks,
                 **kwargs
             )
             for device in kwargs.pop('devices', [])

diff --git a/neutron/tests/unit/ml2/test_port_binding.py b/neutron/tests/unit/ml2/test_port_binding.py
index ae88ab67c2ffbfa623b71549992fdabadbcd1afa..83dca2d2f6788f080bc53592208954788ca1e7ff 100644 (file)
--- a/neutron/tests/unit/ml2/test_port_binding.py
+++ b/neutron/tests/unit/ml2/test_port_binding.py
@@ -116,6 +116,26 @@ class PortBindingTestCase(test_plugin.NeutronDbPluginV2TestCase):
                                 portbindings.VIF_TYPE_OVS,
                                 False, True, network_type='vlan')
 
+    def test_get_bound_port_context_cache_hit(self):
+        ctx = context.get_admin_context()
+        with self.port(name='name') as port:
+            cached_network_id = port['port']['network_id']
+            some_network = {'id': cached_network_id}
+            cached_networks = {cached_network_id: some_network}
+            self.plugin.get_network = mock.Mock(return_value=some_network)
+            self.plugin.get_bound_port_context(ctx, port['port']['id'],
+                                               cached_networks=cached_networks)
+            self.assertFalse(self.plugin.get_network.called)
+
+    def test_get_bound_port_context_cache_miss(self):
+        ctx = context.get_admin_context()
+        with self.port(name='name') as port:
+            some_network = {'id': u'2ac23560-7638-44e2-9875-c1888b02af72'}
+            self.plugin.get_network = mock.Mock(return_value=some_network)
+            self.plugin.get_bound_port_context(ctx, port['port']['id'],
+                                               cached_networks={})
+            self.assertEqual(1, self.plugin.get_network.call_count)
+
     def _test_update_port_binding(self, host, new_host=None):
         with mock.patch.object(self.plugin,
                                '_notify_port_updated') as notify_mock:

diff --git a/neutron/tests/unit/ml2/test_rpcapi.py b/neutron/tests/unit/ml2/test_rpcapi.py
index 0a4c0453f5b2ce0009674f02a3ac630db1de3a87..701b9b731a557a479336652b8146f8fe385cf796 100644 (file)
--- a/neutron/tests/unit/ml2/test_rpcapi.py
+++ b/neutron/tests/unit/ml2/test_rpcapi.py
@@ -116,6 +116,16 @@ class RpcCallbacksTestCase(base.BaseTestCase):
                 self.assertEqual(status == new_status,
                                  not self.plugin.update_port_status.called)
 
+    def test_get_device_details_caching(self):
+        port = collections.defaultdict(lambda: 'fake_port')
+        cached_networks = {}
+        self.plugin.get_bound_port_context().current = port
+        self.plugin.get_bound_port_context().network.current = (
+            {"id": "fake_network"})
+        self.callbacks.get_device_details('fake_context', host='fake_host',
+                                          cached_networks=cached_networks)
+        self.assertTrue('fake_port' in cached_networks)
+
     def test_get_devices_details_list(self):
         devices = [1, 2, 3, 4, 5]
         kwargs = {'host': 'fake_host', 'agent_id': 'fake_agent_id'}
@@ -126,7 +136,8 @@ class RpcCallbacksTestCase(base.BaseTestCase):
                                                           **kwargs)
             self.assertEqual(devices, res)
             self.assertEqual(len(devices), f.call_count)
-            calls = [mock.call('fake_context', device=i, **kwargs)
+            calls = [mock.call('fake_context', device=i,
+                               cached_networks={}, **kwargs)
                      for i in devices]
             f.assert_has_calls(calls)