Batch db segment retrieval

A net-list operation was calling extend_network_dict_provider for
each network individually which would result in a database call for
each network.

This adds a new call in the manager to extend multiple networks at
once and then it adds a bulk version of get_network_segments that
it calls.

Now 1 net list of any number of networks will only result in 1
segment DB call.

Change-Id: I2543b3bdbb178ee4bb8d1288e9a27af1c5c8c8b4
Closes-Bug: #1525423
Partial-Bug: #1513782

diff --git a/neutron/plugins/ml2/db.py b/neutron/plugins/ml2/db.py
index 5ad811e37ca80de54fcb80aecaca923a26d9eea6..6edaa74d4bb731462130d9771395ed4c071cc87e 100644 (file)
--- a/neutron/plugins/ml2/db.py
+++ b/neutron/plugins/ml2/db.py
@@ -65,15 +65,22 @@ def add_network_segment(session, network_id, segment, segment_index=0,
 
 
 def get_network_segments(session, network_id, filter_dynamic=False):
+    return get_networks_segments(
+        session, [network_id], filter_dynamic)[network_id]
+
+
+def get_networks_segments(session, network_ids, filter_dynamic=False):
     with session.begin(subtransactions=True):
         query = (session.query(models.NetworkSegment).
-                 filter_by(network_id=network_id).
+                 filter(models.NetworkSegment.network_id.in_(network_ids)).
                  order_by(models.NetworkSegment.segment_index))
         if filter_dynamic is not None:
             query = query.filter_by(is_dynamic=filter_dynamic)
         records = query.all()
-
-        return [_make_segment_dict(record) for record in records]
+        result = {net_id: [] for net_id in network_ids}
+        for record in records:
+            result[record.network_id].append(_make_segment_dict(record))
+        return result
 
 
 def get_segment_by_id(session, segment_id):

diff --git a/neutron/plugins/ml2/managers.py b/neutron/plugins/ml2/managers.py
index 995c6826073b7263815f770aec3a3e64226f655a..97c63434a69c374673d29d995f05cd5f448bbc25 100644 (file)
--- a/neutron/plugins/ml2/managers.py
+++ b/neutron/plugins/ml2/managers.py
@@ -148,10 +148,20 @@ class TypeManager(stevedore.named.NamedExtensionManager):
         return value
 
     def extend_network_dict_provider(self, context, network):
-        id = network['id']
-        segments = db.get_network_segments(context.session, id)
+        # this method is left for backward compat even though it would be
+        # easy to change the callers in tree to use the bulk function
+        return self.extend_networks_dict_provider(context, [network])
+
+    def extend_networks_dict_provider(self, context, networks):
+        ids = [network['id'] for network in networks]
+        net_segments = db.get_networks_segments(context.session, ids)
+        for network in networks:
+            segments = net_segments[network['id']]
+            self._extend_network_dict_provider(network, segments)
+
+    def _extend_network_dict_provider(self, network, segments):
         if not segments:
-            LOG.error(_LE("Network %s has no segments"), id)
+            LOG.error(_LE("Network %s has no segments"), network['id'])
             for attr in provider.ATTRIBUTES:
                 network[attr] = None
         elif len(segments) > 1:

diff --git a/neutron/plugins/ml2/plugin.py b/neutron/plugins/ml2/plugin.py
index 3c226e56052aad2b67677eb6ae88f64f84bd2f63..158e010e45e84946e760881f574e4e0592b51b81 100644 (file)
--- a/neutron/plugins/ml2/plugin.py
+++ b/neutron/plugins/ml2/plugin.py
@@ -722,8 +722,7 @@ class Ml2Plugin(db_base_plugin_v2.NeutronDbPluginV2,
             nets = super(Ml2Plugin,
                          self).get_networks(context, filters, None, sorts,
                                             limit, marker, page_reverse)
-            for net in nets:
-                self.type_manager.extend_network_dict_provider(context, net)
+            self.type_manager.extend_networks_dict_provider(context, nets)
 
             nets = self._filter_nets_provider(context, nets, filters)
 

diff --git a/neutron/tests/unit/plugins/ml2/test_db.py b/neutron/tests/unit/plugins/ml2/test_db.py
index 3e2fe40459341809e7c6a41acc7dcb6e5e45d223..4d4f9d4a4ec492a37d841a386f031dd4d292a89b 100644 (file)
--- a/neutron/tests/unit/plugins/ml2/test_db.py
+++ b/neutron/tests/unit/plugins/ml2/test_db.py
@@ -57,8 +57,8 @@ class Ml2DBTestCase(testlib_api.SqlTestCase):
                                                     vif_type=vif_type,
                                                     host=host))
 
-    def _create_segments(self, segments, is_seg_dynamic=False):
-        network_id = 'foo-network-id'
+    def _create_segments(self, segments, is_seg_dynamic=False,
+                         network_id='foo-network-id'):
         self._setup_neutron_network(network_id)
         for segment in segments:
             ml2_db.add_network_segment(
@@ -95,6 +95,32 @@ class Ml2DBTestCase(testlib_api.SqlTestCase):
                      api.SEGMENTATION_ID: 2}]
         self._create_segments(segments)
 
+    def test_get_networks_segments(self):
+        segments1 = [{api.NETWORK_TYPE: 'vlan',
+                      api.PHYSICAL_NETWORK: 'physnet1',
+                      api.SEGMENTATION_ID: 1},
+                     {api.NETWORK_TYPE: 'vlan',
+                      api.PHYSICAL_NETWORK: 'physnet1',
+                      api.SEGMENTATION_ID: 2}]
+        segments2 = [{api.NETWORK_TYPE: 'vlan',
+                      api.PHYSICAL_NETWORK: 'physnet1',
+                      api.SEGMENTATION_ID: 3},
+                     {api.NETWORK_TYPE: 'vlan',
+                      api.PHYSICAL_NETWORK: 'physnet1',
+                      api.SEGMENTATION_ID: 4}]
+        net1segs = self._create_segments(segments1, network_id='net1')
+        net2segs = self._create_segments(segments2, network_id='net2')
+        segs = ml2_db.get_networks_segments(self.ctx.session, ['net1', 'net2'])
+        self.assertEqual(net1segs, segs['net1'])
+        self.assertEqual(net2segs, segs['net2'])
+
+    def test_get_networks_segments_no_segments(self):
+        self._create_segments([], network_id='net1')
+        self._create_segments([], network_id='net2')
+        segs = ml2_db.get_networks_segments(self.ctx.session, ['net1', 'net2'])
+        self.assertEqual([], segs['net1'])
+        self.assertEqual([], segs['net2'])
+
     def test_get_segment_by_id(self):
         segment = {api.NETWORK_TYPE: 'vlan',
                    api.PHYSICAL_NETWORK: 'physnet1',

diff --git a/neutron/tests/unit/plugins/ml2/test_plugin.py b/neutron/tests/unit/plugins/ml2/test_plugin.py
index 7fb9a6338e5adbabec4242d1fe29b260029738ca..c0011fb30adb929bb3e557a04cc32946dc22027b 100644 (file)
--- a/neutron/tests/unit/plugins/ml2/test_plugin.py
+++ b/neutron/tests/unit/plugins/ml2/test_plugin.py
@@ -676,10 +676,10 @@ class TestMl2PortsV2(test_plugin.TestPortsV2, Ml2PluginV2TestCase):
         ctx = context.get_admin_context()
         with self.network() as net:
             with self.subnet(network=net) as subnet:
-                segments = ml2_db.get_network_segments(ctx.session,
-                                                       net['network']['id'])
+                segments = ml2_db.get_networks_segments(ctx.session,
+                                                        [net['network']['id']])
                 with mock.patch('neutron.plugins.ml2.plugin.'
-                                'db.get_network_segments') as get_seg_mock:
+                                'db.get_networks_segments') as get_seg_mock:
                     get_seg_mock.side_effect = [db_exc.DBDeadlock, segments,
                                                 segments, segments]
                     with self.port(subnet=subnet) as port: