From 806e67538fbea49530aa0eedb75694cd8bad439d Mon Sep 17 00:00:00 2001 From: Kevin Benton Date: Fri, 11 Dec 2015 10:55:38 -0800 Subject: [PATCH] Batch db segment retrieval A net-list operation was calling extend_network_dict_provider for each network individually which would result in a database call for each network. This adds a new call in the manager to extend multiple networks at once and then it adds a bulk version of get_network_segments that it calls. Now 1 net list of any number of networks will only result in 1 segment DB call. Change-Id: I2543b3bdbb178ee4bb8d1288e9a27af1c5c8c8b4 Closes-Bug: #1525423 Partial-Bug: #1513782 --- neutron/plugins/ml2/db.py | 13 ++++++-- neutron/plugins/ml2/managers.py | 16 ++++++++-- neutron/plugins/ml2/plugin.py | 3 +- neutron/tests/unit/plugins/ml2/test_db.py | 30 +++++++++++++++++-- neutron/tests/unit/plugins/ml2/test_plugin.py | 6 ++-- 5 files changed, 55 insertions(+), 13 deletions(-) diff --git a/neutron/plugins/ml2/db.py b/neutron/plugins/ml2/db.py index 5ad811e37..6edaa74d4 100644 --- a/neutron/plugins/ml2/db.py +++ b/neutron/plugins/ml2/db.py @@ -65,15 +65,22 @@ def add_network_segment(session, network_id, segment, segment_index=0, def get_network_segments(session, network_id, filter_dynamic=False): + return get_networks_segments( + session, [network_id], filter_dynamic)[network_id] + + +def get_networks_segments(session, network_ids, filter_dynamic=False): with session.begin(subtransactions=True): query = (session.query(models.NetworkSegment). - filter_by(network_id=network_id). + filter(models.NetworkSegment.network_id.in_(network_ids)). order_by(models.NetworkSegment.segment_index)) if filter_dynamic is not None: query = query.filter_by(is_dynamic=filter_dynamic) records = query.all() - - return [_make_segment_dict(record) for record in records] + result = {net_id: [] for net_id in network_ids} + for record in records: + result[record.network_id].append(_make_segment_dict(record)) + return result def get_segment_by_id(session, segment_id): diff --git a/neutron/plugins/ml2/managers.py b/neutron/plugins/ml2/managers.py index 995c68260..97c63434a 100644 --- a/neutron/plugins/ml2/managers.py +++ b/neutron/plugins/ml2/managers.py @@ -148,10 +148,20 @@ class TypeManager(stevedore.named.NamedExtensionManager): return value def extend_network_dict_provider(self, context, network): - id = network['id'] - segments = db.get_network_segments(context.session, id) + # this method is left for backward compat even though it would be + # easy to change the callers in tree to use the bulk function + return self.extend_networks_dict_provider(context, [network]) + + def extend_networks_dict_provider(self, context, networks): + ids = [network['id'] for network in networks] + net_segments = db.get_networks_segments(context.session, ids) + for network in networks: + segments = net_segments[network['id']] + self._extend_network_dict_provider(network, segments) + + def _extend_network_dict_provider(self, network, segments): if not segments: - LOG.error(_LE("Network %s has no segments"), id) + LOG.error(_LE("Network %s has no segments"), network['id']) for attr in provider.ATTRIBUTES: network[attr] = None elif len(segments) > 1: diff --git a/neutron/plugins/ml2/plugin.py b/neutron/plugins/ml2/plugin.py index 3c226e560..158e010e4 100644 --- a/neutron/plugins/ml2/plugin.py +++ b/neutron/plugins/ml2/plugin.py @@ -722,8 +722,7 @@ class Ml2Plugin(db_base_plugin_v2.NeutronDbPluginV2, nets = super(Ml2Plugin, self).get_networks(context, filters, None, sorts, limit, marker, page_reverse) - for net in nets: - self.type_manager.extend_network_dict_provider(context, net) + self.type_manager.extend_networks_dict_provider(context, nets) nets = self._filter_nets_provider(context, nets, filters) diff --git a/neutron/tests/unit/plugins/ml2/test_db.py b/neutron/tests/unit/plugins/ml2/test_db.py index 3e2fe4045..4d4f9d4a4 100644 --- a/neutron/tests/unit/plugins/ml2/test_db.py +++ b/neutron/tests/unit/plugins/ml2/test_db.py @@ -57,8 +57,8 @@ class Ml2DBTestCase(testlib_api.SqlTestCase): vif_type=vif_type, host=host)) - def _create_segments(self, segments, is_seg_dynamic=False): - network_id = 'foo-network-id' + def _create_segments(self, segments, is_seg_dynamic=False, + network_id='foo-network-id'): self._setup_neutron_network(network_id) for segment in segments: ml2_db.add_network_segment( @@ -95,6 +95,32 @@ class Ml2DBTestCase(testlib_api.SqlTestCase): api.SEGMENTATION_ID: 2}] self._create_segments(segments) + def test_get_networks_segments(self): + segments1 = [{api.NETWORK_TYPE: 'vlan', + api.PHYSICAL_NETWORK: 'physnet1', + api.SEGMENTATION_ID: 1}, + {api.NETWORK_TYPE: 'vlan', + api.PHYSICAL_NETWORK: 'physnet1', + api.SEGMENTATION_ID: 2}] + segments2 = [{api.NETWORK_TYPE: 'vlan', + api.PHYSICAL_NETWORK: 'physnet1', + api.SEGMENTATION_ID: 3}, + {api.NETWORK_TYPE: 'vlan', + api.PHYSICAL_NETWORK: 'physnet1', + api.SEGMENTATION_ID: 4}] + net1segs = self._create_segments(segments1, network_id='net1') + net2segs = self._create_segments(segments2, network_id='net2') + segs = ml2_db.get_networks_segments(self.ctx.session, ['net1', 'net2']) + self.assertEqual(net1segs, segs['net1']) + self.assertEqual(net2segs, segs['net2']) + + def test_get_networks_segments_no_segments(self): + self._create_segments([], network_id='net1') + self._create_segments([], network_id='net2') + segs = ml2_db.get_networks_segments(self.ctx.session, ['net1', 'net2']) + self.assertEqual([], segs['net1']) + self.assertEqual([], segs['net2']) + def test_get_segment_by_id(self): segment = {api.NETWORK_TYPE: 'vlan', api.PHYSICAL_NETWORK: 'physnet1', diff --git a/neutron/tests/unit/plugins/ml2/test_plugin.py b/neutron/tests/unit/plugins/ml2/test_plugin.py index 7fb9a6338..c0011fb30 100644 --- a/neutron/tests/unit/plugins/ml2/test_plugin.py +++ b/neutron/tests/unit/plugins/ml2/test_plugin.py @@ -676,10 +676,10 @@ class TestMl2PortsV2(test_plugin.TestPortsV2, Ml2PluginV2TestCase): ctx = context.get_admin_context() with self.network() as net: with self.subnet(network=net) as subnet: - segments = ml2_db.get_network_segments(ctx.session, - net['network']['id']) + segments = ml2_db.get_networks_segments(ctx.session, + [net['network']['id']]) with mock.patch('neutron.plugins.ml2.plugin.' - 'db.get_network_segments') as get_seg_mock: + 'db.get_networks_segments') as get_seg_mock: get_seg_mock.side_effect = [db_exc.DBDeadlock, segments, segments, segments] with self.port(subnet=subnet) as port: -- 2.45.2