Improve vxlan type driver initialization performance

Vxlan type driver may take long time to initialize
vxlan allocation table. Optimize db performance by issuing
raw sql inserts coalesced into bulk statements.
Also optimize deleting logic.

Proposed patch gives ~2x performance gain in comparison with
original code on Mysql and Postgesql backends

Change-Id: I801d967e8e3c0260593f289097d17270ef0b391e
Partial-Bug: #1324875

diff --git a/neutron/plugins/ml2/drivers/type_vxlan.py b/neutron/plugins/ml2/drivers/type_vxlan.py
index 3e5d4756791f875b8e98588c99d5f5fafa6ea3f1..2d39d5ff32a0990f847367c8a8590afc8333f7c4 100644 (file)
--- a/neutron/plugins/ml2/drivers/type_vxlan.py
+++ b/neutron/plugins/ml2/drivers/type_vxlan.py
@@ -153,23 +153,33 @@ class VxlanTypeDriver(type_tunnel.TunnelTypeDriver):
         session = db_api.get_session()
         with session.begin(subtransactions=True):
             # remove from table unallocated tunnels not currently allocatable
-            allocs = session.query(VxlanAllocation).with_lockmode("update")
-            for alloc in allocs:
-                try:
-                    # see if tunnel is allocatable
-                    vxlan_vnis.remove(alloc.vxlan_vni)
-                except KeyError:
-                    # it's not allocatable, so check if its allocated
-                    if not alloc.allocated:
-                        # it's not, so remove it from table
-                        LOG.debug(_("Removing tunnel %s from pool"),
-                                  alloc.vxlan_vni)
-                        session.delete(alloc)
-
-            # add missing allocatable tunnels to table
-            for vxlan_vni in sorted(vxlan_vnis):
-                alloc = VxlanAllocation(vxlan_vni=vxlan_vni)
-                session.add(alloc)
+            # fetch results as list via all() because we'll be iterating
+            # through them twice
+            allocs = (session.query(VxlanAllocation).
+                      with_lockmode("update").all())
+            # collect all vnis present in db
+            existing_vnis = set(alloc.vxlan_vni for alloc in allocs)
+            # collect those vnis that needs to be deleted from db
+            vnis_to_remove = [alloc.vxlan_vni for alloc in allocs
+                              if (alloc.vxlan_vni not in vxlan_vnis and
+                                  not alloc.allocated)]
+            # Immediately delete vnis in chunks. This leaves no work for
+            # flush at the end of transaction
+            bulk_size = 100
+            chunked_vnis = (vnis_to_remove[i:i + bulk_size] for i in
+                            range(0, len(vnis_to_remove), bulk_size))
+            for vni_list in chunked_vnis:
+                session.query(VxlanAllocation).filter(
+                    VxlanAllocation.vxlan_vni.in_(vni_list)).delete(
+                        synchronize_session=False)
+            # collect vnis that need to be added
+            vnis = list(vxlan_vnis - existing_vnis)
+            chunked_vnis = (vnis[i:i + bulk_size] for i in
+                            range(0, len(vnis), bulk_size))
+            for vni_list in chunked_vnis:
+                bulk = [{'vxlan_vni': vni, 'allocated': False}
+                        for vni in vni_list]
+                session.execute(VxlanAllocation.__table__.insert(), bulk)
 
     def get_vxlan_allocation(self, session, vxlan_vni):
         with session.begin(subtransactions=True):