From 2093d8727070829064b1a6005a9b411a383d246d Mon Sep 17 00:00:00 2001 From: Ryan Moats Date: Thu, 6 Aug 2015 16:24:55 -0500 Subject: [PATCH] Introduce ItemAllocator class The ItemAllocator class is used as the base class for LinkLocalAllocator in preparation for adding FipRulePriorityAllocator as a child class. Change-Id: I2c77e5a895f750845b46d3e8a2326e01ea87ee78 Partial-Bug: #1414779 Signed-off-by: Ryan Moats --- neutron/agent/l3/item_allocator.py | 104 ++++++++++++++++++ neutron/agent/l3/link_local_allocator.py | 81 ++------------ .../unit/agent/l3/test_item_allocator.py | 29 +++++ 3 files changed, 144 insertions(+), 70 deletions(-) create mode 100644 neutron/agent/l3/item_allocator.py create mode 100644 neutron/tests/unit/agent/l3/test_item_allocator.py diff --git a/neutron/agent/l3/item_allocator.py b/neutron/agent/l3/item_allocator.py new file mode 100644 index 000000000..79e8ee1cf --- /dev/null +++ b/neutron/agent/l3/item_allocator.py @@ -0,0 +1,104 @@ +# Copyright 2015 IBM Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + + +class ItemAllocator(object): + """Manages allocation of items from a pool + + Some of the allocations such as link local addresses used for routing + inside the fip namespaces need to persist across agent restarts to maintain + consistency. Persisting such allocations in the neutron database is + unnecessary and would degrade performance. ItemAllocator utilizes local + file system to track allocations made for objects of a given class. + + The persistent datastore is a file. The records are one per line of + the format: keyvalue. For example if the delimiter is a ',' + (the default value) then the records will be: key,value (one per line) + """ + + def __init__(self, state_file, ItemClass, item_pool, delimiter=','): + """Read the file with previous allocations recorded. + + See the note in the allocate method for more detail. + """ + self.ItemClass = ItemClass + self.state_file = state_file + + self.allocations = {} + + self.remembered = {} + self.pool = item_pool + + for line in self._read(): + key, saved_value = line.strip().split(delimiter) + self.remembered[key] = self.ItemClass(saved_value) + + self.pool.difference_update(self.remembered.values()) + + def allocate(self, key): + """Try to allocate an item of ItemClass type. + + I expect this to work in all cases because I expect the pool size to be + large enough for any situation. Nonetheless, there is some defensive + programming in here. + + Since the allocations are persisted, there is the chance to leak + allocations which should have been released but were not. This leak + could eventually exhaust the pool. + + So, if a new allocation is needed, the code first checks to see if + there are any remembered allocations for the key. If not, it checks + the free pool. If the free pool is empty then it dumps the remembered + allocations to free the pool. This final desperate step will not + happen often in practice. + """ + if key in self.remembered: + self.allocations[key] = self.remembered.pop(key) + return self.allocations[key] + + if not self.pool: + # Desperate times. Try to get more in the pool. + self.pool.update(self.remembered.values()) + self.remembered.clear() + if not self.pool: + # More than 256 routers on a compute node! + raise RuntimeError("Cannot allocate item of type:" + " %s from pool using file %s" + % (self.ItemClass, self.state_file)) + + self.allocations[key] = self.pool.pop() + self._write_allocations() + return self.allocations[key] + + def release(self, key): + self.pool.add(self.allocations.pop(key)) + self._write_allocations() + + def _write_allocations(self): + current = ["%s,%s\n" % (k, v) for k, v in self.allocations.items()] + remembered = ["%s,%s\n" % (k, v) for k, v in self.remembered.items()] + current.extend(remembered) + self._write(current) + + def _write(self, lines): + with open(self.state_file, "w") as f: + f.writelines(lines) + + def _read(self): + if not os.path.exists(self.state_file): + return [] + with open(self.state_file) as f: + return f.readlines() diff --git a/neutron/agent/l3/link_local_allocator.py b/neutron/agent/l3/link_local_allocator.py index 67d7d54a5..af7bca083 100644 --- a/neutron/agent/l3/link_local_allocator.py +++ b/neutron/agent/l3/link_local_allocator.py @@ -13,7 +13,8 @@ # under the License. import netaddr -import os + +from neutron.agent.l3.item_allocator import ItemAllocator class LinkLocalAddressPair(netaddr.IPNetwork): @@ -26,7 +27,7 @@ class LinkLocalAddressPair(netaddr.IPNetwork): netaddr.IPNetwork("%s/%s" % (self.broadcast, self.prefixlen))) -class LinkLocalAllocator(object): +class LinkLocalAllocator(ItemAllocator): """Manages allocation of link local IP addresses. These link local addresses are used for routing inside the fip namespaces. @@ -37,73 +38,13 @@ class LinkLocalAllocator(object): Persisting these in the database is unnecessary and would degrade performance. """ - def __init__(self, state_file, subnet): - """Read the file with previous allocations recorded. - - See the note in the allocate method for more detail. + def __init__(self, data_store_path, subnet): + """Create the necessary pool and item allocator + using ',' as the delimiter and LinkLocalAllocator as the + class type """ - self.state_file = state_file subnet = netaddr.IPNetwork(subnet) - - self.allocations = {} - - self.remembered = {} - for line in self._read(): - key, cidr = line.strip().split(',') - self.remembered[key] = LinkLocalAddressPair(cidr) - - self.pool = set(LinkLocalAddressPair(s) for s in subnet.subnet(31)) - self.pool.difference_update(self.remembered.values()) - - def allocate(self, key): - """Try to allocate a link local address pair. - - I expect this to work in all cases because I expect the pool size to be - large enough for any situation. Nonetheless, there is some defensive - programming in here. - - Since the allocations are persisted, there is the chance to leak - allocations which should have been released but were not. This leak - could eventually exhaust the pool. - - So, if a new allocation is needed, the code first checks to see if - there are any remembered allocations for the key. If not, it checks - the free pool. If the free pool is empty then it dumps the remembered - allocations to free the pool. This final desperate step will not - happen often in practice. - """ - if key in self.remembered: - self.allocations[key] = self.remembered.pop(key) - return self.allocations[key] - - if not self.pool: - # Desperate times. Try to get more in the pool. - self.pool.update(self.remembered.values()) - self.remembered.clear() - if not self.pool: - # More than 256 routers on a compute node! - raise RuntimeError(_("Cannot allocate link local address")) - - self.allocations[key] = self.pool.pop() - self._write_allocations() - return self.allocations[key] - - def release(self, key): - self.pool.add(self.allocations.pop(key)) - self._write_allocations() - - def _write_allocations(self): - current = ["%s,%s\n" % (k, v) for k, v in self.allocations.items()] - remembered = ["%s,%s\n" % (k, v) for k, v in self.remembered.items()] - current.extend(remembered) - self._write(current) - - def _write(self, lines): - with open(self.state_file, "w") as f: - f.writelines(lines) - - def _read(self): - if not os.path.exists(self.state_file): - return [] - with open(self.state_file) as f: - return f.readlines() + pool = set(LinkLocalAddressPair(s) for s in subnet.subnet(31)) + super(LinkLocalAllocator, self).__init__(data_store_path, + LinkLocalAddressPair, + pool) diff --git a/neutron/tests/unit/agent/l3/test_item_allocator.py b/neutron/tests/unit/agent/l3/test_item_allocator.py new file mode 100644 index 000000000..767ad8d5c --- /dev/null +++ b/neutron/tests/unit/agent/l3/test_item_allocator.py @@ -0,0 +1,29 @@ +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from neutron.agent.l3 import item_allocator as ia +from neutron.tests import base + + +class TestItemAllocator(base.BaseTestCase): + def setUp(self): + super(TestItemAllocator, self).setUp() + + def test__init__(self): + test_pool = set(s for s in range(32768, 40000)) + a = ia.ItemAllocator('/file', object, test_pool) + self.assertEqual('/file', a.state_file) + self.assertEqual({}, a.allocations) + self.assertEqual(object, a.ItemClass) + self.assertEqual(test_pool, a.pool) -- 2.45.2