Set IPset hash type to 'net' instead of 'ip'

The previous hash type was 'ip' and this caused a major
issue with the allowed address pairs extension since it
results in CIDRs being passed to ipset. When the hash type
is 'ip', a CIDR is completely enumerated into all of its
addresses so 10.100.0.0/16 results in ~65k entries. This
meant a single allowed_address_pairs entry could easily
exhaust an entire set.

This patch changes the hash type to 'net', which is designed
to handle a CIDRs as a single entry.

This patch also changes the names of the ipsets because
creating an ipset with different parameters will cause an
error and our ipset manager code isn't robust enough to handle
that at this time. There is another ongoing patch to fix
that but it won't be ready in time.[1]

The related bug was closed by increasing the set limit, which
did alleviate the problem. However, this change would also
address the issue because the gate tests run an allowed address
pairs extension test with the CIDR mentioned above.

1. I59e2e1c090cb95ee1bd14dbb53b6ff2c5e2713fd

Related-Bug: #1439817
Closes-Bug: #1444397
Change-Id: I8177699b157cd3eac46e2f481f47b5d966c49b07
(cherry picked from commit a38b5df5cd3c47672705aad4c30e789ae11ec958)

diff --git a/neutron/agent/linux/ipset_manager.py b/neutron/agent/linux/ipset_manager.py
index e5ab7a01e9c5813e4e1035b43158d434968c3a2f..0f764185239e7dbf43ff6f73c29133b77dd08117 100644 (file)
--- a/neutron/agent/linux/ipset_manager.py
+++ b/neutron/agent/linux/ipset_manager.py
@@ -36,7 +36,7 @@ class IpsetManager(object):
         """Returns the given ipset name for an id+ethertype pair.
         This reference can be used from iptables.
         """
-        name = ethertype + id
+        name = 'NET' + ethertype + id
         return name[:IPSET_NAME_MAX_LENGTH]
 
     def set_exists(self, id, ethertype):
@@ -85,7 +85,7 @@ class IpsetManager(object):
     def _refresh_set(self, set_name, member_ips, ethertype):
         new_set_name = set_name + SWAP_SUFFIX
         set_type = self._get_ipset_set_type(ethertype)
-        process_input = ["create %s hash:ip family %s" % (new_set_name,
+        process_input = ["create %s hash:net family %s" % (new_set_name,
                                                           set_type)]
         for ip in member_ips:
             process_input.append("add %s %s" % (new_set_name, ip))
@@ -101,7 +101,7 @@ class IpsetManager(object):
         self.ipset_sets[set_name].remove(member_ip)
 
     def _create_set(self, set_name, ethertype):
-        cmd = ['ipset', 'create', '-exist', set_name, 'hash:ip', 'family',
+        cmd = ['ipset', 'create', '-exist', set_name, 'hash:net', 'family',
                self._get_ipset_set_type(ethertype)]
         self._apply(cmd)
         self.ipset_sets[set_name] = []

diff --git a/neutron/tests/unit/agent/linux/test_ipset_manager.py b/neutron/tests/unit/agent/linux/test_ipset_manager.py
index cbd156218ffe2b7a5deb7929a1aaa5c5aef6a7d9..44840086f608bb2e1980c7c22fa2e13c941fba16 100644 (file)
--- a/neutron/tests/unit/agent/linux/test_ipset_manager.py
+++ b/neutron/tests/unit/agent/linux/test_ipset_manager.py
@@ -36,8 +36,9 @@ class BaseIpsetManagerTest(base.BaseTestCase):
         self.execute.assert_has_calls(self.expected_calls, any_order=False)
 
     def expect_set(self, addresses):
-        temp_input = ['create IPv4fake_sgid-new hash:ip family inet']
-        temp_input.extend('add IPv4fake_sgid-new %s' % ip for ip in addresses)
+        temp_input = ['create NETIPv4fake_sgid-new hash:net family inet']
+        temp_input.extend('add NETIPv4fake_sgid-new %s' % ip
+                          for ip in addresses)
         input = '\n'.join(temp_input)
         self.expected_calls.extend([
             mock.call(['ipset', 'restore', '-exist'],
@@ -65,7 +66,7 @@ class BaseIpsetManagerTest(base.BaseTestCase):
     def expect_create(self):
         self.expected_calls.append(
             mock.call(['ipset', 'create', '-exist', TEST_SET_NAME,
-                       'hash:ip', 'family', 'inet'],
+                       'hash:net', 'family', 'inet'],
                       process_input=None,
                       run_as_root=True))
 

diff --git a/neutron/tests/unit/agent/test_securitygroups_rpc.py b/neutron/tests/unit/agent/test_securitygroups_rpc.py
index feabbcbfdbd4551012a79038e830b0ede0cdb2f2..6a4e2990f414830d5722a1bdc49fb900b261c710 100644 (file)
--- a/neutron/tests/unit/agent/test_securitygroups_rpc.py
+++ b/neutron/tests/unit/agent/test_securitygroups_rpc.py
@@ -1776,7 +1776,7 @@ IPSET_FILTER_1 = """# Generated by iptables_manager
 [0:0] -A %(bn)s-i_port1 -s 10.0.0.2/32 -p udp -m udp --sport 67 --dport 68 \
 -j RETURN
 [0:0] -A %(bn)s-i_port1 -p tcp -m tcp --dport 22 -j RETURN
-[0:0] -A %(bn)s-i_port1 -m set --match-set IPv4security_group1 src -j \
+[0:0] -A %(bn)s-i_port1 -m set --match-set NETIPv4security_group1 src -j \
 RETURN
 [0:0] -A %(bn)s-i_port1 -j %(bn)s-sg-fallback
 [0:0] -A %(bn)s-FORWARD %(physdev_mod)s --physdev-EGRESS tap_port1 \
@@ -1935,7 +1935,7 @@ IPSET_FILTER_2 = """# Generated by iptables_manager
 [0:0] -A %(bn)s-i_%(port1)s -s 10.0.0.2/32 -p udp -m udp --sport 67 \
 --dport 68 -j RETURN
 [0:0] -A %(bn)s-i_%(port1)s -p tcp -m tcp --dport 22 -j RETURN
-[0:0] -A %(bn)s-i_%(port1)s -m set --match-set IPv4security_group1 src -j \
+[0:0] -A %(bn)s-i_%(port1)s -m set --match-set NETIPv4security_group1 src -j \
 RETURN
 [0:0] -A %(bn)s-i_%(port1)s -j %(bn)s-sg-fallback
 [0:0] -A %(bn)s-FORWARD %(physdev_mod)s --physdev-EGRESS tap_%(port1)s \
@@ -1963,7 +1963,7 @@ RETURN
 [0:0] -A %(bn)s-i_%(port2)s -s 10.0.0.2/32 -p udp -m udp --sport 67 \
 --dport 68 -j RETURN
 [0:0] -A %(bn)s-i_%(port2)s -p tcp -m tcp --dport 22 -j RETURN
-[0:0] -A %(bn)s-i_%(port2)s -m set --match-set IPv4security_group1 src -j \
+[0:0] -A %(bn)s-i_%(port2)s -m set --match-set NETIPv4security_group1 src -j \
 RETURN
 [0:0] -A %(bn)s-i_%(port2)s -j %(bn)s-sg-fallback
 [0:0] -A %(bn)s-FORWARD %(physdev_mod)s --physdev-EGRESS tap_%(port2)s \
@@ -2018,7 +2018,7 @@ IPSET_FILTER_2_3 = """# Generated by iptables_manager
 [0:0] -A %(bn)s-i_%(port1)s -s 10.0.0.2/32 -p udp -m udp --sport 67 \
 --dport 68 -j RETURN
 [0:0] -A %(bn)s-i_%(port1)s -p tcp -m tcp --dport 22 -j RETURN
-[0:0] -A %(bn)s-i_%(port1)s -m set --match-set IPv4security_group1 src -j \
+[0:0] -A %(bn)s-i_%(port1)s -m set --match-set NETIPv4security_group1 src -j \
 RETURN
 [0:0] -A %(bn)s-i_%(port1)s -p icmp -j RETURN
 [0:0] -A %(bn)s-i_%(port1)s -j %(bn)s-sg-fallback
@@ -2047,7 +2047,7 @@ RETURN
 [0:0] -A %(bn)s-i_%(port2)s -s 10.0.0.2/32 -p udp -m udp --sport 67 \
 --dport 68 -j RETURN
 [0:0] -A %(bn)s-i_%(port2)s -p tcp -m tcp --dport 22 -j RETURN
-[0:0] -A %(bn)s-i_%(port2)s -m set --match-set IPv4security_group1 src -j \
+[0:0] -A %(bn)s-i_%(port2)s -m set --match-set NETIPv4security_group1 src -j \
 RETURN
 [0:0] -A %(bn)s-i_%(port2)s -p icmp -j RETURN
 [0:0] -A %(bn)s-i_%(port2)s -j %(bn)s-sg-fallback