Set IPset hash type to 'net' instead of 'ip'

The previous hash type was 'ip' and this caused a major
issue with the allowed address pairs extension since it
results in CIDRs being passed to ipset. When the hash type
is 'ip', a CIDR is completely enumerated into all of its
addresses so 10.100.0.0/16 results in ~65k entries. This
meant a single allowed_address_pairs entry could easily
exhaust an entire set.

This patch changes the hash type to 'net', which is designed
to handle a CIDRs as a single entry.

This patch also changes the names of the ipsets because
creating an ipset with different parameters will cause an
error and our ipset manager code isn't robust enough to handle
that at this time. There is another ongoing patch to fix
that but it won't be ready in time.[1]

The related bug was closed by increasing the set limit, which
did alleviate the problem. However, this change would also
address the issue because the gate tests run an allowed address
pairs extension test with the CIDR mentioned above.

1. I59e2e1c090cb95ee1bd14dbb53b6ff2c5e2713fd

Related-Bug: #1439817
Closes-Bug: #1444397
Change-Id: I8177699b157cd3eac46e2f481f47b5d966c49b07

diff --git a/neutron/agent/linux/ipset_manager.py b/neutron/agent/linux/ipset_manager.py
index 8935c72a00cb1ae333f3eda2744bcea87386d475..f80cba979b235dbf7778576dabf7f2662014fb22 100644 (file)
--- a/neutron/agent/linux/ipset_manager.py
+++ b/neutron/agent/linux/ipset_manager.py
@@ -38,7 +38,7 @@ class IpsetManager(object):
         """Returns the given ipset name for an id+ethertype pair.
         This reference can be used from iptables.
         """
-        name = ethertype + id
+        name = 'NET' + ethertype + id
         return name[:IPSET_NAME_MAX_LENGTH]
 
     def set_exists(self, id, ethertype):
@@ -87,7 +87,7 @@ class IpsetManager(object):
     def _refresh_set(self, set_name, member_ips, ethertype):
         new_set_name = set_name + SWAP_SUFFIX
         set_type = self._get_ipset_set_type(ethertype)
-        process_input = ["create %s hash:ip family %s" % (new_set_name,
+        process_input = ["create %s hash:net family %s" % (new_set_name,
                                                           set_type)]
         for ip in member_ips:
             process_input.append("add %s %s" % (new_set_name, ip))
@@ -103,7 +103,7 @@ class IpsetManager(object):
         self.ipset_sets[set_name].remove(member_ip)
 
     def _create_set(self, set_name, ethertype):
-        cmd = ['ipset', 'create', '-exist', set_name, 'hash:ip', 'family',
+        cmd = ['ipset', 'create', '-exist', set_name, 'hash:net', 'family',
                self._get_ipset_set_type(ethertype)]
         self._apply(cmd)
         self.ipset_sets[set_name] = []

diff --git a/neutron/tests/unit/agent/linux/test_ipset_manager.py b/neutron/tests/unit/agent/linux/test_ipset_manager.py
index cbd156218ffe2b7a5deb7929a1aaa5c5aef6a7d9..44840086f608bb2e1980c7c22fa2e13c941fba16 100644 (file)
--- a/neutron/tests/unit/agent/linux/test_ipset_manager.py
+++ b/neutron/tests/unit/agent/linux/test_ipset_manager.py
@@ -36,8 +36,9 @@ class BaseIpsetManagerTest(base.BaseTestCase):
         self.execute.assert_has_calls(self.expected_calls, any_order=False)
 
     def expect_set(self, addresses):
-        temp_input = ['create IPv4fake_sgid-new hash:ip family inet']
-        temp_input.extend('add IPv4fake_sgid-new %s' % ip for ip in addresses)
+        temp_input = ['create NETIPv4fake_sgid-new hash:net family inet']
+        temp_input.extend('add NETIPv4fake_sgid-new %s' % ip
+                          for ip in addresses)
         input = '\n'.join(temp_input)
         self.expected_calls.extend([
             mock.call(['ipset', 'restore', '-exist'],
@@ -65,7 +66,7 @@ class BaseIpsetManagerTest(base.BaseTestCase):
     def expect_create(self):
         self.expected_calls.append(
             mock.call(['ipset', 'create', '-exist', TEST_SET_NAME,
-                       'hash:ip', 'family', 'inet'],
+                       'hash:net', 'family', 'inet'],
                       process_input=None,
                       run_as_root=True))
 

diff --git a/neutron/tests/unit/agent/test_securitygroups_rpc.py b/neutron/tests/unit/agent/test_securitygroups_rpc.py
index 29f305118105a78748afaaab3d1f5aa2813e51da..783c08c5c51a9b45e0005ee9765e0b9b02b2bbdf 100644 (file)
--- a/neutron/tests/unit/agent/test_securitygroups_rpc.py
+++ b/neutron/tests/unit/agent/test_securitygroups_rpc.py
@@ -1775,7 +1775,7 @@ IPSET_FILTER_1 = """# Generated by iptables_manager
 [0:0] -A %(bn)s-i_port1 -s 10.0.0.2/32 -p udp -m udp --sport 67 --dport 68 \
 -j RETURN
 [0:0] -A %(bn)s-i_port1 -p tcp -m tcp --dport 22 -j RETURN
-[0:0] -A %(bn)s-i_port1 -m set --match-set IPv4security_group1 src -j \
+[0:0] -A %(bn)s-i_port1 -m set --match-set NETIPv4security_group1 src -j \
 RETURN
 [0:0] -A %(bn)s-i_port1 -j %(bn)s-sg-fallback
 [0:0] -A %(bn)s-FORWARD %(physdev_mod)s --physdev-EGRESS tap_port1 \
@@ -1934,7 +1934,7 @@ IPSET_FILTER_2 = """# Generated by iptables_manager
 [0:0] -A %(bn)s-i_%(port1)s -s 10.0.0.2/32 -p udp -m udp --sport 67 \
 --dport 68 -j RETURN
 [0:0] -A %(bn)s-i_%(port1)s -p tcp -m tcp --dport 22 -j RETURN
-[0:0] -A %(bn)s-i_%(port1)s -m set --match-set IPv4security_group1 src -j \
+[0:0] -A %(bn)s-i_%(port1)s -m set --match-set NETIPv4security_group1 src -j \
 RETURN
 [0:0] -A %(bn)s-i_%(port1)s -j %(bn)s-sg-fallback
 [0:0] -A %(bn)s-FORWARD %(physdev_mod)s --physdev-EGRESS tap_%(port1)s \
@@ -1962,7 +1962,7 @@ RETURN
 [0:0] -A %(bn)s-i_%(port2)s -s 10.0.0.2/32 -p udp -m udp --sport 67 \
 --dport 68 -j RETURN
 [0:0] -A %(bn)s-i_%(port2)s -p tcp -m tcp --dport 22 -j RETURN
-[0:0] -A %(bn)s-i_%(port2)s -m set --match-set IPv4security_group1 src -j \
+[0:0] -A %(bn)s-i_%(port2)s -m set --match-set NETIPv4security_group1 src -j \
 RETURN
 [0:0] -A %(bn)s-i_%(port2)s -j %(bn)s-sg-fallback
 [0:0] -A %(bn)s-FORWARD %(physdev_mod)s --physdev-EGRESS tap_%(port2)s \
@@ -2017,7 +2017,7 @@ IPSET_FILTER_2_3 = """# Generated by iptables_manager
 [0:0] -A %(bn)s-i_%(port1)s -s 10.0.0.2/32 -p udp -m udp --sport 67 \
 --dport 68 -j RETURN
 [0:0] -A %(bn)s-i_%(port1)s -p tcp -m tcp --dport 22 -j RETURN
-[0:0] -A %(bn)s-i_%(port1)s -m set --match-set IPv4security_group1 src -j \
+[0:0] -A %(bn)s-i_%(port1)s -m set --match-set NETIPv4security_group1 src -j \
 RETURN
 [0:0] -A %(bn)s-i_%(port1)s -p icmp -j RETURN
 [0:0] -A %(bn)s-i_%(port1)s -j %(bn)s-sg-fallback
@@ -2046,7 +2046,7 @@ RETURN
 [0:0] -A %(bn)s-i_%(port2)s -s 10.0.0.2/32 -p udp -m udp --sport 67 \
 --dport 68 -j RETURN
 [0:0] -A %(bn)s-i_%(port2)s -p tcp -m tcp --dport 22 -j RETURN
-[0:0] -A %(bn)s-i_%(port2)s -m set --match-set IPv4security_group1 src -j \
+[0:0] -A %(bn)s-i_%(port2)s -m set --match-set NETIPv4security_group1 src -j \
 RETURN
 [0:0] -A %(bn)s-i_%(port2)s -p icmp -j RETURN
 [0:0] -A %(bn)s-i_%(port2)s -j %(bn)s-sg-fallback