Retry metadata request on connection refused error

This testcase may fail intermittently on 'Connection refused' error.
This could be due to the fact that the metadata proxy setup is not exactly
complete at the time the request is issued; in fact there is no
synchronization between the router being up and the metadata request being
issued, and clearly this may be the reason of accidental but seldom failures.

In order to rule out this possibility and stabilize the test, let's retry
on connection refused only. If we continue to fail, then the next step would
be to dump the content of iptables to figure out why the error occurs.

Closes-bug: #1461172

Change-Id: I65a5bf4fbbcad6ba93a46d36cabe7844ff528d8d

diff --git a/neutron/tests/functional/agent/test_l3_agent.py b/neutron/tests/functional/agent/test_l3_agent.py
index 17b0ee00e9e1456a7da48f6973dda83763141d65..d075266d2485852d136fbeccaf2995a4f5ea2221 100644 (file)
--- a/neutron/tests/functional/agent/test_l3_agent.py
+++ b/neutron/tests/functional/agent/test_l3_agent.py
@@ -16,6 +16,7 @@
 import copy
 import functools
 import os.path
+import time
 
 import mock
 import netaddr
@@ -56,6 +57,7 @@ LOG = logging.getLogger(__name__)
 _uuid = uuidutils.generate_uuid
 
 METADATA_REQUEST_TIMEOUT = 60
+METADATA_REQUEST_SLEEP = 5
 
 
 def get_ovs_bridge(br_name):
@@ -912,6 +914,28 @@ class MetadataL3AgentTestCase(L3AgentTestFramework):
                      self.agent.conf.metadata_proxy_socket,
                      workers=0, backlog=4096, mode=self.SOCKET_MODE)
 
+    def _query_metadata_proxy(self, machine):
+        url = 'http://%(host)s:%(port)s' % {'host': dhcp.METADATA_DEFAULT_IP,
+                                            'port': dhcp.METADATA_PORT}
+        cmd = 'curl', '--max-time', METADATA_REQUEST_TIMEOUT, '-D-', url
+        i = 0
+        CONNECTION_REFUSED_TIMEOUT = METADATA_REQUEST_TIMEOUT // 2
+        while i <= CONNECTION_REFUSED_TIMEOUT:
+            try:
+                raw_headers = machine.execute(cmd)
+                break
+            except RuntimeError as e:
+                if 'Connection refused' in str(e):
+                    time.sleep(METADATA_REQUEST_SLEEP)
+                    i += METADATA_REQUEST_SLEEP
+                else:
+                    self.fail('metadata proxy unreachable '
+                              'on %s before timeout' % url)
+
+        if i > CONNECTION_REFUSED_TIMEOUT:
+            self.fail('Timed out waiting metadata proxy to become available')
+        return raw_headers.splitlines()[0]
+
     def test_access_to_metadata_proxy(self):
         """Test access to the l3-agent metadata proxy.
 
@@ -943,16 +967,9 @@ class MetadataL3AgentTestCase(L3AgentTestFramework):
                 router_ip_cidr.partition('/')[0]))
 
         # Query metadata proxy
-        url = 'http://%(host)s:%(port)s' % {'host': dhcp.METADATA_DEFAULT_IP,
-                                            'port': dhcp.METADATA_PORT}
-        cmd = 'curl', '--max-time', METADATA_REQUEST_TIMEOUT, '-D-', url
-        try:
-            raw_headers = machine.execute(cmd)
-        except RuntimeError:
-            self.fail('metadata proxy unreachable on %s before timeout' % url)
+        firstline = self._query_metadata_proxy(machine)
 
         # Check status code
-        firstline = raw_headers.splitlines()[0]
         self.assertIn(str(webob.exc.HTTPOk.code), firstline.split())