Make sure we return unicode strings for process output

Process output is supposed to be represented with lines, so we should
put Python strings in the queue (not bytes). Just in case, we do it only
for Python 3 environment.

To fix that, we reuse code from utils.execute() linux/windows
implementations.

This fixes the TestAsyncProcess.test_async_process_respawns functional
test for Python 3 environment.

Related-Bug: #1515118
Change-Id: I9efec2290003add44909aab33a0026372a580016

diff --git a/neutron/agent/linux/async_process.py b/neutron/agent/linux/async_process.py
index f94cfa6a2e8eef044d0adec708ab99b6873a9fb5..aa516d4f00915dc8862e91de5417f431fb424386 100644 (file)
--- a/neutron/agent/linux/async_process.py
+++ b/neutron/agent/linux/async_process.py
@@ -21,6 +21,7 @@ from oslo_log import log as logging
 
 from neutron.agent.linux import ip_lib
 from neutron.agent.linux import utils
+from neutron.common import utils as common_utils
 from neutron.i18n import _LE
 
 
@@ -226,7 +227,7 @@ class AsyncProcess(object):
     def _read(self, stream, queue):
         data = stream.readline()
         if data:
-            data = data.strip()
+            data = common_utils.safe_decode_utf8(data.strip())
             queue.put(data)
             return data
 

diff --git a/neutron/agent/linux/utils.py b/neutron/agent/linux/utils.py
index 2148c73e17ddb9e70186b06677c9bf9689f4980c..5bbc1bf0198302f569c1fc267d5c4ffaa33b50cb 100644 (file)
--- a/neutron/agent/linux/utils.py
+++ b/neutron/agent/linux/utils.py
@@ -119,11 +119,8 @@ def execute(cmd, process_input=None, addl_env=None,
             _stdout, _stderr = obj.communicate(_process_input)
             returncode = obj.returncode
             obj.stdin.close()
-        if six.PY3:
-            if isinstance(_stdout, bytes):
-                _stdout = _stdout.decode('utf-8', 'surrogateescape')
-            if isinstance(_stderr, bytes):
-                _stderr = _stderr.decode('utf-8', 'surrogateescape')
+        _stdout = utils.safe_decode_utf8(_stdout)
+        _stderr = utils.safe_decode_utf8(_stderr)
 
         command_str = {
             'cmd': cmd,

diff --git a/neutron/agent/windows/utils.py b/neutron/agent/windows/utils.py
index 046d060601a7212a0a65570830aaeed0b0bd2b91..06795f3e731523b6f6337a1c315cf33abb89515d 100644 (file)
--- a/neutron/agent/windows/utils.py
+++ b/neutron/agent/windows/utils.py
@@ -57,11 +57,8 @@ def execute(cmd, process_input=None, addl_env=None,
         obj, cmd = create_process(cmd, addl_env=addl_env)
         _stdout, _stderr = obj.communicate(_process_input)
         obj.stdin.close()
-        if six.PY3:
-            if isinstance(_stdout, bytes):
-                _stdout = _stdout.decode('utf-8', 'surrogateescape')
-            if isinstance(_stderr, bytes):
-                _stderr = _stderr.decode('utf-8', 'surrogateescape')
+        _stdout = utils.safe_decode_utf8(_stdout)
+        _stderr = utils.safe_decode_utf8(_stderr)
 
         m = _("\nCommand: %(cmd)s\nExit code: %(code)s\nStdin: %(stdin)s\n"
               "Stdout: %(stdout)s\nStderr: %(stderr)s") % \

diff --git a/neutron/common/utils.py b/neutron/common/utils.py
index 83886d5979b48d018143f1042568c1b30530588f..43935e5fcd38690b677f3699a444ac71f4ed92ac 100644 (file)
--- a/neutron/common/utils.py
+++ b/neutron/common/utils.py
@@ -524,3 +524,9 @@ def load_class_by_alias_or_classname(namespace, name):
                       exc_info=True)
             raise ImportError(_("Class not found."))
     return class_to_load
+
+
+def safe_decode_utf8(s):
+    if six.PY3 and isinstance(s, bytes):
+        return s.decode('utf-8', 'surrogateescape')
+    return s

diff --git a/neutron/tests/common/helpers.py b/neutron/tests/common/helpers.py
index 484c0216580e015efe6ea3eb3740d69eecb3550a..b417825393a7827de56cbd90ad470747423077f8 100644 (file)
--- a/neutron/tests/common/helpers.py
+++ b/neutron/tests/common/helpers.py
@@ -16,6 +16,8 @@ import datetime
 import os
 
 from oslo_utils import timeutils
+import six
+import testtools
 
 import neutron
 from neutron.common import constants
@@ -153,3 +155,11 @@ def register_ovs_agent(host=HOST, agent_type=constants.AGENT_TYPE_OVS,
                                 tunneling_ip, interface_mappings,
                                 l2pop_network_types)
     return _register_agent(agent)
+
+
+def requires_py2(testcase):
+    return testtools.skipUnless(six.PY2, "requires python 2.x")(testcase)
+
+
+def requires_py3(testcase):
+    return testtools.skipUnless(six.PY3, "requires python 3.x")(testcase)

diff --git a/neutron/tests/functional/agent/linux/test_async_process.py b/neutron/tests/functional/agent/linux/test_async_process.py
index 46bc93b965df212b88e56a445611305dd5cbc2ed..09bfc027f5d38ebaa46a73794bb0e78e193dfc26 100644 (file)
--- a/neutron/tests/functional/agent/linux/test_async_process.py
+++ b/neutron/tests/functional/agent/linux/test_async_process.py
@@ -13,6 +13,7 @@
 #    under the License.
 
 import eventlet
+import six
 
 from neutron.agent.linux import async_process
 from neutron.agent.linux import utils
@@ -24,7 +25,7 @@ class AsyncProcessTestFramework(base.BaseTestCase):
     def setUp(self):
         super(AsyncProcessTestFramework, self).setUp()
         self.test_file_path = self.get_temp_file_path('test_async_process.tmp')
-        self.data = [str(x) for x in range(4)]
+        self.data = [six.text_type(x) for x in range(4)]
         with open(self.test_file_path, 'w') as f:
             f.writelines('%s\n' % item for item in self.data)
 

diff --git a/neutron/tests/unit/agent/linux/test_utils.py b/neutron/tests/unit/agent/linux/test_utils.py
index 0fd76dd1ae03dc2e10e0e07c00dd8de3cb494f3d..1ca469b9a74a71c2ac29120ced2e66d47f390f87 100644 (file)
--- a/neutron/tests/unit/agent/linux/test_utils.py
+++ b/neutron/tests/unit/agent/linux/test_utils.py
@@ -22,6 +22,7 @@ import oslo_i18n
 
 from neutron.agent.linux import utils
 from neutron.tests import base
+from neutron.tests.common import helpers
 
 
 _marker = object()
@@ -147,7 +148,7 @@ class AgentUtilsExecuteTest(base.BaseTestCase):
         result = utils.execute(['ls', self.test_file], return_stderr=True)
         self.assertEqual((str_data, ''), result)
 
-    @testtools.skipUnless(six.PY3, 'This test makes sense only in Python 3')
+    @helpers.requires_py3
     def test_surrogateescape_in_decoding_out_data(self):
         bytes_err_data = b'\xed\xa0\xbd'
         err_data = bytes_err_data.decode('utf-8', 'surrogateescape')

diff --git a/neutron/tests/unit/common/test_utils.py b/neutron/tests/unit/common/test_utils.py
index 97939ebaff83de3c426363e661fc27369ad146f8..5666a808cc75b17255c62f1d184da3b98510b9c8 100644 (file)
--- a/neutron/tests/unit/common/test_utils.py
+++ b/neutron/tests/unit/common/test_utils.py
@@ -18,6 +18,7 @@ import re
 import eventlet
 import mock
 import netaddr
+import six
 import testtools
 
 from neutron.common import constants
@@ -26,6 +27,7 @@ from neutron.common import utils
 from neutron.plugins.common import constants as p_const
 from neutron.plugins.common import utils as plugin_utils
 from neutron.tests import base
+from neutron.tests.common import helpers
 
 from oslo_log import log as logging
 
@@ -716,3 +718,24 @@ class TestGetRandomString(base.BaseTestCase):
         self.assertEqual(length, len(random_string))
         regex = re.compile('^[0-9a-fA-F]+$')
         self.assertIsNotNone(regex.match(random_string))
+
+
+class TestSafeDecodeUtf8(base.BaseTestCase):
+
+    @helpers.requires_py2
+    def test_py2_does_nothing(self):
+        s = 'test-py2'
+        self.assertIs(s, utils.safe_decode_utf8(s))
+
+    @helpers.requires_py3
+    def test_py3_decoded_valid_bytes(self):
+        s = bytes('test-py2', 'utf-8')
+        decoded_str = utils.safe_decode_utf8(s)
+        self.assertIsInstance(decoded_str, six.text_type)
+        self.assertEqual(s, decoded_str.encode('utf-8'))
+
+    @helpers.requires_py3
+    def test_py3_decoded_invalid_bytes(self):
+        s = bytes('test-py2', 'utf_16')
+        decoded_str = utils.safe_decode_utf8(s)
+        self.assertIsInstance(decoded_str, six.text_type)

diff --git a/neutron/tests/unit/test_wsgi.py b/neutron/tests/unit/test_wsgi.py
index 02166549e72a089f3ad7875d06b9662966f02a82..ff6515e221489e6f27fa2eb6f4bc89889c598a11 100644 (file)
--- a/neutron/tests/unit/test_wsgi.py
+++ b/neutron/tests/unit/test_wsgi.py
@@ -19,7 +19,6 @@ import ssl
 
 import mock
 from oslo_config import cfg
-import six
 import six.moves.urllib.request as urlrequest
 import testtools
 import webob
@@ -28,6 +27,7 @@ import webob.exc
 from neutron.common import exceptions as exception
 from neutron.db import api
 from neutron.tests import base
+from neutron.tests.common import helpers
 from neutron import wsgi
 
 CONF = cfg.CONF
@@ -496,7 +496,7 @@ class JSONDictSerializerTest(base.BaseTestCase):
 
     # The tested behaviour is only meant to be witnessed in Python 2, so it is
     # OK to skip this test with Python 3.
-    @testtools.skipIf(six.PY3, "This test does not make sense in Python 3")
+    @helpers.requires_py2
     def test_json_with_utf8(self):
         input_dict = dict(servers=dict(a=(2, '\xe7\xbd\x91\xe7\xbb\x9c')))
         expected_json = b'{"servers":{"a":[2,"\\u7f51\\u7edc"]}}'