From: Ihar Hrachyshka Date: Wed, 11 Nov 2015 12:59:22 +0000 (+0100) Subject: Make sure we return unicode strings for process output X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=e30d8cead155c119ee2ffde074312591f7938fb0;p=openstack-build%2Fneutron-build.git Make sure we return unicode strings for process output Process output is supposed to be represented with lines, so we should put Python strings in the queue (not bytes). Just in case, we do it only for Python 3 environment. To fix that, we reuse code from utils.execute() linux/windows implementations. This fixes the TestAsyncProcess.test_async_process_respawns functional test for Python 3 environment. Related-Bug: #1515118 Change-Id: I9efec2290003add44909aab33a0026372a580016 --- diff --git a/neutron/agent/linux/async_process.py b/neutron/agent/linux/async_process.py index f94cfa6a2..aa516d4f0 100644 --- a/neutron/agent/linux/async_process.py +++ b/neutron/agent/linux/async_process.py @@ -21,6 +21,7 @@ from oslo_log import log as logging from neutron.agent.linux import ip_lib from neutron.agent.linux import utils +from neutron.common import utils as common_utils from neutron.i18n import _LE @@ -226,7 +227,7 @@ class AsyncProcess(object): def _read(self, stream, queue): data = stream.readline() if data: - data = data.strip() + data = common_utils.safe_decode_utf8(data.strip()) queue.put(data) return data diff --git a/neutron/agent/linux/utils.py b/neutron/agent/linux/utils.py index 2148c73e1..5bbc1bf01 100644 --- a/neutron/agent/linux/utils.py +++ b/neutron/agent/linux/utils.py @@ -119,11 +119,8 @@ def execute(cmd, process_input=None, addl_env=None, _stdout, _stderr = obj.communicate(_process_input) returncode = obj.returncode obj.stdin.close() - if six.PY3: - if isinstance(_stdout, bytes): - _stdout = _stdout.decode('utf-8', 'surrogateescape') - if isinstance(_stderr, bytes): - _stderr = _stderr.decode('utf-8', 'surrogateescape') + _stdout = utils.safe_decode_utf8(_stdout) + _stderr = utils.safe_decode_utf8(_stderr) command_str = { 'cmd': cmd, diff --git a/neutron/agent/windows/utils.py b/neutron/agent/windows/utils.py index 046d06060..06795f3e7 100644 --- a/neutron/agent/windows/utils.py +++ b/neutron/agent/windows/utils.py @@ -57,11 +57,8 @@ def execute(cmd, process_input=None, addl_env=None, obj, cmd = create_process(cmd, addl_env=addl_env) _stdout, _stderr = obj.communicate(_process_input) obj.stdin.close() - if six.PY3: - if isinstance(_stdout, bytes): - _stdout = _stdout.decode('utf-8', 'surrogateescape') - if isinstance(_stderr, bytes): - _stderr = _stderr.decode('utf-8', 'surrogateescape') + _stdout = utils.safe_decode_utf8(_stdout) + _stderr = utils.safe_decode_utf8(_stderr) m = _("\nCommand: %(cmd)s\nExit code: %(code)s\nStdin: %(stdin)s\n" "Stdout: %(stdout)s\nStderr: %(stderr)s") % \ diff --git a/neutron/common/utils.py b/neutron/common/utils.py index 83886d597..43935e5fc 100644 --- a/neutron/common/utils.py +++ b/neutron/common/utils.py @@ -524,3 +524,9 @@ def load_class_by_alias_or_classname(namespace, name): exc_info=True) raise ImportError(_("Class not found.")) return class_to_load + + +def safe_decode_utf8(s): + if six.PY3 and isinstance(s, bytes): + return s.decode('utf-8', 'surrogateescape') + return s diff --git a/neutron/tests/common/helpers.py b/neutron/tests/common/helpers.py index 484c02165..b41782539 100644 --- a/neutron/tests/common/helpers.py +++ b/neutron/tests/common/helpers.py @@ -16,6 +16,8 @@ import datetime import os from oslo_utils import timeutils +import six +import testtools import neutron from neutron.common import constants @@ -153,3 +155,11 @@ def register_ovs_agent(host=HOST, agent_type=constants.AGENT_TYPE_OVS, tunneling_ip, interface_mappings, l2pop_network_types) return _register_agent(agent) + + +def requires_py2(testcase): + return testtools.skipUnless(six.PY2, "requires python 2.x")(testcase) + + +def requires_py3(testcase): + return testtools.skipUnless(six.PY3, "requires python 3.x")(testcase) diff --git a/neutron/tests/functional/agent/linux/test_async_process.py b/neutron/tests/functional/agent/linux/test_async_process.py index 46bc93b96..09bfc027f 100644 --- a/neutron/tests/functional/agent/linux/test_async_process.py +++ b/neutron/tests/functional/agent/linux/test_async_process.py @@ -13,6 +13,7 @@ # under the License. import eventlet +import six from neutron.agent.linux import async_process from neutron.agent.linux import utils @@ -24,7 +25,7 @@ class AsyncProcessTestFramework(base.BaseTestCase): def setUp(self): super(AsyncProcessTestFramework, self).setUp() self.test_file_path = self.get_temp_file_path('test_async_process.tmp') - self.data = [str(x) for x in range(4)] + self.data = [six.text_type(x) for x in range(4)] with open(self.test_file_path, 'w') as f: f.writelines('%s\n' % item for item in self.data) diff --git a/neutron/tests/unit/agent/linux/test_utils.py b/neutron/tests/unit/agent/linux/test_utils.py index 0fd76dd1a..1ca469b9a 100644 --- a/neutron/tests/unit/agent/linux/test_utils.py +++ b/neutron/tests/unit/agent/linux/test_utils.py @@ -22,6 +22,7 @@ import oslo_i18n from neutron.agent.linux import utils from neutron.tests import base +from neutron.tests.common import helpers _marker = object() @@ -147,7 +148,7 @@ class AgentUtilsExecuteTest(base.BaseTestCase): result = utils.execute(['ls', self.test_file], return_stderr=True) self.assertEqual((str_data, ''), result) - @testtools.skipUnless(six.PY3, 'This test makes sense only in Python 3') + @helpers.requires_py3 def test_surrogateescape_in_decoding_out_data(self): bytes_err_data = b'\xed\xa0\xbd' err_data = bytes_err_data.decode('utf-8', 'surrogateescape') diff --git a/neutron/tests/unit/common/test_utils.py b/neutron/tests/unit/common/test_utils.py index 97939ebaf..5666a808c 100644 --- a/neutron/tests/unit/common/test_utils.py +++ b/neutron/tests/unit/common/test_utils.py @@ -18,6 +18,7 @@ import re import eventlet import mock import netaddr +import six import testtools from neutron.common import constants @@ -26,6 +27,7 @@ from neutron.common import utils from neutron.plugins.common import constants as p_const from neutron.plugins.common import utils as plugin_utils from neutron.tests import base +from neutron.tests.common import helpers from oslo_log import log as logging @@ -716,3 +718,24 @@ class TestGetRandomString(base.BaseTestCase): self.assertEqual(length, len(random_string)) regex = re.compile('^[0-9a-fA-F]+$') self.assertIsNotNone(regex.match(random_string)) + + +class TestSafeDecodeUtf8(base.BaseTestCase): + + @helpers.requires_py2 + def test_py2_does_nothing(self): + s = 'test-py2' + self.assertIs(s, utils.safe_decode_utf8(s)) + + @helpers.requires_py3 + def test_py3_decoded_valid_bytes(self): + s = bytes('test-py2', 'utf-8') + decoded_str = utils.safe_decode_utf8(s) + self.assertIsInstance(decoded_str, six.text_type) + self.assertEqual(s, decoded_str.encode('utf-8')) + + @helpers.requires_py3 + def test_py3_decoded_invalid_bytes(self): + s = bytes('test-py2', 'utf_16') + decoded_str = utils.safe_decode_utf8(s) + self.assertIsInstance(decoded_str, six.text_type) diff --git a/neutron/tests/unit/test_wsgi.py b/neutron/tests/unit/test_wsgi.py index 02166549e..ff6515e22 100644 --- a/neutron/tests/unit/test_wsgi.py +++ b/neutron/tests/unit/test_wsgi.py @@ -19,7 +19,6 @@ import ssl import mock from oslo_config import cfg -import six import six.moves.urllib.request as urlrequest import testtools import webob @@ -28,6 +27,7 @@ import webob.exc from neutron.common import exceptions as exception from neutron.db import api from neutron.tests import base +from neutron.tests.common import helpers from neutron import wsgi CONF = cfg.CONF @@ -496,7 +496,7 @@ class JSONDictSerializerTest(base.BaseTestCase): # The tested behaviour is only meant to be witnessed in Python 2, so it is # OK to skip this test with Python 3. - @testtools.skipIf(six.PY3, "This test does not make sense in Python 3") + @helpers.requires_py2 def test_json_with_utf8(self): input_dict = dict(servers=dict(a=(2, '\xe7\xbd\x91\xe7\xbb\x9c'))) expected_json = b'{"servers":{"a":[2,"\\u7f51\\u7edc"]}}'