Ensure to decode bytes or fail

The commit 048316e98114145ad19285cdeedd0040b59a66f8 introduces the
pattern:

  if isinstance(line, bytes):
    try:
      line = line.decode(encoding='utf-8')
    except UnicodeError:
      pass
  # concat line with a string

which is not working in PY3K if an UnicodeError is raised because line
is (silently) not decoded and concatened to a string.

This change ensures to return a text object or to raise an error.

Closes-Bug: #1503415
Blueprint: neutron-python3
Change-Id: I16b8013f33aa3efad65be8040d3210120e047bbd

diff --git a/neutron/agent/linux/utils.py b/neutron/agent/linux/utils.py
index 83e54ef0ec199b1c3884c244ce7d32bc26764434..3ec6af7f2e96b25d02ee1f77d052d952a3957857 100644 (file)
--- a/neutron/agent/linux/utils.py
+++ b/neutron/agent/linux/utils.py
@@ -123,15 +123,9 @@ def execute(cmd, process_input=None, addl_env=None,
             obj.stdin.close()
         if six.PY3:
             if isinstance(_stdout, bytes):
-                try:
-                    _stdout = _stdout.decode(encoding='utf-8')
-                except UnicodeError:
-                    pass
+                _stdout = _stdout.decode('utf-8', 'surrogateescape')
             if isinstance(_stderr, bytes):
-                try:
-                    _stderr = _stderr.decode(encoding='utf-8')
-                except UnicodeError:
-                    pass
+                _stderr = _stderr.decode('utf-8', 'surrogateescape')
 
         command_str = {
             'cmd': cmd,

diff --git a/neutron/agent/windows/utils.py b/neutron/agent/windows/utils.py
index bcbccd3bcd12bc15098136f23419ed830b684961..5707097eacd5bfb616fc707143aeb81cf41257b4 100644 (file)
--- a/neutron/agent/windows/utils.py
+++ b/neutron/agent/windows/utils.py
@@ -59,15 +59,9 @@ def execute(cmd, process_input=None, addl_env=None,
         obj.stdin.close()
         if six.PY3:
             if isinstance(_stdout, bytes):
-                try:
-                    _stdout = _stdout.decode(encoding='utf-8')
-                except UnicodeError:
-                    pass
+                _stdout = _stdout.decode('utf-8', 'surrogateescape')
             if isinstance(_stderr, bytes):
-                try:
-                    _stderr = _stderr.decode(encoding='utf-8')
-                except UnicodeError:
-                    pass
+                _stderr = _stderr.decode('utf-8', 'surrogateescape')
 
         m = _("\nCommand: %(cmd)s\nExit code: %(code)s\nStdin: %(stdin)s\n"
               "Stdout: %(stdout)s\nStderr: %(stderr)s") % \

diff --git a/neutron/tests/unit/agent/linux/test_utils.py b/neutron/tests/unit/agent/linux/test_utils.py
index 71a7b8959a0473370484d0d7f56013afc512c0ef..0fd76dd1ae03dc2e10e0e07c00dd8de3cb494f3d 100644 (file)
--- a/neutron/tests/unit/agent/linux/test_utils.py
+++ b/neutron/tests/unit/agent/linux/test_utils.py
@@ -147,20 +147,15 @@ class AgentUtilsExecuteTest(base.BaseTestCase):
         result = utils.execute(['ls', self.test_file], return_stderr=True)
         self.assertEqual((str_data, ''), result)
 
-    def test_raise_unicodeerror_in_decoding_out_data(self):
-        class m_bytes(bytes):
-            def decode(self, encoding=None):
-                raise UnicodeError
-
-        err_data = 'UnicodeError'
-        bytes_err_data = b'UnicodeError'
+    @testtools.skipUnless(six.PY3, 'This test makes sense only in Python 3')
+    def test_surrogateescape_in_decoding_out_data(self):
+        bytes_err_data = b'\xed\xa0\xbd'
+        err_data = bytes_err_data.decode('utf-8', 'surrogateescape')
         out_data = "%s\n" % self.test_file
-        bytes_out_data = m_bytes(out_data.encode(encoding='utf-8'))
-        if six.PY3:
-            self.mock_popen.return_value = [bytes_out_data, bytes_err_data]
-            result = utils.execute(['ls', self.test_file],
-                                   return_stderr=True)
-            self.assertEqual((bytes_out_data, err_data), result)
+        bytes_out_data = out_data.encode(encoding='utf-8')
+        self.mock_popen.return_value = [bytes_out_data, bytes_err_data]
+        result = utils.execute(['ls', self.test_file], return_stderr=True)
+        self.assertEqual((out_data, err_data), result)
 
 
 class AgentUtilsExecuteEncodeTest(base.BaseTestCase):