Fix Python 3 issues in utils

* Replace string.split(key_file, ',') with key_file.split(','):
  the string.split() function was removed in Python 3, use the
  str.split() method instead
* Fix conf_fixture: import backup_enable_progress_timer and
  backup_swift_enable_progress_timer options. This change is required
  to run tests using testtools (it's now possible yet to run tests on
  Python 3 using testr).
* hash_file(): use b'' (bytes) for the sentinel, not '' (unicode), when
  reading the input binary file
* test_hash_file() requires a binary file (bytes), not a text file
  (unicode)
* Get the builtin open() function from six.moves.builtins instead of
  getting it from __builtin__ to mock the open() function. The
  __builtin__ module was renamed to builtins in Python 3.
* sanitize_hostname(): on Python 3, decode back the hostname from Latin1
  to work on a native string (Unicode)
* replace reduce() with six.moves.reduce(): the Python 2 builtin
  reduce() function was moved to the functools module in Python 3
* convert_version_to_str(): fix integer division, use a//b, not a/b
* convert_version_to_str(): replace reduce() with ''.join() with map(str)

Blueprint cinder-python3
Change-Id: If7b8f50c6a8b0a5044c2c7108b2b0293dddafff3

diff --git a/cinder/ssh_utils.py b/cinder/ssh_utils.py
index e6b73637418b94b5e5e55f5c3f3f08106f916c1f..22f29a52d04b19a6161028d5b9e31988a5e7d555 100644 (file)
--- a/cinder/ssh_utils.py
+++ b/cinder/ssh_utils.py
@@ -19,7 +19,6 @@
 """Utilities related to SSH connection management."""
 
 import os
-import string
 
 from eventlet import pools
 from oslo_config import cfg
@@ -103,7 +102,7 @@ class SSHPool(pools.Pool):
         try:
             ssh = paramiko.SSHClient()
             if ',' in self.hosts_key_file:
-                files = string.split(self.hosts_key_file, ',')
+                files = self.hosts_key_file.split(',')
                 for f in files:
                     ssh.load_host_keys(f)
             else:

diff --git a/cinder/tests/unit/conf_fixture.py b/cinder/tests/unit/conf_fixture.py
index 2898eb1cc3e56c67b63e2f296ce7122f22a9c05d..2685b15f7f3305b5b64377c17abceff82b24df04 100644 (file)
--- a/cinder/tests/unit/conf_fixture.py
+++ b/cinder/tests/unit/conf_fixture.py
@@ -21,6 +21,10 @@ from oslo_config import cfg
 
 CONF = cfg.CONF
 
+CONF.import_opt('backup_enable_progress_timer',
+                'cinder.backup.drivers.nfs')
+CONF.import_opt('backup_swift_enable_progress_timer',
+                'cinder.backup.drivers.swift')
 CONF.import_opt('policy_file', 'cinder.policy')
 CONF.import_opt('volume_driver', 'cinder.volume.manager')
 CONF.import_opt('xiv_ds8k_proxy',

diff --git a/cinder/tests/unit/test_utils.py b/cinder/tests/unit/test_utils.py
index 4631a723b9f7d10574457331ee6838027e8d0444..e117bbe90dd9c5b9c4d75077d1b39a73dda52755 100644 (file)
--- a/cinder/tests/unit/test_utils.py
+++ b/cinder/tests/unit/test_utils.py
@@ -416,8 +416,8 @@ class GenericUtilsTestCase(test.TestCase):
         self.assertEqual(''foo'', utils.xhtml_escape("'foo'"))
 
     def test_hash_file(self):
-        data = 'Mary had a little lamb, its fleece as white as snow'
-        flo = six.StringIO(data)
+        data = b'Mary had a little lamb, its fleece as white as snow'
+        flo = six.BytesIO(data)
         h1 = utils.hash_file(flo)
         h2 = hashlib.sha1(data).hexdigest()
         self.assertEqual(h1, h2)
@@ -1029,7 +1029,7 @@ class FakeTransport(object):
 class SSHPoolTestCase(test.TestCase):
     """Unit test for SSH Connection Pool."""
     @mock.patch('cinder.ssh_utils.CONF')
-    @mock.patch('__builtin__.open')
+    @mock.patch('six.moves.builtins.open')
     @mock.patch('paramiko.SSHClient')
     @mock.patch('os.path.isfile', return_value=True)
     def test_ssh_default_hosts_key_file(self, mock_isfile, mock_sshclient,
@@ -1053,7 +1053,7 @@ class SSHPoolTestCase(test.TestCase):
             '/var/lib/cinder/ssh_known_hosts')
 
     @mock.patch('cinder.ssh_utils.CONF')
-    @mock.patch('__builtin__.open')
+    @mock.patch('six.moves.builtins.open')
     @mock.patch('paramiko.SSHClient')
     @mock.patch('os.path.isfile', return_value=True)
     def test_ssh_host_key_file_kwargs(self, mock_isfile, mock_sshclient,
@@ -1082,7 +1082,7 @@ class SSHPoolTestCase(test.TestCase):
         mock_ssh.assert_has_calls(expected, any_order=True)
 
     @mock.patch('cinder.ssh_utils.CONF')
-    @mock.patch('__builtin__.open')
+    @mock.patch('six.moves.builtins.open')
     @mock.patch('os.path.isfile', return_value=True)
     @mock.patch('paramiko.RSAKey.from_private_key_file')
     @mock.patch('paramiko.SSHClient')
@@ -1121,7 +1121,7 @@ class SSHPoolTestCase(test.TestCase):
                           min_size=1,
                           max_size=1)
 
-    @mock.patch('__builtin__.open')
+    @mock.patch('six.moves.builtins.open')
     @mock.patch('paramiko.SSHClient')
     def test_closed_reopened_ssh_connections(self, mock_sshclient, mock_open):
         mock_sshclient.return_value = eval('FakeSSHClient')()
@@ -1149,7 +1149,7 @@ class SSHPoolTestCase(test.TestCase):
         self.assertNotEqual(first_id, third_id)
 
     @mock.patch('cinder.ssh_utils.CONF')
-    @mock.patch('__builtin__.open')
+    @mock.patch('six.moves.builtins.open')
     @mock.patch('paramiko.SSHClient')
     def test_missing_ssh_hosts_key_config(self, mock_sshclient, mock_open,
                                           mock_conf):
@@ -1165,7 +1165,7 @@ class SSHPoolTestCase(test.TestCase):
                           min_size=1,
                           max_size=1)
 
-    @mock.patch('__builtin__.open')
+    @mock.patch('six.moves.builtins.open')
     @mock.patch('paramiko.SSHClient')
     def test_create_default_known_hosts_file(self, mock_sshclient,
                                              mock_open):
@@ -1187,7 +1187,7 @@ class SSHPoolTestCase(test.TestCase):
             ssh_pool.remove(ssh)
 
     @mock.patch('os.path.isfile', return_value=False)
-    @mock.patch('__builtin__.open')
+    @mock.patch('six.moves.builtins.open')
     @mock.patch('paramiko.SSHClient')
     def test_ssh_missing_hosts_key_file(self, mock_sshclient, mock_open,
                                         mock_isfile):
@@ -1207,7 +1207,7 @@ class SSHPoolTestCase(test.TestCase):
     @mock.patch.multiple('cinder.ssh_utils.CONF',
                          strict_ssh_host_key_policy=True,
                          ssh_hosts_key_file='/var/lib/cinder/ssh_known_hosts')
-    @mock.patch('__builtin__.open')
+    @mock.patch('six.moves.builtins.open')
     @mock.patch('paramiko.SSHClient')
     @mock.patch('os.path.isfile', return_value=True)
     def test_ssh_strict_host_key_policy(self, mock_isfile, mock_sshclient,
@@ -1225,7 +1225,7 @@ class SSHPoolTestCase(test.TestCase):
             self.assertTrue(isinstance(ssh.get_policy(),
                                        paramiko.RejectPolicy))
 
-    @mock.patch('__builtin__.open')
+    @mock.patch('six.moves.builtins.open')
     @mock.patch('paramiko.SSHClient')
     @mock.patch('os.path.isfile', return_value=True)
     def test_ssh_not_strict_host_key_policy(self, mock_isfile, mock_sshclient,

diff --git a/cinder/utils.py b/cinder/utils.py
index 1b2baa59daaecefe6fe34043151cc06099992c07..ce1757ec57e9ed89491a4b8d9ddd73c46b27b869 100644 (file)
--- a/cinder/utils.py
+++ b/cinder/utils.py
@@ -459,8 +459,12 @@ def make_dev_path(dev, partition=None, base='/dev'):
 
 def sanitize_hostname(hostname):
     """Return a hostname which conforms to RFC-952 and RFC-1123 specs."""
-    if isinstance(hostname, six.text_type):
+    if six.PY3:
         hostname = hostname.encode('latin-1', 'ignore')
+        hostname = hostname.decode('latin-1')
+    else:
+        if isinstance(hostname, six.text_type):
+            hostname = hostname.encode('latin-1', 'ignore')
 
     hostname = re.sub('[ _]', '-', hostname)
     hostname = re.sub('[^\w.-]+', '', hostname)
@@ -473,7 +477,7 @@ def sanitize_hostname(hostname):
 def hash_file(file_like_object):
     """Generate a hash for the contents of a file."""
     checksum = hashlib.sha1()
-    any(map(checksum.update, iter(lambda: file_like_object.read(32768), '')))
+    any(map(checksum.update, iter(lambda: file_like_object.read(32768), b'')))
     return checksum.hexdigest()
 
 
@@ -793,7 +797,7 @@ def convert_version_to_int(version):
         if isinstance(version, six.string_types):
             version = convert_version_to_tuple(version)
         if isinstance(version, tuple):
-            return reduce(lambda x, y: (x * 1000) + y, version)
+            return six.moves.reduce(lambda x, y: (x * 1000) + y, version)
     except Exception:
         msg = _("Version %s is invalid.") % version
         raise exception.CinderException(msg)
@@ -805,9 +809,9 @@ def convert_version_to_str(version_int):
     while version_int != 0:
         version_number = version_int - (version_int // factor * factor)
         version_numbers.insert(0, six.text_type(version_number))
-        version_int = version_int / factor
+        version_int = version_int // factor
 
-    return reduce(lambda x, y: "%s.%s" % (x, y), version_numbers)
+    return '.'.join(map(str, version_numbers))
 
 
 def convert_version_to_tuple(version_str):