From 385eaa23149f6780ba6ff2e9a006e0189d810c23 Mon Sep 17 00:00:00 2001 From: Eric Harney Date: Thu, 16 Jan 2014 13:09:05 -0500 Subject: [PATCH] Sync gettextutils from Oslo Update gettextutils as a dependency for RPC updates Requires changes within Cinder as well for "get_localized_message" -> "translate" method rename. 221c37d Allow the Message class to have non-English default locales 0e1af5b Implementation of translation log handler f5686b1 Merge "Translation Message improvements" 8b2b0b7 Use hacking import_exceptions for gettextutils._ 2cfc1a7 Translation Message improvements 45658e2 Fix violations of H302:import only modules 12bcdb7 Remove vim header Oslo version: 7a51572 Merge "Implement cache abstraction layer" Date: Wed Jan 15 19:31:16 2014 +0000 Change-Id: I979114dbebfe2368b071cc6c18951da227b6f74a --- cinder/api/openstack/wsgi.py | 5 +- cinder/openstack/common/gettextutils.py | 432 +++++++++++---------- cinder/tests/api/middleware/test_faults.py | 2 +- cinder/tests/test_wsgi.py | 6 +- 4 files changed, 241 insertions(+), 204 deletions(-) diff --git a/cinder/api/openstack/wsgi.py b/cinder/api/openstack/wsgi.py index 8c97a1db1..e561b7cab 100644 --- a/cinder/api/openstack/wsgi.py +++ b/cinder/api/openstack/wsgi.py @@ -1164,8 +1164,7 @@ class Fault(webob.exc.HTTPException): fault_data = { fault_name: { 'code': code, - 'message': gettextutils.get_localized_message(explanation, - locale)}} + 'message': gettextutils.translate(explanation, locale)}} if code == 413: retry = self.wrapped_exc.headers.get('Retry-After', None) if retry: @@ -1228,7 +1227,7 @@ class OverLimitFault(webob.exc.HTTPException): def translate(msg): locale = request.best_match_language() - return gettextutils.get_localized_message(msg, locale) + return gettextutils.translate(msg, locale) self.content['overLimitFault']['message'] = \ translate(self.content['overLimitFault']['message']) diff --git a/cinder/openstack/common/gettextutils.py b/cinder/openstack/common/gettextutils.py index caa6a0925..4a4b1b697 100644 --- a/cinder/openstack/common/gettextutils.py +++ b/cinder/openstack/common/gettextutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2012 Red Hat, Inc. # Copyright 2013 IBM Corp. # All Rights Reserved. @@ -26,13 +24,10 @@ Usual usage in an openstack.common module: import copy import gettext -import logging +import locale +from logging import handlers import os import re -try: - import UserString as _userString -except ImportError: - import collections as _userString from babel import localedata import six @@ -58,7 +53,7 @@ def enable_lazy(): def _(msg): if USE_LAZY: - return Message(msg, 'cinder') + return Message(msg, domain='cinder') else: if six.PY3: return _t.gettext(msg) @@ -90,11 +85,6 @@ def install(domain, lazy=False): # messages in OpenStack. We override the standard _() function # and % (format string) operation to build Message objects that can # later be translated when we have more information. - # - # Also included below is an example LocaleHandler that translates - # Messages to an associated locale, effectively allowing many logs, - # each with their own locale. - def _lazy_gettext(msg): """Create and return a Message object. @@ -105,7 +95,7 @@ def install(domain, lazy=False): Message encapsulates a string so that we can translate it later when needed. """ - return Message(msg, domain) + return Message(msg, domain=domain) from six import moves moves.builtins.__dict__['_'] = _lazy_gettext @@ -120,182 +110,158 @@ def install(domain, lazy=False): unicode=True) -class Message(_userString.UserString, object): - """Class used to encapsulate translatable messages.""" - def __init__(self, msg, domain): - # _msg is the gettext msgid and should never change - self._msg = msg - self._left_extra_msg = '' - self._right_extra_msg = '' - self._locale = None - self.params = None - self.domain = domain - - @property - def data(self): - # NOTE(mrodden): this should always resolve to a unicode string - # that best represents the state of the message currently - - localedir = os.environ.get(self.domain.upper() + '_LOCALEDIR') - if self.locale: - lang = gettext.translation(self.domain, - localedir=localedir, - languages=[self.locale], - fallback=True) - else: - # use system locale for translations - lang = gettext.translation(self.domain, - localedir=localedir, - fallback=True) +class Message(six.text_type): + """A Message object is a unicode object that can be translated. + + Translation of Message is done explicitly using the translate() method. + For all non-translation intents and purposes, a Message is simply unicode, + and can be treated as such. + """ + def __new__(cls, msgid, msgtext=None, params=None, domain='cinder', *args): + """Create a new Message object. + + In order for translation to work gettext requires a message ID, this + msgid will be used as the base unicode text. It is also possible + for the msgid and the base unicode text to be different by passing + the msgtext parameter. + """ + # If the base msgtext is not given, we use the default translation + # of the msgid (which is in English) just in case the system locale is + # not English, so that the base text will be in that locale by default. + if not msgtext: + msgtext = Message._translate_msgid(msgid, domain) + # We want to initialize the parent unicode with the actual object that + # would have been plain unicode if 'Message' was not enabled. + msg = super(Message, cls).__new__(cls, msgtext) + msg.msgid = msgid + msg.domain = domain + msg.params = params + return msg + + def translate(self, desired_locale=None): + """Translate this message to the desired locale. + + :param desired_locale: The desired locale to translate the message to, + if no locale is provided the message will be + translated to the system's default locale. + + :returns: the translated message in unicode + """ + + translated_message = Message._translate_msgid(self.msgid, + self.domain, + desired_locale) + if self.params is None: + # No need for more translation + return translated_message + + # This Message object may have been formatted with one or more + # Message objects as substitution arguments, given either as a single + # argument, part of a tuple, or as one or more values in a dictionary. + # When translating this Message we need to translate those Messages too + translated_params = _translate_args(self.params, desired_locale) + + translated_message = translated_message % translated_params + + return translated_message + + @staticmethod + def _translate_msgid(msgid, domain, desired_locale=None): + if not desired_locale: + system_locale = locale.getdefaultlocale() + # If the system locale is not available to the runtime use English + if not system_locale[0]: + desired_locale = 'en_US' + else: + desired_locale = system_locale[0] + + locale_dir = os.environ.get(domain.upper() + '_LOCALEDIR') + lang = gettext.translation(domain, + localedir=locale_dir, + languages=[desired_locale], + fallback=True) if six.PY3: - ugettext = lang.gettext - else: - ugettext = lang.ugettext - - full_msg = (self._left_extra_msg + - ugettext(self._msg) + - self._right_extra_msg) - - if self.params is not None: - full_msg = full_msg % self.params - - return six.text_type(full_msg) - - @property - def locale(self): - return self._locale - - @locale.setter - def locale(self, value): - self._locale = value - if not self.params: - return - - # This Message object may have been constructed with one or more - # Message objects as substitution parameters, given as a single - # Message, or a tuple or Map containing some, so when setting the - # locale for this Message we need to set it for those Messages too. - if isinstance(self.params, Message): - self.params.locale = value - return - if isinstance(self.params, tuple): - for param in self.params: - if isinstance(param, Message): - param.locale = value - return - if isinstance(self.params, dict): - for param in self.params.values(): - if isinstance(param, Message): - param.locale = value - - def _save_dictionary_parameter(self, dict_param): - full_msg = self.data - # look for %(blah) fields in string; - # ignore %% and deal with the - # case where % is first character on the line - keys = re.findall('(?:[^%]|^)?%\((\w*)\)[a-z]', full_msg) - - # if we don't find any %(blah) blocks but have a %s - if not keys and re.findall('(?:[^%]|^)%[a-z]', full_msg): - # apparently the full dictionary is the parameter - params = copy.deepcopy(dict_param) + translator = lang.gettext else: - params = {} - for key in keys: - try: - params[key] = copy.deepcopy(dict_param[key]) - except TypeError: - # cast uncopyable thing to unicode string - params[key] = six.text_type(dict_param[key]) + translator = lang.ugettext - return params + translated_message = translator(msgid) + return translated_message - def _save_parameters(self, other): - # we check for None later to see if - # we actually have parameters to inject, - # so encapsulate if our parameter is actually None + def __mod__(self, other): + # When we mod a Message we want the actual operation to be performed + # by the parent class (i.e. unicode()), the only thing we do here is + # save the original msgid and the parameters in case of a translation + unicode_mod = super(Message, self).__mod__(other) + modded = Message(self.msgid, + msgtext=unicode_mod, + params=self._sanitize_mod_params(other), + domain=self.domain) + return modded + + def _sanitize_mod_params(self, other): + """Sanitize the object being modded with this Message. + + - Add support for modding 'None' so translation supports it + - Trim the modded object, which can be a large dictionary, to only + those keys that would actually be used in a translation + - Snapshot the object being modded, in case the message is + translated, it will be used as it was when the Message was created + """ if other is None: - self.params = (other, ) + params = (other,) elif isinstance(other, dict): - self.params = self._save_dictionary_parameter(other) + params = self._trim_dictionary_parameters(other) else: - # fallback to casting to unicode, - # this will handle the problematic python code-like - # objects that cannot be deep-copied - try: - self.params = copy.deepcopy(other) - except TypeError: - self.params = six.text_type(other) - - return self - - # overrides to be more string-like - def __unicode__(self): - return self.data - - def __str__(self): - if six.PY3: - return self.__unicode__() - return self.data.encode('utf-8') + params = self._copy_param(other) + return params - def __getstate__(self): - to_copy = ['_msg', '_right_extra_msg', '_left_extra_msg', - 'domain', 'params', '_locale'] - new_dict = self.__dict__.fromkeys(to_copy) - for attr in to_copy: - new_dict[attr] = copy.deepcopy(self.__dict__[attr]) + def _trim_dictionary_parameters(self, dict_param): + """Return a dict that only has matching entries in the msgid.""" + # NOTE(luisg): Here we trim down the dictionary passed as parameters + # to avoid carrying a lot of unnecessary weight around in the message + # object, for example if someone passes in Message() % locals() but + # only some params are used, and additionally we prevent errors for + # non-deepcopyable objects by unicoding() them. + + # Look for %(param) keys in msgid; + # Skip %% and deal with the case where % is first character on the line + keys = re.findall('(?:[^%]|^)?%\((\w*)\)[a-z]', self.msgid) + + # If we don't find any %(param) keys but have a %s + if not keys and re.findall('(?:[^%]|^)%[a-z]', self.msgid): + # Apparently the full dictionary is the parameter + params = self._copy_param(dict_param) + else: + params = {} + for key in keys: + params[key] = self._copy_param(dict_param[key]) - return new_dict + return params - def __setstate__(self, state): - for (k, v) in state.items(): - setattr(self, k, v) + def _copy_param(self, param): + try: + return copy.deepcopy(param) + except TypeError: + # Fallback to casting to unicode this will handle the + # python code-like objects that can't be deep-copied + return six.text_type(param) - # operator overloads def __add__(self, other): - copied = copy.deepcopy(self) - copied._right_extra_msg += other.__str__() - return copied + msg = _('Message objects do not support addition.') + raise TypeError(msg) def __radd__(self, other): - copied = copy.deepcopy(self) - copied._left_extra_msg += other.__str__() - return copied + return self.__add__(other) - def __mod__(self, other): - # do a format string to catch and raise - # any possible KeyErrors from missing parameters - self.data % other - copied = copy.deepcopy(self) - return copied._save_parameters(other) - - def __mul__(self, other): - return self.data * other - - def __rmul__(self, other): - return other * self.data - - def __getitem__(self, key): - return self.data[key] - - def __getslice__(self, start, end): - return self.data.__getslice__(start, end) - - def __getattribute__(self, name): - # NOTE(mrodden): handle lossy operations that we can't deal with yet - # These override the UserString implementation, since UserString - # uses our __class__ attribute to try and build a new message - # after running the inner data string through the operation. - # At that point, we have lost the gettext message id and can just - # safely resolve to a string instead. - ops = ['capitalize', 'center', 'decode', 'encode', - 'expandtabs', 'ljust', 'lstrip', 'replace', 'rjust', 'rstrip', - 'strip', 'swapcase', 'title', 'translate', 'upper', 'zfill'] - if name in ops: - return getattr(self.data, name) - else: - return _userString.UserString.__getattribute__(self, name) + def __str__(self): + # NOTE(luisg): Logging in python 2.6 tries to str() log records, + # and it expects specifically a UnicodeError in order to proceed. + msg = _('Message objects do not support str() because they may ' + 'contain non-ascii characters. ' + 'Please use unicode() or translate() instead.') + raise UnicodeError(msg) def get_available_languages(domain): @@ -328,46 +294,118 @@ def get_available_languages(domain): return copy.copy(language_list) -def get_localized_message(message, user_locale): - """Gets a localized version of the given message in the given locale. +def translate(obj, desired_locale=None): + """Gets the translated unicode representation of the given object. - If the message is not a Message object the message is returned as-is. - If the locale is None the message is translated to the default locale. + If the object is not translatable it is returned as-is. + If the locale is None the object is translated to the system locale. - :returns: the translated message in unicode, or the original message if + :param obj: the object to translate + :param desired_locale: the locale to translate the message to, if None the + default system locale will be used + :returns: the translated object in unicode, or the original object if it could not be translated """ - translated = message + message = obj + if not isinstance(message, Message): + # If the object to translate is not already translatable, + # let's first get its unicode representation + message = six.text_type(obj) if isinstance(message, Message): - original_locale = message.locale - message.locale = user_locale - translated = six.text_type(message) - message.locale = original_locale - return translated + # Even after unicoding() we still need to check if we are + # running with translatable unicode before translating + return message.translate(desired_locale) + return obj + + +def _translate_args(args, desired_locale=None): + """Translates all the translatable elements of the given arguments object. + This method is used for translating the translatable values in method + arguments which include values of tuples or dictionaries. + If the object is not a tuple or a dictionary the object itself is + translated if it is translatable. -class LocaleHandler(logging.Handler): - """Handler that can have a locale associated to translate Messages. + If the locale is None the object is translated to the system locale. - A quick example of how to utilize the Message class above. - LocaleHandler takes a locale and a target logging.Handler object - to forward LogRecord objects to after translating the internal Message. + :param args: the args to translate + :param desired_locale: the locale to translate the args to, if None the + default system locale will be used + :returns: a new args object with the translated contents of the original """ + if isinstance(args, tuple): + return tuple(translate(v, desired_locale) for v in args) + if isinstance(args, dict): + translated_dict = {} + for (k, v) in six.iteritems(args): + translated_v = translate(v, desired_locale) + translated_dict[k] = translated_v + return translated_dict + return translate(args, desired_locale) - def __init__(self, locale, target): - """Initialize a LocaleHandler + +class TranslationHandler(handlers.MemoryHandler): + """Handler that translates records before logging them. + + The TranslationHandler takes a locale and a target logging.Handler object + to forward LogRecord objects to after translating them. This handler + depends on Message objects being logged, instead of regular strings. + + The handler can be configured declaratively in the logging.conf as follows: + + [handlers] + keys = translatedlog, translator + + [handler_translatedlog] + class = handlers.WatchedFileHandler + args = ('/var/log/api-localized.log',) + formatter = context + + [handler_translator] + class = openstack.common.log.TranslationHandler + target = translatedlog + args = ('zh_CN',) + + If the specified locale is not available in the system, the handler will + log in the default locale. + """ + + def __init__(self, locale=None, target=None): + """Initialize a TranslationHandler :param locale: locale to use for translating messages :param target: logging.Handler object to forward LogRecord objects to after translation """ - logging.Handler.__init__(self) + # NOTE(luisg): In order to allow this handler to be a wrapper for + # other handlers, such as a FileHandler, and still be able to + # configure it using logging.conf, this handler has to extend + # MemoryHandler because only the MemoryHandlers' logging.conf + # parsing is implemented such that it accepts a target handler. + handlers.MemoryHandler.__init__(self, capacity=0, target=target) self.locale = locale - self.target = target + + def setFormatter(self, fmt): + self.target.setFormatter(fmt) def emit(self, record): - if isinstance(record.msg, Message): - # set the locale and resolve to a string - record.msg.locale = self.locale + # We save the message from the original record to restore it + # after translation, so other handlers are not affected by this + original_msg = record.msg + original_args = record.args + + try: + self._translate_and_log_record(record) + finally: + record.msg = original_msg + record.args = original_args + + def _translate_and_log_record(self, record): + record.msg = translate(record.msg, self.locale) + + # In addition to translating the message, we also need to translate + # arguments that were passed to the log method that were not part + # of the main message e.g., log.info(_('Some message %s'), this_one)) + record.args = _translate_args(record.args, self.locale) self.target.emit(record) diff --git a/cinder/tests/api/middleware/test_faults.py b/cinder/tests/api/middleware/test_faults.py index 882ed6895..83100feca 100644 --- a/cinder/tests/api/middleware/test_faults.py +++ b/cinder/tests/api/middleware/test_faults.py @@ -118,7 +118,7 @@ class TestFaults(test.TestCase): return "Mensaje traducido" self.stubs.Set(gettextutils, - "get_localized_message", _mock_translation) + "translate", _mock_translation) @webob.dec.wsgify def raiser(req): diff --git a/cinder/tests/test_wsgi.py b/cinder/tests/test_wsgi.py index ce0ac9776..c63711723 100644 --- a/cinder/tests/test_wsgi.py +++ b/cinder/tests/test_wsgi.py @@ -263,7 +263,7 @@ class ExceptionTest(test.TestCase): resp = webob.Request.blank('/').get_response(api) self.assertEqual(500, resp.status_int) - @mock.patch('cinder.openstack.common.gettextutils.get_localized_message') + @mock.patch('cinder.openstack.common.gettextutils.translate') def test_cinder_exception_with_localized_explanation(self, mock_t9n): msg = 'My Not Found' msg_translation = 'Mi No Encontrado' @@ -289,12 +289,12 @@ class ExceptionTest(test.TestCase): self.assertIn(msg, resp.body) # Test response with localization - def mock_get_localized_message(msgid, locale): + def mock_translate(msgid, locale): if isinstance(msgid, gettextutils.Message): return msg_translation return msgid - mock_t9n.side_effect = mock_get_localized_message + mock_t9n.side_effect = mock_translate api = self._wsgi_app(fail) resp = webob.Request.blank('/').get_response(api) -- 2.45.2