Ticket #2972: text_to_unicode_r3109.diff

trac/config.py

@@ -19 +19 @@
 import sys
 
 from trac.core import *
-from trac.util import doctrim
+from trac.util import doctrim, text_to_unicode
 
 __all__ = ['IConfigurable', 'ConfigSection', 'ConfigOption', 'Configuration',
            'ConfigurationError', 'default_dir']
@@ -211 +211 @@
             if default is None:
                 return self.config._defaults.get((self.name, name), '')
             return default
-        return self.config.parser.get(self.name, name)
+        return text_to_unicode(self.config.parser.get(self.name, name))
 
     def getbool(self, name, default=None):
         """Return the value of the specified option as boolean.
@@ -268 +268 @@
         """
         if not self.config.parser.has_section(self.name):
             self.config.parser.add_section(self.name)
-        return self.config.parser.set(self.name, name, value)
+        return self.config.parser.set(self.name, name, value.decode('utf-8'))
 
 
 def default_dir(name):

trac/web/clearsilver.py

@@ -15 +15 @@
 # Author: Christopher Lenz <cmlenz@gmx.de>
 
 from trac.core import TracError
-from trac.util import markup
+from trac.util import markup, text_to_unicode
 
 
 class HDFWrapper:
@@ -221 +221 @@
             elif isinstance(value, str):
                 if escape:
                     # Assume UTF-8 here, for backward compatibility reasons
-                    set_unicode(prefix, markup.escape(unicode(value, 'utf-8',
-                                                              'replace')))
+                    set_unicode(prefix, markup.escape(text_to_unicode(value)))
                 else:
                     set_str(prefix, value)
             elif isinstance(value, unicode):

trac/util/__init__.py

@@ -75 +75 @@
 
     If no charset is specified or if the decoding fails, then fallback
     to 'iso-8859-15', which will always work as there will be one Unicode
-    character for each byte of the input.
+    character for each byte of the input. So this function can also be
+    used for arbitrary binary data.
 
     If that unicode string is later displayed, this might result in some
     garbled characters, but at least there will be something to show...
@@ -87 +88 @@
     except UnicodeError:
         return unicode(text, 'iso-8859-15')
 
+def text_to_unicode(text):
+    """Convert a textual string to `unicode`.
+
+    This is useful when we don't know anything about the string's encoding,
+    but contrary to the `to_unicode` function aboe, we know at least that
+    `text` is some kind of text and is not binary data.
+
+    We first try to encode using UTF-8, and if this fails, we try
+    the locale preferred encoding, in 'replace' mode.
+    """
+    if isinstance(text, unicode):
+        return text
+    try:
+        return unicode(text, 'utf-8')
+    except UnicodeError:
+        return unicode(text, locale.getpreferredencoding(), 'replace')
+
 def shorten_line(text, maxlen = 75):
     if len(text or '') < maxlen:
         return text