Ticket #3332: mimeview_refactoring-r3507.diff

trac/attachment.py

@@ -26 +26 @@
 from trac.config import BoolOption, IntOption
 from trac.core import *
 from trac.env import IEnvironmentSetupParticipant
-from trac.mimeview import *
+from trac.mimeview.api import Mimeview, MimeType, FileMimeContent
 from trac.util import get_reporter_id, create_unique_file
 from trac.util.datefmt import format_datetime, pretty_timedelta
 from trac.util.markup import Markup, html
@@ -124 +124 @@
     def parent_href(self, req):
         return req.href(self.parent_type, self.parent_id)
 
-    def _get_title(self):
+    def _get_title(self): # TODO: should extend to other `parent_type`s
         return '%s%s: %s' % (self.parent_type == 'ticket' and '#' or '',
                              self.parent_id, self.filename)
     title = property(_get_title)
@@ -227 +227 @@
 
     select = classmethod(select)
 
-    def open(self):
+    def open(self): # deprecate?
         self.env.log.debug('Trying to open attachment at %s', self.path)
         try:
             fd = open(self.path, 'rb')
@@ -505 +505 @@
                                  'author': get_reporter_id(req)}
 
     def _render_view(self, req, attachment):
+        # FIXME: perm_map should extend to other `parent_type`s
         perm_map = {'ticket': 'TICKET_VIEW', 'wiki': 'WIKI_VIEW'}
         req.perm.assert_permission(perm_map[attachment.parent_type])
 
@@ -522 +523 @@
         if req.perm.has_permission(perm_map[attachment.parent_type]):
             req.hdf['attachment.can_delete'] = 1
 
-        fd = attachment.open()
-        try:
-            mimeview = Mimeview(self.env)
+        format = req.args.get('format')
+        raw_href = attachment.href(req, format='raw')
 
-            # MIME type detection
-            str_data = fd.read(1000)
-            fd.seek(0)
-            
-            binary = is_binary(str_data)
-            mime_type = mimeview.get_mimetype(attachment.filename, str_data)
+        mimecontent = FileMimeContent(self.env, attachment.path, raw_href,
+                                      'Attachment')
+        can_render_as_txt = self.render_unsafe_content and \
+                             not mimecontent.is_binary
 
-            # Eventually send the file directly
-            format = req.args.get('format')
-            if format in ('raw', 'txt'):
-                if not self.render_unsafe_content and not binary:
-                    # Force browser to download HTML/SVG/etc pages that may
-                    # contain malicious code enabling XSS attacks
-                    req.send_header('Content-Disposition', 'attachment;' +
-                                    'filename=' + attachment.filename)
-                if not mime_type or (self.render_unsafe_content and \
-                                     not binary and format == 'txt'):
-                    mime_type = 'text/plain'
-                if 'charset=' not in mime_type:
-                    charset = mimeview.get_charset(str_data, mime_type)
-                    mime_type = mime_type + '; charset=' + charset
-                req.send_file(attachment.path, mime_type)
+        # Eventually send the file directly
+        if format in ('raw', 'txt'):
+            if not self.render_unsafe_content and not mimecontent.is_binary:
+                # Force browser to download HTML/SVG/etc pages that may
+                # contain malicious code enabling XSS attacks
+                req.send_header('Content-Disposition',
+                                'attachment;filename=%s' % attachment.filename)
+            if not mimecontent.type.is_known or \
+                   (format == 'txt' and can_render_as_txt):
+                # Force the content to be displayed as text
+                type = MimeType('text/plain', mimecontent.encoding)
+            else:
+                type = mimecontent.type
+            req.send_file(attachment.path, type.mimetype_charset)
 
-            # add ''Plain Text'' alternate link if needed
-            if self.render_unsafe_content and not binary and \
-               mime_type and not mime_type.startswith('text/plain'):
-                plaintext_href = attachment.href(req, format='txt')
-                add_link(req, 'alternate', plaintext_href, 'Plain Text',
-                         mime_type)
+        mimetype = mimecontent.type.mimetype
 
-            # add ''Original Format'' alternate link (always)
-            raw_href = attachment.href(req, format='raw')
-            add_link(req, 'alternate', raw_href, 'Original Format', mime_type)
+        # add ''Plain Text'' alternate link if needed
+        if can_render_as_txt and mimecontent.type.is_known and \
+               mimetype != 'text/plain':
+            add_link(req, 'alternate', attachment.href(req, format='txt'),
+                     'Plain Text', 'text/plain')
 
-            self.log.debug("Rendering preview of file %s with mime-type %s"
-                           % (attachment.filename, mime_type))
+        # add ''Original Format'' alternate link (always)
+        add_link(req, 'alternate', raw_href, 'Original Format', mimetype)
 
-            req.hdf['attachment'] = mimeview.preview_to_hdf(
-                req, fd, os.fstat(fd.fileno()).st_size, mime_type,
-                attachment.filename, raw_href, annotations=['lineno'])
-        finally:
-            fd.close()
+        self.log.debug("Rendering preview of file %s with mime-type %s" % \
+                       (attachment.filename, mimetype))
 
+        req.hdf['attachment'] = Mimeview(self.env).preview_to_hdf(
+            req, mimecontent, annotations=['lineno'])
+
     def _render_list(self, req, p_type, p_id):
         self._parent_to_hdf(req, p_type, p_id)
         req.hdf['attachment'] = {

trac/mimeview/rst.py

@@ -28 +28 @@
 import re
 
 from trac.core import *
-from trac.mimeview.api import IHTMLPreviewRenderer, content_to_unicode
+from trac.mimeview.api import IHTMLPreviewRenderer
 from trac.web.href import Href
 from trac.wiki.formatter import WikiProcessor
 from trac.wiki import WikiSystem
@@ -223 +223 @@
 
         _inliner = rst.states.Inliner()
         _parser = rst.Parser(inliner=_inliner)
-        content = content_to_unicode(self.env, content, mimetype)
+        content = unicode(content)
         content = content.encode('utf-8')
         html = publish_string(content, writer_name='html', parser=_parser,
                               settings_overrides={'halt_level': 6})

trac/mimeview/api.py

@@ -19 +19 @@
 #         Christian Boos <cboos@neuf.fr>
 
 """
-The `trac.mimeview` module centralize the intelligence related to
-file metadata, principally concerning the `type` (MIME type) of the content
-and, if relevant, concerning the text encoding (charset) used by the content.
+The `trac.mimeview` module centralizes the intelligence related to file
+metadata, principally concerning the `type` (MIME type) and the text
+encoding (charset) used by the content, if the latter one is relevant.
 
 There are primarily two approaches for getting the MIME type of a given file:
  * taking advantage of existing conventions for the file name
  * examining the file content and applying various heuristics
 
-The module also knows how to convert the file content from one type
-to another type.
+The module also knows about conversions from one data type to another type,
+like conversions to text/html (this is no more a special case).
 
-In some cases, only the `url` pointing to the file's content is actually
-needed, that's why we avoid to read the file's content when it's not needed.
-
-The actual `content` to be converted might be a `unicode` object,
-but it can also be the raw byte string (`str`) object, or simply
-an object that can be `read()`.
+In order to keep the API simple, we deal with a few classes,
+each encapsulating a part of the knowledge related to content.
+ * the `MimeType`, for storing the mime type string, the charset,
+   but also eventually the name and the commonly used file extension
+   for that type
+ * the `MimeContent` and `FileMimeContent`, which provide a flexible
+   API for handling string and file content, respectively.
+   Those classes inherit from the abstract `MimeContentBase`, which
+   can be used to make new wrapper classes for any kind of content.
+ * the `Conversion` class, which is use to specify conversion from
+   one `MimeType` to another
 """
 
+import os
 import re
 from StringIO import StringIO
 
 from trac.config import IntOption, ListOption, Option
 from trac.core import *
 from trac.util import sorted
-from trac.util.text import to_utf8, to_unicode
+from trac.util.text import to_utf8, to_unicode, IDENTITY_CHARSET
 from trac.util.markup import escape, Markup, Fragment, html
 
 
-__all__ = ['get_mimetype', 'is_binary', 'detect_unicode', 'Mimeview',
-           'content_to_unicode']
+__all__ = ['get_mimetype', 'is_binary', 'detect_unicode',
+           'Mimeview', 'MimeType', 'Conversion',
+           'MimeContentBase', 'MimeContent', 'FileMimeContent',
+           'TEXT_PLAIN', 'TEXT_HTML', 'APPLICATION_OCTET_STREAM']
 
 
 # Some common MIME types and their associated keywords and/or file extensions
 
+APPLICATION_OCTET_STREAM_STR = 'application/octet-stream'
+
 KNOWN_MIME_TYPES = {
     'application/pdf':        ['pdf'],
     'application/postscript': ['ps'],
@@ -114 +124 @@
     for e in exts:
         MIME_MAP[e] = t
 
-# Simple builtin autodetection from the content using a regexp
-MODE_RE = re.compile(
-    r"#!(?:[/\w.-_]+/)?(\w+)|"               # look for shebang
-    r"-\*-\s*(?:mode:\s*)?([\w+-]+)\s*-\*-|" # look for Emacs' -*- mode -*-
-    r"vim:.*?syntax=(\w+)"                   # look for VIM's syntax=<n>
-    )
 
-def get_mimetype(filename, content=None, mime_map=MIME_MAP):
-    """Guess the most probable MIME type of a file with the given name.
+# -- a few functions for dealing with MIME types / binary / text content
+#    in a simple way (get_mimetype, is_binary, detect_unicode)
 
+def get_mimetype_from_filename(filename, mime_map=MIME_MAP):
+    """Guess the most probable MIME type of file with the given `filename`.
+
     `filename` is either a filename (the lookup will then use the suffix)
     or some arbitrary keyword.
-    
-    `content` is either a `str` or an `unicode` string.
+    `mime_map` maps keywords to MIME types.
+
+    Return the MIME type as a string, or `None` if not detected.
     """
     suffix = filename.split('.')[-1]
     if suffix in mime_map:
@@ -141 +149 @@
             mimetype = mimetypes.guess_type(filename)[0]
         except:
             pass
-        if not mimetype and content:
-            match = re.search(MODE_RE, content[:1000])
-            if match:
-                mode = match.group(1) or match.group(3) or \
-                    match.group(2).lower()
-                if mode in mime_map:
-                    # 3) mimetype from the content, using the `MODE_RE`
-                    return mime_map[mode]
-            else:
-                if is_binary(content):
-                    # 4) mimetype from the content, using`is_binary`
-                    return 'application/octet-stream'
         return mimetype
 
+# Simple builtin autodetection from the content using a regexp
+MODE_RE = re.compile(
+    r"#!(?:[/\w.-_]+/)?(\w+)|"               # look for shebang
+    r"-\*-\s*(?:mode:\s*)?([\w+-]+)\s*-\*-|" # look for Emacs' -*- mode -*-
+    r"vim:.*?syntax=(\w+)"                   # look for VIM's syntax=<n>
+    )
+
+def get_mimetype_from_content(content):
+    """Guess the most probable MIME type of file with the given `filename`.
+
+    `content` is either a `str` or an `unicode` string 
+
+    Return the MIME type as a string, or `None` if not detected.
+    """
+    match = re.search(MODE_RE, content[:1000])
+    if match:
+        mode = match.group(1) or match.group(3) or \
+            match.group(2).lower()
+        if mode in mime_map:
+            # 3) mimetype from the content, using the `MODE_RE`
+            return mime_map[mode]
+    else:
+        if is_binary(content):
+            # 4) mimetype from the content, using `is_binary`
+            return APPLICATION_OCTET_STREAM_STR
+
+def get_mimetype(filename, content=None, mime_map=MIME_MAP):
+    """Auto-detect MIME type either from the `filename` or from the `content`.
+    """
+    mimetype = get_mimetype_from_filename(filename, mime_map)
+    if not mimetype and content:
+        mimetype = get_mimetype_from_content(content)
+    return mimetype
+
 def is_binary(data):
     """Detect binary content by checking the first thousand bytes for zeroes.
 
@@ -165 +195 @@
     return '\0' in data[:1000]
 
 def detect_unicode(data):
-    """Detect different unicode charsets by looking for BOMs (Byte Order Marks).
+    """Detect different unicode charsets by looking for Byte Order Marks.
 
     Operate obviously only on `str` objects.
     """
@@ -178 +208 @@
     else:
         return None
 
-def content_to_unicode(env, content, mimetype):
-    """Retrieve an `unicode` object from a `content` to be previewed"""
-    mimeview = Mimeview(env)
-    if hasattr(content, 'read'):
-        content = content.read(mimeview.max_preview_size)
-    return mimeview.to_unicode(content, mimetype)
 
+# -- Classes for mimetype, content and conversion
 
+class MimeType(object):
+    """Representation of a MIME-Type.
+
+    If the MIME type correspond to text content, the object can also
+    store a `charset` information.
+
+    A MIME type has a `name` and has an `extension`
+    that can be used for storing the converted data in a file.
+
+    All the properties of this class are read-only.
+    """
+    
+    def __init__(self, mimetype, charset=None, name=None, extension=None):
+        """The `mimetype` string can eventually embed the `charset`."""
+        self._mimetype = mimetype
+        # determine charset
+        self._charset = charset
+        if not self._charset and self._mimetype:
+            sep_idx = mimetype.find(';')
+            if sep_idx >= 0:
+                self._mimetype = mimetype[:sep_idx].strip()
+                charset_idx = mimetype.find('charset=', sep_idx)
+                if charset_idx >= 0:
+                    self._charset = mimetype[charset_idx+8:].strip()
+        self._extension = extension
+        self._name = name
+
+    def __repr__(self):
+        return self.mimetype_charset
+
+    def _get_extension(self):
+        if not self._extension:
+            self._extension = KNOWN_MIME_TYPES.get(self.mimetype)
+            if not self._extension:
+                detail = self.mimetype.split('/', 1)[1]
+                if detail.startswith('x-'):
+                    self._extension = detail[2:]
+        return self._extension
+
+    def _get_mimetype_charset(self):
+        """Combine the MIME type and charset information in a single string.
+        """
+        if self._mimetype and self._charset:
+            return '%s; charset=%s' % (self.mimetype, self.charset)
+        else:
+            return self.mimetype
+
+    is_known = property(lambda x: x._mimetype is not None)
+    mimetype = property(lambda x: x._mimetype or APPLICATION_OCTET_STREAM_STR,
+                        doc="MIME Type string (without charset information)")
+    charset = property(lambda x: x._charset,
+                       doc="Eventual charset information")
+    name = property(lambda x: x._name or x._extension)
+    extension = property(_get_extension)
+    mimetype_charset = property(_get_mimetype_charset)
+
+    def match(self, other, regexp=False):
+        """Compare MIME type string only.
+
+        If `regexp` is set, `self.mimetype` is used as a regexp.
+        """
+        if not isinstance(other, MimeType):
+            return False
+        if regexp:
+            return re.match(self.mimetype, other.mimetype)
+        else:
+            return self.mimetype == other.mimetype
+
+
+TEXT_PLAIN = MimeType('text/plain', 'utf-8', 'Plain Text', 'txt')
+TEXT_HTML = MimeType('text/html', 'utf-8', 'HTML', 'html')
+APPLICATION_OCTET_STREAM = MimeType(APPLICATION_OCTET_STREAM_STR,
+                                    IDENTITY_CHARSET,
+                                    'Undefined (binary)', 'bin') 
+
+
+class MimeContentBase(object):
+    """An abstract MIME content, with an associated MimeType.
+
+    Such an object has means to auto-detect both the MIME Type and
+    the `encoding` of its content.
+
+    That `encoding` is more reliable than the `type.charset` information.
+    There are additional consistency checks that are performed, and it
+    can be `None` if the content is an `unicode` object.
+
+    The content itself can be accessed in various ways: through
+    the iterator protocol, the len() and unicode() operators...
+    """
+
+    def __init__(self, env, mimetype=None, filename=None, url=None):
+        """
+        `mimetype` can be specified as a `MimeType` object,
+        or as string, which will then be a hint about the content.
+
+        If the mimetype is not specified or equal to
+        "application/octet-stream", then it will be auto-detected when needed.
+
+        In case auto-detection fails, APPLICATION_OCTET_STREAM will be the
+        corresponding MIME type.
+
+        The `filename` is simply a suggested basename for that content.
+
+        The `url` is a link for retrieving the raw content directly
+        from the server. This can be useful for converters that can
+        provide links to objects, instead of having to expand the
+        content inline.
+        """
+        self.env = env
+        if isinstance(mimetype, basestring):
+            mimetype = MimeType(mimetype)
+        self._type = mimetype
+        self._filename = filename
+        self._url = url
+        self._binary = None
+        self._encoding = False
+
+    def __repr__(self):
+        return '<%s %s "%s">' % (self.__class__.__name__, self._type,
+                                 self._filename or self._url)
+
+    def __unicode__(self):
+        """Return the `unicode` object corresponding to the content."""
+        return to_unicode(self.content, self.encoding)
+        # Note: this does the right thing if the content is already `unicode`
+
+
+    def _is_binary(self):
+        """An heuristic to guess whether the content is binary data or not.
+
+        This will trigger the retrieval of an `excerpt` of the content.
+        """
+        if self._binary is None:
+            mimetype = self.type.mimetype
+            self._binary = is_binary(self.excerpt) or \
+                           (self.type.is_known and 
+                            mimetype == APPLICATION_OCTET_STREAM_STR) or \
+                            mimetype in Mimeview(self.env).treat_as_binary
+        return self._binary
+
+    def _get_type(self):
+        """Get or determine the MimeType corresponding to this content.
+
+        An `excerpt` of the content will be examined if needed.
+        """
+        if self._type is None: # not set
+            mimetype = None
+            if self.filename:
+                mimemap = Mimeview(self.env).mimemap
+                mimetype = get_mimetype_from_filename(self.filename, mimemap)
+            if not mimetype:
+                mimetype = get_mimetype_from_content(self.excerpt)
+            if not mimetype:
+                pass # TODO 0.11: go through IMimeTypeDetectors
+            self._type = MimeType(mimetype)
+        return self._type
+
+    def _set_type(self, type):
+        """Simply replace the existing `type` by the given `MimeType` object.
+
+        If `None` is given, this will force auto-detection the next time
+        `type` will be accessed.
+
+        Can be used for in-place conversion (e.g. ''any'' to text/plain).
+        """
+        self._type = type
+
+    def _get_encoding(self):
+        """Get or determine the current encoding of that `content`.
+
+        The encoding will be determined using this order:
+         * from the charset information present in the mimetype information
+         * auto-detection of the charset from the `content`
+         * if nothing else worked, use the configured `default_charset`
+
+        If the `content` happens to be a genuine `unicode` object, then
+        this returns `None`.
+        If the `content` is binary, then the encoding will be the identity
+        charset (ISO Latin 1).
+        """
+        if self._encoding is False:
+            charset = self.type.charset
+            if charset:
+                self._encoding = charset
+            elif isinstance(self.excerpt, str):
+                utf_encoding = detect_unicode(self.excerpt)
+                if utf_encoding is not None:
+                    self._encoding = utf_encoding
+                elif self.is_binary:
+                    self._encoding = IDENTITY_CHARSET
+            elif isinstance(self.excerpt, unicode):
+                self._encoding = None
+            if self._encoding is False:
+                pass # TODO 0.11: go through ICharsetDetectors here
+            if self._encoding is False:
+                self._encoding = Mimeview(self.env).default_charset
+        return self._encoding
+    
+    def _get_content(self):
+        """Retrieve all the content.
+
+        Default implementation based on iterator. If the iterator itself
+        is implemented based on the content... reimplement this one!
+        """
+        return "".join(self.__iter__())
+
+    def read(self): # TODO: remove in 0.11
+        return self.content # (compatibility with IHTMLPreviewRenderer)
+
+    # Methods that need to be reimplemented by subclasses:
+
+    def __iter__(self):
+        """Iterate on chunks of raw content."""
+        raise NotImplementedError
+
+    def __len__(self):
+        """Length of the raw content, in bytes."""
+        raise NotImplementedError
+
+    def _get_excerpt(self, len=1000):
+        """Extracts the first `len` characters from the content."""
+        raise NotImplementedError
+            
+    type = property(fget=lambda x: x._get_type(),
+                    fset=lambda x, y: x._set_type(y))
+    is_binary = property(lambda x: x._is_binary())
+    encoding = property(lambda x: x._get_encoding())
+    excerpt = property(lambda x: x._get_excerpt())
+    content = property(lambda x: x._get_content())
+    filename = property(lambda x: x._filename)
+    url = property(lambda x: x._url)
+
+
+class MimeContent(MimeContentBase):
+    """MIME-typed content wrapper for a basestring."""
+
+    def __init__(self, env, content, mimetype, filename='file', url=None):
+        MimeContentBase.__init__(self, env, mimetype, filename, url)
+        self._content = content
+
+    # Reimplemented methods
+
+    def _get_content(self):
+        """Retrieve the wrapped content.
+
+        Note: therefore this *might* be an `unicode` object.
+        Remember that in this case, `encoding` will be `None`.
+        """
+        return self._content
+
+    def __iter__(self):
+        """Iterate on chunks of content.
+
+        If the content `is_binary` property is `False`, those chunks will
+        be lines, with the line endings kept.
+        """
+        if self.is_binary:
+            buf = StringIO(self.content)
+            chunk = buf.read(1000)
+            while chunk:
+                yield chunk
+                chunk = buf.read(1000)
+        else:
+            for line in self.content.splitlines(True):
+                yield line
+
+    def __len__(self):
+        """Length of the content, in characters."""
+        return len(self.content)
+
+    def _get_excerpt(self, len=1000):
+        """Extracts the first `len` characters from the content."""
+        return self._content[:len]
+
+
+class FileMimeContent(MimeContentBase):
+    """MIME-typed content wrapper for a file."""
+
+    def __init__(self, env, path, url=None, kind='File', mimetype=None):
+        self._fd = None
+        self._path = path
+        self._kind = kind
+        self._excerpt = None
+        MimeContentBase.__init__(self, env, mimetype, os.path.basename(path),
+                                 url)
+    def __del__(self):
+        if self._fd:
+            self._fd.close()
+
+    def _ensure_open(self):
+        if not self._fd:
+            try:
+                self._fd = open(self._path)
+            except IOError:
+                raise TracError('%s "%s" not found' % (self._kind,
+                                                       self._filename))
+    # Reimplemented methods
+    
+    def __iter__(self):
+        """Iterate on chunks of raw content."""
+        chunk = self.excerpt
+        while chunk:
+            yield chunk
+            chunk = self._fd.read(1000)
+
+    def __len__(self):
+        """Length of the raw content, in bytes."""
+        if self._fd:
+            stat = os.fstat(self._fd.fileno())
+        else:
+            stat = os.stat(self._path)
+        return stat.st_size
+
+    def _get_excerpt(self, len=1000):
+        """Extracts the `len` first bytes from the content."""
+        if self._excerpt is None:
+            self._ensure_open()
+            self._excerpt = self._fd.read(1000)
+        return self._excerpt
+
+        
+class NoConversion(TracError):
+    def __init__(self, msg, from_, to):
+        self.msg = msg
+        self.from_ = from_
+        self.to = to
+
+    def message(self):
+        return '%s, from %s to %s' % \
+               (self.msg, self.from_.mimetype, self.to.mimetype)
+
+class Conversion(object):
+    """A specification for performing a data conversion.
+
+    Each conversion is identified by a `key` and targets an output `mimetype`.
+
+    A conversion also specifies a `quality` ranking, which is a number
+    in the range 0 to 9, where 0 means no support and 9 means "perfect"
+    support (try to keep 9 available for user defined conversions,
+    though nothing will prevent them from using 10 or 100...)
+
+    Finally, `expand_tabs` indicates whether a tab expansion should precede
+    the conversion attempt.
+
+    e.g. Conversion(key='latex', quality=8, mimetype=MimeType('text/x-tex'))
+    """
+
+    def __init__(self, key, quality=1, mimetype=TEXT_HTML, expand_tabs=False):
+        self.key = key
+        self.quality = quality
+        self.mimetype = mimetype
+        self.expand_tabs = expand_tabs
+
+    def __repr__(self):
+        return '<Conversion "%s" to %s>' % (self.key, self.mimetype) 
+
+
+# -- Deprecated (TODO: remove in 0.11)
+
 class IHTMLPreviewRenderer(Interface):
     """Extension point interface for components that add HTML renderers of
     specific content types to the `Mimeview` component.
 
-    (Deprecated)
+    Deprecated in 0.10. Implement `IContentConverter` instead.
     """
 
     # implementing classes should set this property to True if they
@@ -198 +582 @@
     expand_tabs = False
 
     def get_quality_ratio(mimetype):
-        """Return the level of support this renderer provides for the `content`
-        of the specified MIME type. The return value must be a number between
-        0 and 9, where 0 means no support and 9 means "perfect" support.
-        """
+        """Return the level of support this renderer provides"""
 
     def render(req, mimetype, content, filename=None, url=None):
-        """Render an XHTML preview of the raw `content`.
+        """Render an XHTML preview of the raw `content`."""
 
-        The `content` might be:
-         * a `str` object
-         * an `unicode` string
-         * any object with a `read` method, returning one of the above
 
-        It is assumed that the content will correspond to the given `mimetype`.
+# -- Interfaces for the extension points
 
-        Besides the `content` value, the same content may eventually
-        be available through the `filename` or `url` parameters.
-        This is useful for renderers that embed objects, using <object> or
-        <img> instead of including the content inline.
-        
-        Can return the generated XHTML text as a single string or as an
-        iterable that yields strings. In the latter case, the list will
-        be considered to correspond to lines of text in the original content.
+class IContentConverter(Interface):
+    """An extension point interface for generic content conversion.
+    """
+
+    def get_supported_conversions(mimetype): 
+        """Check if conversion of `mimetype` is supported by this converter.
+
+        Return an iterable of `Conversion` objects for which this is
+        the case.
         """
 
+    def convert_content(context, conversion, content): 
+        """Convert the given `content` using the specified `conversion`.
+
+        The conversion takes place in the given formatting `context`.
+
+        A `context` can provide at least a `req` property.
+        
+        Return the converted content as a new `MimeContent` object.
+        """ 
+
 class IHTMLPreviewAnnotator(Interface):
     """Extension point interface for components that can annotate an XHTML
     representation of file contents with additional information."""
@@ -240 +628 @@
         annotation data."""
 
 
-class IContentConverter(Interface):
-    """An extension point interface for generic MIME based content
-    conversion."""
+# -- The main Mimeview component
 
-    def get_supported_conversions():
-        """Return an iterable of tuples in the form (key, name, extension,
-        in_mimetype, out_mimetype, quality) representing the MIME conversions
-        supported and
-        the quality ratio of the conversion in the range 0 to 9, where 0 means
-        no support and 9 means "perfect" support. eg. ('latex', 'LaTeX', 'tex',
-        'text/x-trac-wiki', 'text/plain', 8)"""
-
-    def convert_content(req, mimetype, content, key):
-        """Convert the given content from mimetype to the output MIME type
-        represented by key. Returns a tuple in the form (content,
-        output_mime_type) or None if conversion is not possible."""
-
-
 class Mimeview(Component):
     """A generic class to prettify data, typically source code."""
 
-    renderers = ExtensionPoint(IHTMLPreviewRenderer)
+    renderers = ExtensionPoint(IHTMLPreviewRenderer) # TODO: remove in 0.11
     annotators = ExtensionPoint(IHTMLPreviewAnnotator)
     converters = ExtensionPoint(IContentConverter)
 
@@ -275 +647 @@
         """Maximum file size for HTML preview. (''since 0.9'').""")
 
     mime_map = ListOption('mimeviewer', 'mime_map',
-        'text/x-dylan:dylan,text/x-idl:ice,text/x-ada:ads:adb',
+        'text/x-dylan:dylan,text/x-idl:ice,text/x-ada:ads:adb', doc=
         """List of additional MIME types and keyword mappings.
-        Mappings are comma-separated, and for each MIME type,
-        there's a colon (":") separated list of associated keywords
-        or file extensions. (''since 0.10'').""")
 
+        Mappings are comma-separated. Each mapping starts with the mimetype,
+        followed by a colon (":") and the (colon separated) list of associated
+        keywords or file extensions. (''since 0.10'').""")
+
+    treat_as_binary = ListOption('mimeviewer', 'treat_as_binary',
+        'application/pdf,application/postscript,application/rtf', doc=
+        """List of MIME types that should always be treated as binary content.
+
+        Accounts for the fact that our binary detection heuristic can't
+        always work for some kind of binary data. (''since 0.10'').""")
+
     def __init__(self):
         self._mime_map = None
-        
-    # Public API
 
-    def get_supported_conversions(self, mimetype):
-        """Return a list of target MIME types in same form as
-        `IContentConverter.get_supported_conversions()`, but with the converter
-        component appended. Output is ordered from best to worst quality."""
+    def _get_mimemap(self):
+        """Extend default extension to MIME type mappings"""
+        if not self._mime_map:
+            self._mime_map = {}
+            self._mime_map.update(MIME_MAP)
+            for mapping in self.config['mimeviewer'].getlist('mime_map'):
+                if ':' in mapping:
+                    assocations = mapping.split(':')
+                    mimetype = assocations[0]
+                    for keyword in assocations: # mimetype->mimetype on purpose
+                        self._mime_map[keyword] = mimetype
+        return self._mime_map
+
+    mimemap = property(_get_mimemap)
+
+    def lookup(self, keyword, charset=None):
+        mimetype = self.mimemap.get(keyword, APPLICATION_OCTET_STREAM_STR)
+        return MimeType(mimetype, charset, extension=keyword)
+
+    # -- MIME type conversion
+    
+    def get_conversions(self, input):
+        """Return a list of possible conversions for the `input` MimeType.
+
+        The returned list contains pair of (conversion, converter) objects,
+        ordered from best to worst quality.
+        """
+        # Build list of possible conversions
         converters = []
         for converter in self.converters:
-            for k, n, e, im, om, q in converter.get_supported_conversions():
-                if im == mimetype and q > 0:
-                    converters.append((k, n, e, im, om, q, converter))
-        converters = sorted(converters, key=lambda i: i[-1], reverse=True)
-        return converters
+            for conversion in converter.get_supported_conversions(input):
+                if conversion.quality > 0:
+                    converters.append((conversion, converter))
 
-    def convert_content(self, req, mimetype, content, key, filename=None,
-                        url=None):
-        """Convert the given content to the target MIME type represented by
-        `key`, which can be either a MIME type or a key. Returns a tuple of
-        (content, output_mime_type, extension)."""
-        if not content:
-            return ('', 'text/plain;charset=utf-8')
+        # ---- Backward compatibility support for IHTMLPreviewRenderer
+        class IHTMLPreviewRendererWrapper(object):
+            def __init__(self, renderer):
+                self.renderer = renderer
+            def __repr__(self):
+                return repr(self.renderer)
+            def convert_content(self, context, conversion, mimecontent):
+                return self.renderer.render(
+                    context.req, mimecontent.type.mimetype,
+                    mimecontent, # which is read()able 
+                    mimecontent.filename, mimecontent.url)
+        for renderer in self.renderers:
+            qr = renderer.get_quality_ratio(input.mimetype)
+            if qr > 0:
+                expand_tabs = getattr(renderer, 'expand_tabs', False)
+                converters.append(
+                    (Conversion(key='', quality=qr, mimetype=TEXT_HTML,
+                                expand_tabs=expand_tabs),
+                     IHTMLPreviewRendererWrapper(renderer)))
+        # ---- (to be removed in 0.11)
 
-        # Ensure we have a MIME type for this content
-        full_mimetype = mimetype
-        if not full_mimetype:
-            if hasattr(content, 'read'):
-                content = content.read(self.max_preview_size)
-            full_mimetype = self.get_mimetype(filename, content)
-        if full_mimetype:
-            mimetype = full_mimetype.split(';')[0].strip() # split off charset
-        else:
-            mimetype = full_mimetype = 'text/plain' # fallback if not binary
+        return sorted(converters, key=lambda c: c[0].quality, reverse=True)
 
-        # Choose best converter
-        candidates = list(self.get_supported_conversions(mimetype))
-        candidates = [c for c in candidates if key in (c[0], c[4])]
+    def convert(self, context, selector, mimecontent):
+        """Convert the `mimecontent` to another MIME type.
+
+        The conversion to be done is determined by `selector`,
+        which can be either directly the desired output MIME type or
+        a key identifying the Conversion object.
+
+        Returns a new MimeContent.
+        """
+        # Ensure we have the mimetype and the charset information
+        output = self.lookup(selector)
+        if not output:
+            self.log.debug('No output MIME type selected by "%s"' % selector)
+
+        # Get all available conversions for the input
+        candidates = self.get_conversions(mimecontent.type)
+        # Filter those which are matching `selector`
+        candidates = [cn_cr for cn_cr in candidates
+                      if selector in (cn_cr[0].key, output.mimetype)]
         if not candidates:
-            raise TracError('No available MIME conversions from %s to %s' %
-                            (mimetype, key))
+            raise NoConversion('No available MIME conversions',
+                               mimecontent.type, output)
 
-        # First successful conversion wins
-        for ck, name, ext, input_mimettype, output_mimetype, quality, \
-                converter in candidates:
-            output = converter.convert_content(req, mimetype, content, ck)
-            if not output:
-                continue
-            return (output[0], output[1], ext)
-        raise TracError('No available MIME conversions from %s to %s' %
-                        (mimetype, key))
+        tab_expanded = None # we don't want to expand tabs more than once.
 
+        # First candidate which converts successfully wins.
+        for conversion, converter in candidates:
+            if conversion.expand_tabs and not tab_expanded:
+                tab_expanded = unicode(mimecontent).expandtabs(self.tab_width)
+                mimecontent = MimeContent(self.env, tab_expanded,
+                                          mimecontent.type)
+                self.log.debug('tab expansion performed.')
+            try:
+                self.log.debug('Attempt conversion %s using %s' %
+                               (conversion, converter))
+                res = converter.convert_content(context, conversion,
+                                                mimecontent)
+                if not res:
+                    continue
+                return res
+            except Exception, e:
+                self.log.warning('MIME conversion using %s failed (%s)'
+                                 % (converter, e), exc_info=True)
+        raise NoConversion('No MIME conversions succeeded',
+                           mimecontent.type, output)
+
+    # -- XHTML rendering and annotations (based on the conversion API)
+    
     def get_annotation_types(self):
         """Generator that returns all available annotation types."""
         for annotator in self.annotators:
             yield annotator.get_annotation_type()
 
-    def render(self, req, mimetype, content, filename=None, url=None,
-               annotations=None):
-        """Render an XHTML preview of the given `content`.
+    def render(self, req, mimecontent, annotations=None):
+        """Render an XHTML preview of the given `mimecontent`.
 
-        `content` is the same as an `IHTMLPreviewRenderer.render`'s
-        `content` argument.
-
-        The specified `mimetype` will be used to select the most appropriate
-        `IHTMLPreviewRenderer` implementation available for this MIME type.
-        If not given, the MIME type will be infered from the filename or the
-        content.
-
-        Return a string containing the XHTML text.
+        Some `annotations` might be requested as well.
         """
-        if not content:
-            return ''
+        class ToplevelContext(object):
+            def __init__(self, req):
+                self.req = req
+        result = self.convert(ToplevelContext(req), 'text/html', mimecontent)
 
-        # Ensure we have a MIME type for this content
-        full_mimetype = mimetype
-        if not full_mimetype:
-            if hasattr(content, 'read'):
-                content = content.read(self.max_preview_size)
-            full_mimetype = self.get_mimetype(filename, content)
-        if full_mimetype:
-            mimetype = full_mimetype.split(';')[0].strip() # split off charset
-        else:
-            mimetype = full_mimetype = 'text/plain' # fallback if not binary
+        if isinstance(result, Fragment):
+            return result             # might be processed further
+        elif isinstance(result, basestring):
+            self.log.warning('HTML rendering: got %s' %
+                             result.__class__.__name__)
+            return Markup(to_unicode(result)) # needed for compatibility 
 
-        # Determine candidate `IHTMLPreviewRenderer`s
-        candidates = []
-        for renderer in self.renderers:
-            qr = renderer.get_quality_ratio(mimetype)
-            if qr > 0:
-                candidates.append((qr, renderer))
-        candidates.sort(lambda x,y: cmp(y[0], x[0]))
+        # otherwise, it's an iterable yielding lines
+        if annotations:
+            return Markup(self._annotate(result, annotations))
 
-        # First candidate which renders successfully wins.
-        # Also, we don't want to expand tabs more than once.
-        expanded_content = None
-        for qr, renderer in candidates:
-            try:
-                self.log.debug('Trying to render HTML preview using %s'
-                               % renderer.__class__.__name__)
-                # check if we need to perform a tab expansion
-                rendered_content = content
-                if getattr(renderer, 'expand_tabs', False):
-                    if expanded_content is None:
-                        content = content_to_unicode(self.env, content,
-                                                     full_mimetype)
-                        expanded_content = content.expandtabs(self.tab_width)
-                    rendered_content = expanded_content
-                result = renderer.render(req, full_mimetype, rendered_content,
-                                         filename, url)
-                if not result:
-                    continue
-                elif isinstance(result, Fragment):
-                    return result
-                elif isinstance(result, basestring):
-                    return Markup(to_unicode(result))
-                elif annotations:
-                    return Markup(self._annotate(result, annotations))
-                else:
-                    buf = StringIO()
-                    buf.write('<div class="code"><pre>')
-                    for line in result:
-                        buf.write(line + '\n')
-                    buf.write('</pre></div>')
-                    return Markup(buf.getvalue())
-            except Exception, e:
-                self.log.warning('HTML preview using %s failed (%s)'
-                                 % (renderer, e), exc_info=True)
+        return html.DIV(html.PRE(Markup(''.join(result))), class_="code")
 
     def _annotate(self, lines, annotations):
+        """Add requested `annotations` to the lines' content."""
         buf = StringIO()
         buf.write('<table class="code"><thead><tr>')
         annotators = []
@@ -445 +832 @@
         buf.write('</tbody></table>')
         return buf.getvalue()
 
+    # -- Deprecated API (TODO: remove in 0.11)
+
     def get_max_preview_size(self):
         """Deprecated: use `max_preview_size` attribute directly."""
         return self.max_preview_size
 
-    def get_charset(self, content='', mimetype=None):
-        """Infer the character encoding from the `content` or the `mimetype`.
-
-        `content` is either a `str` or an `unicode` object.
-        
-        The charset will be determined using this order:
-         * from the charset information present in the `mimetype` argument
-         * auto-detection of the charset from the `content`
-         * the configured `default_charset` 
-        """
-        if mimetype:
-            ctpos = mimetype.find('charset=')
-            if ctpos >= 0:
-                return mimetype[ctpos + 8:].strip()
-        if isinstance(content, str):
-            utf = detect_unicode(content)
-            if utf is not None:
-                return utf
-        return self.default_charset
-
-    def get_mimetype(self, filename, content=None):
-        """Infer the MIME type from the `filename` or the `content`.
-
-        `content` is either a `str` or an `unicode` object.
-
-        Return the detected MIME type, augmented by the
-        charset information (i.e. "<mimetype>; charset=..."),
-        or `None` if detection failed.
-        """
-        # Extend default extension to MIME type mappings with configured ones
-        if not self._mime_map:
-            self._mime_map = MIME_MAP
-            for mapping in self.config['mimeviewer'].getlist('mime_map'):
-                if ':' in mapping:
-                    assocations = mapping.split(':')
-                    for keyword in assocations: # Note: [0] kept on purpose
-                        self._mime_map[keyword] = assocations[0]
-
-        mimetype = get_mimetype(filename, content, self._mime_map)
-        charset = None
-        if mimetype:
-            charset = self.get_charset(content, mimetype)
-        if mimetype and charset and not 'charset' in mimetype:
-            mimetype += '; charset=' + charset
-        return mimetype
-
     def to_utf8(self, content, mimetype=None):
         """Convert an encoded `content` to utf-8.
 
         ''Deprecated in 0.10. You should use `unicode` strings only.''
         """
-        return to_utf8(content, self.get_charset(content, mimetype))
+        return to_utf8(content)
 
-    def to_unicode(self, content, mimetype=None, charset=None):
-        """Convert `content` (an encoded `str` object) to an `unicode` object.
+    # -- Utilities
 
-        This calls `trac.util.to_unicode` with the `charset` provided,
-        or the one obtained by `Mimeview.get_charset()`.
+    def configured_modes_mapping(self, renderer):
+        """Utility for configurable custom converters
+
+        Return a MIME type to `(mode,quality)` mapping for given `option`,
+        assuming a format of comma-separated <mimetype>:<mode>:<quality>
+        associations.
+
+        See EnscriptConverter and SilverCityConverter.
         """
-        if not charset:
-            charset = self.get_charset(content, mimetype)
-        return to_unicode(content, charset)
-
-    def configured_modes_mapping(self, renderer):
-        """Return a MIME type to `(mode,quality)` mapping for given `option`"""
         types, option = {}, '%s_modes' % renderer
         for mapping in self.config['mimeviewer'].getlist(option):
             if not mapping:
@@ -526 +868 @@
                                  "option." % (mapping, option))
         return types
     
-    def preview_to_hdf(self, req, content, length, mimetype, filename,
-                       url=None, annotations=None):
-        """Prepares a rendered preview of the given `content`.
-
-        Note: `content` will usually be an object with a `read` method.
-        """        
-        if length >= self.max_preview_size:
+    def preview_to_hdf(self, req, mimecontent, annotations=None):
+        """Prepares a rendered preview of the given `mimecontent`."""        
+        if len(mimecontent) >= self.max_preview_size:
             return {'max_file_size_reached': True,
                     'max_file_size': self.max_preview_size,
-                    'raw_href': url}
+                    'raw_href': mimecontent.url}
         else:
-            return {'preview': self.render(req, mimetype, content, filename,
-                                           url, annotations),
-                    'raw_href': url}
+            preview_error = preview = None
+            try:
+                preview = self.render(req, mimecontent, annotations)
+            except NoConversion, e:
+                preview_error = e.message()
+            return {'preview': preview,
+                    'preview_error': preview_error,
+                    'raw_href': mimecontent.url}
 
-    def send_converted(self, req, in_type, content, selector, filename='file'):
-        """Helper method for converting `content` and sending it directly.
+    def send_converted(self, req, selector, mimecontent):
+        """Helper method for converting `mimecontent` and sending it directly.
 
-        `selector` can be either a key or a MIME Type."""
+        `selector` can be either a key or the expected output MIME Type.
+        """
         from trac.web import RequestDone
-        content, output_type, ext = self.convert_content(req, in_type,
-                                                         content, selector)
+        result = self.convert(req, selector, mimecontent)
         req.send_response(200)
-        req.send_header('Content-Type', output_type)
-        req.send_header('Content-Disposition', 'filename=%s.%s' % (filename,
-                                                                   ext))
+        req.send_header('Content-Type', self.get_content_type(result))
+        req.send_header('Content-Disposition',
+                        self.get_content_disposition(result))
         req.end_headers()
-        req.write(content)
-        raise RequestDone        
+
+        self.send_content(req, result)
+        raise RequestDone
+
+    def get_content_type(self, result):
+        if isinstance(result, unicode):
+            return 'utf-8'
+        elif isinstance(result, str):
+            return IDENTITY_CHARSET # FIXME: could probably do better...
+        else: # MimeContentBase
+            return result.encoding or 'utf-8'
+
+    def get_content_disposition(self, result):
+        if isinstance(result, MimeContentBase):
+            return 'filename=%s.%s' % (result.filename, result.type.extension)
+        else:
+            return 'file.'+(is_binary(result) and 'bin' or 'txt')
+
+    def send_content(self, req, result):
+        if isinstance(result, unicode):
+            return result.encode('utf-8')
+        elif isinstance(result, str):
+            return result
+        else: # MimeContentBase
+            if result.encoding:
+                req.write(result.content)
+            else:
+                req.write(unicode(result).encode('utf-8'))
+            # or: for chunk in result: req.write(chunk)
+            # TODO: check what's the fastest solution
         
 
+# utility for Mimeview._annotate
 def _html_splitlines(lines):
     """Tracks open and close tags in lines of HTML text and yields lines that
     have no tags spanning more than one line."""
@@ -604 +976 @@
     def annotate_line(self, number, content):
         return '<th id="L%s"><a href="#L%s">%s</a></th>' % (number, number,
                                                             number)
+#        return html.TH(html.A(number, href="#L%s" % number), id=number)
 
 
-# -- Default renderers
+# -- Default TEXT_HTML converters (previously ''IHTMLPreviewRenderer'')
 
 class PlainTextRenderer(Component):
-    """HTML preview renderer for plain text, and fallback for any kind of text
-    for which no more specific renderer is available.
+    """Convert text to HTML-escaped text.
+
+    Will be used as a fallback for any kind of text
+    for which no more specific HTML converter is available.
     """
-    implements(IHTMLPreviewRenderer)
+    implements(IContentConverter)
 
-    expand_tabs = True
-
-    TREAT_AS_BINARY = [
-        'application/pdf',
-        'application/postscript',
-        'application/rtf'
-    ]
-
-    def get_quality_ratio(self, mimetype):
-        if mimetype in self.TREAT_AS_BINARY:
-            return 0
-        return 1
-
-    def render(self, req, mimetype, content, filename=None, url=None):
-        if is_binary(content):
+    def get_supported_conversions(self, input):
+        if input.mimetype in Mimeview(self.env).treat_as_binary:
+            return
+        yield Conversion(key='default',
+                         quality=TEXT_PLAIN.match(input) and 8 or 1,
+                         mimetype=TEXT_HTML, expand_tabs=True)
+        
+    def convert_content(self, context, conversion, mimecontent):
+        if mimecontent.is_binary:
             self.env.log.debug("Binary data; no preview available")
-            return
+        else:
+            if not TEXT_PLAIN.match(mimecontent.type):
+                self.env.log.debug("Fallback to plain text renderer.")
+            mimecontent.type = MimeType('text/plain', mimecontent.encoding)
+            return mimecontent
 
-        self.env.log.debug("Using default plain text mimeviewer")
-        content = content_to_unicode(self.env, content, mimetype)
-        for line in content.splitlines():
-            yield escape(line)
 
-
 class ImageRenderer(Component):
-    """Inline image display. Here we don't need the `content` at all."""
-    implements(IHTMLPreviewRenderer)
+    """Inline image display.
 
-    def get_quality_ratio(self, mimetype):
-        if mimetype.startswith('image/'):
-            return 8
-        return 0
+    This renderer doesn't need the actual data at all, only the url.
+    """
+    implements(IContentConverter)
 
-    def render(self, req, mimetype, content, filename=None, url=None):
-        if url:
-            return html.DIV(html.IMG(src=url,alt=filename),
+    def get_supported_conversions(self, input):
+        if MimeType('^image/').match(input, regexp=True):
+            yield Conversion(key='image', quality=8, mimetype=TEXT_HTML)
+
+    def convert_content(self, context, conversion, mimecontent):
+        if mimecontent.url:
+            return html.DIV(html.IMG(src=mimecontent.url,
+                                     alt=mimecontent.filename),
                             class_="image-file")
 
 
 class WikiTextRenderer(Component):
     """Render files containing Trac's own Wiki formatting markup."""
-    implements(IHTMLPreviewRenderer)
 
-    def get_quality_ratio(self, mimetype):
-        if mimetype in ('text/x-trac-wiki', 'application/x-trac-wiki'):
-            return 8
-        return 0
+    implements(IContentConverter)
 
-    def render(self, req, mimetype, content, filename=None, url=None):
+    def get_supported_conversions(self, input):
+        from trac.wiki import TEXT_X_TRAC_WIKI, APPLICATION_X_TRAC_WIKI
+        if TEXT_X_TRAC_WIKI.match(input) or \
+               APPLICATION_X_TRAC_WIKI.match(input):
+            yield Conversion(key='wiki', quality=8, mimetype=TEXT_HTML)
+
+    def convert_content(self, context, conversion, mimecontent):
         from trac.wiki import wiki_to_html
-        return wiki_to_html(content_to_unicode(self.env, content, mimetype),
-                            self.env, req)
+        return MimeContent(self.env, wiki_to_html(unicode(mimecontent),
+                                                  self.env, context.req),
+                           TEXT_HTML)

trac/mimeview/silvercity.py

@@ -108 +108 @@
             raise Exception, err
 
         # SilverCity does not like unicode strings
+        content = content.read()
         content = content.encode('utf-8')
         
         # SilverCity generates extra empty line against some types of

trac/mimeview/patch.py

@@ -16 +16 @@
 #         Ludvig Strigeus
 
 from trac.core import *
-from trac.mimeview.api import content_to_unicode, IHTMLPreviewRenderer, Mimeview
+from trac.mimeview.api import IHTMLPreviewRenderer, Mimeview
 from trac.util.markup import escape, Markup
 from trac.web.chrome import add_stylesheet
 
@@ -65 +65 @@
     def render(self, req, mimetype, content, filename=None, rev=None):
         from trac.web.clearsilver import HDFWrapper
 
-        content = content_to_unicode(self.env, content, mimetype)
+        content = unicode(content)
         d = self._diff_to_hdf(content.splitlines(),
                               Mimeview(self.env).tab_width)
         if not d:

trac/mimeview/enscript.py

@@ -135 +135 @@
         cmdline += ' --color -h -q --language=html -p - -E%s' % mode
         self.env.log.debug("Enscript command line: %s" % cmdline)
 
+        content = unicode(content)
         np = NaivePopen(cmdline, content.encode('utf-8'), capturestderr=1)
         if np.errorlevel or np.err:
             err = 'Running (%s) failed: %s, %s.' % (cmdline, np.errorlevel,

trac/mimeview/php.py

@@ -20 +20 @@
 
 from trac.core import *
 from trac.config import Option
-from trac.mimeview.api import IHTMLPreviewRenderer, content_to_unicode
+from trac.mimeview.api import IHTMLPreviewRenderer
 from trac.util import NaivePopen
 from trac.util.markup import Deuglifier
 
@@ -79 +79 @@
         cmdline += ' -sn'
         self.env.log.debug("PHP command line: %s" % cmdline)
         
-        content = content_to_unicode(self.env, content, mimetype)
+        content = unicode(content)
         content = content.encode('utf-8')
         np = NaivePopen(cmdline, content, capturestderr=1)
         if np.errorlevel or np.err:

trac/versioncontrol/web_ui/util.py

@@ -16 +16 @@
 # Author: Jonas Borgström <jonas@edgewall.com>
 #         Christian Boos <cboos@neuf.fr>
 
+import posixpath
 import re
 import urllib
 
 from trac.core import TracError
+from trac.mimeview.api import MimeContentBase
 from trac.util.datefmt import format_datetime, pretty_timedelta
 from trac.util.text import shorten_line
 from trac.util.markup import escape, html, Markup
@@ -27 +29 @@
 from trac.wiki import wiki_to_html, wiki_to_oneliner
 
 __all__ = ['get_changes', 'get_path_links', 'get_path_rev_line',
-           'get_existing_node', 'render_node_property']
+           'get_existing_node', 'render_node_property',
+           'NodeMimeContent']
 
+
+# MimeContent wrapper for repository files
+#
+# -- This could be part of trac.versioncontrol.api itself,
+#    if introducing a dependency on trac.mimeview.api is
+#    considered to be OK.
+
+CHUNK_SIZE = 4096
+
+class NodeMimeContent(MimeContentBase):
+    """Encapsulation of the content of a file in the repository."""
+            
+    def __init__(self, env, node, url=None):
+        self.node = node
+        MimeContentBase.__init__(self, env, node.content_type,
+                                 posixpath.basename(node.path), url)
+        self._excerpt = None
+
+    def __iter__(self):
+        """Get chunks of the byte content"""
+        chunk = self.excerpt
+        self._excerpt = None
+        while True:
+            if not chunk:
+                return
+            yield chunk
+            chunk = self._content.read(CHUNK_SIZE)
+
+    def __len__(self):
+        return self.node.get_content_length()
+    
+    def _get_excerpt(self):
+        if not self._excerpt:
+            self._content = self.node.get_content()
+            self._excerpt = self._content.read(CHUNK_SIZE)
+        return self._excerpt
+            
+
 def get_changes(env, repos, revs, full=None, req=None, format=None):
     db = env.get_db_cnx()
     changes = {}

trac/versioncontrol/web_ui/changeset.py

@@ -26 +26 @@
 from trac import util
 from trac.config import BoolOption, IntOption
 from trac.core import *
-from trac.mimeview import Mimeview, is_binary
 from trac.perm import IPermissionRequestor
 from trac.Search import ISearchSource, search_to_sql, shorten_result
 from trac.Timeline import ITimelineEventProvider
@@ -36 +35 @@
 from trac.versioncontrol import Changeset, Node
 from trac.versioncontrol.diff import get_diff_options, hdf_diff, unified_diff
 from trac.versioncontrol.svn_authz import SubversionAuthorizer
-from trac.versioncontrol.web_ui.util import render_node_property
+from trac.versioncontrol.web_ui.util import *
 from trac.web import IRequestHandler
 from trac.web.chrome import INavigationContributor, add_link, add_stylesheet
 from trac.wiki import wiki_to_html, wiki_to_oneliner, IWikiSyntaxProvider, \
@@ -439 +438 @@
             The list is empty when no differences between comparable files
             are detected, but the return value is None for non-comparable files.
             """
-            old_content = old_node.get_content().read()
-            if is_binary(old_content):
+            old_content = NodeMimeContent(self.env, old_node)
+            if old_content.is_binary:
                 return None
 
-            new_content = new_node.get_content().read()
-            if is_binary(new_content):
+            new_content = NodeMimeContent(self.env, new_content)
+            if new_content.is_binary:
                 return None
 
-            mview = Mimeview(self.env)
-            old_content = mview.to_unicode(old_content, old_node.content_type)
-            new_content = mview.to_unicode(new_content, new_node.content_type)
+            old_content = unicode(old_content)
+            new_content = unicode(new_content)
 
             if old_content != new_content:
                 context = 3
@@ -526 +524 @@
                         'filename=%s.diff' % filename)
         req.end_headers()
 
-        mimeview = Mimeview(self.env)
         for old_node, new_node, kind, change in repos.get_changes(**diff):
             # TODO: Property changes
 
@@ -536 +533 @@
 
             new_content = old_content = ''
             new_node_info = old_node_info = ('','')
-            mimeview = Mimeview(self.env)
 
             if old_node:
-                old_content = old_node.get_content().read()
-                if is_binary(old_content):
+                old_content = NodeMimeContent(self.env, old_node)
+                if old_content.is_binary:
                     continue
                 old_node_info = (old_node.path, old_node.rev)
-                old_content = mimeview.to_unicode(old_content,
-                                                  old_node.content_type)
+                old_content = unicode(old_content)
             if new_node:
-                new_content = new_node.get_content().read()
-                if is_binary(new_content):
+                new_content = NodeMimeContent(self.env, new_node)
+                if new_content.is_binary:
                     continue
                 new_node_info = (new_node.path, new_node.rev)
                 new_path = new_node.path
-                new_content = mimeview.to_unicode(new_content,
-                                                  new_node.content_type)
+                new_content = unicode(new_content)
             else:
                 old_node_path = repos.normalize_path(old_node.path)
                 diff_old_path = repos.normalize_path(diff.old_path)

trac/versioncontrol/web_ui/browser.py

@@ -23 +23 @@
 from trac import util
 from trac.config import ListOption, Option
 from trac.core import *
-from trac.mimeview import Mimeview, is_binary, get_mimetype
+from trac.mimeview import Mimeview, MimeType
 from trac.perm import IPermissionRequestor
 from trac.util import sorted, embedded_numbers
 from trac.util.datefmt import http_date, format_datetime, pretty_timedelta
@@ -36 +36 @@
 from trac.versioncontrol.web_ui.util import *
 
 
-CHUNK_SIZE = 4096
-
-
 class BrowserModule(Component):
 
     implements(INavigationContributor, IPermissionRequestor, IRequestHandler,
@@ -199 +196 @@
     def _render_file(self, req, repos, node, rev=None):
         req.perm.assert_permission('FILE_VIEW')
 
-        mimeview = Mimeview(self.env)
+        format = req.args.get('format')
 
-        # MIME type detection
-        content = node.get_content()
-        chunk = content.read(CHUNK_SIZE)
-        mime_type = node.content_type
-        if not mime_type or mime_type == 'application/octet-stream':
-            mime_type = mimeview.get_mimetype(node.name, chunk) or \
-                        mime_type or 'text/plain'
+        raw_href = req.href.browser(node.path, rev=rev, format='raw')
 
+        mimecontent = NodeMimeContent(self.env, node, raw_href)
+
         # Eventually send the file directly
-        format = req.args.get('format')
-        if format in ['raw', 'txt']:
+        if format in ('raw', 'txt'):
+            if format == 'txt':
+                type = MimeType('text/plain', mimecontent.encoding)
+            else:
+                type = mimecontent.type
             req.send_response(200)
-            req.send_header('Content-Type',
-                            format == 'txt' and 'text/plain' or mime_type)
-            req.send_header('Content-Length', node.content_length)
+            req.send_header('Content-Type', type.mimetype_charset)
+            req.send_header('Content-Length', len(mimecontent))
             req.send_header('Last-Modified', http_date(node.last_modified))
             req.end_headers()
 
-            while 1:
-                if not chunk:
-                    raise RequestDone
+            for chunk in mimecontent:
                 req.write(chunk)
-                chunk = content.read(CHUNK_SIZE)
+            raise RequestDone
         else:
             # The changeset corresponding to the last change on `node` 
             # is more interesting than the `rev` changeset.
-            changeset = repos.get_changeset(node.rev)
+            changeset = repos.get_changeset(node.created_rev)
 
             message = changeset.message or '--'
             if self.config['changeset'].getbool('wiki_format_messages'):
@@ -238 +231 @@
 
             req.hdf['file'] = {
                 'rev': node.rev,
-                'changeset_href': req.href.changeset(node.rev),
+                'changeset_href': req.href.changeset(node.created_rev),
                 'date': format_datetime(changeset.date),
                 'age': pretty_timedelta(changeset.date),
                 'size': pretty_size(node.content_length),
@@ -246 +239 @@
                 'message': message
             } 
 
+            mimetype = mimecontent.type.mimetype
+            
             # add ''Plain Text'' alternate link if needed
-            if not is_binary(chunk) and mime_type != 'text/plain':
+            if not mimecontent.is_binary and mimecontent.type.is_known and \
+                   mimetype != 'text/plain':
                 plain_href = req.href.browser(node.path, rev=rev, format='txt')
                 add_link(req, 'alternate', plain_href, 'Plain Text',
                          'text/plain')
 
             # add ''Original Format'' alternate link (always)
-            raw_href = req.href.browser(node.path, rev=rev, format='raw')
-            add_link(req, 'alternate', raw_href, 'Original Format', mime_type)
+            add_link(req, 'alternate', raw_href, 'Original Format', mimetype)
 
             self.log.debug("Rendering preview of node %s@%s with mime-type %s"
-                           % (node.name, str(rev), mime_type))
+                           % (node.name, str(rev), mimetype))
 
-            del content # the remainder of that content is not needed
+            req.hdf['file'] = Mimeview(self.env).preview_to_hdf(
+                req, mimecontent, annotations=['lineno'])
 
-            req.hdf['file'] = mimeview.preview_to_hdf(
-                req, node.get_content(), node.get_content_length(), mime_type,
-                node.created_path, raw_href, annotations=['lineno'])
-
             add_stylesheet(req, 'common/css/code.css')
 
     # IWikiSyntaxProvider methods

trac/wiki/web_ui.py

@@ -34 +34 @@
 from trac.web import HTTPNotFound, IRequestHandler
 from trac.wiki.api import IWikiPageManipulator, WikiSystem
 from trac.wiki.model import WikiPage
-from trac.wiki.formatter import wiki_to_html, wiki_to_oneliner
-from trac.mimeview.api import Mimeview, IContentConverter
+from trac.wiki.formatter import wiki_to_html, wiki_to_oneliner, \
+                                TEXT_X_TRAC_WIKI, APPLICATION_X_TRAC_WIKI
+from trac.mimeview.api import Mimeview, IContentConverter, Conversion, \
+                              MimeContent, TEXT_PLAIN
 
 
 class InvalidWikiPage(TracError):
@@ -50 +52 @@
     page_manipulators = ExtensionPoint(IWikiPageManipulator)
 
     # IContentConverter methods
-    def get_supported_conversions(self):
-        yield ('txt', 'Plain Text', 'txt', 'text/x-trac-wiki', 'text/plain', 9)
 
-    def convert_content(self, req, mimetype, content, key):
-        return (content, 'text/plain;charset=utf-8')
+    def get_supported_conversions(self, input):
+        if TEXT_X_TRAC_WIKI.match(input) or \
+               APPLICATION_X_TRAC_WIKI.match(input):
+            yield Conversion('txt', quality=8, mimetype=TEXT_PLAIN)
 
+    def convert_content(self, context, conversion, content):
+        return content # identity transform
+
     # INavigationContributor methods
 
     def get_active_navigation_item(self, req):
@@ -128 +133 @@
         else:
             format = req.args.get('format')
             if format:
-                Mimeview(self.env).send_converted(req, 'text/x-trac-wiki',
-                                                  page.text, format, page.name)
+                wiki_content = MimeContent(self.env, page.text,
+                                           TEXT_X_TRAC_WIKI,
+                                           filename=page.name)
+                Mimeview(self.env).send_converted(req, format, wiki_content)
             self._render_view(req, db, page)
 
         req.hdf['wiki.action'] = action
@@ -429 +436 @@
             req.hdf['html.norobots'] = 1
 
         # Add registered converters
-        for conversion in Mimeview(self.env).get_supported_conversions(
-                                             'text/x-trac-wiki'):
+        for conv, _ in Mimeview(self.env).get_conversions(TEXT_X_TRAC_WIKI):
+            if conv.key in ('wiki', 'default'):
+                continue
             conversion_href = req.href.wiki(page.name, version=version,
-                                            format=conversion[0])
-            add_link(req, 'alternate', conversion_href, conversion[1],
-                     conversion[3])
+                                            format=conv.key)
+            add_link(req, 'alternate', conversion_href,
+                     conv.mimetype.name, conv.mimetype.mimetype_charset)
 
         latest_page = WikiPage(self.env, page.name)
         req.hdf['wiki'] = {'exists': page.exists,

trac/wiki/formatter.py

@@ -31 +31 @@
 from trac.util.markup import escape, Markup, Element, html
 
 __all__ = ['wiki_to_html', 'wiki_to_oneliner', 'wiki_to_outline',
-           'wiki_to_link', 'Formatter' ]
+           'wiki_to_link', 'Formatter',
+           'TEXT_X_TRAC_WIKI', 'APPLICATION_X_TRAC_WIKI']
 
 
+TEXT_X_TRAC_WIKI = MimeType('text/x-trac-wiki',
+                            name='Trac Wiki Text', extension='txt')
+
+APPLICATION_X_TRAC_WIKI = MimeType('application/x-trac-wiki',
+                                   name='Trac Wiki Text', extension='txt')
+
+
 def system_message(msg, text=None):
     return html.DIV(html.STRONG(msg), text and html.PRE(text),
                     class_="system-message")
@@ -43 +51 @@
 
     _code_block_re = re.compile('^<div(?:\s+class="([^"]+)")?>(.*)</div>$')
 
-    def __init__(self, env, name):
-        # TODO: transmit `formatter` argument
-        self.env = env
+    def __init__(self, formatter, name):
+        self.formatter = formatter
+        self.env = formatter.env
         self.name = name
         self.error = None
         self.macro_provider = None
+        self.mimetype = None
 
         builtin_processors = {'html': self._html_processor,
                               'default': self._default_processor,
@@ -65 +74 @@
                         break
         if not self.processor:
             # Find a matching mimeview renderer
-            from trac.mimeview.api import Mimeview
-            mimetype = Mimeview(self.env).get_mimetype(self.name)
-            if mimetype:
-                self.name = mimetype
+            mimetype = Mimeview(self.env).lookup(self.name)
+            if not APPLICATION_OCTET_STREAM.match(mimetype):
+                self.mimetype = mimetype
                 self.processor = self._mimeview_processor
             else:
                 self.processor = self._default_processor
@@ -94 +102 @@
     # generic processors
 
     def _macro_processor(self, req, text):
-        # TODO: macro should take a `formatter` argument
+        # TODO: macro should take a `formatter` argument (0.11)
         self.env.log.debug('Executing Wiki macro %s by provider %s'
                            % (self.name, self.macro_provider))
         return self.macro_provider.render_macro(req, self.name, text)
 
     def _mimeview_processor(self, req, text):
-        # TODO: transmit context from `formatter`
-        return Mimeview(self.env).render(req, self.name, text)
+        class FormatterContext(object):
+            def __init__(self, req):
+                self.req = req
+        blockcontent = MimeContent(self.env, text, self.mimetype)
+        return Mimeview(self.env).convert(self.formatter, 'text/html',
+                                          blockcontent)
+                                         
 
     def process(self, req, text, in_paragraph=False):
         if self.error:
@@ -429 +442 @@
             return '<br />'
         args = fullmatch.group('macroargs')
         try:
-            macro = WikiProcessor(self.env, name)
+            macro = WikiProcessor(self, name)
             return macro.process(self.req, args, True)
         except Exception, e:
             self.env.log.error('Macro %s(%s) failed' % (name, args),
@@ -714 +727 @@
             else:
                 self.code_text += line + os.linesep
                 if not self.code_processor:
-                    self.code_processor = WikiProcessor(self.env, 'default')
+                    self.code_processor = WikiProcessor(self, 'default')
         elif line.strip() == Formatter.ENDBLOCK:
             self.in_code_block -= 1
             if self.in_code_block == 0 and self.code_processor:
@@ -728 +741 @@
             match = Formatter._processor_re.search(line)
             if match:
                 name = match.group(1)
-                self.code_processor = WikiProcessor(self.env, name)
+                self.code_processor = WikiProcessor(self, name)
             else:
                 self.code_text += line + os.linesep 
-                self.code_processor = WikiProcessor(self.env, 'default')
+                self.code_processor = WikiProcessor(self, 'default')
         else:
             self.code_text += line + os.linesep
 

trac/web/chrome.py

@@ -17 +17 @@
 import os
 import re
 
-from trac import mimeview
 from trac.config import *
 from trac.core import *
 from trac.env import IEnvironmentSetupParticipant
+from trac.mimeview import get_mimetype
 from trac.util.markup import html
 from trac.web.api import IRequestHandler, HTTPNotFound
 from trac.web.href import Href
@@ -288 +288 @@
                     icon = href.chrome(icon)
                 else:
                     icon = href.chrome('common', icon)
-            mimetype = mimeview.get_mimetype(icon)
+            mimetype = get_mimetype(icon)
             add_link(req, 'icon', icon, mimetype=mimetype)
             add_link(req, 'shortcut icon', icon, mimetype=mimetype)
 

trac/util/text.py

@@ -25 +25 @@
 
 
 CRLF = '\r\n'
+IDENTITY_CHARSET = 'iso-8859-1' # not iso-8859-15
 
 # -- Unicode
 
@@ -40 +41 @@
     If the `lossy` argument is `True`, which is the default, then
     we use the 'replace' mode:
 
-    If the `lossy` argument is `False`, we fallback to the 'iso-8859-15'
-    charset in case of an error (encoding a `str` using 'iso-8859-15'
+    If the `lossy` argument is `False`, we fallback to the 'iso-8859-1'
+    charset in case of an error (encoding a `str` using 'iso-8859-1'
     will always work, as there's one Unicode character for each byte of
     the input).
     """
@@ -65 +66 @@
             except UnicodeError:
                 return unicode(text, locale.getpreferredencoding(), errors)
     except UnicodeError:
-        return unicode(text, 'iso-8859-15')
+        return unicode(text, IDENTITY_CHARSET)
 
 def unicode_quote(value):
     """A unicode aware version of urllib.quote"""
@@ -85 +86 @@
     return urlencode([(k, isinstance(v, unicode) and v.encode('utf-8') or v)
                       for k, v in params])
 
-def to_utf8(text, charset='iso-8859-15'):
+def to_utf8(text, charset=IDENTITY_CHARSET):
     """Convert a string to UTF-8, assuming the encoding is either UTF-8, ISO
     Latin-1, or as specified by the optional `charset` parameter.
 
@@ -101 +102 @@
             u = unicode(text, charset)
         except UnicodeError:
             # This should always work
-            u = unicode(text, 'iso-8859-15')
+            u = unicode(text, IDENTITY_CHARSET)
         return u.encode('utf-8')
 
 

templates/browser.cs

@@ -111 +111 @@
   </table><?cs
  /if ?><?cs
  
- if:!browser.is_dir ?>
-  <div id="preview"><?cs
-   if:file.preview ?><?cs
-    var:file.preview ?><?cs
-   elif:file.max_file_size_reached ?>
-    <strong>HTML preview not available</strong>, since the file size exceeds
-    <?cs var:file.max_file_size ?> bytes. Try <a href="<?cs
-    var:file.raw_href ?>">downloading</a> the file instead.<?cs
-   else ?><strong>HTML preview not available</strong>. To view, <a href="<?cs
-    var:file.raw_href ?>">download</a> the file.<?cs
-   /if ?>
-  </div><?cs
+ if:!browser.is_dir ?><?cs
+  call:html_preview(file) ?><?cs
  /if ?>
 
  <div id="help">

templates/macros.cs

@@ -194 +194 @@
 
 def:plural(base, count) ?><?cs
  var:base ?><?cs if:count != 1 ?>s<?cs /if ?><?cs
+/def ?><?cs
+
+def:html_preview(base) ?>
+ <div id="preview"><?cs
+  if:base.preview ?>
+   <?cs var:base.preview ?><?cs
+  elif:base.max_file_size_reached ?>
+   <strong>HTML preview not available</strong>, since the file size exceeds
+   <?cs var:base.max_file_size  ?> bytes. You may <a href="<?cs
+     var:base.raw_href ?>">download the file</a> instead.<?cs
+  else ?>
+   <strong>HTML preview not available</strong>. To view the file,
+   <a href="<?cs var:base.raw_href ?>">download the file</a>.<?cs
+  /if ?>
+ </div><?cs
 /def ?>

templates/attachment.cs

@@ -66 +66 @@
    </th></tr><tr>
    <td class="message"><?cs var:attachment.description ?></td>
   </tr>
- </tbody></table>
- <div id="preview"><?cs
-  if:attachment.preview ?>
-   <?cs var:attachment.preview ?><?cs
-  elif:attachment.max_file_size_reached ?>
-   <strong>HTML preview not available</strong>, since the file size exceeds
-   <?cs var:attachment.max_file_size  ?> bytes. You may <a href="<?cs
-     var:attachment.raw_href ?>">download the file</a> instead.<?cs
-  else ?>
-   <strong>HTML preview not available</strong>. To view the file,
-   <a href="<?cs var:attachment.raw_href ?>">download the file</a>.<?cs
-  /if ?>
- </div>
+ </tbody></table><?cs
+ call:html_preview(attachment) ?>
  <?cs if:attachment.can_delete ?><div class="buttons">
   <form method="get" action=""><div id="delete">
    <input type="hidden" name="action" value="delete" />