Context Navigation

Back to Ticket #3332

Ticket #3332: mimeview_api_refactoring-r3612.2.diff

File mimeview_api_refactoring-r3612.2.diff, 119.1 KB (added by cboos, 5 years ago)
Slightly improved version

trac/attachment.py

 from trac.config import BoolOption, IntOption
 from trac.core import *
 from trac.env import IEnvironmentSetupParticipant
 from trac.mimeview import *
+from trac.mimeview.api import Mimeview, MimeType, FileMimeContent, TEXT_PLAIN
 from trac.util import get_reporter_id, create_unique_file
 from trac.util.datefmt import format_datetime, pretty_timedelta
 from trac.util.html import Markup, html
 …
     def parent_href(self, req):
         return req.href(self.parent_type, self.parent_id)
     def _get_title(self):
+    def _get_title(self): # TODO: should extend to other `parent_type`s
         return '%s%s: %s' % (self.parent_type == 'ticket' and '#' or '',
                              self.parent_id, self.filename)
     title = property(_get_title)
 …
     select = classmethod(select)
     delete_all = classmethod(delete_all)
     def open(self):
+    def open(self): # deprecate, use FileMimeContent if necessary
         self.env.log.debug('Trying to open attachment at %s', self.path)
         try:
             fd = open(self.path, 'rb')
 …
                                  'author': get_reporter_id(req)}
     def _render_view(self, req, attachment):
+        # FIXME: perm_map should extend to other `parent_type`s
         perm_map = {'ticket': 'TICKET_VIEW', 'wiki': 'WIKI_VIEW'}
         req.perm.assert_permission(perm_map[attachment.parent_type])
 …
         req.hdf['attachment'] = attachment_to_hdf(self.env, req, None,
                                                   attachment)
         # Override the 'oneliner'
         req.hdf['attachment.description'] = wiki_to_html(attachment.description,
                                                          self.env, req)
+        req.hdf['attachment.description'] = wiki_to_html(
+            attachment.description, self.env, req)
         perm_map = {'ticket': 'TICKET_ADMIN', 'wiki': 'WIKI_DELETE'}
         if req.perm.has_permission(perm_map[attachment.parent_type]):
             req.hdf['attachment.can_delete'] = 1
         fd = attachment.open()
         try:
             mimeview = Mimeview(self.env)
+        content = FileMimeContent(self.env, attachment.path,
+                                  url=attachment.href(req, format='raw'),
+                                  kind='Attachment')
+            # MIME type detection
+            str_data = fd.read(1000)
+            fd.seek(0)
+            binary = is_binary(str_data)
+            mime_type = mimeview.get_mimetype(attachment.filename, str_data)
+        # Eventually send the file directly
+        format = req.args.get('format')
+        if format in ('raw', 'txt'):
+            disposition = 'inline'
+            if self.render_unsafe_content:
+                if content.type.is_unknown or \
+                       (format == 'txt' and not content.is_binary):
+                    # Force the content to be displayed as text
+                    content.type = TEXT_PLAIN ### SHOULD BE ENOUGH
+                    content.type = MimeType('text/plain', content.encoding)
+            elif not content.is_binary:
+                # Force browser to download HTML/SVG/etc pages that may
+                # contain malicious code enabling XSS aattacks
+                disposition = 'attachment'
+            content.send(req, disposition)
+            # Eventually send the file directly
+            format = req.args.get('format')
+            if format in ('raw', 'txt'):
+                if not self.render_unsafe_content and not binary:
+                    # Force browser to download HTML/SVG/etc pages that may
+                    # contain malicious code enabling XSS attacks
+                    req.send_header('Content-Disposition', 'attachment;' +
+                                    'filename=' + attachment.filename)
+                if not mime_type or (self.render_unsafe_content and \
+                                     not binary and format == 'txt'):
+                    mime_type = 'text/plain'
+                if 'charset=' not in mime_type:
+                    charset = mimeview.get_charset(str_data, mime_type)
+                    mime_type = mime_type + '; charset=' + charset
+                req.send_file(attachment.path, mime_type)
+        # add ''Plain Text'' alternate link if needed
+        if self.render_unsafe_content and not \
+               (content.is_binary or
+                content.type.is_unknown or
+                TEXT_PLAIN.match(content.type)):
+            add_link(req, 'alternate', attachment.href(req, format='txt'),
+                     TEXT_PLAIN.name, TEXT_PLAIN.mimetype)
+            # add ''Plain Text'' alternate link if needed
+            if self.render_unsafe_content and not binary and \
+               mime_type and not mime_type.startswith('text/plain'):
+                plaintext_href = attachment.href(req, format='txt')
+                add_link(req, 'alternate', plaintext_href, 'Plain Text',
+                         mime_type)
+        # add ''Original Format'' alternate link (always)
+        add_link(req, 'alternate', content.url,
+                 'Original Format', content.type.mimetype)
+            # add ''Original Format'' alternate link (always)
+            raw_href = attachment.href(req, format='raw')
+            add_link(req, 'alternate', raw_href, 'Original Format', mime_type)
+        req.hdf['attachment'] = Mimeview(self.env).preview_to_hdf(
+            req, content, annotations=['lineno'])
-            self.log.debug("Rendering preview of file %s with mime-type %s"
-                           % (attachment.filename, mime_type))
-            req.hdf['attachment'] = mimeview.preview_to_hdf(
-                req, fd, os.fstat(fd.fileno()).st_size, mime_type,
-                attachment.filename, raw_href, annotations=['lineno'])
-        finally:
-            fd.close()
     def _render_list(self, req, p_type, p_id):
         self._parent_to_hdf(req, p_type, p_id)
         req.hdf['attachment'] = {

trac/mimeview/rst.py

 import re
 from trac.core import *
+from trac.mimeview.api import IHTMLPreviewRenderer, content_to_unicode
+from trac.mimeview.api import IContentConverter, MimeType, TEXT_HTML, \
+                              Conversion, StringMimeContent
 from trac.util.html import Element
 from trac.web.href import Href
 from trac.wiki.formatter import WikiProcessor
 …
     """
     Renders plain text in reStructuredText format as HTML.
     """
     implements(IHTMLPreviewRenderer)
+    implements(IContentConverter)
+    def get_quality_ratio(self, mimetype):
+        if mimetype == 'text/x-rst':
+            return 8
+        return 0
+    def get_supported_conversions(self, input):
+        if input.match(MimeType('text/x-rst')):
+            yield Conversion('html', 8, TEXT_HTML)
+    def render(self, req, mimetype, content, filename=None, rev=None):
+    def convert_content(self, context, conversion, content):
+        req = context.req
         try:
             from docutils import nodes
             from docutils.core import publish_parts
 …
         _inliner = rst.states.Inliner()
         _parser = rst.Parser(inliner=_inliner)
         content = content_to_unicode(self.env, content, mimetype)
         parts = publish_parts(content, writer_name='html', parser=_parser,
+        parts = publish_parts(unicode(content),
+                              writer_name='html', parser=_parser,
                               settings_overrides={'halt_level': 6,
                                                   'file_insertion_enabled': 0,
                                                   'raw_enabled': 0})
+        return parts['html_body']
+        return StringMimeContent(self.env, parts['html_body'],
+                                 content.type, filename=content.filename,
+                                 annotable=False)

trac/mimeview/api.py

 #         Christian Boos <cboos@neuf.fr>
 """
+The `trac.mimeview` module centralize the intelligence related to
+file metadata, principally concerning the `type` (MIME type) of the content
+and, if relevant, concerning the text encoding (charset) used by the content.
+The `trac.mimeview` module centralizes the intelligence about typed content.
+There are primarily two approaches for getting the MIME type of a given file:
+ * taking advantage of existing conventions for the file name
+ * examining the file content and applying various heuristics
+Originally, this was about file metadata, principally  its MIME type and
+eventually the text encoding (charset) used by that content.
+The module also knows how to convert the file content from one type
+to another type.
+Now, this has evolved into managing any kind of typed content,
+and deals also with converting a content from one type to another.
+A common situation which is now handled part of the general case
+is the conversion of any kind of content to a text/html representation.
+In some cases, only the `url` pointing to the file's content is actually
+needed, that's why we avoid to read the file's content when it's not needed.
+In order to keep the API of conversion interface IContentConverter simple,
+we introduced a few classes, each encapsulating a part of the knowledge
+related to the content and the conversion:
+The actual `content` to be converted might be a `unicode` object,
+but it can also be the raw byte string (`str`) object, or simply
+an object that can be `read()`.
+ * the `ContentType` is used to describe the type of a content.
+   This is an abstract superclass for:
+   - `MimeType`, used for storing the mime type string, the charset,
+     and eventually the name and the commonly used file extension
+     for that type
+   - `ObjectType`, used when wrapping an arbitrary Python object
+ * the `AbstractContent`, which wraps access to the actual content,
+   and provides a few generic methods, among which `convert()` is
+   the most important.
+   - the `ObjectContent`, when the content is an arbitrary Python object
+   - the `MimeContent` abstract class, which offers an uniform API to
+     access the data over a wide range of containers:
+      * the `StringMimeContent`, for handling string content
+        (like `str`, `unicode` or `Markup` objects)
+      * the `StructuredMimeContent`, for handling structured content
+        (like `Element` objects)
+      * the `LineIteratorMimeContent`, for handling line-oriented string
+        content (like string iterators or string arrays)
+      * the `FileMimeContent`, for handling file content
+      * etc. (as an example, the repository layer defines a `NodeMimeContent`)
+     That API provides different ways to get access to the data:
+      * chunks(), an iterator for reading chunks of raw content
+      * lines(), an iterator for reading lines from text content
+      * markup(), when it makes sense to interpret the content as markup
+      * `__unicode()__`, for converting the content to an `unicode` object
+      * encode(), for converting the content to a `str` object. Note that
+        we don't use `__str__` for that on purpose, as we need to be able
+        to specify the charset.
+      * size(), for getting a hint about the content size without actually
+        reading it.
+ * the `Conversion` class, which is used to specify how a given
+   `IContentConverter` component will perform a content conversion.
 """
+import os
 import re
 from StringIO import StringIO
 from trac.config import IntOption, ListOption, Option
 from trac.core import *
 from trac.util import sorted
+from trac.util.text import to_utf8, to_unicode
+from trac.util.datefmt import http_date
+from trac.util.text import to_utf8, to_unicode, IDENTITY_CHARSET
 from trac.util.html import escape, Markup, Fragment, html
+__all__ = ['get_mimetype', 'is_binary', 'detect_unicode', 'Mimeview',
+           'content_to_unicode']
+__all__ = ['get_mimetype', 'is_binary', 'detect_unicode',
+           'IContentConverter', 'IHTMLPreviewAnnotator',
+           'IHTMLPreviewRenderer', # deprecated
+           'ObjectType', 'MimeType',
+           'ObjectContent',
+           'MimeContent', 'FileMimeContent', 'StringMimeContent',
+           'StructuredMimeContent', 'LineIteratorMimeContent',
+           'Mimeview', 'Conversion',
+           'TEXT_PLAIN', 'TEXT_HTML', 'TEXT_CSV', 'TEXT_TSV',
+           'APPLICATION_RSS_XML', 'APPLICATION_OCTET_STREAM']
 # Some common MIME types and their associated keywords and/or file extensions
+APPLICATION_OCTET_STREAM_STR = 'application/octet-stream'
 KNOWN_MIME_TYPES = {
     'application/pdf':        ['pdf'],
     'application/postscript': ['ps'],
 …
     for e in exts:
         MIME_MAP[e] = t
-# Simple builtin autodetection from the content using a regexp
-MODE_RE = re.compile(
-    r"#!(?:[/\w.-_]+/)?(\w+)|"               # look for shebang
-    r"-\*-\s*(?:mode:\s*)?([\w+-]+)\s*-\*-|" # look for Emacs' -*- mode -*-
-    r"vim:.*?syntax=(\w+)"                   # look for VIM's syntax=<n>
+    )
+def get_mimetype(filename, content=None, mime_map=MIME_MAP):
+    """Guess the most probable MIME type of a file with the given name.
+# -- get_mimetype, is_binary, detect_unicode: a few functions for dealing
+#    with MIME types, binary and text content in a simple way
+def get_mimetype_from_filename(filename, mime_map=MIME_MAP):
+    """Guess the most probable MIME type of file with the given `filename`.
     `filename` is either a filename (the lookup will then use the suffix)
     or some arbitrary keyword.
+    `content` is either a `str` or an `unicode` string.
+    `mime_map` maps keywords to MIME types.
+    Return the MIME type as a string, or `None` if nothing was detected.
     """
     suffix = filename.split('.')[-1]
     if suffix in mime_map:
 …
             mimetype = mimetypes.guess_type(filename)[0]
         except:
             pass
-        if not mimetype and content:
-            match = re.search(MODE_RE, content[:1000])
-            if match:
-                mode = match.group(1) or match.group(3) or \
-                    match.group(2).lower()
-                if mode in mime_map:
-                    # 3) mimetype from the content, using the `MODE_RE`
-                    return mime_map[mode]
-            else:
-                if is_binary(content):
-                    # 4) mimetype from the content, using`is_binary`
-                    return 'application/octet-stream'
         return mimetype
+# Simple builtin autodetection from the content using a regexp
+MODE_RE = re.compile(
+    # look for /usr/bin/prog1 or /usr/bin/env prog2
+    r"#!(?:[/\w.-_]+/)?(?P<prog1>\w+)(?:\s+(?P<prog2>\w+))?|"
+    # look for Emacs' -*- mode -*-
+    r"-\*-\s*(?:mode:\s*)?(?P<emacsmode>[\w+-]+)\s*-\*-|"
+    # look for VIM's syntax=<n>
+    r"vim:.*?syntax=(?P<vimsyntax>\w+)"
+    )
+def get_mimetype_from_content(content, mime_map=MIME_MAP):
+    """Guess the most probable MIME type of file with the given `filename`.
+    `content` is either a `str` or an `unicode` string
+    Return the MIME type as a string, or `None` if not detected.
+    """
+    match = re.search(MODE_RE, content[:1000])
+    if match:
+        mode = match.group('prog1')
+        if mode and mode == 'env':
+            mode = match.group('prog2')
+        if not mode:
+            mode = match.group('vimsyntax') or \
+                   match.group('emacsmode').lower()
+        if mode in mime_map:
+            # 3) mimetype from the content, using the `MODE_RE`
+            return mime_map[mode]
+    else:
+        if is_binary(content):
+            # 4) mimetype from the content, using `is_binary`
+            return APPLICATION_OCTET_STREAM_STR
+def get_mimetype(filename, content=None, mime_map=MIME_MAP):
+    """Auto-detect MIME type either from the `filename` or from the `content`.
+    """
+    mimetype = get_mimetype_from_filename(filename, mime_map)
+    if not mimetype and content:
+        mimetype = get_mimetype_from_content(content, mime_map)
+    return mimetype
 def is_binary(data):
     """Detect binary content by checking the first thousand bytes for zeroes.
 …
     return '\0' in data[:1000]
 def detect_unicode(data):
     """Detect different unicode charsets by looking for BOMs (Byte Order Marks).
+    """Detect different unicode charsets by looking for Byte Order Marks.
     Operate obviously only on `str` objects.
     """
 …
     else:
         return None
-def content_to_unicode(env, content, mimetype):
-    """Retrieve an `unicode` object from a `content` to be previewed"""
-    mimeview = Mimeview(env)
-    if hasattr(content, 'read'):
-        content = content.read(mimeview.max_preview_size)
-    return mimeview.to_unicode(content, mimetype)
+# -- ContentType and subclasses
+class IHTMLPreviewRenderer(Interface):
+    """Extension point interface for components that add HTML renderers of
+    specific content types to the `Mimeview` component.
+class ContentType(object):
+    """Abstract representation of the "type" of some content."""
+    (Deprecated)
+    def match(self, other):
+        """Compare this instance with another `ContentType` instance.
+        Return True if `other` can be say to ''match'' our type.
+        """
+        raise NotImplementedError
+    is_binary = property(lambda x: x._is_binary())
+    mimetype = property(lambda x: x._get_mimetype(),
+                        doc="MIME Type string (without charset information)")
+    name = 'unknown'
+class ObjectType(ContentType):
+    """Represent the "type" of a content by using a Python class.
+    The corresponding content is then expected to be an `ObjectContent`,
+    wrapping a Python instance of that class.
     """
+    # implementing classes should set this property to True if they
+    # support text content where Trac should expand tabs into spaces
+    expand_tabs = False
+    def __init__(self, obj):
+        self._class = isinstance(obj, type) and obj or obj.__class__
+        self._mimetype = self._charset = None
+        self.name = self._class.__name__
+    def get_quality_ratio(mimetype):
+        """Return the level of support this renderer provides for the `content`
+        of the specified MIME type. The return value must be a number between
+and 9, where 0 means no support and 9 means "perfect" support.
+    def __repr__(self):
+        return "<ObjectType %s>" % self.name
+    # Reimplemented methods
+    def _is_binary(self): return True
+    def _get_mimetype(self): return None
+    # or return 'application/x-python-'+self.name ?
+    def match(self, other, regexp=False):
+        other_class = isinstance(other, type) and other or \
+                      isinstance(other, ObjectType) and other._class or \
+                      other.__class__
+        return other_class == self._class
+class MimeType(ContentType):
+    """Represent the "type" of a content by using a MIME type string.
+    If the type is not binary (i.e. is some kind of text), the `charset`
+    information can also be given, when creating the instance.
+    A MIME type has a `name` and has an `extension` that can be used
+    for storing the converted data in a file.
+    All the properties of this class are read-only.
+    """
+    def __init__(self, mimetype, charset=None, name=None, extension=None,
+                 env=None):
+        """Create a MimeType based on a `mimetype` string.
+        That string can eventually contain a `charset=...`, which will be
+        retained as the charset for this instance, unless there's an
+        explicit `charset` parameter given.
+        Another possibility is to give a pattern object for the `mimetype`
+        argument, in which case this instance can be used to do pattern
+        matching when using `match()`.
+        When optional `env` parameter is given, additional knowledge is
+        used for determining whether this type is binary or not.
         """
+        self.env = env
+        self._mimetype = mimetype
+        # determine charset
+        self._charset = charset
+        if not self._charset and isinstance(self._mimetype, basestring):
+            sep_idx = mimetype.find(';')
+            if sep_idx >= 0:
+                self._mimetype = mimetype[:sep_idx].strip()
+                charset_idx = mimetype.find('charset=', sep_idx)
+                if charset_idx >= 0:
+                    self._charset = mimetype[charset_idx+8:].strip()
+        if extension and not extension.startswith('.'):
+            extension = '.' + extension
+        self._extension = extension
+        self._name = name
     def render(req, mimetype, content, filename=None, url=None):
         """Render an XHTML preview of the raw `content`.
+    def __repr__(self):
+        return '<MimeType "%s">' % self.mimetype_charset
+        The `content` might be:
+         * a `str` object
+         * an `unicode` string
+         * any object with a `read` method, returning one of the above
+    def _get_extension(self):
+        if self._extension is None:
+            self._extension = KNOWN_MIME_TYPES.get(self.mimetype, [''])[0]
+            if not self._extension:
+                detail = self.mimetype.split('/', 1)[1]
+                if detail.startswith('x-'):
+                    self._extension = detail[2:]
+            self._extension = '.' + self._extension
+        return self._extension
+        It is assumed that the content will correspond to the given `mimetype`.
+    def _get_mimetype_charset(self):
+        """Combine in a single string the MIME type and charset information."""
+        if self._mimetype and self._charset:
+            return '%s; charset=%s' % (self.mimetype, self.charset)
+        else:
+            return self.mimetype ### or default to utf-8 if not binary?
+    charset = property(lambda x: x._charset,
+                       doc="Charset information if available")
+        Besides the `content` value, the same content may eventually
+        be available through the `filename` or `url` parameters.
+        This is useful for renderers that embed objects, using <object> or
+        <img> instead of including the content inline.
+    name = property(lambda x: x._name or x._get_extension())
+    extension = property(lambda x: x._get_extension())
+    mimetype_charset = property(lambda x: x._get_mimetype_charset(),
+                                doc="MIME Type plus charset information")
+    is_unknown = property(lambda x: x._mimetype is None)
+    # Reimplemented methods
+    def _is_binary(self):
+        return (self._mimetype == APPLICATION_OCTET_STREAM_STR or
+                (self.env and self.mimetype in
+                 Mimeview(self.env).treat_as_binary) or
+                self.is_unknown)
+    def _get_mimetype(self):
+        return self._mimetype or APPLICATION_OCTET_STREAM_STR
+    def match(self, other):
+        if isinstance(other, MimeType):
+            other = other.mimetype
+        if not isinstance(other, basestring):
+            return False
+        if hasattr(self.mimetype, 'match'):
+            return self.mimetype.match(other)
+        else:
+            return self.mimetype == other
+# A few commonly used MIME types:
+TEXT_PLAIN = MimeType('text/plain', 'utf-8', 'Plain Text', 'txt')
+TEXT_HTML = MimeType('text/html', 'utf-8', 'HTML', 'html')
+TEXT_CSV = MimeType('text/csv', 'utf-8', 'Comma-delimited Text', 'csv')
+TEXT_TSV = MimeType('text/tab-separated-values', 'utf-8',
+                    'Tab-delimited Text', 'tsv')
+APPLICATION_RSS_XML = MimeType('application/rss+xml', 'utf-8',
+                               'RSS Feed', 'xml')
+APPLICATION_OCTET_STREAM = MimeType(APPLICATION_OCTET_STREAM_STR,
+                                    IDENTITY_CHARSET,
+                                    'Undefined (binary)', 'bin')
+# -- AbstractContent and subclasses
+class AbstractContent(object):
+    """An abstract content, with an associated content type.
+    There can also be a `filename` eventually associated to this content.
+    """
+    def __init__(self, env, content, type, filename=None):
+        self.env = env
+        self.content = content
+        self._type = type
+        self.filename = filename
+    def _is_binary(self): raise NotImplementedError
+    def _get_type(self): return self._type
+    def _set_type(self, type): self._type = type
+    # Properties
+    type = property(fget=lambda x: x._get_type(),
+                    fset=lambda x, y: x._set_type(y), doc=
+                    """The corresponding `ContentType` for this content.""")
+    is_binary = property(lambda x: x._is_binary(), doc=
+                         "True if content should be considered to be binary")
+    # Methods
+    def basename(self):
+        """Return a possible file basename for this content."""
+        return self.filename and os.path.splitext(self.filename)[0] or ''
+    def convert(self, req, format=None, typespec=None, context=None):
+        """Convert this content to a `MimeContent`.
+        The converter can be selected by either:
+        Can return the generated XHTML text as a single string or as an
+        iterable that yields strings. In the latter case, the list will
+        be considered to correspond to lines of text in the original content.
+         - specifying the desired output `format`, which must match the
+           `format` of one of the `Conversion` object returned by
+           `Mimeview.get_conversions(self.type)`.
+           There's usually only one such converter.
+         - giving a `typespec`, which is a `MimeType` object.
+           All matching converters are tried in turn, until one succeeds.
+           This is quite flexible, as one can use a regexp match ### FIXME
+        Either succeeds or raise `NoConversion` error.
         """
+        if not typespec and not format:
+            self.env.log.error("Convert called but no conversion specified.")
+            return self
+        if isinstance(format, MimeType): # or signal a programming error?
+            typespec = format
+            format = None
+        # Get all available conversions for our `type`, and keep those
+        # which are matching either the `format` or the `typespec`.
+        mimeview = Mimeview(self.env)
+        candidates = []
+        for conversion, converter in mimeview.get_converters(self.type):
+            if conversion.format == format or \
+                   (typespec and typespec.match(conversion.output)):
+                candidates.append((conversion, converter))
+        if not candidates:
+            raise NoConversion('No available MIME conversions',
+                               self.type, typespec, format)
+        tab_expanded = None # we don't want to expand tabs more than once.
+        # First candidate which converts successfully wins.
+        self.env.log.debug('Converting %s' % repr(self))
+        if not context:
+            context = IContentConverter.ToplevelContext(req)
+        content = self
+        for conversion, converter in candidates:
+            self.env.log.debug('Trying converter %s using %s' %
+                               (repr(converter), repr(conversion)))
+            if conversion.expand_tabs and not tab_expanded:
+                expanded = unicode(content).expandtabs(mimeview.tab_width)
+                content = StringMimeContent(self.env, expanded, content.type,
+                                            filename=content.filename)
+                self.env.log.debug('(tab expansion performed)')
+            try:
+                result = converter.convert_content(context, conversion, content)
+                if result is not None:
+                    return result
+            except Exception, e:
+                self.env.log.warning('MIME conversion using %s failed (%s)'
+                                      % (repr(converter), e), exc_info=True)
+        raise NoConversion('No MIME conversions succeeded',
+                           content.type, typespec, format)
+class ObjectContent(AbstractContent):
+    """Content is a Python a object, and its type is an `ObjectType`.
+    Only supports the bare minimum of the `AbstractContent` methods.
+    `filename` in this case will be the suggested base name to be used
+    when creating a filename for a converted content.
+    """
+    def __init__(self, env, obj, filename=None):
+        AbstractContent.__init__(self, env, obj, ObjectType(obj),
+                                 filename=filename)
+    # Reimplemented property readers
+    def _is_binary(self):
+        return True
+    def basename(self):
+        return AbstractContent.basename(self) or self.type.name
+class MimeContent(AbstractContent):
+    """Content associated with a MIME type.
+    Such an object has ways to auto-detect both the MIME Type and
+    the `encoding` of its content.
+    That `encoding` is more reliable than the `type.charset` information.
+    There are additional consistency checks that are performed, and it
+    can be `None` if the content is an `unicode` object.
+    The content itself can be accessed in various ways: through
+    the iterator protocol, the len() and unicode() operators...
+    This class may eventually be instantiated directly when we don't
+    care about the actual content but only about the MimeType detection.
+    """
+    EXCERPT_LEN = 1000
+    disposition = 'inline'
+    last_modified = None # means unknown, otherwise is a time value
+    def __init__(self, env, content=None, mimetype=None,
+                 filename=None, url=None, annotable=True):
+        """
+        `mimetype` can be specified as a `MimeType` object or as string,
+        which will then be used as a hint for the content.
+        If the mimetype is not specified or is "application/octet-stream",
+        then it will be auto-detected when needed.
+        In case auto-detection fails, the MIME type will be set
+        to APPLICATION_OCTET_STREAM.
+        The `filename` is the corresponding file name for that content,
+        or if there's none, a possible basename for that content.
+        The `url` is a link for retrieving the raw content directly
+        from the server. This can be useful for converters that can
+        provide links to objects instead of having to expand the content.
+        """
+        if isinstance(mimetype, basestring):
+            mimetype = MimeType(mimetype, env=env)
+        AbstractContent.__init__(self, env, content, mimetype, filename)
+        self.url = url
+        self.annotable = annotable
+        self._binary = None
+        self._encoding = False # i.e. not yet determined
+        self._excerpt = None
+    def __repr__(self):
+        return '<%s %s "%s">' % (self.__class__.__name__, self._type,
+                                 self.filename or self.url)
+    # Reimplemented property accessors
+    def _is_binary(self):
+        """An heuristic for guessing whether the content is binary or not.
+        This will eventually fetch an `excerpt` of the content.
+        """
+        if self._binary is None:
+            self._binary = self.type.is_binary
+            if not self._binary: # double-check using the content
+                self._binary = is_binary(self.excerpt) or \
+                               (self.type.mimetype in \
+                                Mimeview(self.env).treat_as_binary)
+        return self._binary
+    def _get_type(self):
+        """Get or determine the MimeType corresponding to this content.
+        An `excerpt` of the content will be examined if needed.
+        """
+        if self._type is None: # not set
+            mimetype = None
+            if self.filename:
+                mimemap = Mimeview(self.env).mimemap
+                mimetype = get_mimetype_from_filename(self.filename, mimemap)
+            if not mimetype:
+                mimetype = get_mimetype_from_content(self.excerpt, mimemap)
+            if not mimetype:
+                pass # TODO 0.11: go through IMimeTypeDetectors
+            self._type = MimeType(mimetype, env=self.env)
+        return self._type
+    def _set_type(self, type):
+        """Simply replace the existing `type` by the given `MimeType` object.
+        If `None` is given, this will force auto-detection the next time
+        `type` will be accessed.
+        Can be used for in-place conversion (e.g. ''any'' to text/plain).
+        """
+        self._type = type
+    # Helper methods
+    def _get_encoding(self):
+        """Get or determine the current encoding of that `content`.
+        The encoding will be determined using this order:
+         * from the charset information present in the mimetype information
+         * auto-detection of the charset from the `content`
+         * if nothing else worked, use the configured `default_charset`
+        If the `content` happens to be a genuine `unicode` object, then
+        this returns `None` (and this, whatever the `type.charset` says).
+        ### XXX
+        If the `content` is binary, then the encoding will be the identity
+        charset (ISO Latin 1).
+        """
+        if self._encoding is False:
+            if isinstance(self.excerpt, unicode):
+                self._encoding = None
+            else:
+                charset = self.type.charset
+                if charset:
+                    # we have external knowledge about the charset,
+                    # this always override what we think it could be...
+                    self._encoding = charset
+                elif isinstance(self.excerpt, str):
+                    utf_encoding = detect_unicode(self.excerpt)
+                    if utf_encoding is not None:
+                        self._encoding = utf_encoding
+                    elif self.is_binary:
+                        self._encoding = IDENTITY_CHARSET
+                if self._encoding is False:
+                    pass # TODO 0.11: go through ICharsetDetectors here
+                if self._encoding is False:
+                    self._encoding = Mimeview(self.env).default_charset
+        return self._encoding
+    def _get_excerpt(self):
+        """Get once the initial content of the file."""
+        if self._excerpt is None:
+            self._excerpt = self._retrieve_excerpt()
+        return self._excerpt
+    # Public API Methods
+    def __unicode__(self):
+        """Return the `unicode` object corresponding to the content.
+        Note: this should do the right thing if the content is already
+        `unicode`, i.e. it should be directly returned.
+        """
+        return to_unicode(self.read(), self.encoding)
+    def encode(self, charset=None):
+        """Return a `str`, corresponding to the `charset` encoded content.
+        If `charset` is not specified or is the same as the original encoding,
+        the raw content is directly returned.
+        """
+        if not charset or self.encoding == charset:
+            content = self.read()
+            if isinstance(content, unicode):
+                return content.encode('utf-8')
+            else:
+                return content
+        else:
+            return unicode(self).encode(charset)
+    def read(self):
+        """Should return a `basestring` containing the whole content.
+        This is either a `str` or an `unicode` object, depending on what's
+        most appropriate for the wrapped content.
+        A more specialized way to access the content is often more adequate
+        or efficient than this one and is of course '''absolutely''' required
+        when `chunck()` is implemented with `read()`...
+        """
+        return ''.join(self.chunks())
+    def chunks(self):
+        """Iterator on chunks of raw content or None if there's no raw content.
+        """
+        return None
+    def lines(self):
+        """Iterator on lines of text content or None if there are no lines."""
+        return None
+    def markup(self):
+        """Return content as Markup."""
+        return None
+    def size(self):
+        """Size of the raw content, in bytes or `-1` if unknown beforehand."""
+        return -1
+    def send(self, req, disposition=None):
+        """Send this content to the requester.
+        When specified, the `disposition` argument will override the content's
+        own disposition and will be used for the 'Content-Disposition' header.
+        If the 'filename=' info is not given, it will be automatically added
+        based on the already available information.
+        `send()` Never returns, as it raises `RequestDone` upon completion.
+        """
+        from trac.web import RequestDone
+        req.send_response(200)
+        req.send_header('Content-Type', self.type.mimetype_charset)
+        if self.size >= 0: # size is known
+            req.send_header('Content-Length', self.size())
+        if self.last_modified:
+            req.send_header('Last-Modified', http_date(self.last_modified))
+        if not disposition:
+            disposition = self.disposition
+        if self.filename and 'filename=' not in disposition:
+            extension = os.path.splitext(self.filename)[1]
+            if not extension:
+                extension = self.type.extension
+            disposition += '; filename=' + self.basename() + extension
+        req.send_header('Content-Disposition', disposition)
+        req.end_headers()
+        if self.is_binary:
+            for chunk in self.chunks():
+                req.write(chunk)
+        else:
+            req.write(self.encode())
+        raise RequestDone
+    # Properties
+    encoding = property(lambda x: x._get_encoding())
+    excerpt = property(lambda x: x._get_excerpt())
+    # Methods that needs to be reimplemented by subclasses
+    def _retrieve_excerpt(self):
+        """Extracts an excerpts of the initial content of the file.
+        '''Note: Be sure to ''not'' use `self.encoding` while doing that,
+        as precisely an excerpt of the content is taken in that method.'''
+        """
+        raise NotImplementedError
+class StringMimeContent(MimeContent):
+    """MIME-typed content wrapper for a basestring."""
+    # Reimplemented MimeContent methods
+    def read(self):
+        return self.content
+    def chunks(self):
+        """Iterator on chunks of content."""
+        buf = StringIO(self.read()) # can be used as we reimplement it
+        chunk = buf.read(1000)
+        while chunk:
+            yield chunk
+            chunk = buf.read(1000)
+    def lines(self):
+        """Iterator on lines."""
+        for line in self.read().splitlines(): # don't keep eols?
+            yield line
+    def markup(self):
+        return Markup(self.read())
+    def size(self):
+        """Length of the content, in characters."""
+        return len(self.read())
+    def _retrieve_excerpt(self):
+        return self.read()[:self.EXCERPT_LEN]
+class LineIteratorMimeContent(StringMimeContent):
+    """MIME-typed content wrapper for an iterable of lines."""
+    # Reimplemented methods
+    def __unicode__(self): return u'\n'.join(self.content)
+    def read(self): return unicode(self)
+    def lines(self):
+        yield self.excerpt
+        for line in self.content:
+            yield line
+    def _retrieve_excerpt(self):
+        for first_line in self.content:
+            return first_line
+class StructuredMimeContent(StringMimeContent):
+    """MIME-typed content wrapper for an Element.
+    Nearly identical to the `StringMimeContent`, except the content
+    property will access to actual Element for structured processing.
+    """
+    # Reimplemented methods
+    def read(self): return unicode(self.content)
+    def lines(self): return None
+    def markup(self): return self.content
+class FileMimeContent(MimeContent):
+    """MIME-typed content wrapper for a file."""
+    def __init__(self, env, path, mimetype=None, url=None, kind='File'):
+        MimeContent.__init__(self, env, path, mimetype,
+                             os.path.basename(path), url)
+        self._fd = None
+        self._path = path
+        self._kind = kind
+    def __del__(self):
+        if self._fd:
+            self._fd.close()
+    # Reimplemented methods
+    def chunks(self):
+        """Iterate on chunks of raw content."""
+        chunk = self.excerpt
+        while chunk:
+            yield chunk
+            chunk = self._fd.read(self.EXCERPT_LEN)
+        self._fd.seek(self.EXCERPT_LEN) # make it possible to iter again
+    def size(self):
+        """Length of the raw content, in bytes."""
+        if self._fd:
+            stat = os.fstat(self._fd.fileno())
+        else:
+            try:
+                stat = os.stat(self._path)
+            except OSError:
+                raise TracError('%s "%s" not found' % (self._kind,
+                                                       self.filename))
+        return stat.st_size
+    def send(self, req, disposition=None):
+        """Directly send the file."""
+        ### self.encoding should be used when charset not available
+        ### maybe do that in a self.mimetype_charset method
+        req.send_file(self._path, self.type.mimetype_charset)
+    def _retrieve_excerpt(self):
+        if not self._fd:
+            try:
+                self._fd = open(self._path)
+            except IOError:
+                raise TracError('%s "%s" not found' % (self._kind,
+                                                       self.filename))
+        return self._fd.read(self.EXCERPT_LEN)
+# -- Conversion class and related
+class NoConversion(TracError):
+    def __init__(self, msg, from_, output, key):
+        TracError.__init__(self, '%s while converting from %s to %s' %
+                           (msg, repr(from_), output and repr(output) or key),
+                           'MIME Content Conversion')
+class Conversion(object):
+    """Used to specify how a Converter will perform a conversion.
+    Each conversion is identified by a `format`, which must be unique for
+    a given input `type` in `IContentConverter.get_supported_conversions()`.
+    A conversion object can tell:
+     - what will be the precise `output` MimeType (as opposed to the
+       specification given, which can be vague)
+     - what will be the `quality` of the conversion. This is a number
+       in the range 0 to 9, where 0 means no support and 9 means "perfect"
+       support (try to keep 9 available for user defined conversions,
+       though nothing will prevent them from using 10 or 100...)
+     - whether a tab expansion should precede the conversion itself,
+       (`expand_tabs` flag which defaults to False).
+    e.g. Conversion(format='latex', quality=8, type=MimeType('text/x-tex'))
+    """
+    def __init__(self, format, quality=1, output=TEXT_HTML, expand_tabs=False):
+        self.format = format
+        self.quality = quality
+        self.output = output
+        self.expand_tabs = expand_tabs
+    def __repr__(self):
+        return '<Conversion %s %s (quality %s)>' % \
+               (self.format, self.output, self.quality)
+# -- Interfaces for the extension points
+class IContentConverter(Interface):
+    """An extension point interface for generic content conversion."""
+    def get_supported_conversions(input):
+        """Tells whether this converter can handle this `input` type.
+        Return an iterable of `Conversion` objects, each describing
+        how the conversion should be done and what will be the output type.
+        """
+    def convert_content(context, conversion, content):
+        """Convert the given `AbstractContent` as specified by `Conversion`.
+        The conversion takes place in the given formatting `context`.
+        A `context` provides at least a `req` property.
+        If no other specific context object is available, a
+        `ToplevelContext` can be used to wrap the `req` instance.
+        Return the converted content, which ''must'' be a `MimeContent` object.
+        """
+    class ToplevelContext(object):
+        def __init__(self, req):
+            self.req = req
 class IHTMLPreviewAnnotator(Interface):
     """Extension point interface for components that can annotate an XHTML
     representation of file contents with additional information."""
     def get_annotation_type():
+        """Return a (type, label, description) tuple that defines the type of
+        annotation and provides human readable names. The `type` element should
+        be unique to the annotator. The `label` element is used as column
+        heading for the table, while `description` is used as a display name to
+        let the user toggle the appearance of the annotation type.
+        """Defines the type of annotation and provides human readable names.
+        Return a (type, label, description) tuple, where:
+         - `type` element should be unique to the annotator.
+         - `label` element is used as column heading for the table
+         - `description` is used as a display name to let the user toggle
+            the appearance of the annotation type.
         """
     def annotate_line(number, content):
+        """Return the XHTML markup for the table cell that contains the
+        annotation data."""
+        """Return the annotation data for the given line.
+        The return value must be XHTML markup that fit in a table cell.
+        """
-class IContentConverter(Interface):
-    """An extension point interface for generic MIME based content
-    conversion."""
+    def get_supported_conversions():
+        """Return an iterable of tuples in the form (key, name, extension,
+        in_mimetype, out_mimetype, quality) representing the MIME conversions
+        supported and
+        the quality ratio of the conversion in the range 0 to 9, where 0 means
+        no support and 9 means "perfect" support. eg. ('latex', 'LaTeX', 'tex',
+        'text/x-trac-wiki', 'text/plain', 8)"""
+# -- The main Mimeview component
-    def convert_content(req, mimetype, content, key):
-        """Convert the given content from mimetype to the output MIME type
-        represented by key. Returns a tuple in the form (content,
-        output_mime_type) or None if conversion is not possible."""
 class Mimeview(Component):
     """A generic class to prettify data, typically source code."""
-    renderers = ExtensionPoint(IHTMLPreviewRenderer)
     annotators = ExtensionPoint(IHTMLPreviewAnnotator)
     converters = ExtensionPoint(IContentConverter)
 …
         """Maximum file size for HTML preview. (''since 0.9'').""")
     mime_map = ListOption('mimeviewer', 'mime_map',
         'text/x-dylan:dylan,text/x-idl:ice,text/x-ada:ads:adb',
+        'text/x-dylan:dylan,text/x-idl:ice,text/x-ada:ads:adb', doc=
         """List of additional MIME types and keyword mappings.
-        Mappings are comma-separated, and for each MIME type,
-        there's a colon (":") separated list of associated keywords
-        or file extensions. (''since 0.10'').""")
+        Mappings are comma-separated. Each mapping starts with the mimetype,
+        followed by a colon (":") and the (colon separated) list of associated
+        keywords or file extensions. (''since 0.10'').""")
+    treat_as_binary = ListOption('mimeviewer', 'treat_as_binary',
+        'application/pdf,application/postscript,application/rtf', doc=
+        """List of MIME types that should always be treated as binary content.
+        Accounts for the fact that our binary detection heuristic can't
+        always work for some kind of binary data. (''since 0.10'').""")
     def __init__(self):
         self._mime_map = None
-    # Public API
+    def get_supported_conversions(self, mimetype):
+        """Return a list of target MIME types in same form as
+        `IContentConverter.get_supported_conversions()`, but with the converter
+        component appended. Output is ordered from best to worst quality."""
+        converters = []
+        for converter in self.converters:
+            for k, n, e, im, om, q in converter.get_supported_conversions():
+                if im == mimetype and q > 0:
+                    converters.append((k, n, e, im, om, q, converter))
+        converters = sorted(converters, key=lambda i: i[-1], reverse=True)
+        return converters
+    def _get_mimemap(self):
+        """Extend default extension to MIME type mappings"""
+        if not self._mime_map:
+            self._mime_map = {}
+            self._mime_map.update(MIME_MAP)
+            for mapping in self.config['mimeviewer'].getlist('mime_map'):
+                if ':' in mapping:
+                    assocations = mapping.split(':')
+                    mimetype = assocations[0]
+                    for keyword in assocations:
+                        self._mime_map[keyword] = mimetype
+                    # Note: 'mimetype' is associated to itself
+        return self._mime_map
+    def convert_content(self, req, mimetype, content, key, filename=None,
+                        url=None):
+        """Convert the given content to the target MIME type represented by
+        `key`, which can be either a MIME type or a key. Returns a tuple of
+        (content, output_mime_type, extension)."""
+        if not content:
+            return ('', 'text/plain;charset=utf-8')
+    mimemap = property(_get_mimemap)
+        # Ensure we have a MIME type for this content
+        full_mimetype = mimetype
+        if not full_mimetype:
+            if hasattr(content, 'read'):
+                content = content.read(self.max_preview_size)
+            full_mimetype = self.get_mimetype(filename, content)
+        if full_mimetype:
+            mimetype = full_mimetype.split(';')[0].strip() # split off charset
+        else:
+            mimetype = full_mimetype = 'text/plain' # fallback if not binary
+    def get_mimetype(self, filename, charset=None):
+        """Lookup for given `filename`, among known MIME Types.
+        # Choose best converter
+        candidates = list(self.get_supported_conversions(mimetype))
+        candidates = [c for c in candidates if key in (c[0], c[4])]
+        if not candidates:
+            raise TracError('No available MIME conversions from %s to %s' %
+                            (mimetype, key))
+        `filename` is either a file name, or simply a keyword.
+        # First successful conversion wins
+        for ck, name, ext, input_mimettype, output_mimetype, quality, \
+                converter in candidates:
+            output = converter.convert_content(req, mimetype, content, ck)
+            if not output:
+                continue
+            return (output[0], output[1], ext)
+        raise TracError('No available MIME conversions from %s to %s' %
+                        (mimetype, key))
+        Return a `MimeType` object if found, `None` otherwise.
+        """
+        mimetype = get_mimetype_from_filename(filename, self.mimemap)
+        if mimetype:
+            ext = os.path.splitext(filename)[1]
+            return MimeType(mimetype, charset, extension=ext, env=self.env)
     def get_annotation_types(self):
         """Generator that returns all available annotation types."""
         for annotator in self.annotators:
             yield annotator.get_annotation_type()
+    # -- MIME type conversion
+    def get_converters(self, input):
+        """Return a list of conversions for the `input` `ContentType`.
+    def render(self, req, mimetype, content, filename=None, url=None,
+               annotations=None):
+        """Render an XHTML preview of the given `content`.
+        The returned list contains pair of `(conversion, converter)` objects,
+        ordered from best to worst quality.
+        """
+        converters = []
+        for converter in self.converters:
+            for conversion in converter.get_supported_conversions(input):
+                if conversion.quality > 0:
+                    converters.append((conversion, converter))
+        `content` is the same as an `IHTMLPreviewRenderer.render`'s
+        `content` argument.
+        return sorted(converters, key=lambda c: c[0].quality, reverse=True)
+        The specified `mimetype` will be used to select the most appropriate
+        `IHTMLPreviewRenderer` implementation available for this MIME type.
+        If not given, the MIME type will be infered from the filename or the
+        content.
+    def get_conversions(self, input):
+        """Convenience method, which return only the `Conversion` information.
         Return a string containing the XHTML text.
+        Otherwise, it's the same as `get_converters`.
         """
+        if not content:
+            return ''
+        return [conversion for conversion, _ in self.get_converters(input)]
+        # Ensure we have a MIME type for this content
+        full_mimetype = mimetype
+        if not full_mimetype:
+            if hasattr(content, 'read'):
+                content = content.read(self.max_preview_size)
+            full_mimetype = self.get_mimetype(filename, content)
+        if full_mimetype:
+            mimetype = full_mimetype.split(';')[0].strip() # split off charset
+        else:
+            mimetype = full_mimetype = 'text/plain' # fallback if not binary
+    # -- XHTML rendering and annotations (based on the conversion API)
+        # Determine candidate `IHTMLPreviewRenderer`s
+        candidates = []
+        for renderer in self.renderers:
+            qr = renderer.get_quality_ratio(mimetype)
+            if qr > 0:
+                candidates.append((qr, renderer))
+        candidates.sort(lambda x,y: cmp(y[0], x[0]))
+    def render(self, context, content, annotations=None):
+        """Render an XHTML preview of the given `content`.
+        # First candidate which renders successfully wins.
+        # Also, we don't want to expand tabs more than once.
+        expanded_content = None
+        for qr, renderer in candidates:
+            try:
+                self.log.debug('Trying to render HTML preview using %s'
+                               % renderer.__class__.__name__)
+                # check if we need to perform a tab expansion
+                rendered_content = content
+                if getattr(renderer, 'expand_tabs', False):
+                    if expanded_content is None:
+                        content = content_to_unicode(self.env, content,
+                                                     full_mimetype)
+                        expanded_content = content.expandtabs(self.tab_width)
+                    rendered_content = expanded_content
+                result = renderer.render(req, full_mimetype, rendered_content,
+                                         filename, url)
+                if not result:
+                    continue
+                elif isinstance(result, Fragment):
+                    return result
+                elif isinstance(result, basestring):
+                    return Markup(to_unicode(result))
+                elif annotations:
+                    return Markup(self._annotate(result, annotations))
+                else:
+                    buf = StringIO()
+                    buf.write('<div class="code"><pre>')
+                    for line in result:
+                        buf.write(line + '\n')
+                    buf.write('</pre></div>')
+                    return Markup(buf.getvalue())
+            except Exception, e:
+                self.log.warning('HTML preview using %s failed (%s)'
+                                 % (renderer, e), exc_info=True)
+        Some `annotations` might be requested as well. This argument
+        is a list of annotation keys, each of them should match a
+        `type` as returned by `IHTMLPreviewAnnotator.get_annotation_type`.
+        """
+        result = content.convert(context, typespec=TEXT_HTML)
+        if annotations and result.annotable:
+            lines = result.lines()
+            if lines:
+                return Markup(self._annotate(lines, annotations))
+            # i.e. don't annotate contents that are not line oriented
+            ### don't annotate content which should not be annotated...
+        if isinstance(result, LineIteratorMimeContent):
+            return html.DIV(html.PRE(result.markup()), class_="code")
+        else:
+            return result.markup()
+    def get_annotation_types(self):
+        """Generator that returns all available annotation types."""
+        for annotator in self.annotators:
+            yield annotator.get_annotation_type()
     def _annotate(self, lines, annotations):
+        """Add requested `annotations` to the lines' content."""
         buf = StringIO()
         buf.write('<table class="code"><thead><tr>')
         annotators = []
 …
         buf.write('</tbody></table>')
         return buf.getvalue()
+    # -- Deprecated API (TODO: remove in 0.11)
     def get_max_preview_size(self):
         """Deprecated: use `max_preview_size` attribute directly."""
         return self.max_preview_size
-    def get_charset(self, content='', mimetype=None):
-        """Infer the character encoding from the `content` or the `mimetype`.
-        `content` is either a `str` or an `unicode` object.
-        The charset will be determined using this order:
-         * from the charset information present in the `mimetype` argument
-         * auto-detection of the charset from the `content`
-         * the configured `default_charset`
-        """
-        if mimetype:
-            ctpos = mimetype.find('charset=')
-            if ctpos >= 0:
-                return mimetype[ctpos + 8:].strip()
-        if isinstance(content, str):
-            utf = detect_unicode(content)
-            if utf is not None:
-                return utf
-        return self.default_charset
-    def get_mimetype(self, filename, content=None):
-        """Infer the MIME type from the `filename` or the `content`.
-        `content` is either a `str` or an `unicode` object.
-        Return the detected MIME type, augmented by the
-        charset information (i.e. "<mimetype>; charset=..."),
-        or `None` if detection failed.
-        """
-        # Extend default extension to MIME type mappings with configured ones
-        if not self._mime_map:
-            self._mime_map = MIME_MAP
-            for mapping in self.config['mimeviewer'].getlist('mime_map'):
-                if ':' in mapping:
-                    assocations = mapping.split(':')
-                    for keyword in assocations: # Note: [0] kept on purpose
-                        self._mime_map[keyword] = assocations[0]
-        mimetype = get_mimetype(filename, content, self._mime_map)
-        charset = None
-        if mimetype:
-            charset = self.get_charset(content, mimetype)
-        if mimetype and charset and not 'charset' in mimetype:
-            mimetype += '; charset=' + charset
-        return mimetype
     def to_utf8(self, content, mimetype=None):
         """Convert an encoded `content` to utf-8.
         ''Deprecated in 0.10. You should use `unicode` strings only.''
         """
         return to_utf8(content, self.get_charset(content, mimetype))
+        return to_utf8(content)
+    def to_unicode(self, content, mimetype=None, charset=None):
+        """Convert `content` (an encoded `str` object) to an `unicode` object.
+    # -- Utilities
+        This calls `trac.util.to_unicode` with the `charset` provided,
+        or the one obtained by `Mimeview.get_charset()`.
+    def configured_modes_mapping(self, renderer):
+        """Utility for configurable custom converters
+        Return a MIME type to `(mode,quality)` mapping for given `option`,
+        assuming a format of comma-separated <mimetype>:<mode>:<quality>
+        associations.
+        See EnscriptConverter and SilverCityConverter.
         """
-        if not charset:
-            charset = self.get_charset(content, mimetype)
-        return to_unicode(content, charset)
-    def configured_modes_mapping(self, renderer):
-        """Return a MIME type to `(mode,quality)` mapping for given `option`"""
         types, option = {}, '%s_modes' % renderer
         for mapping in self.config['mimeviewer'].getlist(option):
             if not mapping:
 …
                                  "option." % (mapping, option))
         return types
+    def preview_to_hdf(self, req, content, length, mimetype, filename,
+                       url=None, annotations=None):
+        """Prepares a rendered preview of the given `content`.
+    def preview_to_hdf(self, req, content, annotations=None):
+        """Prepares a rendered preview to text/html of the given `content`.
+        Note: `content` will usually be an object with a `read` method.
+        """
+        if length >= self.max_preview_size:
+        That content is eventually annotated with the specified `annotations`.
+        """
+        self.log.debug("Rendering preview of %s" % repr(content))
+        if content.size() >= self.max_preview_size:
             return {'max_file_size_reached': True,
                     'max_file_size': self.max_preview_size,
                     'raw_href': url}
+                    'raw_href': content.url}
         else:
+            return {'preview': self.render(req, mimetype, content, filename,
+                                           url, annotations),
+                    'raw_href': url}
+            try:
+                preview = self.render(IContentConverter.ToplevelContext(req),
+                                      content, annotations)
+            except NoConversion, e:
+                preview = None
+            return {'preview': preview,
+                    'raw_href': content.url}
-    def send_converted(self, req, in_type, content, selector, filename='file'):
-        """Helper method for converting `content` and sending it directly.
-        `selector` can be either a key or a MIME Type."""
-        from trac.web import RequestDone
-        content, output_type, ext = self.convert_content(req, in_type,
-                                                         content, selector)
-        req.send_response(200)
-        req.send_header('Content-Type', output_type)
-        req.send_header('Content-Disposition', 'filename=%s.%s' % (filename,
-                                                                   ext))
-        req.end_headers()
-        req.write(content)
-        raise RequestDone
+# utility for Mimeview._annotate
 def _html_splitlines(lines):
     """Tracks open and close tags in lines of HTML text and yields lines that
     have no tags spanning more than one line."""
 …
     def annotate_line(self, number, content):
         return '<th id="L%s"><a href="#L%s">%s</a></th>' % (number, number,
                                                             number)
+#        return html.TH(html.A(number, href="#L%s" % number), id=number)
 # -- Default renderers
+# -- Default TEXT_HTML converters
 class PlainTextRenderer(Component):
+    """HTML preview renderer for plain text, and fallback for any kind of text
+    for which no more specific renderer is available.
+    """Convert text to HTML-escaped text.
+    Will be used as a fallback for any kind of text for which no more
+    specific HTML converter is available.
     """
     implements(IHTMLPreviewRenderer)
+    implements(IContentConverter)
+    expand_tabs = True
+    def get_supported_conversions(self, input):
+        if input.is_binary:
+            return
+        yield Conversion(format='default', ### TODO Conversion.output
+                         quality=TEXT_PLAIN.match(input) and 8 or 1,
+                         output=TEXT_HTML, expand_tabs=True)
+    def convert_content(self, context, conversion, content):
+        if content.is_binary:
+            self.log.debug("Binary data; can't be rendered as plain text.")
+        else:
+            if not TEXT_PLAIN.match(content.type):
+                self.log.debug("Using plain text renderer as a fallback.")
+            def escape_lines():
+                for line in content.lines():
+                    yield escape(line)
+            plaintext_same_encoding = MimeType('text/plain', content.encoding)
+            return LineIteratorMimeContent(self.env, escape_lines(),
+                                           plaintext_same_encoding)
-    TREAT_AS_BINARY = [
-        'application/pdf',
-        'application/postscript',
-        'application/rtf'
+    ]
+    def get_quality_ratio(self, mimetype):
+        if mimetype in self.TREAT_AS_BINARY:
+            return 0
+        return 1
+class ImageRenderer(Component):
+    """Inline image display.
+    def render(self, req, mimetype, content, filename=None, url=None):
+        if is_binary(content):
+            self.env.log.debug("Binary data; no preview available")
+            return
+    This renderer doesn't need the actual data at all, only the url.
+    """
+        self.env.log.debug("Using default plain text mimeviewer")
+        content = content_to_unicode(self.env, content, mimetype)
+        for line in content.splitlines():
+            yield escape(line)
+    implements(IContentConverter)
+    IMAGE_TYPE = MimeType(re.compile(r'image/'))
 class ImageRenderer(Component):
     """Inline image display. Here we don't need the `content` at all."""
     implements(IHTMLPreviewRenderer)
+    def get_supported_conversions(self, input):
+        if self.IMAGE_TYPE.match(input):
+            yield Conversion(format='image', quality=8, output=TEXT_HTML)
+    def get_quality_ratio(self, mimetype):
+        if mimetype.startswith('image/'):
+            return 8
+        return 0
+    def convert_content(self, context, conversion, content):
+        if content.url:
+            img = html.DIV(html.IMG(src=content.url, alt=content.filename),
+                           class_="image-file")
+            return StructuredMimeContent(self.env, img, TEXT_HTML)
-    def render(self, req, mimetype, content, filename=None, url=None):
-        if url:
-            return html.DIV(html.IMG(src=url,alt=filename),
-                            class_="image-file")
+# ---- Backward compatibility support for IHTMLPreviewRenderer
+#
+# (TODO: remove in 0.11)
+#
 class WikiTextRenderer(Component):
     """Render files containing Trac's own Wiki formatting markup."""
     implements(IHTMLPreviewRenderer)
+class IHTMLPreviewRenderer(Interface):
+    """Extension point interface for components that add HTML renderers of
+    specific content types to the `Mimeview` component.
+    def get_quality_ratio(self, mimetype):
+        if mimetype in ('text/x-trac-wiki', 'application/x-trac-wiki'):
+            return 8
+        return 0
+    Deprecated in 0.10, implement `IContentConverter` instead.
+    """
+    def render(self, req, mimetype, content, filename=None, url=None):
+        from trac.wiki import wiki_to_html
+        return wiki_to_html(content_to_unicode(self.env, content, mimetype),
+                            self.env, req)
+    # implementing classes should set this property to True if they
+    # support text content where Trac should expand tabs into spaces
+    expand_tabs = False
+    def get_quality_ratio(mimetype):
+        """Return the level of support this renderer provides"""
+    def render(req, mimetype, content, filename=None, url=None):
+        """Render an XHTML preview of the raw `content`."""
+class PreviewRendererAdapter(Component):
+    """Single IContentConverter which wraps legacy IHTMLPreviewRenderer"""
+    renderers = ExtensionPoint(IHTMLPreviewRenderer)
+    implements(IContentConverter)
+    # IContentConverter methods
+    def get_supported_conversions(self, input):
+        if not isinstance(input, MimeType):
+            return
+        for renderer in self.renderers:
+            qr = renderer.get_quality_ratio(input.mimetype)
+            if qr > 0:
+                expand_tabs = getattr(renderer, 'expand_tabs', False)
+                yield Conversion(format=renderer.__class__.__name__,
+                                 quality=qr, output=TEXT_HTML,
+                                 expand_tabs=expand_tabs)
+    def convert_content(self, context, conversion, content):
+        for renderer in self.renderers:
+            if conversion.format == renderer.__class__.__name__:
+                self.log.debug('Rendering using %s' % renderer)
+                try:
+                    result = renderer.render(context.req,
+                                           content.type.mimetype,
+                                           content, # ...which is read()able
+                                           content.filename, content.url)
+                    if isinstance(result, Fragment):
+                        return StructuredMimeContent(self.env, result)
+                    elif isinstance(result, basestring):
+                        self.log.warning('IHTMLPreviewRenderer: string %s' %
+                                         result.__class__.__name__)
+                        return StringMimeContent(self.env, result)
+                    else: # something else... assume it's an iterable of lines
+                        self.log.warning('IHTMLPreviewRenderer: type %s' %
+                                         result.__class__.__name__)
+                        result = LineIteratorMimeContent(self.env, result)
+                        # Check if the conversion is really valid, by trying to
+                        # access an excerpt of the result. This might trigger
+                        # errors with legacy renderers returning an iterator...
+                        check_result = result.excerpt
+                        return result
+                except Exception, e:
+                    self.env.log.warning('HTML renderer %s failed (%s)'
+                                          % (repr(renderer), e))
+                    self.env.log.debug('HTML renderer:',exc_info=True)

trac/mimeview/silvercity.py

 from trac.core import *
 from trac.config import ListOption
+from trac.mimeview.api import IHTMLPreviewRenderer, Mimeview
+from trac.mimeview.api import IContentConverter, Mimeview, StringMimeContent, \
+                              Conversion, TEXT_HTML
 __all__ = ['SilverCityRenderer']
 …
 CRLF_RE = re.compile('\r$', re.MULTILINE)
 class SilverCityRenderer(Component):
+class SilverCityConverter(Component):
     """Syntax highlighting based on SilverCity."""
     implements(IHTMLPreviewRenderer)
+    implements(IContentConverter)
+    enscript_modes = ListOption('mimeviewer', 'silvercity_modes',
+        '',
+    silvercity_modes = ListOption('mimeviewer', 'silvercity_modes', doc=
         """List of additional MIME types known by SilverCity.
         For each, a tuple `mimetype:mode:quality` has to be
         specified, where `mimetype` is the MIME type,
         `mode` is the corresponding SilverCity mode to be used
 …
     def __init__(self):
         self._types = None
     def get_quality_ratio(self, mimetype):
+    def get_supported_conversions(self, input):
         # Extend default MIME type to mode mappings with configured ones
         if not self._types:
             self._types = {}
             self._types.update(types)
+            self._types.update(
+                Mimeview(self.env).configured_modes_mapping('silvercity'))
+        return self._types.get(mimetype, (None, 0))[1]
+            ctypes = Mimeview(self.env).configured_modes_mapping('silvercity')
+            self._types.update(ctypes)
+        quality_ratio = self._types.get(input.mimetype, (None, 0))[1]
+        if quality_ratio:
+            yield Conversion('silvercity', quality_ratio, TEXT_HTML)
     def render(self, req, mimetype, content, filename=None, rev=None):
+    def convert_content(self, context, conversion, input):
         import SilverCity
         try:
+            mimetype = mimetype.split(';', 1)[0]
+            typelang = self._types[mimetype]
+            typelang = self._types[input.type.mimetype]
             lang = typelang[0]
             module = getattr(SilverCity, lang)
             generator = getattr(module, lang + "HTMLGenerator")
 …
             raise Exception, err
         # SilverCity does not like unicode strings
         content = content.encode('utf-8')
+        content = input.encode('utf-8')
         # SilverCity generates extra empty line against some types of
         # the line such as comment or #include with CRLF. So we
 …
         span_default_re = re.compile(r'<span class="\w+_default">(.*?)</span>',
                                      re.DOTALL)
         html = span_default_re.sub(r'\1', br_re.sub('', buf.getvalue()))
-        # Convert the output back to a unicode string
-        html = html.decode('utf-8')
         # SilverCity generates _way_ too many non-breaking spaces...
         # We don't need them anyway, so replace them by normal spaces
+        return html.replace('&nbsp;', ' ').splitlines()
+        return StringMimeContent(self.env, html.replace('&nbsp;', ' '),
+                                 TEXT_HTML)

trac/mimeview/patch.py

 #         Ludvig Strigeus
 from trac.core import *
 from trac.mimeview.api import content_to_unicode, IHTMLPreviewRenderer, Mimeview
+from trac.mimeview.api import IHTMLPreviewRenderer, Mimeview
 from trac.util.html import escape, Markup
 from trac.web.chrome import add_stylesheet
 …
     def render(self, req, mimetype, content, filename=None, rev=None):
         from trac.web.clearsilver import HDFWrapper
         content = content_to_unicode(self.env, content, mimetype)
+        content = unicode(content)
         d = self._diff_to_hdf(content.splitlines(),
                               Mimeview(self.env).tab_width)
         if not d:

trac/mimeview/enscript.py

         cmdline += ' --color -h -q --language=html -p - -E%s' % mode
         self.env.log.debug("Enscript command line: %s" % cmdline)
+        content = unicode(content)
         np = NaivePopen(cmdline, content.encode('utf-8'), capturestderr=1)
         if np.errorlevel or np.err:
             err = 'Running (%s) failed: %s, %s.' % (cmdline, np.errorlevel,

trac/mimeview/php.py

 from trac.core import *
 from trac.config import Option
 from trac.mimeview.api import IHTMLPreviewRenderer, content_to_unicode
+from trac.mimeview.api import IHTMLPreviewRenderer
 from trac.util import NaivePopen
 from trac.util.html import Deuglifier
 …
         cmdline += ' -sn'
         self.env.log.debug("PHP command line: %s" % cmdline)
         content = content_to_unicode(self.env, content, mimetype)
+        content = unicode(content)
         content = content.encode('utf-8')
         np = NaivePopen(cmdline, content, capturestderr=1)
         if np.errorlevel or np.err:

trac/ticket/web_ui.py

 from trac.config import BoolOption, Option
 from trac.core import *
 from trac.env import IEnvironmentSetupParticipant
+from trac.mimeview.api import *
 from trac.ticket import Milestone, Ticket, TicketSystem, ITicketManipulator
 from trac.ticket.notification import TicketNotifyEmail
 from trac.Timeline import ITimelineEventProvider
 …
 from trac.web import IRequestHandler
 from trac.web.chrome import add_link, add_stylesheet, INavigationContributor
 from trac.wiki import wiki_to_html, wiki_to_oneliner
-from trac.mimeview.api import Mimeview, IContentConverter
 class InvalidTicket(TracError):
 …
     # IContentConverter methods
+    def get_supported_conversions(self):
+        yield ('csv', 'Comma-delimited Text', 'csv',
+               'trac.ticket.Ticket', 'text/csv', 8)
+        yield ('tab', 'Tab-delimited Text', 'tsv',
+               'trac.ticket.Ticket', 'text/tab-separated-values', 8)
+        yield ('rss', 'RSS Feed', 'xml',
+               'trac.ticket.Ticket', 'application/rss+xml', 8)
+    def get_supported_conversions(self, input):
+        if input.match(Ticket):
+            yield Conversion('csv', 8, TEXT_CSV)
+            yield Conversion('tab', 8, TEXT_TSV)
+            yield Conversion('rss', 8, APPLICATION_RSS_XML)
+    def convert_content(self, req, mimetype, ticket, key):
+    def convert_content(self, context, conversion, content):
+        ticket = content.content
+        key = conversion.format
         if key == 'csv':
             return self.export_csv(ticket, mimetype='text/csv')
+            data = self.export_csv(ticket)
         elif key == 'tab':
+            return self.export_csv(ticket, sep='\t',
+                                   mimetype='text/tab-separated-values')
+            data = self.export_csv(ticket, sep='\t')
         elif key == 'rss':
+            return self.export_rss(req, ticket)
+            data = self.export_rss(context.req, ticket)
+        return StringMimeContent(self.env, data, conversion.output,
+                                 filename=content.basename())
     # INavigationContributor methods
 …
         ticket = Ticket(self.env, id, db=db)
+        # Send as alternate content if required
+        format = req.args.get('format')
+        if format:
+            content = ObjectContent(self.env, ticket,
+                                    filename='ticket_%d' % ticket.id)
+            content.convert(req, format).send(req)
         if req.method == 'POST':
             if not req.args.has_key('preview'):
                 self._do_save(req, db, ticket)
 …
         self._insert_ticket_data(req, db, ticket,
                                  get_reporter_id(req, 'author'))
-        mime = Mimeview(self.env)
-        format = req.args.get('format')
-        if format:
-            mime.send_converted(req, 'trac.ticket.Ticket', ticket, format,
-                                'ticket_%d' % ticket.id)
         # If the ticket is being shown in the context of a query, add
         # links to help navigate in the query result set
         if 'query_tickets' in req.session:
 …
         add_stylesheet(req, 'common/css/ticket.css')
         # Add registered converters
         for conversion in mime.get_supported_conversions('trac.ticket.Ticket'):
             conversion_href = req.href.ticket(ticket.id, format=conversion[0])
             add_link(req, 'alternate', conversion_href, conversion[1],
                      conversion[3])
+        for c in Mimeview(self.env).get_conversions(ObjectType(Ticket)):
+            add_link(req, 'alternate',
+                     req.href.ticket(ticket.id, format=c.format),
+                     c.output.name, c.output.mimetype)
         return 'ticket.cs', None
 …
     # Internal methods
+    def export_csv(self, ticket, sep=',', mimetype='text/plain'):
+        content = StringIO()
+        content.write(sep.join(['id'] + [f['name'] for f in ticket.fields])
+                      + CRLF)
+        content.write(sep.join([unicode(ticket.id)] +
+                                [ticket.values.get(f['name'], '')
+                                 .replace(sep, '_').replace('\\', '\\\\')
+                                 .replace('\n', '\\n').replace('\r', '\\r')
+                                 for f in ticket.fields]) + CRLF)
+        return (content.getvalue(), '%s;charset=utf-8' % mimetype)
+    def export_csv(self, ticket, sep=','):
+        csv = StringIO()
+        csv.write(sep.join(['id'] + [f['name'] for f in ticket.fields]) + CRLF)
+        csv.write(sep.join([unicode(ticket.id)] +
+                           [ticket.values.get(f['name'], '')
+                            .replace(sep, '_').replace('\\', '\\\\')
+                            .replace('\n', '\\n').replace('\r', '\\r')
+                            for f in ticket.fields]) + CRLF)
+        return csv.getvalue()
     def export_rss(self, req, ticket):
         db = self.env.get_db_cnx()
         changes = []
 …
             change['title'] = '; '.join(['%s %s' % (', '.join(v), k) for k, v \
                                          in change_summary.iteritems()])
         req.hdf['ticket.changes'] = changes
         return (req.hdf.render('ticket_rss.cs'), 'application/rss+xml')
+        return req.hdf.render('ticket_rss.cs')
     def _do_save(self, req, db, ticket):
         if req.perm.has_permission('TICKET_CHGPROP'):
             # TICKET_CHGPROP gives permission to edit the ticket

trac/ticket/tests/conversion.py

 from trac.test import EnvironmentStub, Mock
 from trac.util import sorted
+from trac.util import sorted, CRLF
 from trac.ticket.model import Ticket
 from trac.ticket.web_ui import TicketModule
 from trac.mimeview.api import Mimeview
+from trac.mimeview.api import *
 from trac.web.clearsilver import HDFWrapper
 from trac.web.href import Href
 …
     def setUp(self):
         self.env = EnvironmentStub()
         self.ticket_module = TicketModule(self.env)
-        self.mimeview = Mimeview(self.env)
         self.req = Mock(hdf=HDFWrapper(['./templates']),
                         base_path='/trac.cgi', path_info='',
+                        href=Href('/trac.cgi'))
+                        href=Href('/trac.cgi'),
+                        abs_href=Href('http://example.org/trac.cgi'))
     def _create_a_ticket(self):
+    def _create_a_ticket_content(self):
         # 1. Creating ticket
         ticket = Ticket(self.env)
         ticket['reporter'] = 'santa'
         ticket['summary'] = 'Foo'
         ticket['description'] = 'Bar'
         ticket['foo'] = 'This is a custom field'
+        return ticket
+        # 2. Creating ObjectContent wrapper
+        return ObjectContent(self.env, ticket)
     def test_conversions(self):
+        conversions = self.mimeview.get_supported_conversions(
+            'trac.ticket.Ticket')
+        expected = sorted([('csv', 'Comma-delimited Text', 'csv',
+                           'trac.ticket.Ticket', 'text/csv', 8,
+                           self.ticket_module),
+                          ('tab', 'Tab-delimited Text', 'tsv',
+                           'trac.ticket.Ticket', 'text/tab-separated-values', 8,
+                           self.ticket_module),
+                           ('rss', 'RSS Feed', 'xml',
+                            'trac.ticket.Ticket', 'application/rss+xml', 8,
+                            self.ticket_module)],
+                          key=lambda i: i[-1], reverse=True)
+        self.assertEqual(expected, conversions)
+        conversions = Mimeview(self.env).get_conversions(ObjectType(Ticket))
+        expected = [Conversion('csv', 8, TEXT_CSV),
+                           Conversion('rss', 8, APPLICATION_RSS_XML),
+                           Conversion('tab', 8, TEXT_TSV)]
+        for expected, actual in zip(expected, sorted(conversions,
+                                                     key=lambda c: c.format)):
+            for attr in ('format', 'output', 'quality', 'expand_tabs'):
+                self.assertEqual(getattr(expected, attr),
+                                 getattr(actual, attr))
     def test_csv_conversion(self):
+        ticket = self._create_a_ticket()
+        csv = self.mimeview.convert_content(self.req, 'trac.ticket.Ticket',
+                                            ticket, 'csv')
+        self.assertEqual((u'id,summary,reporter,owner,description,keywords,cc'
+                          '\r\nNone,Foo,santa,,Bar,,\r\n',
+                          'text/csv;charset=utf-8', 'csv'), csv)
+        content = self._create_a_ticket_content()
+        expected = [u'id,summary,reporter,owner,description,keywords,cc',
+                    u'None,Foo,santa,,Bar,,',
+                    '']
+        csv = content.convert(self.req, 'csv')
+        self.assertEqual(CRLF.join(expected), csv.read())
+        self.assertEqual(CRLF.join(expected), unicode(csv))
+        self.assertEqual(TEXT_CSV, csv.type)
+        self.assertEqual('text/csv; charset=utf-8', csv.type.mimetype_charset)
+        csv = content.convert(self.req, TEXT_CSV)
+        self.assertEqual(CRLF.join(expected), csv.read())
+        self.assertEqual(CRLF.join(expected), unicode(csv))
+        self.assertEqual(TEXT_CSV, csv.type)
+        self.assertEqual('text/csv; charset=utf-8', csv.type.mimetype_charset)
     def test_tab_conversion(self):
+        ticket = self._create_a_ticket()
+        csv = self.mimeview.convert_content(self.req, 'trac.ticket.Ticket',
+                                            ticket, 'tab')
+        self.assertEqual((u'id\tsummary\treporter\towner\tdescription\tkeywords'
+                          '\tcc\r\nNone\tFoo\tsanta\t\tBar\t\t\r\n',
+                          'text/tab-separated-values;charset=utf-8', 'tsv'),
+                         csv)
+        content = self._create_a_ticket_content()
+        expected = [u'id\tsummary\treporter\towner\tdescription\tkeywords\tcc',
+                    u'None\tFoo\tsanta\t\tBar\t\t',
+                    '']
+        tsv = content.convert(self.req, 'tab')
+        self.assertEqual(CRLF.join(expected), tsv.read())
+        self.assertEqual(CRLF.join(expected), unicode(tsv))
+        self.assertEqual(TEXT_TSV, tsv.type)
+        self.assertEqual('text/tab-separated-values; charset=utf-8',
+                         tsv.type.mimetype_charset)
+        tsv = content.convert(self.req, TEXT_TSV)
+        self.assertEqual(CRLF.join(expected), tsv.read())
+        self.assertEqual(CRLF.join(expected), unicode(tsv))
+        self.assertEqual(TEXT_TSV, tsv.type)
+        self.assertEqual('text/tab-separated-values; charset=utf-8',
+                         tsv.type.mimetype_charset)
     def test_rss_conversion(self):
+        ticket = self._create_a_ticket()
+        content, mimetype, ext = self.mimeview.convert_content(
+            self.req, 'trac.ticket.Ticket', ticket, 'rss')
+        self.assertEqual(('<?xml version="1.0"?>\n<!-- RSS generated by Trac v '
+                          'on  -->\n<rss version="2.0">\n <channel>\n   '
+                          '<title>Ticket </title>\n  <link></link>\n  '
+                          '<description>&lt;p&gt;\nBar\n&lt;/p&gt;\n'
+                          '</description>\n  <language>en-us</language>\n  '
+                          '<generator>Trac v</generator>\n </channel>\n</rss>\n',
+                          'application/rss+xml', 'xml'),
+                         (content.replace('\r', ''), mimetype, ext))
+        content = self._create_a_ticket_content()
+        expected = ['<?xml version="1.0"?>',
+                    '<!-- RSS generated by Trac v on  -->',
+                    '<rss version="2.0">',
+                    ' <channel>',
+                    '   <title>Ticket </title>',
+                    '  <link></link>',
+                    '  <description>&lt;p&gt;\r',
+                    'Bar\r',
+                    '&lt;/p&gt;\r',
+                    '</description>',
+                    '  <language>en-us</language>',
+                    '  <generator>Trac v</generator>',
+                    ' </channel>',
+                    '</rss>',
+                    '']
+        rss = content.convert(self.req, 'rss')
+        self.assertEqual('\n'.join(expected), rss.read())
+        self.assertEqual('\n'.join(expected), unicode(rss))
+        self.assertEqual(APPLICATION_RSS_XML, rss.type)
+        self.assertEqual('application/rss+xml; charset=utf-8',
+                         rss.type.mimetype_charset)
+        rss = content.convert(self.req, APPLICATION_RSS_XML)
+        self.assertEqual('\n'.join(expected), rss.read())
+        self.assertEqual('\n'.join(expected), unicode(rss))
+        self.assertEqual(APPLICATION_RSS_XML, rss.type)
+        self.assertEqual('application/rss+xml; charset=utf-8',
+                         rss.type.mimetype_charset)
 def suite():
     return unittest.makeSuite(TicketConversionTestCase, 'test')

trac/ticket/roadmap.py

     # Internal methods
     def render_ics(self, req, db, milestones):
+    def render_ics(self, req, db, milestones): # FIXME: use IContentConverter
         req.send_response(200)
         req.send_header('Content-Type', 'text/calendar;charset=utf-8')
         req.end_headers()

trac/ticket/query.py

 from trac.core import *
 from trac.db import get_column_names
+from trac.mimeview.api import *
 from trac.perm import IPermissionRequestor
 from trac.ticket import Ticket, TicketSystem
 from trac.util.datefmt import format_datetime, http_date
 …
                             INavigationContributor
 from trac.wiki import wiki_to_html, wiki_to_oneliner, IWikiSyntaxProvider
 from trac.wiki.macros import WikiMacroBase
-from trac.mimeview.api import Mimeview, IContentConverter
 class QuerySyntaxError(Exception):
     """Exception raised when a ticket query cannot be parsed from a string."""
 …
                IContentConverter)
     # IContentConverter methods
+    def get_supported_conversions(self):
+        yield ('rss', 'RSS Feed', 'xml',
+               'trac.ticket.Query', 'application/rss+xml', 8)
+        yield ('csv', 'Comma-delimited Text', 'csv',
+               'trac.ticket.Query', 'text/csv', 8)
+        yield ('tab', 'Tab-delimited Text', 'tsv',
+               'trac.ticket.Query', 'text/tab-separated-values', 8)
+    def get_supported_conversions(self, input):
+        if input.match(Query):
+            yield Conversion('rss', 8, APPLICATION_RSS_XML)
+            yield Conversion('csv', 8, TEXT_CSV)
+            yield Conversion('tab', 8, TEXT_TSV)
+    def convert_content(self, req, mimetype, query, key):
+    def convert_content(self, context, conversion, content):
+        query = content.content
+        req = context.req
+        key = conversion.format
         if key == 'rss':
+            return self.export_rss(req, query)
+        elif key == 'csv':
+            return self.export_csv(req, query, mimetype='text/csv')
+            data = self.export_rss(req, query)
         elif key == 'tab':
+            return self.export_csv(req, query, '\t', 'text/tab-separated-values')
+            data = self.export_csv(req, query, '\t')
+        else: # key == 'tab':
+            data = self.export_csv(req, query)
+        return StringMimeContent(self.env, data, conversion.output,
+                                 filename=content.basename())
     # INavigationContributor methods
 …
                     del req.session[var]
             req.redirect(query.get_href(req))
+        # Add registered converters
+        for conversion in Mimeview(self.env).get_supported_conversions(
+                                             'trac.ticket.Query'):
+            add_link(req, 'alternate',
+                     query.get_href(req, format=conversion[0]),
+                     conversion[1], conversion[3])
+        # Send as alternate content if required
+        format = req.args.get('format')
+        if format:
+            ObjectContent(self.env, query).convert(req, format).send(req)
+            # quite Rubyesque, isn't it? ;)
         constraints = {}
         for k, v in query.constraints.items():
 …
             constraints[k] = constraint
         req.hdf['query.constraints'] = constraints
-        format = req.args.get('format')
-        if format:
-            Mimeview(self.env).send_converted(req, 'trac.ticket.Query', query,
-                                              format, 'query')
         self.display_html(req, query)
         return 'query.cs', None
 …
            self.env.is_component_enabled(ReportModule):
             req.hdf['query.report_href'] = req.href.report()
+    def export_csv(self, req, query, sep=',', mimetype='text/plain'):
+        content = StringIO()
+        # Add registered converters
+        for c in Mimeview(self.env).get_conversions(ObjectType(query)):
+            add_link(req, 'alternate', query.get_href(req, format=c.format),
+                     c.output.name, c.output.mimetype)
+    def export_csv(self, req, query, sep=','):
+        csv = StringIO()
         cols = query.get_columns()
         content.write(sep.join([col for col in cols]) + CRLF)
+        csv.write(sep.join([col for col in cols]) + CRLF)
         results = query.execute(req, self.env.get_db_cnx())
         for result in results:
             content.write(sep.join([unicode(result[col]).replace(sep, '_')
                                                         .replace('\n', ' ')
                                                         .replace('\r', ' ')
+        lines = query.execute(req, self.env.get_db_cnx())
+        for line in lines:
+            csv.write(sep.join([unicode(line[col]).replace(sep, '_')
+                                                  .replace('\n', ' ')
+                                                  .replace('\r', ' ')
                                     for col in cols]) + CRLF)
         return (content.getvalue(), '%s;charset=utf-8' % mimetype)
+        return csv.getvalue()
     def export_rss(self, req, query):
         query.verbose = True
 …
                 groupdesc=query.groupdesc and 1 or None,
                 verbose=query.verbose and 1 or None,
                 **query.constraints)
         return (req.hdf.render('query_rss.cs'), 'application/rss+xml')
+        return req.hdf.render('query_rss.cs')
     # IWikiSyntaxProvider methods

trac/versioncontrol/web_ui/util.py

 # Author: Jonas Borgström <jonas@edgewall.com>
 #         Christian Boos <cboos@neuf.fr>
+import posixpath
 import re
 import urllib
 from trac.core import TracError
+from trac.mimeview.api import MimeContent
 from trac.util.datefmt import format_datetime, pretty_timedelta
 from trac.util.html import escape, html, Markup
 from trac.util.text import shorten_line
 …
 from trac.wiki import wiki_to_html, wiki_to_oneliner
 __all__ = ['get_changes', 'get_path_links', 'get_path_rev_line',
+           'get_existing_node', 'render_node_property']
+           'get_existing_node', 'render_node_property',
+           'NodeMimeContent']
+# MimeContent wrapper for repository files
+#
+# -- This could be part of trac.versioncontrol.api itself,
+#    if introducing a dependency on trac.mimeview.api is
+#    considered to be OK.
+CHUNK_SIZE = 4096
+class NodeMimeContent(MimeContent):
+    """MimeContent wrapper for a file Node in the repository."""
+    def __init__(self, env, node, url=None):
+        MimeContent.__init__(self, env, node, node.content_type,
+                             posixpath.basename(node.path), url)
+        self.last_modified = node.last_modified
+        self._stream = None
+    def chunks(self):
+        """Get chunks of the byte content"""
+        chunk = self.excerpt
+        self._excerpt = None
+        while True:
+            if not chunk:
+                return
+            yield chunk
+            chunk = self._stream.read(CHUNK_SIZE)
+    def size(self):
+        return self.content.get_content_length()
+    def _retrieve_excerpt(self):
+        self._stream = self.content.get_content()
+        return self._stream.read(CHUNK_SIZE)
 def get_changes(env, repos, revs, full=None, req=None, format=None):
     db = env.get_db_cnx()
     changes = {}

trac/versioncontrol/web_ui/changeset.py

 from trac import util
 from trac.config import BoolOption, IntOption
 from trac.core import *
-from trac.mimeview import Mimeview, is_binary
 from trac.perm import IPermissionRequestor
 from trac.Search import ISearchSource, search_to_sql, shorten_result
 from trac.Timeline import ITimelineEventProvider
 …
 from trac.versioncontrol import Changeset, Node
 from trac.versioncontrol.diff import get_diff_options, hdf_diff, unified_diff
 from trac.versioncontrol.svn_authz import SubversionAuthorizer
 from trac.versioncontrol.web_ui.util import render_node_property
+from trac.versioncontrol.web_ui.util import *
 from trac.web import IRequestHandler
 from trac.web.chrome import INavigationContributor, add_link, add_stylesheet
 from trac.wiki import wiki_to_html, wiki_to_oneliner, IWikiSyntaxProvider, \
 …
             The list is empty when no differences between comparable files
             are detected, but the return value is None for non-comparable files.
             """
             old_content = old_node.get_content().read()
             if is_binary(old_content):
+            old_content = NodeMimeContent(self.env, old_node)
+            if old_content.is_binary:
                 return None
             new_content = new_node.get_content().read()
             if is_binary(new_content):
+            new_content = NodeMimeContent(self.env, new_node)
+            if new_content.is_binary:
                 return None
+            mview = Mimeview(self.env)
+            old_content = mview.to_unicode(old_content, old_node.content_type)
+            new_content = mview.to_unicode(new_content, new_node.content_type)
+            old_content = unicode(old_content)
+            new_content = unicode(new_content)
+            # We could theoretically avoid converting to unicode() in case
+            # both files share the same encoding, i.e.
+            #
+            #   old_content.type.encoding == new_content.type.encoding
+            #
+            # however it's not safe to call splitlines on arbitrary str
+            # see r3236. A workaround could be to use re.split(r'[\r\n]+') ?
             if old_content != new_content:
                 context = 3
 …
                         'filename=%s.diff' % filename)
         req.end_headers()
-        mimeview = Mimeview(self.env)
         for old_node, new_node, kind, change in repos.get_changes(**diff):
             # TODO: Property changes
 …
             new_content = old_content = ''
             new_node_info = old_node_info = ('','')
-            mimeview = Mimeview(self.env)
             if old_node:
                 old_content = old_node.get_content().read()
                 if is_binary(old_content):
+                old_content = NodeMimeContent(self.env, old_node)
+                if old_content.is_binary:
                     continue
                 old_node_info = (old_node.path, old_node.rev)
+                old_content = mimeview.to_unicode(old_content,
+                                                  old_node.content_type)
+                old_content = unicode(old_content)
             if new_node:
                 new_content = new_node.get_content().read()
                 if is_binary(new_content):
+                new_content = NodeMimeContent(self.env, new_node)
+                if new_content.is_binary:
                     continue
                 new_node_info = (new_node.path, new_node.rev)
                 new_path = new_node.path
+                new_content = mimeview.to_unicode(new_content,
+                                                  new_node.content_type)
+                new_content = unicode(new_content)
             else:
                 old_node_path = repos.normalize_path(old_node.path)
                 diff_old_path = repos.normalize_path(diff.old_path)

trac/versioncontrol/web_ui/browser.py

 from trac import util
 from trac.config import ListOption, Option
 from trac.core import *
 from trac.mimeview import Mimeview, is_binary, get_mimetype
+from trac.mimeview import Mimeview, MimeType, TEXT_PLAIN
 from trac.perm import IPermissionRequestor
 from trac.util import sorted, embedded_numbers
 from trac.util.datefmt import http_date, format_datetime, pretty_timedelta
+from trac.util.datefmt import format_datetime, pretty_timedelta
 from trac.util.html import escape, html, Markup
 from trac.util.text import pretty_size
 from trac.web import IRequestHandler, RequestDone
 …
 from trac.versioncontrol.web_ui.util import *
-CHUNK_SIZE = 4096
 class BrowserModule(Component):
     implements(INavigationContributor, IPermissionRequestor, IRequestHandler,
 …
     def _render_file(self, req, repos, node, rev=None):
         req.perm.assert_permission('FILE_VIEW')
+        mimeview = Mimeview(self.env)
+        # MIME type detection
+        content = node.get_content()
+        chunk = content.read(CHUNK_SIZE)
+        mime_type = node.content_type
+        if not mime_type or mime_type == 'application/octet-stream':
+            mime_type = mimeview.get_mimetype(node.name, chunk) or \
+                        mime_type or 'text/plain'
+        content = NodeMimeContent(self.env, node,
+                                  req.href.browser(node.path, rev=rev,
+                                                   format='raw'))
         # Eventually send the file directly
         format = req.args.get('format')
+        if format in ['raw', 'txt']:
+            req.send_response(200)
+            req.send_header('Content-Type',
+                            format == 'txt' and 'text/plain' or mime_type)
+            req.send_header('Content-Length', node.content_length)
+            req.send_header('Last-Modified', http_date(node.last_modified))
+            req.end_headers()
+            while 1:
+                if not chunk:
+                    raise RequestDone
+                req.write(chunk)
+                chunk = content.read(CHUNK_SIZE)
+        if format in ('raw', 'txt'):
+            if format == 'txt':
+                content.type = TEXT_PLAIN ### SHOULD BE ENOUGH
+                content.type = MimeType('text/plain', content.encoding)
+            content.send(req)
         else:
             # The changeset corresponding to the last change on `node`
             # is more interesting than the `rev` changeset.
             changeset = repos.get_changeset(node.rev)
+            changeset = repos.get_changeset(node.created_rev)
             message = changeset.message or '--'
             if self.config['changeset'].getbool('wiki_format_messages'):
 …
             req.hdf['file'] = {
                 'rev': node.rev,
                 'changeset_href': req.href.changeset(node.rev),
+                'changeset_href': req.href.changeset(node.created_rev),
                 'date': format_datetime(changeset.date),
                 'age': pretty_timedelta(changeset.date),
                 'size': pretty_size(node.content_length),
                 'author': changeset.author or 'anonymous',
                 'message': message
+            }
             # add ''Plain Text'' alternate link if needed
             if not is_binary(chunk) and mime_type != 'text/plain':
+            if not content.is_binary and not TEXT_PLAIN.match(content.type):
                 plain_href = req.href.browser(node.path, rev=rev, format='txt')
                 add_link(req, 'alternate', plain_href, 'Plain Text',
                          'text/plain')
+                add_link(req, 'alternate', plain_href,
+                         TEXT_PLAIN.name, TEXT_PLAIN.mimetype)
             # add ''Original Format'' alternate link (always)
             raw_href = req.href.browser(node.path, rev=rev, format='raw')
             add_link(req, 'alternate', raw_href, 'Original Format', mime_type)
+            add_link(req, 'alternate', content.url,
+                     'Original Format', content.type.mimetype)
             self.log.debug("Rendering preview of node %s@%s with mime-type %s"
                            % (node.name, str(rev), mime_type))
+            req.hdf['file'] = Mimeview(self.env).preview_to_hdf(
+                req, content, annotations=['lineno'])
-            del content # the remainder of that content is not needed
-            req.hdf['file'] = mimeview.preview_to_hdf(
-                req, node.get_content(), node.get_content_length(), mime_type,
-                node.created_path, raw_href, annotations=['lineno'])
             add_stylesheet(req, 'common/css/code.css')
     # IWikiSyntaxProvider methods

trac/wiki/web_ui.py

 from trac.attachment import attachments_to_hdf, Attachment, AttachmentModule
 from trac.core import *
+from trac.mimeview.api import *
 from trac.perm import IPermissionRequestor
 from trac.Search import ISearchSource, search_to_sql, shorten_result
 from trac.Timeline import ITimelineEventProvider
 …
 from trac.web import HTTPNotFound, IRequestHandler
 from trac.wiki.api import IWikiPageManipulator, WikiSystem
 from trac.wiki.model import WikiPage
 from trac.wiki.formatter import wiki_to_html, wiki_to_oneliner
+from trac.mimeview.api import Mimeview, IContentConverter
+from trac.wiki.formatter import wiki_to_html, wiki_to_oneliner, \
+                                TEXT_X_TRAC_WIKI, APPLICATION_X_TRAC_WIKI
 class InvalidWikiPage(TracError):
 …
     page_manipulators = ExtensionPoint(IWikiPageManipulator)
     # IContentConverter methods
-    def get_supported_conversions(self):
-        yield ('txt', 'Plain Text', 'txt', 'text/x-trac-wiki', 'text/plain', 9)
+    def convert_content(self, req, mimetype, content, key):
+        return (content, 'text/plain;charset=utf-8')
+    def get_supported_conversions(self, input):
+        if TEXT_X_TRAC_WIKI.match(input) or \
+               APPLICATION_X_TRAC_WIKI.match(input):
+            yield Conversion(format='txt', quality=8, output=TEXT_PLAIN)
+            yield Conversion(format='html', quality=8, output=TEXT_HTML)
+    def convert_content(self, context, conversion, content):
+        if conversion.format == 'txt':
+            return content # identity transform
+        elif conversion.format == 'html':
+            # TODO 0.11: give the context to `wiki_to_html`
+            html = wiki_to_html(unicode(content), self.env, context.req)
+            return StringMimeContent(self.env, html, TEXT_HTML)
     # INavigationContributor methods
     def get_active_navigation_item(self, req):
 …
         else:
             format = req.args.get('format')
             if format:
+                Mimeview(self.env).send_converted(req, 'text/x-trac-wiki',
+                                                  page.text, format, page.name)
+                content = StringMimeContent(self.env, page.text,
+                                            TEXT_X_TRAC_WIKI, page.name)
+                content.convert(req, format).send(req)
             self._render_view(req, db, page)
         req.hdf['wiki.action'] = action
 …
             req.hdf['html.norobots'] = 1
         # Add registered converters
         for conversion in Mimeview(self.env).get_supported_conversions(
                                              'text/x-trac-wiki'):
             conversion_href = req.href.wiki(page.name, version=version,
                                             format=conversion[0])
             add_link(req, 'alternate', conversion_href, conversion[1],
                      conversion[3])
+        for c in Mimeview(self.env).get_conversions(TEXT_X_TRAC_WIKI):
+            if c.format in ('default', 'html'):
+                continue
+            add_link(req, 'alternate',
+                     req.href.wiki(page.name, version=version, format=c.format),
+                     c.output.name, c.output.mimetype_charset)
         latest_page = WikiPage(self.env, page.name)
         req.hdf['wiki'] = {'exists': page.exists,

trac/wiki/formatter.py

 from trac.util.text import shorten_line, to_unicode
 __all__ = ['wiki_to_html', 'wiki_to_oneliner', 'wiki_to_outline',
+           'wiki_to_link', 'Formatter' ]
+           'wiki_to_link', 'Formatter',
+           'TEXT_X_TRAC_WIKI', 'APPLICATION_X_TRAC_WIKI']
+TEXT_X_TRAC_WIKI = MimeType('text/x-trac-wiki',
+                            name='Trac Wiki Text', extension='txt')
+APPLICATION_X_TRAC_WIKI = MimeType('application/x-trac-wiki',
+                                   name='Trac Wiki Text', extension='txt')
 def system_message(msg, text=None):
     return html.DIV(html.STRONG(msg), text and html.PRE(text),
                     class_="system-message")
 …
     _code_block_re = re.compile('^<div(?:\s+class="([^"]+)")?>(.*)</div>$')
     def __init__(self, env, name):
         # TODO: transmit `formatter` argument
         self.env = env
+    def __init__(self, formatter, name):
+        self.formatter = formatter
+        self.env = formatter.env
         self.name = name
         self.error = None
         self.macro_provider = None
+        self.mimetype = None
         builtin_processors = {'html': self._html_processor,
                               'default': self._default_processor,
 …
             # Find a matching wiki macro
             for macro_provider in WikiSystem(self.env).macro_providers:
                 for macro_name in macro_provider.get_macros():
                     if self.name == macro_name:
+                    if name == macro_name:
                         self.processor = self._macro_processor
                         self.macro_provider = macro_provider
                         break
         if not self.processor:
             # Find a matching mimeview renderer
+            from trac.mimeview.api import Mimeview
+            mimetype = Mimeview(self.env).get_mimetype(self.name)
+            if mimetype:
+                self.name = mimetype
+            mimetype = Mimeview(self.env).get_mimetype(name)
+            if mimetype and not mimetype.is_binary:
+                self.mimetype = mimetype
                 self.processor = self._mimeview_processor
             else:
                 self.processor = self._default_processor
 …
     # generic processors
     def _macro_processor(self, req, text):
         # TODO: macro should take a `formatter` argument
+        # TODO: macro should take a `formatter` argument (0.11)
         self.env.log.debug('Executing Wiki macro %s by provider %s'
                            % (self.name, self.macro_provider))
         return self.macro_provider.render_macro(req, self.name, text)
     def _mimeview_processor(self, req, text):
         # TODO: transmit context from `formatter`
         return Mimeview(self.env).render(req, self.name, text)
+        content = StringMimeContent(self.env, text, self.mimetype)
+        return Mimeview(self.env).render(self.formatter, content)
     def process(self, req, text, in_paragraph=False):
         if self.error:
 …
     ENDBLOCK_TOKEN = r"\}\}\}"
     ENDBLOCK = "}}}"
+    LINK_SCHEME = r"[\w.+-]+" # as per RFC 2396
+    LINK_SCHEME = r"[a-zA-Z][\w.+-]*" # RFC 2396, except upper case is also ok
     INTERTRAC_SCHEME = r"[a-zA-Z.+-]*?" # no digits (support for shorthand links)
     QUOTED_STRING = r"'[^']+'|\"[^\"]+\""