ChristianBoos: mimeview_conversion.diff

trac/attachment.py

@@ -432 +432 @@
             fd.seek(0)
             
             binary = is_binary(str_data)
-            mime_type = mimeview.get_mimetype(attachment.filename, str_data)
+            mimetype = mimeview.get_mimetype(attachment.filename, str_data)
 
             # Eventually send the file directly
             format = req.args.get('format')
@@ -442 +442 @@
                     # contain malicious code enabling XSS attacks
                     req.send_header('Content-Disposition', 'attachment;' +
                                     'filename=' + attachment.filename)
-                if not mime_type or (self.render_unsafe_content and \
+                if not mimetype or (self.render_unsafe_content and \
                                      not binary and format == 'txt'):
-                    mime_type = 'text/plain'
-                if 'charset=' not in mime_type:
-                    charset = mimeview.get_charset(str_data, mime_type)
-                    mime_type = mime_type + '; charset=' + charset
-                req.send_file(attachment.path, mime_type)
+                    mimetype = 'text/plain'
+                full_mimetype = mimeview.get_mimetype_charset(
+                    attachment.filename, str_data, mimetype)
+                req.send_file(attachment.path, full_mimetype)
 
             # add ''Plain Text'' alternate link if needed
             if self.render_unsafe_content and not binary and \
-               not mime_type.startswith('text/plain'):
+               not mimetype.startswith('text/plain'):
                 plaintext_href = attachment.href(req, format='txt')
                 add_link(req, 'alternate', plaintext_href, 'Plain Text',
-                         mime_type)
+                         mimetype)
 
             # add ''Original Format'' alternate link (always)
             raw_href = attachment.href(req, format='raw')
-            add_link(req, 'alternate', raw_href, 'Original Format', mime_type)
+            add_link(req, 'alternate', raw_href, 'Original Format', mimetype)
 
             self.log.debug("Rendering preview of file %s with mime-type %s"
-                           % (attachment.filename, mime_type))
+                           % (attachment.filename, mimetype))
 
             req.hdf['attachment'] = mimeview.preview_to_hdf(
-                req, fd, os.fstat(fd.fileno()).st_size, mime_type,
+                req, fd, os.fstat(fd.fileno()).st_size, mimetype,
                 attachment.filename, raw_href, annotations=['lineno'])
         finally:
             fd.close()

trac/mimeview/api.py

@@ -27 +27 @@
  * taking advantage of existing conventions for the file name
  * examining the file content and applying various heuristics
 
-The module also knows how to convert the file content from one type
-to another type.
+The module also knows about conversions from one data type to another type,
+like conversions to text/html (this is no more a special case).
 
 In some cases, only the `url` pointing to the file's content is actually
 needed, that's why we avoid to read the file's content when it's not needed.
@@ -49 +49 @@
 
 
 __all__ = ['get_mimetype', 'is_binary', 'detect_unicode', 'Mimeview',
-           'content_to_unicode']
+           'content_to_unicode',
+           'combine_mimetype_charset', 'split_mimetype_charset']
 
 
 # Some common MIME types and their associated keywords and/or file extensions
@@ -153 +154 @@
                     return 'application/octet-stream'
         return mimetype
 
+def combine_mimetype_charset(mimetype, charset):
+    """Combine the MIME type and charset information in a single string."""
+    if mimetype and charset and not 'charset' in mimetype:
+        return '%s; charset=%s' % (mimetype, charset)
+    else:
+        return mimetype
+
+def split_mimetype_charset(full_mimetype):
+    """Return (mimetype, charset) from the combined information"""
+    mimetype = full_mimetype
+    charset = None
+    idx = full_mimetype.find(';')
+    if idx >= 0:
+        mimetype = full_mimetype[:idx].strip()
+        idx = full_mimetype.find('charset=', idx)
+        if idx >= -1:
+            charset = full_mimetype[idx+8:].strip()
+    return mimetype, charset
+
 def is_binary(data):
     """Detect binary content by checking the first thousand bytes for zeroes.
 
@@ -176 +196 @@
     else:
         return None
 
+# Deprecated (TODO: remove in 0.11)
+
 def content_to_unicode(env, content, mimetype):
-    """Retrieve an `unicode` object from a `content` to be previewed"""
-    mimeview = Mimeview(env)
-    if hasattr(content, 'read'):
-        content = content.read(mimeview.max_preview_size)
-    return mimeview.to_unicode(content, mimetype)
+    """Retrieve an `unicode` object from a `content` to be previewed.
+    ''Deprecated in 0.10.''
+    """
+    return Mimeview(env).fetch_content(content, mimetype)
 
 
 class IHTMLPreviewRenderer(Interface):
     """Extension point interface for components that add HTML renderers of
     specific content types to the `Mimeview` component.
 
-    (Deprecated)
+    Deprecated in 0.10. Implement `IContentConverter` instead.
     """
 
     # implementing classes should set this property to True if they
@@ -196 +217 @@
     expand_tabs = False
 
     def get_quality_ratio(mimetype):
-        """Return the level of support this renderer provides for the `content`
-        of the specified MIME type. The return value must be a number between
-        0 and 9, where 0 means no support and 9 means "perfect" support.
-        """
+        """Return the level of support this renderer provides"""
 
     def render(req, mimetype, content, filename=None, url=None):
-        """Render an XHTML preview of the raw `content`.
+        """Render an XHTML preview of the raw `content`."""
 
-        The `content` might be:
-         * a `str` object
-         * an `unicode` string
-         * any object with a `read` method, returning one of the above
 
-        It is assumed that the content will correspond to the given `mimetype`.
-
-        Besides the `content` value, the same content may eventually
-        be available through the `filename` or `url` parameters.
-        This is useful for renderers that embed objects, using <object> or
-        <img> instead of including the content inline.
-        
-        Can return the generated XHTML text as a single string or as an
-        iterable that yields strings. In the latter case, the list will
-        be considered to correspond to lines of text in the original content.
-        """
-
 class IHTMLPreviewAnnotator(Interface):
     """Extension point interface for components that can annotate an XHTML
     representation of file contents with additional information."""
@@ -238 +240 @@
         annotation data."""
 
 
+class Conversion(object):
+    """A data conversion specification.
+
+    The conversion goes from an `in_type` to an `out_type`.
+    A conversion is identified by a `key`, has a `name` and proposes
+    an `extension` that can be used for storing the converted data in a file.
+
+    The `quality` ratio of the conversion is a number in the range 0 to 9,
+    where 0 means no support and 9 means "perfect" support.
+
+    Finally, `expand_tabs` indicates whether a tab expansion should precede
+    the conversion attempt.
+
+    e.g. Conversion(key='latex', name='LaTeX', extension='tex',
+                    in_type='text/x-trac-wiki', out_type='text/x-tex',
+                    quality=8)
+    """
+
+    def __init__(self, key, name=None, extension='',
+                 in_type=None, out_type='text/html',
+                 quality=1, expand_tabs=False):
+        self.key = key
+        self.name = name or key
+        self.extension = extension
+        self.in_type = in_type
+        self.out_type = out_type
+        self.quality = quality
+        self.expand_tabs = expand_tabs
+
+    def __repr__(self):
+        return '<Conversion "%s" %s -> %s>' % \
+               (self.key, self.in_type, self.out_type) 
+
+
 class IContentConverter(Interface):
     """An extension point interface for generic MIME based content
     conversion."""
 
-    def get_supported_conversions():
-        """Return an iterable of tuples in the form (key, name, extension,
-        in_mimetype, out_mimetype, quality) representing the MIME conversions
-        supported and
-        the quality ratio of the conversion in the range 0 to 9, where 0 means
-        no support and 9 means "perfect" support. eg. ('latex', 'LaTeX', 'tex',
-        'text/x-trac-wiki', 'text/plain', 8)"""
+    def get_supported_conversions(mimetype):
+        """Check if conversion of `mimetype` is supported by this converter.
 
-    def convert_content(req, mimetype, content, key):
-        """Convert the given content from mimetype to the output MIME type
-        represented by key. Returns a tuple in the form (content,
-        output_mime_type)."""
+        Return an iterable of `Conversion` objects if this is the case.
+        """
 
+    def convert_content(req, conversion, content, filename, url):
+        """Convert the given `content` using the specified `conversion` object.
 
+        If not directly available through the `content` value,
+        the content may be available through the `filename` or `url`
+        arguments.
+        This can be useful for converters that can provide links to objects,
+        instead of having to inline the content.
+
+        Return the converted content.
+        """
+
+
 class Mimeview(Component):
     """A generic class to prettify data, typically source code."""
 
-    renderers = ExtensionPoint(IHTMLPreviewRenderer)
+    renderers = ExtensionPoint(IHTMLPreviewRenderer) # TODO: remove in 0.11
     annotators = ExtensionPoint(IHTMLPreviewAnnotator)
     converters = ExtensionPoint(IContentConverter)
 
@@ -282 +323 @@
     def __init__(self):
         self._mime_map = None
         
-    # Public API
+    # -- MIME type conversion
+    
+    def get_supported_conversions(self, mimetype, content=None, filename=None):
+        """Return a list of possible conversions for the given `content`.
 
-    def get_supported_conversions(self, mimetype):
-        """Return a list of target MIME types in same form as
-        `IContentConverter.get_supported_conversions()`, but with the converter
-        component appended. Output is ordered from best to worst quality."""
+        The input `mimetype` is inferred from the `content` and/or the
+        `filename`, if not given.
+
+        Return a list of (conversion,converter), ordered from best
+        to worst quality.
+        """
+        # Ensure we have a mimetype and only the mimetype, without the charset
+        if mimetype:
+            mimetype, charset = split_mimetype_charset(mimetype)
+        else:
+            mimetype = self.get_mimetype(filename, content) or 'text/plain'
+
+        # Build list of possible conversions, with their associated converters
         converters = []
         for converter in self.converters:
-            for k, n, e, im, om, q in converter.get_supported_conversions():
-                if im == mimetype and q > 0:
-                    converters.append((k, n, e, im, om, q, converter))
-        converters = sorted(converters, key=lambda i: i[-1], reverse=True)
-        return converters
+            print converter
+            for conversion in converter.get_supported_conversions(mimetype):
+                if conversion.quality > 0:
+                    converters.append((conversion, converter))
 
-    def convert_content(self, req, mimetype, content, key, filename=None,
-                        url=None):
-        """Convert the given content to the target MIME type represented by
-        `key`, which can be either a MIME type or a key. Returns a tuple of
-        (content, output_mime_type, extension)."""
+        # ---- Backward compatibility support for IHTMLPreviewRenderer
+        class RendererWrapper(object):
+            def __init__(self, renderer):
+                self.renderer = renderer
+            def convert_content(self, req, conversion, content,
+                                filename=None, url=None):
+                return self.renderer.render(req, conversion.in_type,
+                                            content, filename, url)
+        for renderer in self.renderers:
+            qr = renderer.get_quality_ratio(mimetype)
+            if qr > 0:
+                expand_tabs = getattr(renderer, 'expand_tabs', False)
+                converters.append(
+                    (Conversion(key='', name='', extension=None,
+                                in_type=mimetype, out_type='text/html',
+                                quality=8, expand_tabs=expand_tabs),
+                     RendererWrapper(renderer)))
+        # ---- (to be removed in 0.11)
+
+        return sorted(converters, key=lambda c: c[0].quality, reverse=True)
+
+    def convert_content(self, req, content, mimetype, selector,
+                        filename=None, url=None):
+        """Convert the `content` to targeted MIME type specified by 'selector'.
+
+        The content has the MIME type `mimetype` and the target MIME type
+        is determined by `selector`, which can be either directly the
+        output MIME type or a key identifying the Conversion.
+
+        Returns a tuple of (content, output_mime_type, extension).
+        """
         if not content:
-            return ('', 'text/plain;charset=utf-8')
+            return ('', 'text/plain; charset=utf-8', '')
 
-        # Ensure we have a MIME type for this content
-        full_mimetype = mimetype
-        if not full_mimetype:
-            if hasattr(content, 'read'):
-                content = content.read(self.max_preview_size)
-            full_mimetype = self.get_mimetype(filename, content)
-        if full_mimetype:
-            mimetype = full_mimetype.split(';')[0].strip() # split off charset
-        else:
-            mimetype = full_mimetype = 'text/plain' # fallback if not binary
+        # Ensure we have the mimetype and the charset information
+        print `('cc', filename, content, mimetype)`
+        full_mimetype = self.get_mimetype_charset(filename, content, mimetype)
+        mimetype, charset = split_mimetype_charset(full_mimetype)
 
-        # Choose best converter
-        candidates = self.get_supported_conversions(mimetype)
-        candidates = [c for c in candidates if key in (c[0], c[4])]
+        # Filter the converters of `mimetype` that are matching `selector`
+        candidates = self.get_supported_conversions(mimetype, content, filename)
+        candidates = [c for c in candidates
+                      if selector in (c[0].key, c[0].out_type)]
         if not candidates:
             raise TracError('No available MIME conversions from %s to %s' %
-                            (mimetype, key))
+                            (mimetype, selector))
 
+        tab_expanded = False # we don't want to expand tabs more than once.
+
         # First candidate which converts successfully wins.
-        for ck, name, ext, input_mimettype, output_mimetype, quality, \
-                converter in candidates:
+        for conversion, converter in candidates:
+            if conversion.expand_tabs and not tab_expanded:
+                content = self.fetch_content(content, full_mimetype)
+                content = content.expandtabs(self.tab_width)
+                tab_expanded = True
             try:
-                output = converter.convert_content(req, mimetype, content, ck)
+                output = converter.convert_content(req, conversion, content,
+                                                   filename, url)
                 if not output:
                     continue
-                return (output[0], output[1], ext)
+                return (output[0], output[1], conversion.extension)
             except Exception, e:
                 self.log.warning('MIME conversion using %s failed (%s)'
                                  % (converter, e), exc_info=True)
-        raise TracError('No available MIME conversions from %s to %s' %
-                        (mimetype, key))
+        raise TracError('No MIME conversions from %s to %s succeeded' %
+                        (mimetype, selector))
 
+    # -- XHTML rendering and annotations (based on the conversion API)
+    
     def get_annotation_types(self):
         """Generator that returns all available annotation types."""
         for annotator in self.annotators:
@@ -343 +424 @@
 
     def render(self, req, mimetype, content, filename=None, url=None,
                annotations=None):
-        """Render an XHTML preview of the given `content`.
-
-        `content` is the same as an `IHTMLPreviewRenderer.render`'s
-        `content` argument.
-
-        The specified `mimetype` will be used to select the most appropriate
-        `IHTMLPreviewRenderer` implementation available for this MIME type.
-        If not given, the MIME type will be infered from the filename or the
-        content.
-
-        Return a string containing the XHTML text.
+        """Render an XHTML preview of the given `content`, with `annotations`.
         """
-        if not content:
-            return ''
-
-        # Ensure we have a MIME type for this content
-        full_mimetype = mimetype
-        if not full_mimetype:
-            if hasattr(content, 'read'):
-                content = content.read(self.max_preview_size)
-            full_mimetype = self.get_mimetype(filename, content)
-        if full_mimetype:
-            mimetype = full_mimetype.split(';')[0].strip() # split off charset
+        result, output_type, ext = self.convert_content(
+            req, content, mimetype, 'text/html', filename, url)
+        print `result`
+        if isinstance(result, Fragment):
+            return result
+        elif isinstance(result, basestring):
+            return Markup(to_unicode(result))
+        elif annotations:
+            return Markup(self._annotate(result, annotations))
         else:
-            mimetype = full_mimetype = 'text/plain' # fallback if not binary
+            buf = StringIO()
+            buf.write('<div class="code"><pre>')
+            for line in result:
+                buf.write(line + '\n')
+            buf.write('</pre></div>')
+            return Markup(buf.getvalue())
 
-        # Determine candidate `IHTMLPreviewRenderer`s
-        candidates = []
-        for renderer in self.renderers:
-            qr = renderer.get_quality_ratio(mimetype)
-            if qr > 0:
-                candidates.append((qr, renderer))
-        candidates.sort(lambda x,y: cmp(y[0], x[0]))
-
-        # First candidate which renders successfully wins.
-        # Also, we don't want to expand tabs more than once.
-        expanded_content = None
-        for qr, renderer in candidates:
-            try:
-                self.log.debug('Trying to render HTML preview using %s'
-                               % renderer.__class__.__name__)
-                # check if we need to perform a tab expansion
-                rendered_content = content
-                if getattr(renderer, 'expand_tabs', False):
-                    if expanded_content is None:
-                        content = content_to_unicode(self.env, content,
-                                                     full_mimetype)
-                        expanded_content = content.expandtabs(self.tab_width)
-                    rendered_content = expanded_content
-                result = renderer.render(req, full_mimetype, rendered_content,
-                                         filename, url)
-                if not result:
-                    continue
-                elif isinstance(result, Fragment):
-                    return result
-                elif isinstance(result, basestring):
-                    return Markup(to_unicode(result))
-                elif annotations:
-                    return Markup(self._annotate(result, annotations))
-                else:
-                    buf = StringIO()
-                    buf.write('<div class="code"><pre>')
-                    for line in result:
-                        buf.write(line + '\n')
-                    buf.write('</pre></div>')
-                    return Markup(buf.getvalue())
-            except Exception, e:
-                self.log.warning('HTML preview using %s failed (%s)'
-                                 % (renderer, e), exc_info=True)
-
     def _annotate(self, lines, annotations):
         buf = StringIO()
         buf.write('<table class="code"><thead><tr>')
@@ -447 +477 @@
         buf.write('</tbody></table>')
         return buf.getvalue()
 
-    def get_max_preview_size(self):
-        """Deprecated: use `max_preview_size` attribute directly."""
-        return self.max_preview_size
-
+    # -- MIME type and charset detection
+    
     def get_charset(self, content='', mimetype=None):
         """Infer the character encoding from the `content` or the `mimetype`.
 
@@ -459 +487 @@
         The charset will be determined using this order:
          * from the charset information present in the `mimetype` argument
          * auto-detection of the charset from the `content`
-         * the configured `default_charset` 
+         * the configured `default_charset`
         """
         if mimetype:
-            ctpos = mimetype.find('charset=')
-            if ctpos >= 0:
-                return mimetype[ctpos + 8:].strip()
+            mimetype, charset = split_mimetype_charset(mimetype)
+            if charset:
+                return charset
         if isinstance(content, str):
             utf = detect_unicode(content)
             if utf is not None:
                 return utf
+        # TODO: ICharsetDetector
         return self.default_charset
 
     def get_mimetype(self, filename, content=None):
         """Infer the MIME type from the `filename` or the `content`.
 
-        `content` is either a `str` or an `unicode` object.
+        `content` is either a `str` or an `unicode` object,
+        or something that can be `read`.
 
-        Return the detected MIME type, augmented by the
-        charset information (i.e. "<mimetype>; charset=..."),
-        or `None` if detection failed.
+        Return the detected MIME type or `None` if detection failed.
         """
         # Extend default extension to MIME type mappings with configured ones
         if not self._mime_map:
@@ -489 +517 @@
                     for keyword in assocations: # Note: [0] kept on purpose
                         self._mime_map[keyword] = assocations[0]
 
-        mimetype = get_mimetype(filename, content, self._mime_map)
+        # read the content only if there's no other way to get the mimetype
+        if hasattr(content, 'read'):
+            # first try to get the mimetype from the filename only
+            mimetype = get_mimetype(filename, None, self._mime_map)
+            if mimetype:
+                return mimetype
+            content = self.fetch_content(content, mimetype)
+        return get_mimetype(filename, content, self._mime_map)
+
+    def get_mimetype_charset(self, filename, content=None, mimetype=None):
+        """Retrieve combined mimetype and charset information.
+
+        If `mimetype` is given, we check if it provides the needed information,
+        otherwise we try to detect the mimetype and/or the charset.
+        """
+        print `('gmc', filename, content, mimetype)`
         charset = None
+        if not mimetype:
+            mimetype = self.get_mimetype(filename, content)
+        print `('gmc2', mimetype)`
         if mimetype:
+            if 'charset=' in mimetype:
+                return mimetype
             charset = self.get_charset(content, mimetype)
-        if mimetype and charset and not 'charset' in mimetype:
-            mimetype += '; charset=' + charset
-        return mimetype
+        return combine_mimetype_charset(mimetype, charset)
 
-    def to_utf8(self, content, mimetype=None):
-        """Convert an encoded `content` to utf-8.
-
-        ''Deprecated in 0.10. You should use `unicode` strings only.''
-        """
-        return to_utf8(content, self.get_charset(content, mimetype))
-
+    # -- Charset conversion
+    
     def to_unicode(self, content, mimetype=None, charset=None):
         """Convert `content` (an encoded `str` object) to an `unicode` object.
 
@@ -514 +555 @@
             charset = self.get_charset(content, mimetype)
         return to_unicode(content, charset)
 
+    def fetch_content(self, content, mimetype):
+        if hasattr(content, 'read'):
+            content = content.read(self.max_preview_size)
+        return self.to_unicode(content, mimetype)
+
+    # -- Deprecated API (TODO: remove in 0.11)
+
+    def get_max_preview_size(self):
+        """Deprecated: use `max_preview_size` attribute directly."""
+        return self.max_preview_size
+
+    def to_utf8(self, content, mimetype=None):
+        """Convert an encoded `content` to utf-8.
+
+        ''Deprecated in 0.10. You should use `unicode` strings only.''
+        """
+        return to_utf8(content, self.get_charset(content, mimetype))
+
+    # -- Utilities
+
     def configured_modes_mapping(self, renderer):
-        """Return a MIME type to `(mode,quality)` mapping for given `option`"""
+        """Utility for configurable custom converters
+
+        Return a MIME type to `(mode,quality)` mapping for given `option`,
+        assuming a format of comma-separated <mimetype>:<mode>:<quality>
+        associations.
+
+        See EnscriptConverter and SilverCityConverter.
+        """
         types, option = {}, '%s_modes' % renderer
         for mapping in self.config['mimeviewer'].getlist(option):
             if not mapping:
@@ -543 +611 @@
                                            url, annotations),
                     'raw_href': url}
 
-    def send_converted(self, req, in_type, content, selector, filename='file'):
+    def send_converted(self, req, content, mimetype, selector,
+                       filename='file'):
         """Helper method for converting `content` and sending it directly.
 
-        `selector` can be either a key or a MIME Type."""
+        `mimetype` is the type of the content.
+        `selector` can be either a key or the expected output MIME Type.
+        """
         from trac.web import RequestDone
-        content, output_type, ext = self.convert_content(req, in_type,
-                                                         content, selector)
+        content, output_type, ext = self.convert_content(
+            req, content, mimetype, selector, filename)
         req.send_response(200)
         req.send_header('Content-Type', output_type)
-        req.send_header('Content-Disposition', 'filename=%s.%s' % (filename,
-                                                                   ext))
+        req.send_header('Content-Disposition', 'filename=%s.%s' % 
+                        (filename, ext))
         req.end_headers()
         req.write(content)
         raise RequestDone        
         
 
+# utility for Mimeview._annotate
 def _html_splitlines(lines):