Ticket #4246: trac-pygments.3.diff

htdocs/css/code.css

@@ -50 +50 @@
 table.code tbody th :link:hover, table.code tbody th :visited:hover {
  color: #000;
 }
-table.code tbody td {
- background: #fff;
+table.code td {
  font: normal 11px monospace;
  overflow: hidden;
  padding: 1px 2px;
  vertical-align: top;
 }
-table.code tbody tr.hilite th {
+table.code tr.hilite th {
  background: #ccf;
 }
-table.code tbody tr.hilite td {
+table.code tr.hilite td {
  background: #ddf;
 }
 .image-file { background: #eee; padding: .3em }
 .image-file img { background: url(../imggrid.png) }
 
 /* Default */
-.code-block span {
- font-family: monospace;
-}
+.code-block span { font-family: monospace; }
 
 /* Comments */
 .code-comment, .css_comment, .c_comment, .c_commentdoc, .c_commentline,

trac/db_default.py

@@ -396 +396 @@
                       'trac.attachment', 'trac.db.mysql_backend',
                       'trac.db.postgres_backend', 'trac.db.sqlite_backend',
                       'trac.mimeview.enscript', 'trac.mimeview.patch',
-                      'trac.mimeview.php', 'trac.mimeview.rst',
-                      'trac.mimeview.silvercity', 'trac.mimeview.txtl',
+                      'trac.mimeview.php', 'trac.mimeview.pygment',
+                      'trac.mimeview.rst', 'trac.mimeview.silvercity',
+                      'trac.mimeview.txtl',
                       'trac.prefs.web_ui', 'trac.search.web_ui',
                       'trac.ticket.admin', 'trac.ticket.query',
                       'trac.ticket.report', 'trac.ticket.roadmap',

trac/mimeview/api.py

@@ -59 +59 @@
 import re
 from StringIO import StringIO
 
-from genshi.core import escape, Markup, Stream
+from genshi import escape, Markup, Stream
+from genshi.core import TEXT, START, END, START_NS, END_NS
 from genshi.builder import Fragment, tag
+from genshi.input import HTMLParser
 
 from trac.config import IntOption, ListOption, Option
 from trac.core import *
@@ -230 +232 @@
     # support text content where Trac should expand tabs into spaces
     expand_tabs = False
 
+    # indicate whether the output of this renderer is source code that can
+    # be decorated with annotations
+    returns_source = False
+
     def get_quality_ratio(mimetype):
         """Return the level of support this renderer provides for the `content`
         of the specified MIME type. The return value must be a number between
@@ -256 +262 @@
         be considered to correspond to lines of text in the original content.
         """
 
+
 class IHTMLPreviewAnnotator(Interface):
     """Extension point interface for components that can annotate an XHTML
     representation of file contents with additional information."""
@@ -268 +275 @@
         let the user toggle the appearance of the annotation type.
         """
 
-    def annotate_line(number, content):
+    def annotate_row(number, content):
         """Return the XHTML markup for the table cell that contains the
         annotation data."""
 
@@ -438 +445 @@
             try:
                 self.log.debug('Trying to render HTML preview using %s'
                                % renderer.__class__.__name__)
+
                 # check if we need to perform a tab expansion
                 rendered_content = content
                 if getattr(renderer, 'expand_tabs', False):
@@ -446 +454 @@
                                                      full_mimetype)
                         expanded_content = content.expandtabs(self.tab_width)
                     rendered_content = expanded_content
+
                 result = renderer.render(req, full_mimetype, rendered_content,
                                          filename, url)
                 if not result:
                     continue
-                elif isinstance(result, (Fragment, Stream)):
-                    return result
-                elif isinstance(result, basestring):
-                    return Markup(to_unicode(result))
-                elif annotations:
+
+                if not getattr(renderer, 'returns_source', False):
+                    if isinstance(result, basestring):
+                        if not isinstance(result, unicode):
+                            result = to_unicode(result)
+                        return Markup(to_unicode(result))
+                    elif isinstance(result, Fragment):
+                        return result.generate()
+                    else:
+                        return result
+
+                if annotations:
                     m = req.args.get('marks')
-                    return Markup(self._annotate(result, annotations,
-                                                 m and Ranges(m)))
+                    return self._annotate(result, annotations, m and Ranges(m))
                 else:
-                    buf = StringIO()
-                    buf.write('<div class="code"><pre>')
-                    for line in result:
-                        buf.write(line + '\n')
-                    buf.write('</pre></div>')
-                    return Markup(buf.getvalue())
+                    return tag.div(class_='code')(tag.pre(result)).generate()
+
             except Exception, e:
                 self.log.warning('HTML preview using %s failed (%s)'
                                  % (renderer, e), exc_info=True)
 
-    def _annotate(self, lines, annotations, marks=None):
-        buf = StringIO()
-        buf.write('<table class="code"><thead><tr>')
-        annotators = []
+    def _annotate(self, stream, annotations, marks=None):
+        annotators, annotypes = [], []
         for annotator in self.annotators:
-            atype, alabel, adesc = annotator.get_annotation_type()
+            atype, alabel, _ = annotator.get_annotation_type()
             if atype in annotations:
-                buf.write('<th class="%s">%s</th>' % (atype, alabel))
+                annotypes.append((atype, alabel))
                 annotators.append(annotator)
-        buf.write('<th class="content">&nbsp;</th>')
-        buf.write('</tr></thead><tbody>')
 
-        space_re = re.compile('(?P<spaces> (?: +))|'
-                              '^(?P<tag><\w+.*?>)?( )')
-        def htmlify(match):
-            m = match.group('spaces')
-            if m:
-                div, mod = divmod(len(m), 2)
-                return div * '&nbsp; ' + mod * '&nbsp;'
-            return (match.group('tag') or '') + '&nbsp;'
+        if isinstance(stream, list):
+            stream = HTMLParser(StringIO('\n'.join(stream)))
 
-        num = -1
-        for num, line in enumerate(_html_splitlines(lines)):
-            cells = []
-            for annotator in annotators:
-                cells.append(annotator.annotate_line(num + 1, line))
-            cells.append('<td>%s</td>\n' % space_re.sub(htmlify, line))
-            if marks and num+1 in marks:
-                buf.write('<tr class="%s">' % ('hilite',) +
-                          '\n'.join(cells) + '</tr>')
-            else:
-                buf.write('<tr>' + '\n'.join(cells) + '</tr>')
-        else:
-            if num < 0:
-                return ''
-        buf.write('</tbody></table>')
-        return buf.getvalue()
+        def _head_row():
+            return tag.re(
+                [tag.th(alabel, class_=atype) for atype, alabel in annotypes] +
+                [tag.th(u'\xa0', class_='content')]
+            )
 
+        def _body_rows():
+            for idx, line in enumerate(_group_lines(stream)):
+                row = tag.tr()
+                if marks and idx + 1 in marks:
+                    row(class_='hilite')
+                for annotator in annotators:
+                    annotator.annotate_row(row, idx + 1, line)
+                row.append(tag.td(line))
+                yield row
+
+        return tag.table(class_='code')(
+            tag.thead(_head_row()),
+            tag.tbody(_body_rows())
+        )
+
     def get_max_preview_size(self):
         """Deprecated: use `max_preview_size` attribute directly."""
         return self.max_preview_size
@@ -616 +621 @@
                                                                    ext))
         req.end_headers()
         req.write(content)
-        raise RequestDone        
-        
+        raise RequestDone
 
-def _html_splitlines(lines):
-    """Tracks open and close tags in lines of HTML text and yields lines that
-    have no tags spanning more than one line."""
-    open_tag_re = re.compile(r'<(\w+)(\s.*?)?[^/]?>')
-    close_tag_re = re.compile(r'</(\w+)>')
-    open_tags = []
-    for line in lines:
-        # Reopen tags still open from the previous line
-        for tag in open_tags:
-            line = tag.group(0) + line
-        open_tags = []
 
-        # Find all tags opened on this line
-        for tag in open_tag_re.finditer(line):
-            open_tags.append(tag)
+def _group_lines(stream):
+    space_re = re.compile('(?P<spaces> (?: +))|^(?P<tag><\w+.*?>)?( )')
+    def pad_spaces(match):
+        m = match.group('spaces')
+        if m:
+            div, mod = divmod(len(m), 2)
+            return div * u'\xa0 ' + mod * u'\xa0'
+        return (match.group('tag') or '') + u'\xa0'
 
-        open_tags.reverse()
+    def _generate():
+        stack = []
+        def _reverse():
+            for event in reversed(stack):
+                if event[0] is START:
+                    yield END, event[1][0], event[2]
+                else:
+                    yield END_NS, event[1][0], event[2]
 
-        # Find all tags closed on this line
-        for ctag in close_tag_re.finditer(line):
-            for otag in open_tags:
-                if otag.group(1) == ctag.group(1):
-                    open_tags.remove(otag)
-                    break
+        for kind, data, pos in stream:
+            if kind is TEXT:
+                lines = data.splitlines(True)
+                for e in stack:
+                    yield e
+                yield kind, lines.pop(0).rstrip('\n'), pos
+                for e in _reverse():
+                    yield e
+                if '\n' in data:
+                    yield TEXT, '\n', pos
+                    for line in lines:
+                        for event in stack:
+                            yield event
+                        yield kind, line.rstrip('\n'), pos
+                        if line.endswith('\n'):
+                            for e in _reverse():
+                                yield e
+                            yield TEXT, '\n', pos
+            else:
+                if kind is START or kind is START_NS:
+                    stack.append((kind, data, pos))
+                elif kind is END or kind is END_NS:
+                    stack.pop()
+                else:
+                    yield kind, data, pos
 
-        # Close all tags still open at the end of line, they'll get reopened at
-        # the beginning of the next line
-        for tag in open_tags:
-            line += '</%s>' % tag.group(1)
+    buf = []
+    for kind, data, pos in _generate():
+        if kind is TEXT and data == '\n':
+            yield Stream(buf[:])
+            del buf[:]
+        else:
+            if kind is TEXT:
+                data = space_re.sub(pad_spaces, data)
+            buf.append((kind, data, pos))
+    if buf:
+        yield Stream(buf[:])
 
-        yield line
 
-
 # -- Default annotators
 
 class LineNumberAnnotator(Component):
@@ -663 +692 @@
     def get_annotation_type(self):
         return 'lineno', 'Line', 'Line numbers'
 
-    def annotate_line(self, number, content):
-        return '<th id="L%s"><a href="#L%s">%s</a></th>' % (number, number,
-                                                            number)
+    def annotate_row(self, row, lineno, content):
+        row.append(tag.th(id='L%s' % lineno)(
+            tag.a(lineno, href='#L%s' % lineno)
+        ))
 
 
 # -- Default renderers
@@ -677 +707 @@
     implements(IHTMLPreviewRenderer)
 
     expand_tabs = True
+    returns_source = True
 
     TREAT_AS_BINARY = [
         'application/pdf',
@@ -696 +727 @@
 
         self.env.log.debug("Using default plain text mimeviewer")
         content = content_to_unicode(self.env, content, mimetype)
-        for line in content.splitlines():
-            yield escape(line)
+        for line in content.splitlines(True):
+            yield TEXT, line, (None, -1, -1)
 
 
 class ImageRenderer(Component):
@@ -712 +743 @@
     def render(self, req, mimetype, content, filename=None, url=None):
         if url:
             return tag.div(tag.img(src=url, alt=filename),
-                           class_="image-file")
+                           class_='image-file')
 
 
 class WikiTextRenderer(Component):

trac/mimeview/tests/api.py

@@ -15 +15 @@
 
 from trac.core import *
 from trac.test import EnvironmentStub
-from trac.mimeview.api import get_mimetype, _html_splitlines, \
-                              Mimeview, IContentConverter
+from trac.mimeview.api import get_mimetype, IContentConverter, Mimeview
 
+
 class GetMimeTypeTestCase(unittest.TestCase):
 
     def test_from_suffix_using_MIME_MAP(self):
@@ -53 +53 @@
     def test_from_content_using_is_binary(self):
         self.assertEqual('application/octet-stream',
                          get_mimetype('xxx', "abc\0xyz"))
-        
 
-class Converter0(Component):
-    implements(IContentConverter)
-    def get_supported_conversions(self):
-        yield ('key0', 'Format 0', 'c0', 'text/x-sample', 'text/html', 8)
 
-class Converter2(Component):
-    implements(IContentConverter)
-    def get_supported_conversions(self):
-        yield ('key2', 'Format 2', 'c2', 'text/x-sample', 'text/html', 2)
-
-class Converter1(Component):
-    implements(IContentConverter)
-    def get_supported_conversions(self):
-        yield ('key1', 'Format 1', 'c1', 'text/x-sample', 'text/html', 4)
-
 class MimeviewTestCase(unittest.TestCase):
 
     def setUp(self):
         self.env = EnvironmentStub(default_data=True)
 
-    def test_html_splitlines_without_markup(self):
-        lines = ['line 1', 'line 2']
-        self.assertEqual(lines, list(_html_splitlines(lines)))
+        # Make sure we have no external components hanging around in the
+        # component registry
+        from trac.core import ComponentMeta
+        self.old_registry = ComponentMeta._registry
+        ComponentMeta._registry = {}
 
-    def test_html_splitlines_with_markup(self):
-        lines = ['<p><b>Hi', 'How are you</b></p>']
-        result = list(_html_splitlines(lines))
-        self.assertEqual('<p><b>Hi</b></p>', result[0])
-        self.assertEqual('<p><b>How are you</b></p>', result[1])
+    def tearDown(self):
+        # Restore the original component registry
+        from trac.core import ComponentMeta
+        ComponentMeta._registry = self.old_registry
 
-    def test_html_splitlines_with_multiline(self):
-        """
-        Regression test for http://trac.edgewall.org/ticket/2655
-        """
-        lines = ['<span class="p_tripledouble">"""',
-                'a <a href="http://google.com">http://google.com</a>/',
-                'Test', 'Test', '"""</span>']
-        result = list(_html_splitlines(lines))
-        self.assertEqual('<span class="p_tripledouble">"""</span>', result[0])
-        self.assertEqual('<span class="p_tripledouble">a '
-                         '<a href="http://google.com">http://google.com</a>/'
-                         '</span>', result[1])
-        self.assertEqual('<span class="p_tripledouble">Test</span>', result[2])
-        self.assertEqual('<span class="p_tripledouble">Test</span>', result[3])
-        self.assertEqual('<span class="p_tripledouble">"""</span>', result[4])
-
     def test_get_supported_conversions(self):
+        class Converter0(Component):
+            implements(IContentConverter)
+            def get_supported_conversions(self):
+                yield 'key0', 'Format 0', 'c0', 'text/x-sample', 'text/html', 8
+
+        class Converter2(Component):
+            implements(IContentConverter)
+            def get_supported_conversions(self):
+                yield 'key2', 'Format 2', 'c2', 'text/x-sample', 'text/html', 2
+
+        class Converter1(Component):
+            implements(IContentConverter)
+            def get_supported_conversions(self):
+                yield 'key1', 'Format 1', 'c1', 'text/x-sample', 'text/html', 4
+
         mimeview = Mimeview(self.env)
         conversions = mimeview.get_supported_conversions('text/x-sample')
         self.assertEqual(Converter0(self.env), conversions[0][-1])
         self.assertEqual(Converter1(self.env), conversions[1][-1])
         self.assertEqual(Converter2(self.env), conversions[2][-1])
 
+
 def suite():
     suite = unittest.TestSuite()
     suite.addTest(unittest.makeSuite(GetMimeTypeTestCase, 'test'))

trac/mimeview/silvercity.py

@@ -41 +41 @@
     'text/x-chdr':              ('CPP', 3),
     'text/x-csrc':              ('CPP', 3),
     'text/x-perl':              ('Perl', 3),
-    'text/x-php':               ('HyperText', 3, {'asp.default.language':4}),
-    'application/x-httpd-php':  ('HyperText', 3, {'asp.default.language':4}),
-    'application/x-httpd-php4': ('HyperText', 3, {'asp.default.language':4}),
-    'application/x-httpd-php3': ('HyperText', 3, {'asp.default.language':4}),
+    'text/x-php':               ('HyperText', 3, {'asp.default.language': 4}),
+    'application/x-httpd-php':  ('HyperText', 3, {'asp.default.language': 4}),
+    'application/x-httpd-php4': ('HyperText', 3, {'asp.default.language': 4}),
+    'application/x-httpd-php3': ('HyperText', 3, {'asp.default.language': 4}),
     'text/x-javascript':        ('CPP', 3), # Kludgy.
-    'text/x-psp':               ('HyperText', 3, {'asp.default.language':3}),
+    'text/x-psp':               ('HyperText', 3, {'asp.default.language': 3}),
     'text/x-python':            ('Python', 3),
     'text/x-ruby':              ('Ruby', 3),
     'text/x-sql':               ('SQL', 3),
@@ -76 +76 @@
         (''since 0.10'').""")
 
     expand_tabs = True
+    returns_source = True
 
     def __init__(self):
         self._types = None

trac/mimeview/pygment.py

@@ +1 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2006 Edgewall Software
+# Copyright (C) 2006 Matthew Good <matt@matt-good.net>
+# All rights reserved.
+#
+# This software is licensed as described in the file COPYING, which
+# you should have received as part of this distribution. The terms
+# are also available at http://trac.edgewall.org/wiki/TracLicense.
+#
+# Author: Matthew Good <matt@matt-good.net>
+
+"""Syntax highlighting based on Pygments."""
+
+from datetime import datetime
+import os
+from pkg_resources import resource_filename
+import re
+
+from trac.core import *
+from trac.config import ListOption, Option
+from trac.mimeview.api import IHTMLPreviewRenderer, Mimeview
+from trac.prefs import IPreferencePanelProvider
+from trac.util.datefmt import http_date, localtz
+from trac.web import IRequestHandler
+from trac.web.chrome import add_stylesheet
+
+from genshi import QName, Stream
+from genshi.core import Attrs, START, END, TEXT
+
+try:
+    from pygments import get_lexer_by_name
+    from pygments.formatters.html import HtmlFormatter
+    from pygments.styles import get_style_by_name
+    have_pygments = True
+except ImportError, e:
+    have_pygments = False
+else:
+    have_pygments = True
+
+__all__ = ['PygmentsRenderer']
+
+
+class PygmentsRenderer(Component):
+    """Syntax highlighting based on Pygments."""
+
+    implements(IHTMLPreviewRenderer, IPreferencePanelProvider, IRequestHandler)
+
+    default_style = Option('mimeviewer', 'pygments_default_style', 'trac',
+        """The default style to use for Pygments syntax highlighting.""")
+
+    pygments_modes = ListOption('mimeviewer', 'pygments_modes',
+        '', doc=
+        """List of additional MIME types known by Pygments.
+        
+        For each, a tuple `mimetype:mode:quality` has to be
+        specified, where `mimetype` is the MIME type,
+        `mode` is the corresponding Pygments mode to be used
+        for the conversion and `quality` is the quality ratio
+        associated to this conversion. That can also be used
+        to override the default quality ratio used by the
+        Pygments render.""")
+
+    expand_tabs = True
+    returns_source = True
+
+    QUALITY_RATIO = 7
+
+    EXAMPLE = """<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <title>Hello, world!</title>
+    <script>
+      $(document).ready(function() {
+        $("h1").fadeIn("slow");
+      });
+    </script>
+  </head>
+  <body>
+    <h1>Hello, world!</h1>
+  </body>
+</html>"""
+
+    def __init__(self):
+        self.log.debug("Pygments installed? %r", have_pygments)
+        self._types = None
+
+    # IHTMLPreviewRenderer implementation
+
+    def get_quality_ratio(self, mimetype):
+        # Extend default MIME type to mode mappings with configured ones
+        if self._types is None:
+            self._init_types()
+        try:
+            return self._types[mimetype][1]
+        except KeyError:
+            return 0
+
+    def render(self, req, mimetype, content, filename=None, rev=None):
+        if self._types is None:
+            self._init_types()
+        add_stylesheet(req, '/pygments/%s.css' %
+                       req.session.get('pygments_style', self.default_style))
+        try:
+            mimetype = mimetype.split(';', 1)[0]
+            language = self._types[mimetype][0]
+            return self._generate(language, content)
+        except (KeyError, ValueError):
+            raise Exception("No Pygments lexer found for mime-type '%s'."
+                            % mimetype)
+
+    # IPreferencePanelProvider implementation
+
+    def get_preference_panels(self, req):
+        if have_pygments:
+            yield ('pygments', 'Pygments Theme')
+
+    def render_preference_panel(self, req, panel):
+        styles = list(get_all_styles())
+
+        if req.method == 'POST':
+            style = req.args.get('style')
+            if style and style in styles:
+                req.session['pygments_style'] = style
+            req.redirect(req.href.prefs(panel or None))
+
+        output = self._generate('html', self.EXAMPLE)
+        return 'prefs_pygments.html', {
+            'output': output,
+            'selection': req.session.get('pygments_style', self.default_style),
+            'styles': styles
+        }
+
+    # IRequestHandler implementation
+
+    def match_request(self, req):
+        if have_pygments:
+            match = re.match(r'/pygments/(\w+)\.css', req.path_info)
+            if match:
+                req.args['style'] = match.group(1)
+                return True
+
+    def process_request(self, req):
+        style = req.args['style']
+        try:
+            style_cls = get_style_by_name(style)
+        except ValueError, e:
+            raise HTTPNotFound(e)
+
+        parts = style_cls.__module__.split('.')
+        filename = resource_filename('.'.join(parts[:-1]), parts[-1] + '.py')
+        mtime = datetime.fromtimestamp(os.path.getmtime(filename), localtz)
+        last_modified = http_date(mtime)
+        if last_modified == req.get_header('If-Modified-Since'):
+            req.send_response(304)
+            req.end_headers()
+            return
+
+        formatter = HtmlFormatter(style=style_cls)
+        content = u'\n\n'.join([
+            formatter.get_style_defs('div.code pre'),
+            formatter.get_style_defs('table.code td')
+        ]).encode('utf-8')
+
+        req.send_response(200)
+        req.send_header('Content-Type', 'text/css; charset=utf-8')
+        req.send_header('Last-Modified', last_modified)
+        req.send_header('Content-Length', len(content))
+        req.write(content)
+
+    # Internal methods
+
+    def _init_types(self):
+        self._types = {}
+        if have_pygments:
+            for _, aliases, _, mimetypes in get_all_lexers():
+                for mimetype in mimetypes:
+                    self._types[mimetype] = (aliases[0], self.QUALITY_RATIO)
+            self._types.update(
+                Mimeview(self.env).configured_modes_mapping('pygments')
+            )
+
+    def _generate(self, language, content):
+        lexer = get_lexer_by_name(language)
+        return GenshiHtmlFormatter().generate(lexer.get_tokens(content))
+
+
+def get_all_lexers():
+    from pygments.lexers._mapping import LEXERS
+    from pygments.plugin import find_plugin_lexers
+
+    for item in LEXERS.itervalues():
+        yield item[1:]
+    for cls in find_plugin_lexers():
+        yield cls.name, cls.aliases, cls.filenames, cls.mimetypes
+
+def get_all_styles():
+    from pygments.styles import find_plugin_styles, STYLE_MAP
+    for name in STYLE_MAP:
+        yield name
+    for name, _ in find_plugin_styles():
+        yield name