Index: trac/attachment.py =================================================================== --- trac/attachment.py (revision 3605) +++ trac/attachment.py (working copy) @@ -26,7 +26,7 @@ from trac.config import BoolOption, IntOption from trac.core import * from trac.env import IEnvironmentSetupParticipant -from trac.mimeview import * +from trac.mimeview.api import Mimeview, MimeType, FileMimeContent, TEXT_PLAIN from trac.util import get_reporter_id, create_unique_file from trac.util.datefmt import format_datetime, pretty_timedelta from trac.util.html import Markup, html @@ -124,7 +124,7 @@ def parent_href(self, req): return req.href(self.parent_type, self.parent_id) - def _get_title(self): + def _get_title(self): # TODO: should extend to other `parent_type`s return '%s%s: %s' % (self.parent_type == 'ticket' and '#' or '', self.parent_id, self.filename) title = property(_get_title) @@ -245,7 +245,7 @@ select = classmethod(select) delete_all = classmethod(delete_all) - def open(self): + def open(self): # deprecate? self.env.log.debug('Trying to open attachment at %s', self.path) try: fd = open(self.path, 'rb') @@ -523,6 +523,7 @@ 'author': get_reporter_id(req)} def _render_view(self, req, attachment): + # FIXME: perm_map should extend to other `parent_type`s perm_map = {'ticket': 'TICKET_VIEW', 'wiki': 'WIKI_VIEW'} req.perm.assert_permission(perm_map[attachment.parent_type]) @@ -533,60 +534,48 @@ req.hdf['attachment'] = attachment_to_hdf(self.env, req, None, attachment) # Override the 'oneliner' - req.hdf['attachment.description'] = wiki_to_html(attachment.description, - self.env, req) + req.hdf['attachment.description'] = wiki_to_html( + attachment.description, self.env, req) perm_map = {'ticket': 'TICKET_ADMIN', 'wiki': 'WIKI_DELETE'} if req.perm.has_permission(perm_map[attachment.parent_type]): req.hdf['attachment.can_delete'] = 1 - fd = attachment.open() - try: - mimeview = Mimeview(self.env) + content = FileMimeContent(self.env, attachment.path, + url=attachment.href(req, format='raw'), + kind='Attachment') - # MIME type detection - str_data = fd.read(1000) - fd.seek(0) - - binary = is_binary(str_data) - mime_type = mimeview.get_mimetype(attachment.filename, str_data) + # Eventually send the file directly + format = req.args.get('format') + if format in ('raw', 'txt'): + disposition = 'inline' + if self.render_unsafe_content: + if content.type.is_unknown or \ + (format == 'txt' and not content.is_binary): + # Force the content to be displayed as text + content.type = TEXT_PLAIN ### SHOULD BE ENOUGH + content.type = MimeType('text/plain', content.encoding) + elif not content.is_binary: + # Force browser to download HTML/SVG/etc pages that may + # contain malicious code enabling XSS aattacks + disposition = 'attachment' + content.send(req, disposition) - # Eventually send the file directly - format = req.args.get('format') - if format in ('raw', 'txt'): - if not self.render_unsafe_content and not binary: - # Force browser to download HTML/SVG/etc pages that may - # contain malicious code enabling XSS attacks - req.send_header('Content-Disposition', 'attachment;' + - 'filename=' + attachment.filename) - if not mime_type or (self.render_unsafe_content and \ - not binary and format == 'txt'): - mime_type = 'text/plain' - if 'charset=' not in mime_type: - charset = mimeview.get_charset(str_data, mime_type) - mime_type = mime_type + '; charset=' + charset - req.send_file(attachment.path, mime_type) + # add ''Plain Text'' alternate link if needed + if self.render_unsafe_content and not \ + (content.is_binary or + content.type.is_unknown or + TEXT_PLAIN.match(content.type)): + add_link(req, 'alternate', attachment.href(req, format='txt'), + TEXT_PLAIN.name, TEXT_PLAIN.mimetype) - # add ''Plain Text'' alternate link if needed - if self.render_unsafe_content and not binary and \ - mime_type and not mime_type.startswith('text/plain'): - plaintext_href = attachment.href(req, format='txt') - add_link(req, 'alternate', plaintext_href, 'Plain Text', - mime_type) + # add ''Original Format'' alternate link (always) + add_link(req, 'alternate', content.url, + 'Original Format', content.type.mimetype) - # add ''Original Format'' alternate link (always) - raw_href = attachment.href(req, format='raw') - add_link(req, 'alternate', raw_href, 'Original Format', mime_type) + req.hdf['attachment'] = Mimeview(self.env).preview_to_hdf( + req, content, annotations=['lineno']) - self.log.debug("Rendering preview of file %s with mime-type %s" - % (attachment.filename, mime_type)) - - req.hdf['attachment'] = mimeview.preview_to_hdf( - req, fd, os.fstat(fd.fileno()).st_size, mime_type, - attachment.filename, raw_href, annotations=['lineno']) - finally: - fd.close() - def _render_list(self, req, p_type, p_id): self._parent_to_hdf(req, p_type, p_id) req.hdf['attachment'] = { Index: trac/mimeview/rst.py =================================================================== --- trac/mimeview/rst.py (revision 3611) +++ trac/mimeview/rst.py (working copy) @@ -28,7 +28,7 @@ import re from trac.core import * -from trac.mimeview.api import IHTMLPreviewRenderer, content_to_unicode +from trac.mimeview.api import IHTMLPreviewRenderer from trac.util.html import Element from trac.web.href import Href from trac.wiki.formatter import WikiProcessor @@ -180,7 +180,7 @@ _inliner = rst.states.Inliner() _parser = rst.Parser(inliner=_inliner) - content = content_to_unicode(self.env, content, mimetype) + content = unicode(content) parts = publish_parts(content, writer_name='html', parser=_parser, settings_overrides={'halt_level': 6, 'file_insertion_enabled': 0, Index: trac/mimeview/api.py =================================================================== --- trac/mimeview/api.py (revision 3605) +++ trac/mimeview/api.py (working copy) @@ -19,41 +19,89 @@ # Christian Boos """ -The `trac.mimeview` module centralize the intelligence related to -file metadata, principally concerning the `type` (MIME type) of the content -and, if relevant, concerning the text encoding (charset) used by the content. +The `trac.mimeview` module centralizes the intelligence about typed content. -There are primarily two approaches for getting the MIME type of a given file: - * taking advantage of existing conventions for the file name - * examining the file content and applying various heuristics +Originally, this was about file metadata, principally its MIME type and +eventually the text encoding (charset) used by that content. -The module also knows how to convert the file content from one type -to another type. +Now, this has evolved into managing any kind of typed content, +and deals also with converting a content from one type to another. +A common situation which is now handled part of the general case +is the conversion of any kind of content to a text/html representation. -In some cases, only the `url` pointing to the file's content is actually -needed, that's why we avoid to read the file's content when it's not needed. +In order to keep the API of conversion interface IContentConverter simple, +we introduced a few classes, each encapsulating a part of the knowledge +related to the content and the conversion: -The actual `content` to be converted might be a `unicode` object, -but it can also be the raw byte string (`str`) object, or simply -an object that can be `read()`. + * the `ContentType` is used to describe the type of a content. + This is an abstract superclass for: + + - `MimeType`, used for storing the mime type string, the charset, + and eventually the name and the commonly used file extension + for that type + + - `ObjectType`, used when wrapping an arbitrary Python object + + * the `AbstractContent`, which wraps access to the actual content, + and provides a few generic methods, among which `convert()` is + the most important. + + - the `ObjectContent`, when the content is an arbitrary Python object + + - the `MimeContent` abstract class, which offers an uniform API to + access the data over a wide range of containers: + * the `StringMimeContent`, for handling string content + (like `str`, `unicode` or `Markup` objects) + * the `StructuredMimeContent`, for handling structured content + (like `Element` objects) + * the `LineIteratorMimeContent`, for handling line-oriented string + content (like string iterators or string arrays) + * the `FileMimeContent`, for handling file content + * etc. (as an example, the repository layer defines a `NodeMimeContent`) + + That API provides different ways to get access to the data: + * chunks(), an iterator for reading chunks of raw content + * lines(), an iterator for reading lines from text content + * markup(), when it makes sense to interpret the content as markup + * `__unicode()__`, for converting the content to an `unicode` object + * encode(), for converting the content to a `str` object. Note that + we don't use `__str__` for that on purpose, as we need to be able + to specify the charset. + * size(), for getting a hint about the content size without actually + reading it. + + * the `Conversion` class, which is used to specify how a given + `IContentConverter` component will perform a content conversion. """ +import os import re from StringIO import StringIO from trac.config import IntOption, ListOption, Option from trac.core import * from trac.util import sorted -from trac.util.text import to_utf8, to_unicode +from trac.util.datefmt import http_date +from trac.util.text import to_utf8, to_unicode, IDENTITY_CHARSET from trac.util.html import escape, Markup, Fragment, html -__all__ = ['get_mimetype', 'is_binary', 'detect_unicode', 'Mimeview', - 'content_to_unicode'] +__all__ = ['get_mimetype', 'is_binary', 'detect_unicode', + 'IContentConverter', 'IHTMLPreviewAnnotator', + 'IHTMLPreviewRenderer', # deprecated + 'ObjectType', 'MimeType', + 'ObjectContent', + 'MimeContent', 'FileMimeContent', 'StringMimeContent', + 'StructuredMimeContent', 'LineIteratorMimeContent', + 'Mimeview', 'Conversion', + 'TEXT_PLAIN', 'TEXT_HTML', 'TEXT_CSV', 'TEXT_TSV', + 'APPLICATION_RSS_XML', 'APPLICATION_OCTET_STREAM'] # Some common MIME types and their associated keywords and/or file extensions +APPLICATION_OCTET_STREAM_STR = 'application/octet-stream' + KNOWN_MIME_TYPES = { 'application/pdf': ['pdf'], 'application/postscript': ['ps'], @@ -114,20 +162,18 @@ for e in exts: MIME_MAP[e] = t -# Simple builtin autodetection from the content using a regexp -MODE_RE = re.compile( - r"#!(?:[/\w.-_]+/)?(\w+)|" # look for shebang - r"-\*-\s*(?:mode:\s*)?([\w+-]+)\s*-\*-|" # look for Emacs' -*- mode -*- - r"vim:.*?syntax=(\w+)" # look for VIM's syntax= - ) -def get_mimetype(filename, content=None, mime_map=MIME_MAP): - """Guess the most probable MIME type of a file with the given name. +# -- get_mimetype, is_binary, detect_unicode: a few functions for dealing +# with MIME types, binary and text content in a simple way +def get_mimetype_from_filename(filename, mime_map=MIME_MAP): + """Guess the most probable MIME type of file with the given `filename`. + `filename` is either a filename (the lookup will then use the suffix) or some arbitrary keyword. - - `content` is either a `str` or an `unicode` string. + `mime_map` maps keywords to MIME types. + + Return the MIME type as a string, or `None` if nothing was detected. """ suffix = filename.split('.')[-1] if suffix in mime_map: @@ -141,20 +187,49 @@ mimetype = mimetypes.guess_type(filename)[0] except: pass - if not mimetype and content: - match = re.search(MODE_RE, content[:1000]) - if match: - mode = match.group(1) or match.group(3) or \ - match.group(2).lower() - if mode in mime_map: - # 3) mimetype from the content, using the `MODE_RE` - return mime_map[mode] - else: - if is_binary(content): - # 4) mimetype from the content, using`is_binary` - return 'application/octet-stream' return mimetype +# Simple builtin autodetection from the content using a regexp +MODE_RE = re.compile( + # look for /usr/bin/prog1 or /usr/bin/env prog2 + r"#!(?:[/\w.-_]+/)?(?P\w+)(?:\s+(?P\w+))?|" + # look for Emacs' -*- mode -*- + r"-\*-\s*(?:mode:\s*)?(?P[\w+-]+)\s*-\*-|" + # look for VIM's syntax= + r"vim:.*?syntax=(?P\w+)" + ) + +def get_mimetype_from_content(content, mime_map=MIME_MAP): + """Guess the most probable MIME type of file with the given `filename`. + + `content` is either a `str` or an `unicode` string + + Return the MIME type as a string, or `None` if not detected. + """ + match = re.search(MODE_RE, content[:1000]) + if match: + mode = match.group('prog1') + if mode and mode == 'env': + mode = match.group('prog2') + if not mode: + mode = match.group('vimsyntax') or \ + match.group('emacsmode').lower() + if mode in mime_map: + # 3) mimetype from the content, using the `MODE_RE` + return mime_map[mode] + else: + if is_binary(content): + # 4) mimetype from the content, using `is_binary` + return APPLICATION_OCTET_STREAM_STR + +def get_mimetype(filename, content=None, mime_map=MIME_MAP): + """Auto-detect MIME type either from the `filename` or from the `content`. + """ + mimetype = get_mimetype_from_filename(filename, mime_map) + if not mimetype and content: + mimetype = get_mimetype_from_content(content, mime_map) + return mimetype + def is_binary(data): """Detect binary content by checking the first thousand bytes for zeroes. @@ -165,7 +240,7 @@ return '\0' in data[:1000] def detect_unicode(data): - """Detect different unicode charsets by looking for BOMs (Byte Order Marks). + """Detect different unicode charsets by looking for Byte Order Marks. Operate obviously only on `str` objects. """ @@ -178,90 +253,734 @@ else: return None -def content_to_unicode(env, content, mimetype): - """Retrieve an `unicode` object from a `content` to be previewed""" - mimeview = Mimeview(env) - if hasattr(content, 'read'): - content = content.read(mimeview.max_preview_size) - return mimeview.to_unicode(content, mimetype) +# -- ContentType and subclasses -class IHTMLPreviewRenderer(Interface): - """Extension point interface for components that add HTML renderers of - specific content types to the `Mimeview` component. +class ContentType(object): + """Abstract representation of the "type" of some content.""" - (Deprecated) + def match(self, other): + """Compare this instance with another `ContentType` instance. + + Return True if `other` can be say to ''match'' our type. + """ + raise NotImplementedError + + is_binary = property(lambda x: x._is_binary()) + + mimetype = property(lambda x: x._get_mimetype(), + doc="MIME Type string (without charset information)") + + name = 'unknown' + + +class ObjectType(ContentType): + """Represent the "type" of a content by using a Python class. + + The corresponding content is then expected to be an `ObjectContent`, + wrapping a Python instance of that class. """ - # implementing classes should set this property to True if they - # support text content where Trac should expand tabs into spaces - expand_tabs = False + def __init__(self, obj): + self._class = isinstance(obj, type) and obj or obj.__class__ + self._mimetype = self._charset = None + self.name = self._class.__name__ - def get_quality_ratio(mimetype): - """Return the level of support this renderer provides for the `content` - of the specified MIME type. The return value must be a number between - 0 and 9, where 0 means no support and 9 means "perfect" support. + def __repr__(self): + return "" % self.name + + # Reimplemented methods + + def _is_binary(self): return True + def _get_mimetype(self): return None + # or return 'application/x-python-'+self.name ? + + def match(self, other, regexp=False): + other_class = isinstance(other, type) and other or \ + isinstance(other, ObjectType) and other._class or \ + other.__class__ + return other_class == self._class + + +class MimeType(ContentType): + """Represent the "type" of a content by using a MIME type string. + + If the type is not binary (i.e. is some kind of text), the `charset` + information can also be given, when creating the instance. + + A MIME type has a `name` and has an `extension` that can be used + for storing the converted data in a file. + + All the properties of this class are read-only. + """ + + def __init__(self, mimetype, charset=None, name=None, extension=None, + env=None): + """Create a MimeType based on a `mimetype` string. + + That string can eventually contain a `charset=...`, which will be + retained as the charset for this instance, unless there's an + explicit `charset` parameter given. + + Another possibility is to give a pattern object for the `mimetype` + argument, in which case this instance can be used to do pattern + matching when using `match()`. + + When optional `env` parameter is given, additional knowledge is + used for determining whether this type is binary or not. """ + self.env = env + self._mimetype = mimetype + # determine charset + self._charset = charset + if not self._charset and isinstance(self._mimetype, basestring): + sep_idx = mimetype.find(';') + if sep_idx >= 0: + self._mimetype = mimetype[:sep_idx].strip() + charset_idx = mimetype.find('charset=', sep_idx) + if charset_idx >= 0: + self._charset = mimetype[charset_idx+8:].strip() + if extension and not extension.startswith('.'): + extension = '.' + extension + self._extension = extension + self._name = name - def render(req, mimetype, content, filename=None, url=None): - """Render an XHTML preview of the raw `content`. + def __repr__(self): + return '' % self.mimetype_charset - The `content` might be: - * a `str` object - * an `unicode` string - * any object with a `read` method, returning one of the above + def _get_extension(self): + if self._extension is None: + self._extension = KNOWN_MIME_TYPES.get(self.mimetype) ### use env + if not self._extension: + detail = self.mimetype.split('/', 1)[1] + if detail.startswith('x-'): + detail = detail[2:] + self._extension = '.' + detail + return self._extension - It is assumed that the content will correspond to the given `mimetype`. + def _get_mimetype_charset(self): + """Combine in a single string the MIME type and charset information.""" + if self._mimetype and self._charset: + return '%s; charset=%s' % (self.mimetype, self.charset) + else: + return self.mimetype ### or default to utf-8 if not binary? + + charset = property(lambda x: x._charset, + doc="Charset information if available") - Besides the `content` value, the same content may eventually - be available through the `filename` or `url` parameters. - This is useful for renderers that embed objects, using or - instead of including the content inline. + name = property(lambda x: x._name or x._get_extension()) + extension = property(lambda x: x._get_extension()) + mimetype_charset = property(lambda x: x._get_mimetype_charset(), + doc="MIME Type plus charset information") + + is_unknown = property(lambda x: x._mimetype is None) + + # Reimplemented methods + + def _is_binary(self): + return (self._mimetype == APPLICATION_OCTET_STREAM_STR or + (self.env and self.mimetype in + Mimeview(self.env).treat_as_binary) or + self.is_unknown) + + def _get_mimetype(self): + return self._mimetype or APPLICATION_OCTET_STREAM_STR + + def match(self, other): + if isinstance(other, MimeType): + other = other.mimetype + if not isinstance(other, basestring): + return False + if hasattr(self.mimetype, 'match'): + return self.mimetype.match(other) + else: + return self.mimetype == other + + +# A few commonly used MIME types: + +TEXT_PLAIN = MimeType('text/plain', 'utf-8', 'Plain Text', 'txt') +TEXT_HTML = MimeType('text/html', 'utf-8', 'HTML', 'html') +TEXT_CSV = MimeType('text/csv', 'utf-8', 'Comma-delimited Text', 'csv') +TEXT_TSV = MimeType('text/tab-separated-values', 'utf-8', + 'Tab-delimited Text', 'tsv') + +APPLICATION_RSS_XML = MimeType('application/rss+xml', 'utf-8', + 'RSS Feed', 'xml') +APPLICATION_OCTET_STREAM = MimeType(APPLICATION_OCTET_STREAM_STR, + IDENTITY_CHARSET, + 'Undefined (binary)', 'bin') + + +# -- AbstractContent and subclasses + +class AbstractContent(object): + """An abstract content, with an associated content type. + + There can also be a `filename` eventually associated to this content. + """ + + def __init__(self, env, content, type, filename=None): + self.env = env + self.content = content + self._type = type + self.filename = filename + + def _is_binary(self): raise NotImplementedError + + def _get_type(self): return self._type + def _set_type(self, type): self._type = type + + # Properties + + type = property(fget=lambda x: x._get_type(), + fset=lambda x, y: x._set_type(y), doc= + """The corresponding `ContentType` for this content.""") + + is_binary = property(lambda x: x._is_binary(), doc= + "True if content should be considered to be binary") + + # Methods + + def basename(self): + """Return a possible file basename for this content.""" + return self.filename and os.path.splitext(self.filename)[0] or '' + + def convert(self, req, format=None, typespec=None, context=None): + """Convert this content to a `MimeContent`. + + The converter can be selected by either: - Can return the generated XHTML text as a single string or as an - iterable that yields strings. In the latter case, the list will - be considered to correspond to lines of text in the original content. + - specifying the desired output `format`, which must match the + `format` of one of the `Conversion` object returned by + `Mimeview.get_conversions(self.type)`. + There's usually only one such converter. + + - giving a `typespec`, which is a `MimeType` object. + All matching converters are tried in turn, until one succeeds. + This is quite flexible, as one can use a regexp match ### FIXME + + Either succeeds or raise `NoConversion` error. """ + if not typespec and not format: + self.env.log.error("Convert called but no conversion specified.") + return self + if isinstance(format, MimeType): # or signal a programming error? + typespec = format + format = None + # Get all available conversions for our `type`, and keep those + # which are matching either the `format` or the `typespec`. + mimeview = Mimeview(self.env) + candidates = [] + for conversion, converter in mimeview.get_converters(self.type): + if conversion.format == format or \ + (typespec and typespec.match(conversion.output)): + candidates.append((conversion, converter)) + if not candidates: + raise NoConversion('No available MIME conversions', + self.type, typespec, format) + + tab_expanded = None # we don't want to expand tabs more than once. + + # First candidate which converts successfully wins. + self.env.log.debug('Converting %s' % repr(self)) + if not context: + class ToplevelContext(object): + def __init__(self, req): + self.req = req + context = ToplevelContext(req) + content = self + for conversion, converter in candidates: + self.env.log.debug('Trying converter %s using %s' % + (repr(converter), repr(conversion))) + if conversion.expand_tabs and not tab_expanded: + expanded = unicode(content).expandtabs(mimeview.tab_width) + content = StringMimeContent(self.env, expanded, content.type, + filename=content.filename) + self.env.log.debug('(tab expansion performed)') + try: + result = converter.convert_content(context, conversion, content) + if result is not None: + # Check if the conversion is really valid, by trying to + # access an excerpt of the result. This might trigger + # errors with legacy renderers returning an iterator... + check_result = result.excerpt + return result + except Exception, e: + self.env.log.warning('MIME conversion using %s failed (%s)' + % (repr(converter), e), exc_info=True) + raise NoConversion('No MIME conversions succeeded', + content.type, typespec, format) + + +class ObjectContent(AbstractContent): + """Content is a Python a object, and its type is an `ObjectType`. + + Only supports the bare minimum of the `AbstractContent` methods. + `filename` in this case will be the suggested base name to be used + when creating a filename for a converted content. + """ + + def __init__(self, env, obj, filename=None): + AbstractContent.__init__(self, env, obj, ObjectType(obj), + filename=filename) + + # Reimplemented property readers + + def _is_binary(self): + return True + + def basename(self): + return AbstractContent.basename(self) or self.type.name + + +class MimeContent(AbstractContent): + """Content associated with a MIME type. + + Such an object has ways to auto-detect both the MIME Type and + the `encoding` of its content. + + That `encoding` is more reliable than the `type.charset` information. + There are additional consistency checks that are performed, and it + can be `None` if the content is an `unicode` object. + + The content itself can be accessed in various ways: through + the iterator protocol, the len() and unicode() operators... + + This class may eventually be instantiated directly when we don't + care about the actual content but only about the MimeType detection. + """ + + EXCERPT_LEN = 1000 + + disposition = 'inline' + last_modified = None # means unknown, otherwise is a time value + + def __init__(self, env, content=None, mimetype=None, + filename=None, url=None): + """ + `mimetype` can be specified as a `MimeType` object or as string, + which will then be used as a hint for the content. + + If the mimetype is not specified or is "application/octet-stream", + then it will be auto-detected when needed. + + In case auto-detection fails, the MIME type will be set + to APPLICATION_OCTET_STREAM. + + The `filename` is the corresponding file name for that content, + or if there's none, a possible basename for that content. + + The `url` is a link for retrieving the raw content directly + from the server. This can be useful for converters that can + provide links to objects instead of having to expand the content. + """ + if isinstance(mimetype, basestring): + mimetype = MimeType(mimetype, env=env) + AbstractContent.__init__(self, env, content, mimetype, filename) + self.url = url + self._binary = None + self._encoding = False # i.e. not yet determined + self._excerpt = None + + def __repr__(self): + return '<%s %s "%s">' % (self.__class__.__name__, self._type, + self.filename or self.url) + + # Reimplemented property accessors + + def _is_binary(self): + """An heuristic for guessing whether the content is binary or not. + + This will eventually fetch an `excerpt` of the content. + """ + if self._binary is None: + self._binary = self.type.is_binary + if not self._binary: # double-check using the content + self._binary = is_binary(self.excerpt) or \ + (self.type.mimetype in \ + Mimeview(self.env).treat_as_binary) + return self._binary + + def _get_type(self): + """Get or determine the MimeType corresponding to this content. + + An `excerpt` of the content will be examined if needed. + """ + if self._type is None: # not set + mimetype = None + if self.filename: + mimemap = Mimeview(self.env).mimemap + mimetype = get_mimetype_from_filename(self.filename, mimemap) + if not mimetype: + mimetype = get_mimetype_from_content(self.excerpt, mimemap) + if not mimetype: + pass # TODO 0.11: go through IMimeTypeDetectors + self._type = MimeType(mimetype, env=self.env) + return self._type + + def _set_type(self, type): + """Simply replace the existing `type` by the given `MimeType` object. + + If `None` is given, this will force auto-detection the next time + `type` will be accessed. + + Can be used for in-place conversion (e.g. ''any'' to text/plain). + """ + self._type = type + + # Helper methods + + def _get_encoding(self): + """Get or determine the current encoding of that `content`. + + The encoding will be determined using this order: + * from the charset information present in the mimetype information + * auto-detection of the charset from the `content` + * if nothing else worked, use the configured `default_charset` + + If the `content` happens to be a genuine `unicode` object, then + this returns `None` (and this, whatever the `type.charset` says). + ### XXX + + If the `content` is binary, then the encoding will be the identity + charset (ISO Latin 1). + """ + if self._encoding is False: + if isinstance(self.excerpt, unicode): + self._encoding = None + else: + charset = self.type.charset + if charset: + # we have external knowledge about the charset, + # this always override what we think it could be... + self._encoding = charset + elif isinstance(self.excerpt, str): + utf_encoding = detect_unicode(self.excerpt) + if utf_encoding is not None: + self._encoding = utf_encoding + elif self.is_binary: + self._encoding = IDENTITY_CHARSET + if self._encoding is False: + pass # TODO 0.11: go through ICharsetDetectors here + if self._encoding is False: + self._encoding = Mimeview(self.env).default_charset + return self._encoding + + def _get_excerpt(self): + """Get once the initial content of the file.""" + if self._excerpt is None: + self._excerpt = self._retrieve_excerpt() + return self._excerpt + + # Public API Methods + + def __unicode__(self): + """Return the `unicode` object corresponding to the content. + + Note: this should do the right thing if the content is already + `unicode`, i.e. it should be directly returned. + """ + return to_unicode(self.read(), self.encoding) + + def encode(self, charset=None): + """Return a `str`, corresponding to the `charset` encoded content. + + If `charset` is not specified or is the same as the original encoding, + the raw content is directly returned. + """ + if not charset or self.encoding == charset: + content = self.read() + if isinstance(content, unicode): + return content.encode('utf-8') + else: + return content + else: + return unicode(self).encode(charset) + + def read(self): + """Should return a `basestring` containing the whole content. + + This is either a `str` or an `unicode` object, depending on what's + most appropriate for the wrapped content. + + A more specialized way to access the content is often more adequate + or efficient than this one and is of course '''absolutely''' required + when `chunck()` is implemented with `read()`... + """ + return ''.join(self.chunks()) + + def chunks(self): + """Iterator on chunks of raw content or None if there's no raw content. + """ + return None + + def lines(self): + """Iterator on lines of text content or None if there are no lines.""" + return None + + def markup(self): + """Return content as Markup.""" + return None + + def size(self): + """Size of the raw content, in bytes or `-1` if unknown beforehand.""" + return -1 + + def send(self, req, disposition=None): + """Send this content to the requester. + + When specified, the `disposition` argument will override the content's + own disposition and will be used for the 'Content-Disposition' header. + If the 'filename=' info is not given, it will be automatically added + based on the already available information. + + `send()` Never returns, as it raises `RequestDone` upon completion. + """ + from trac.web import RequestDone + req.send_response(200) + req.send_header('Content-Type', self.type.mimetype_charset) + if self.size >= 0: # size is known + req.send_header('Content-Length', self.size()) + if self.last_modified: + req.send_header('Last-Modified', http_date(self.last_modified)) + if not disposition: + disposition = self.disposition + if self.filename and 'filename=' not in disposition: + extension = os.path.splitext(self.filename)[1] + if not extension: + extension = self.type.extension + disposition += '; filename=' + self.basename() + extension + req.send_header('Content-Disposition', disposition) + req.end_headers() + + if self.is_binary: + for chunk in self.chunks(): + req.write(chunk) + else: + req.write(self.encode()) + raise RequestDone + + # Properties + + encoding = property(lambda x: x._get_encoding()) + excerpt = property(lambda x: x._get_excerpt()) + + # Methods that needs to be reimplemented by subclasses + + def _retrieve_excerpt(self): + """Extracts an excerpts of the initial content of the file. + + '''Note: Be sure to ''not'' use `self.encoding` while doing that, + as precisely an excerpt of the content is taken in that method.''' + """ + raise NotImplementedError + + +class StringMimeContent(MimeContent): + """MIME-typed content wrapper for a basestring.""" + + # Reimplemented MimeContent methods + + def read(self): + return self.content + + def chunks(self): + """Iterator on chunks of content.""" + buf = StringIO(self.read()) # can be used as we reimplement it + chunk = buf.read(1000) + while chunk: + yield chunk + chunk = buf.read(1000) + + def lines(self): + """Iterator on lines.""" + for line in self.read().splitlines(): # don't keep eols? + yield line + + def markup(self): + return Markup(self.read()) + + def size(self): + """Length of the content, in characters.""" + return len(self.read()) + + def _retrieve_excerpt(self): + return self.read()[:self.EXCERPT_LEN] + + +class LineIteratorMimeContent(StringMimeContent): + """MIME-typed content wrapper for an iterable of lines.""" + + # Reimplemented methods + + def __unicode__(self): return u'\n'.join(self.content) + def read(self): return unicode(self) + + def lines(self): + yield self.excerpt + for line in self.content: + yield line + + def _retrieve_excerpt(self): + for first_line in self.content: + return first_line + + +class StructuredMimeContent(StringMimeContent): + """MIME-typed content wrapper for an Element. + + Nearly identical to the `StringMimeContent`, except the content + property will access to actual Element for structured processing. + """ + + # Reimplemented methods + + def read(self): return unicode(self.content) + def lines(self): return None + def markup(self): return self.content + + +class FileMimeContent(MimeContent): + """MIME-typed content wrapper for a file.""" + + def __init__(self, env, path, mimetype=None, url=None, kind='File'): + MimeContent.__init__(self, env, path, mimetype, + os.path.basename(path), url) + self._fd = None + self._path = path + self._kind = kind + + def __del__(self): + if self._fd: + self._fd.close() + + # Reimplemented methods + + def chunks(self): + """Iterate on chunks of raw content.""" + chunk = self.excerpt + while chunk: + yield chunk + chunk = self._fd.read(self.EXCERPT_LEN) + self._fd.seek(self.EXCERPT_LEN) # make it possible to iter again + + def size(self): + """Length of the raw content, in bytes.""" + if self._fd: + stat = os.fstat(self._fd.fileno()) + else: + try: + stat = os.stat(self._path) + except OSError: + raise TracError('%s "%s" not found' % (self._kind, + self.filename)) + return stat.st_size + + def send(self, req, disposition=None): + """Directly send the file.""" + ### self.encoding should be used when charset not available + ### maybe do that in a self.mimetype_charset method + req.send_file(self._path, self.type.mimetype_charset) + + def _retrieve_excerpt(self): + if not self._fd: + try: + self._fd = open(self._path) + except IOError: + raise TracError('%s "%s" not found' % (self._kind, + self.filename)) + return self._fd.read(self.EXCERPT_LEN) + + +# -- Conversion class and related + +class NoConversion(TracError): + def __init__(self, msg, from_, output, key): + TracError.__init__(self, '%s while converting from %s to %s' % + (msg, repr(from_), output and repr(output) or key), + 'MIME Content Conversion') + +class Conversion(object): + """Used to specify how a Converter will perform a conversion. + + Each conversion is identified by a `format`, which must be unique for + a given input `type` in `IContentConverter.get_supported_conversions()`. + + A conversion object can tell: + - what will be the precise `output` MimeType (as opposed to the + specification given, which can be vague) + - what will be the `quality` of the conversion. This is a number + in the range 0 to 9, where 0 means no support and 9 means "perfect" + support (try to keep 9 available for user defined conversions, + though nothing will prevent them from using 10 or 100...) + - whether a tab expansion should precede the conversion itself, + (`expand_tabs` flag which defaults to False). + + e.g. Conversion(format='latex', quality=8, type=MimeType('text/x-tex')) + """ + + def __init__(self, format, quality=1, output=TEXT_HTML, expand_tabs=False): + self.format = format + self.quality = quality + self.output = output + self.expand_tabs = expand_tabs + + def __repr__(self): + return '' % \ + (self.format, self.output, self.quality) + + +# -- Interfaces for the extension points + +class IContentConverter(Interface): + """An extension point interface for generic content conversion.""" + + def get_supported_conversions(input): + """Tells whether this converter can handle this `input` type. + + Return an iterable of `Conversion` objects, each describing + how the conversion should be done and what will be the output type. + """ + + def convert_content(context, conversion, content): + """Convert the given `AbstractContent` as specified by `Conversion`. + + ### FIXME + The conversion takes place in the given formatting `context`. + A `context` provides at least a `req` property. + + Return the converted content, which ''must'' be a `MimeContent` object. + """ + + class IHTMLPreviewAnnotator(Interface): """Extension point interface for components that can annotate an XHTML representation of file contents with additional information.""" def get_annotation_type(): - """Return a (type, label, description) tuple that defines the type of - annotation and provides human readable names. The `type` element should - be unique to the annotator. The `label` element is used as column - heading for the table, while `description` is used as a display name to - let the user toggle the appearance of the annotation type. + """Defines the type of annotation and provides human readable names. + + Return a (type, label, description) tuple, where: + - `type` element should be unique to the annotator. + - `label` element is used as column heading for the table + - `description` is used as a display name to let the user toggle + the appearance of the annotation type. """ def annotate_line(number, content): - """Return the XHTML markup for the table cell that contains the - annotation data.""" + """Return the annotation data for the given line. + The return value must be XHTML markup that fit in a table cell. + """ -class IContentConverter(Interface): - """An extension point interface for generic MIME based content - conversion.""" - def get_supported_conversions(): - """Return an iterable of tuples in the form (key, name, extension, - in_mimetype, out_mimetype, quality) representing the MIME conversions - supported and - the quality ratio of the conversion in the range 0 to 9, where 0 means - no support and 9 means "perfect" support. eg. ('latex', 'LaTeX', 'tex', - 'text/x-trac-wiki', 'text/plain', 8)""" +# -- The main Mimeview component - def convert_content(req, mimetype, content, key): - """Convert the given content from mimetype to the output MIME type - represented by key. Returns a tuple in the form (content, - output_mime_type) or None if conversion is not possible.""" - - class Mimeview(Component): """A generic class to prettify data, typically source code.""" - renderers = ExtensionPoint(IHTMLPreviewRenderer) annotators = ExtensionPoint(IHTMLPreviewAnnotator) converters = ExtensionPoint(IContentConverter) @@ -275,143 +994,102 @@ """Maximum file size for HTML preview. (''since 0.9'').""") mime_map = ListOption('mimeviewer', 'mime_map', - 'text/x-dylan:dylan,text/x-idl:ice,text/x-ada:ads:adb', + 'text/x-dylan:dylan,text/x-idl:ice,text/x-ada:ads:adb', doc= """List of additional MIME types and keyword mappings. - Mappings are comma-separated, and for each MIME type, - there's a colon (":") separated list of associated keywords - or file extensions. (''since 0.10'').""") + Mappings are comma-separated. Each mapping starts with the mimetype, + followed by a colon (":") and the (colon separated) list of associated + keywords or file extensions. (''since 0.10'').""") + + treat_as_binary = ListOption('mimeviewer', 'treat_as_binary', + 'application/pdf,application/postscript,application/rtf', doc= + """List of MIME types that should always be treated as binary content. + + Accounts for the fact that our binary detection heuristic can't + always work for some kind of binary data. (''since 0.10'').""") + def __init__(self): self._mime_map = None - - # Public API - def get_supported_conversions(self, mimetype): - """Return a list of target MIME types in same form as - `IContentConverter.get_supported_conversions()`, but with the converter - component appended. Output is ordered from best to worst quality.""" - converters = [] - for converter in self.converters: - for k, n, e, im, om, q in converter.get_supported_conversions(): - if im == mimetype and q > 0: - converters.append((k, n, e, im, om, q, converter)) - converters = sorted(converters, key=lambda i: i[-1], reverse=True) - return converters + def _get_mimemap(self): + """Extend default extension to MIME type mappings""" + if not self._mime_map: + self._mime_map = {} + self._mime_map.update(MIME_MAP) + for mapping in self.config['mimeviewer'].getlist('mime_map'): + if ':' in mapping: + assocations = mapping.split(':') + mimetype = assocations[0] + for keyword in assocations: + self._mime_map[keyword] = mimetype + # Note: 'mimetype' is associated to itself + return self._mime_map - def convert_content(self, req, mimetype, content, key, filename=None, - url=None): - """Convert the given content to the target MIME type represented by - `key`, which can be either a MIME type or a key. Returns a tuple of - (content, output_mime_type, extension).""" - if not content: - return ('', 'text/plain;charset=utf-8') + mimemap = property(_get_mimemap) - # Ensure we have a MIME type for this content - full_mimetype = mimetype - if not full_mimetype: - if hasattr(content, 'read'): - content = content.read(self.max_preview_size) - full_mimetype = self.get_mimetype(filename, content) - if full_mimetype: - mimetype = full_mimetype.split(';')[0].strip() # split off charset - else: - mimetype = full_mimetype = 'text/plain' # fallback if not binary + def get_mimetype(self, filename, charset=None): + """Lookup for given `filename`, among known MIME Types. - # Choose best converter - candidates = list(self.get_supported_conversions(mimetype)) - candidates = [c for c in candidates if key in (c[0], c[4])] - if not candidates: - raise TracError('No available MIME conversions from %s to %s' % - (mimetype, key)) + `filename` is either a file name, or simply a keyword. - # First successful conversion wins - for ck, name, ext, input_mimettype, output_mimetype, quality, \ - converter in candidates: - output = converter.convert_content(req, mimetype, content, ck) - if not output: - continue - return (output[0], output[1], ext) - raise TracError('No available MIME conversions from %s to %s' % - (mimetype, key)) + Return a `MimeType` object if found, `None` otherwise. + """ + mimetype = get_mimetype_from_filename(filename, self.mimemap) + if mimetype: + ext = os.path.splitext(filename)[1] + return MimeType(mimetype, charset, extension=ext, env=self.env) - def get_annotation_types(self): - """Generator that returns all available annotation types.""" - for annotator in self.annotators: - yield annotator.get_annotation_type() + # -- MIME type conversion + + def get_converters(self, input): + """Return a list of conversions for the `input` `ContentType`. - def render(self, req, mimetype, content, filename=None, url=None, - annotations=None): - """Render an XHTML preview of the given `content`. + The returned list contains pair of `(conversion, converter)` objects, + ordered from best to worst quality. + """ + converters = [] + for converter in self.converters: + for conversion in converter.get_supported_conversions(input): + if conversion.quality > 0: + converters.append((conversion, converter)) - `content` is the same as an `IHTMLPreviewRenderer.render`'s - `content` argument. + return sorted(converters, key=lambda c: c[0].quality, reverse=True) - The specified `mimetype` will be used to select the most appropriate - `IHTMLPreviewRenderer` implementation available for this MIME type. - If not given, the MIME type will be infered from the filename or the - content. + def get_conversions(self, input): + """Convenience method, which return only the `Conversion` information. - Return a string containing the XHTML text. + Otherwise, it's the same as `get_converters`. """ - if not content: - return '' + return [conversion for conversion, _ in self.get_converters(input)] - # Ensure we have a MIME type for this content - full_mimetype = mimetype - if not full_mimetype: - if hasattr(content, 'read'): - content = content.read(self.max_preview_size) - full_mimetype = self.get_mimetype(filename, content) - if full_mimetype: - mimetype = full_mimetype.split(';')[0].strip() # split off charset - else: - mimetype = full_mimetype = 'text/plain' # fallback if not binary + # -- XHTML rendering and annotations (based on the conversion API) - # Determine candidate `IHTMLPreviewRenderer`s - candidates = [] - for renderer in self.renderers: - qr = renderer.get_quality_ratio(mimetype) - if qr > 0: - candidates.append((qr, renderer)) - candidates.sort(lambda x,y: cmp(y[0], x[0])) + def render(self, req, content, annotations=None): + """Render an XHTML preview of the given `content`. - # First candidate which renders successfully wins. - # Also, we don't want to expand tabs more than once. - expanded_content = None - for qr, renderer in candidates: - try: - self.log.debug('Trying to render HTML preview using %s' - % renderer.__class__.__name__) - # check if we need to perform a tab expansion - rendered_content = content - if getattr(renderer, 'expand_tabs', False): - if expanded_content is None: - content = content_to_unicode(self.env, content, - full_mimetype) - expanded_content = content.expandtabs(self.tab_width) - rendered_content = expanded_content - result = renderer.render(req, full_mimetype, rendered_content, - filename, url) - if not result: - continue - elif isinstance(result, Fragment): - return result - elif isinstance(result, basestring): - return Markup(to_unicode(result)) - elif annotations: - return Markup(self._annotate(result, annotations)) - else: - buf = StringIO() - buf.write('
')
-                    for line in result:
-                        buf.write(line + '\n')
-                    buf.write('
') - return Markup(buf.getvalue()) - except Exception, e: - self.log.warning('HTML preview using %s failed (%s)' - % (renderer, e), exc_info=True) + Some `annotations` might be requested as well. This argument + is a list of annotation keys, each of them should match a + `type` as returned by `IHTMLPreviewAnnotator.get_annotation_type`. + """ + result = content.convert(req, typespec=TEXT_HTML) + if annotations: + lines = result.lines() + if lines: + return Markup(self._annotate(lines, annotations)) + # i.e. don't annotate contents that are not line oriented + if isinstance(result, LineIteratorMimeContent): + return html.DIV(html.PRE(unicode(result)), class_="code") + else: + return result.content + + def get_annotation_types(self): + """Generator that returns all available annotation types.""" + for annotator in self.annotators: + yield annotator.get_annotation_type() + def _annotate(self, lines, annotations): + """Add requested `annotations` to the lines' content.""" buf = StringIO() buf.write('') annotators = [] @@ -445,75 +1123,30 @@ buf.write('
') return buf.getvalue() + # -- Deprecated API (TODO: remove in 0.11) + def get_max_preview_size(self): """Deprecated: use `max_preview_size` attribute directly.""" return self.max_preview_size - def get_charset(self, content='', mimetype=None): - """Infer the character encoding from the `content` or the `mimetype`. - - `content` is either a `str` or an `unicode` object. - - The charset will be determined using this order: - * from the charset information present in the `mimetype` argument - * auto-detection of the charset from the `content` - * the configured `default_charset` - """ - if mimetype: - ctpos = mimetype.find('charset=') - if ctpos >= 0: - return mimetype[ctpos + 8:].strip() - if isinstance(content, str): - utf = detect_unicode(content) - if utf is not None: - return utf - return self.default_charset - - def get_mimetype(self, filename, content=None): - """Infer the MIME type from the `filename` or the `content`. - - `content` is either a `str` or an `unicode` object. - - Return the detected MIME type, augmented by the - charset information (i.e. "; charset=..."), - or `None` if detection failed. - """ - # Extend default extension to MIME type mappings with configured ones - if not self._mime_map: - self._mime_map = MIME_MAP - for mapping in self.config['mimeviewer'].getlist('mime_map'): - if ':' in mapping: - assocations = mapping.split(':') - for keyword in assocations: # Note: [0] kept on purpose - self._mime_map[keyword] = assocations[0] - - mimetype = get_mimetype(filename, content, self._mime_map) - charset = None - if mimetype: - charset = self.get_charset(content, mimetype) - if mimetype and charset and not 'charset' in mimetype: - mimetype += '; charset=' + charset - return mimetype - def to_utf8(self, content, mimetype=None): """Convert an encoded `content` to utf-8. ''Deprecated in 0.10. You should use `unicode` strings only.'' """ - return to_utf8(content, self.get_charset(content, mimetype)) + return to_utf8(content) - def to_unicode(self, content, mimetype=None, charset=None): - """Convert `content` (an encoded `str` object) to an `unicode` object. + # -- Utilities - This calls `trac.util.to_unicode` with the `charset` provided, - or the one obtained by `Mimeview.get_charset()`. + def configured_modes_mapping(self, renderer): + """Utility for configurable custom converters + + Return a MIME type to `(mode,quality)` mapping for given `option`, + assuming a format of comma-separated :: + associations. + + See EnscriptConverter and SilverCityConverter. """ - if not charset: - charset = self.get_charset(content, mimetype) - return to_unicode(content, charset) - - def configured_modes_mapping(self, renderer): - """Return a MIME type to `(mode,quality)` mapping for given `option`""" types, option = {}, '%s_modes' % renderer for mapping in self.config['mimeviewer'].getlist(option): if not mapping: @@ -526,37 +1159,28 @@ "option." % (mapping, option)) return types - def preview_to_hdf(self, req, content, length, mimetype, filename, - url=None, annotations=None): - """Prepares a rendered preview of the given `content`. + def preview_to_hdf(self, req, content, annotations=None): + """Prepares a rendered preview to text/html of the given `content`. - Note: `content` will usually be an object with a `read` method. - """ - if length >= self.max_preview_size: + That content is eventually annotated with the specified `annotations`. + """ + self.log.debug("Rendering preview of %s" % repr(content)) + + if content.size() >= self.max_preview_size: return {'max_file_size_reached': True, 'max_file_size': self.max_preview_size, - 'raw_href': url} + 'raw_href': content.url} else: - return {'preview': self.render(req, mimetype, content, filename, - url, annotations), - 'raw_href': url} + try: + preview = self.render(req, content, annotations) + except NoConversion, e: + preview = None + return {'preview': preview, + 'raw_href': content.url} - def send_converted(self, req, in_type, content, selector, filename='file'): - """Helper method for converting `content` and sending it directly. - `selector` can be either a key or a MIME Type.""" - from trac.web import RequestDone - content, output_type, ext = self.convert_content(req, in_type, - content, selector) - req.send_response(200) - req.send_header('Content-Type', output_type) - req.send_header('Content-Disposition', 'filename=%s.%s' % (filename, - ext)) - req.end_headers() - req.write(content) - raise RequestDone - +# utility for Mimeview._annotate def _html_splitlines(lines): """Tracks open and close tags in lines of HTML text and yields lines that have no tags spanning more than one line.""" @@ -604,65 +1228,121 @@ def annotate_line(self, number, content): return '%s' % (number, number, number) +# return html.TH(html.A(number, href="#L%s" % number), id=number) -# -- Default renderers +# -- Default TEXT_HTML converters class PlainTextRenderer(Component): - """HTML preview renderer for plain text, and fallback for any kind of text - for which no more specific renderer is available. + """Convert text to HTML-escaped text. + + Will be used as a fallback for any kind of text for which no more + specific HTML converter is available. """ - implements(IHTMLPreviewRenderer) + implements(IContentConverter) - expand_tabs = True + def get_supported_conversions(self, input): + if input.is_binary: + return + yield Conversion(format='default', ### TODO Conversion.output + quality=TEXT_PLAIN.match(input) and 8 or 1, + output=TEXT_HTML, expand_tabs=True) + + def convert_content(self, context, conversion, content): + if content.is_binary: + self.log.debug("Binary data; can't be rendered as plain text.") + else: + if not TEXT_PLAIN.match(content.type): + self.log.debug("Using plain text renderer as a fallback.") + def escape_lines(): + for line in content.lines(): + yield escape(line) + plaintext_same_encoding = MimeType('text/plain', content.encoding) + return LineIteratorMimeContent(self.env, escape_lines(), + plaintext_same_encoding) - TREAT_AS_BINARY = [ - 'application/pdf', - 'application/postscript', - 'application/rtf' - ] - def get_quality_ratio(self, mimetype): - if mimetype in self.TREAT_AS_BINARY: - return 0 - return 1 +class ImageRenderer(Component): + """Inline image display. - def render(self, req, mimetype, content, filename=None, url=None): - if is_binary(content): - self.env.log.debug("Binary data; no preview available") - return + This renderer doesn't need the actual data at all, only the url. + """ - self.env.log.debug("Using default plain text mimeviewer") - content = content_to_unicode(self.env, content, mimetype) - for line in content.splitlines(): - yield escape(line) + implements(IContentConverter) + IMAGE_TYPE = MimeType(re.compile(r'image/')) -class ImageRenderer(Component): - """Inline image display. Here we don't need the `content` at all.""" - implements(IHTMLPreviewRenderer) + def get_supported_conversions(self, input): + if self.IMAGE_TYPE.match(input): + yield Conversion(format='image', quality=8, output=TEXT_HTML) - def get_quality_ratio(self, mimetype): - if mimetype.startswith('image/'): - return 8 - return 0 + def convert_content(self, context, conversion, content): + if content.url: + img = html.DIV(html.IMG(src=content.url, alt=content.filename), + class_="image-file") + return StructuredMimeContent(self.env, img, TEXT_HTML) - def render(self, req, mimetype, content, filename=None, url=None): - if url: - return html.DIV(html.IMG(src=url,alt=filename), - class_="image-file") +# ---- Backward compatibility support for IHTMLPreviewRenderer +# +# (TODO: remove in 0.11) +# -class WikiTextRenderer(Component): - """Render files containing Trac's own Wiki formatting markup.""" - implements(IHTMLPreviewRenderer) +class IHTMLPreviewRenderer(Interface): + """Extension point interface for components that add HTML renderers of + specific content types to the `Mimeview` component. - def get_quality_ratio(self, mimetype): - if mimetype in ('text/x-trac-wiki', 'application/x-trac-wiki'): - return 8 - return 0 + Deprecated in 0.10, implement `IContentConverter` instead. + """ - def render(self, req, mimetype, content, filename=None, url=None): - from trac.wiki import wiki_to_html - return wiki_to_html(content_to_unicode(self.env, content, mimetype), - self.env, req) + # implementing classes should set this property to True if they + # support text content where Trac should expand tabs into spaces + expand_tabs = False + + def get_quality_ratio(mimetype): + """Return the level of support this renderer provides""" + + def render(req, mimetype, content, filename=None, url=None): + """Render an XHTML preview of the raw `content`.""" + + +class PreviewRendererAdapter(Component): + """Single IContentConverter which wraps legacy IHTMLPreviewRenderer""" + + renderers = ExtensionPoint(IHTMLPreviewRenderer) + + implements(IContentConverter) + + # IContentConverter methods + + def get_supported_conversions(self, input): + if not isinstance(input, MimeType): + return + for renderer in self.renderers: + qr = renderer.get_quality_ratio(input.mimetype) + if qr > 0: + expand_tabs = getattr(renderer, 'expand_tabs', False) + yield Conversion(format=renderer.__class__.__name__, + quality=qr, output=TEXT_HTML, + expand_tabs=expand_tabs) + + def convert_content(self, context, conversion, content): + for renderer in self.renderers: + if conversion.format == renderer.__class__.__name__: + self.log.debug('Rendering using %s' % renderer) + # FIXME: try/except + result = renderer.render(context.req, + content.type.mimetype, + content, # ...which is read()able + content.filename, content.url) + + if isinstance(result, Fragment): + return StructuredMimeContent(self.env, result) + elif isinstance(result, basestring): + self.log.warning('IHTMLPreviewRenderer: string %s' % + result.__class__.__name__) + return StringMimeContent(self.env, result) + else: # something else... assume it's an iterable of lines + self.log.warning('IHTMLPreviewRenderer: type %s' % + result.__class__.__name__) + return LineIteratorMimeContent(self.env, result) Index: trac/mimeview/silvercity.py =================================================================== --- trac/mimeview/silvercity.py (revision 3605) +++ trac/mimeview/silvercity.py (working copy) @@ -24,7 +24,8 @@ from trac.core import * from trac.config import ListOption -from trac.mimeview.api import IHTMLPreviewRenderer, Mimeview +from trac.mimeview.api import IContentConverter, Mimeview, StringMimeContent, \ + Conversion, TEXT_HTML __all__ = ['SilverCityRenderer'] @@ -57,14 +58,14 @@ CRLF_RE = re.compile('\r$', re.MULTILINE) -class SilverCityRenderer(Component): +class SilverCityConverter(Component): """Syntax highlighting based on SilverCity.""" - implements(IHTMLPreviewRenderer) + implements(IContentConverter) - enscript_modes = ListOption('mimeviewer', 'silvercity_modes', - '', + silvercity_modes = ListOption('mimeviewer', 'silvercity_modes', doc= """List of additional MIME types known by SilverCity. + For each, a tuple `mimetype:mode:quality` has to be specified, where `mimetype` is the MIME type, `mode` is the corresponding SilverCity mode to be used @@ -79,20 +80,21 @@ def __init__(self): self._types = None - def get_quality_ratio(self, mimetype): + def get_supported_conversions(self, input): # Extend default MIME type to mode mappings with configured ones if not self._types: self._types = {} self._types.update(types) - self._types.update( - Mimeview(self.env).configured_modes_mapping('silvercity')) - return self._types.get(mimetype, (None, 0))[1] + ctypes = Mimeview(self.env).configured_modes_mapping('silvercity') + self._types.update(ctypes) + quality_ratio = self._types.get(input.mimetype, (None, 0))[1] + if quality_ratio: + yield Conversion('silvercity', quality_ratio, TEXT_HTML) - def render(self, req, mimetype, content, filename=None, rev=None): + def convert_content(self, context, conversion, input): import SilverCity try: - mimetype = mimetype.split(';', 1)[0] - typelang = self._types[mimetype] + typelang = self._types[input.type.mimetype] lang = typelang[0] module = getattr(SilverCity, lang) generator = getattr(module, lang + "HTMLGenerator") @@ -108,7 +110,7 @@ raise Exception, err # SilverCity does not like unicode strings - content = content.encode('utf-8') + content = input.encode('utf-8') # SilverCity generates extra empty line against some types of # the line such as comment or #include with CRLF. So we @@ -122,10 +124,8 @@ span_default_re = re.compile(r'(.*?)', re.DOTALL) html = span_default_re.sub(r'\1', br_re.sub('', buf.getvalue())) - - # Convert the output back to a unicode string - html = html.decode('utf-8') # SilverCity generates _way_ too many non-breaking spaces... # We don't need them anyway, so replace them by normal spaces - return html.replace(' ', ' ').splitlines() + return StringMimeContent(self.env, html.replace(' ', ' '), + TEXT_HTML) Index: trac/mimeview/patch.py =================================================================== --- trac/mimeview/patch.py (revision 3605) +++ trac/mimeview/patch.py (working copy) @@ -16,7 +16,7 @@ # Ludvig Strigeus from trac.core import * -from trac.mimeview.api import content_to_unicode, IHTMLPreviewRenderer, Mimeview +from trac.mimeview.api import IHTMLPreviewRenderer, Mimeview from trac.util.html import escape, Markup from trac.web.chrome import add_stylesheet @@ -65,7 +65,7 @@ def render(self, req, mimetype, content, filename=None, rev=None): from trac.web.clearsilver import HDFWrapper - content = content_to_unicode(self.env, content, mimetype) + content = unicode(content) d = self._diff_to_hdf(content.splitlines(), Mimeview(self.env).tab_width) if not d: Index: trac/mimeview/enscript.py =================================================================== --- trac/mimeview/enscript.py (revision 3605) +++ trac/mimeview/enscript.py (working copy) @@ -135,6 +135,7 @@ cmdline += ' --color -h -q --language=html -p - -E%s' % mode self.env.log.debug("Enscript command line: %s" % cmdline) + content = unicode(content) np = NaivePopen(cmdline, content.encode('utf-8'), capturestderr=1) if np.errorlevel or np.err: err = 'Running (%s) failed: %s, %s.' % (cmdline, np.errorlevel, Index: trac/mimeview/php.py =================================================================== --- trac/mimeview/php.py (revision 3605) +++ trac/mimeview/php.py (working copy) @@ -20,7 +20,7 @@ from trac.core import * from trac.config import Option -from trac.mimeview.api import IHTMLPreviewRenderer, content_to_unicode +from trac.mimeview.api import IHTMLPreviewRenderer from trac.util import NaivePopen from trac.util.html import Deuglifier @@ -79,7 +79,7 @@ cmdline += ' -sn' self.env.log.debug("PHP command line: %s" % cmdline) - content = content_to_unicode(self.env, content, mimetype) + content = unicode(content) content = content.encode('utf-8') np = NaivePopen(cmdline, content, capturestderr=1) if np.errorlevel or np.err: Index: trac/ticket/web_ui.py =================================================================== --- trac/ticket/web_ui.py (revision 3605) +++ trac/ticket/web_ui.py (working copy) @@ -23,6 +23,7 @@ from trac.config import BoolOption, Option from trac.core import * from trac.env import IEnvironmentSetupParticipant +from trac.mimeview.api import * from trac.ticket import Milestone, Ticket, TicketSystem, ITicketManipulator from trac.ticket.notification import TicketNotifyEmail from trac.Timeline import ITimelineEventProvider @@ -33,7 +34,6 @@ from trac.web import IRequestHandler from trac.web.chrome import add_link, add_stylesheet, INavigationContributor from trac.wiki import wiki_to_html, wiki_to_oneliner -from trac.mimeview.api import Mimeview, IContentConverter class InvalidTicket(TracError): @@ -211,22 +211,23 @@ # IContentConverter methods - def get_supported_conversions(self): - yield ('csv', 'Comma-delimited Text', 'csv', - 'trac.ticket.Ticket', 'text/csv', 8) - yield ('tab', 'Tab-delimited Text', 'tsv', - 'trac.ticket.Ticket', 'text/tab-separated-values', 8) - yield ('rss', 'RSS Feed', 'xml', - 'trac.ticket.Ticket', 'application/rss+xml', 8) + def get_supported_conversions(self, input): + if input.match(Ticket): + yield Conversion('csv', 8, TEXT_CSV) + yield Conversion('tab', 8, TEXT_TSV) + yield Conversion('rss', 8, APPLICATION_RSS_XML) - def convert_content(self, req, mimetype, ticket, key): + def convert_content(self, context, conversion, content): + ticket = content.content + key = conversion.format if key == 'csv': - return self.export_csv(ticket, mimetype='text/csv') + data = self.export_csv(ticket) elif key == 'tab': - return self.export_csv(ticket, sep='\t', - mimetype='text/tab-separated-values') + data = self.export_csv(ticket, sep='\t') elif key == 'rss': - return self.export_rss(req, ticket) + data = self.export_rss(context.req, ticket) + return StringMimeContent(self.env, data, conversion.output, + filename=content.basename()) # INavigationContributor methods @@ -254,6 +255,13 @@ ticket = Ticket(self.env, id, db=db) + # Send as alternate content if required + format = req.args.get('format') + if format: + content = ObjectContent(self.env, ticket, + filename='ticket_%d' % ticket.id) + content.convert(req, format).send(req) + if req.method == 'POST': if not req.args.has_key('preview'): self._do_save(req, db, ticket) @@ -281,12 +289,6 @@ self._insert_ticket_data(req, db, ticket, get_reporter_id(req, 'author')) - mime = Mimeview(self.env) - format = req.args.get('format') - if format: - mime.send_converted(req, 'trac.ticket.Ticket', ticket, format, - 'ticket_%d' % ticket.id) - # If the ticket is being shown in the context of a query, add # links to help navigate in the query result set if 'query_tickets' in req.session: @@ -308,10 +310,10 @@ add_stylesheet(req, 'common/css/ticket.css') # Add registered converters - for conversion in mime.get_supported_conversions('trac.ticket.Ticket'): - conversion_href = req.href.ticket(ticket.id, format=conversion[0]) - add_link(req, 'alternate', conversion_href, conversion[1], - conversion[3]) + for c in Mimeview(self.env).get_conversions(ObjectType(Ticket)): + add_link(req, 'alternate', + req.href.ticket(ticket.id, format=c.format), + c.output.name, c.output.mimetype) return 'ticket.cs', None @@ -429,17 +431,16 @@ # Internal methods - def export_csv(self, ticket, sep=',', mimetype='text/plain'): - content = StringIO() - content.write(sep.join(['id'] + [f['name'] for f in ticket.fields]) - + CRLF) - content.write(sep.join([unicode(ticket.id)] + - [ticket.values.get(f['name'], '') - .replace(sep, '_').replace('\\', '\\\\') - .replace('\n', '\\n').replace('\r', '\\r') - for f in ticket.fields]) + CRLF) - return (content.getvalue(), '%s;charset=utf-8' % mimetype) - + def export_csv(self, ticket, sep=','): + csv = StringIO() + csv.write(sep.join(['id'] + [f['name'] for f in ticket.fields]) + CRLF) + csv.write(sep.join([unicode(ticket.id)] + + [ticket.values.get(f['name'], '') + .replace(sep, '_').replace('\\', '\\\\') + .replace('\n', '\\n').replace('\r', '\\r') + for f in ticket.fields]) + CRLF) + return csv.getvalue() + def export_rss(self, req, ticket): db = self.env.get_db_cnx() changes = [] @@ -471,9 +472,8 @@ change['title'] = '; '.join(['%s %s' % (', '.join(v), k) for k, v \ in change_summary.iteritems()]) req.hdf['ticket.changes'] = changes - return (req.hdf.render('ticket_rss.cs'), 'application/rss+xml') + return req.hdf.render('ticket_rss.cs') - def _do_save(self, req, db, ticket): if req.perm.has_permission('TICKET_CHGPROP'): # TICKET_CHGPROP gives permission to edit the ticket Index: trac/ticket/tests/conversion.py =================================================================== --- trac/ticket/tests/conversion.py (revision 3605) +++ trac/ticket/tests/conversion.py (working copy) @@ -1,8 +1,8 @@ from trac.test import EnvironmentStub, Mock -from trac.util import sorted +from trac.util import sorted, CRLF from trac.ticket.model import Ticket from trac.ticket.web_ui import TicketModule -from trac.mimeview.api import Mimeview +from trac.mimeview.api import * from trac.web.clearsilver import HDFWrapper from trac.web.href import Href @@ -14,67 +14,103 @@ def setUp(self): self.env = EnvironmentStub() self.ticket_module = TicketModule(self.env) - self.mimeview = Mimeview(self.env) self.req = Mock(hdf=HDFWrapper(['./templates']), base_path='/trac.cgi', path_info='', - href=Href('/trac.cgi')) + href=Href('/trac.cgi'), + abs_href=Href('http://example.org/trac.cgi')) - def _create_a_ticket(self): + def _create_a_ticket_content(self): # 1. Creating ticket ticket = Ticket(self.env) ticket['reporter'] = 'santa' ticket['summary'] = 'Foo' ticket['description'] = 'Bar' ticket['foo'] = 'This is a custom field' - return ticket + # 2. Creating ObjectContent wrapper + return ObjectContent(self.env, ticket) def test_conversions(self): - conversions = self.mimeview.get_supported_conversions( - 'trac.ticket.Ticket') - expected = sorted([('csv', 'Comma-delimited Text', 'csv', - 'trac.ticket.Ticket', 'text/csv', 8, - self.ticket_module), - ('tab', 'Tab-delimited Text', 'tsv', - 'trac.ticket.Ticket', 'text/tab-separated-values', 8, - self.ticket_module), - ('rss', 'RSS Feed', 'xml', - 'trac.ticket.Ticket', 'application/rss+xml', 8, - self.ticket_module)], - key=lambda i: i[-1], reverse=True) - self.assertEqual(expected, conversions) + conversions = Mimeview(self.env).get_conversions(ObjectType(Ticket)) + expected = [Conversion('csv', 8, TEXT_CSV), + Conversion('rss', 8, APPLICATION_RSS_XML), + Conversion('tab', 8, TEXT_TSV)] + for expected, actual in zip(expected, sorted(conversions, + key=lambda c: c.format)): + for attr in ('format', 'output', 'quality', 'expand_tabs'): + self.assertEqual(getattr(expected, attr), + getattr(actual, attr)) def test_csv_conversion(self): - ticket = self._create_a_ticket() - csv = self.mimeview.convert_content(self.req, 'trac.ticket.Ticket', - ticket, 'csv') - self.assertEqual((u'id,summary,reporter,owner,description,keywords,cc' - '\r\nNone,Foo,santa,,Bar,,\r\n', - 'text/csv;charset=utf-8', 'csv'), csv) + content = self._create_a_ticket_content() + expected = [u'id,summary,reporter,owner,description,keywords,cc', + u'None,Foo,santa,,Bar,,', + ''] + csv = content.convert(self.req, 'csv') + self.assertEqual(CRLF.join(expected), csv.read()) + self.assertEqual(CRLF.join(expected), unicode(csv)) + self.assertEqual(TEXT_CSV, csv.type) + self.assertEqual('text/csv; charset=utf-8', csv.type.mimetype_charset) + csv = content.convert(self.req, TEXT_CSV) + self.assertEqual(CRLF.join(expected), csv.read()) + self.assertEqual(CRLF.join(expected), unicode(csv)) + self.assertEqual(TEXT_CSV, csv.type) + self.assertEqual('text/csv; charset=utf-8', csv.type.mimetype_charset) + + def test_tab_conversion(self): - ticket = self._create_a_ticket() - csv = self.mimeview.convert_content(self.req, 'trac.ticket.Ticket', - ticket, 'tab') - self.assertEqual((u'id\tsummary\treporter\towner\tdescription\tkeywords' - '\tcc\r\nNone\tFoo\tsanta\t\tBar\t\t\r\n', - 'text/tab-separated-values;charset=utf-8', 'tsv'), - csv) + content = self._create_a_ticket_content() + expected = [u'id\tsummary\treporter\towner\tdescription\tkeywords\tcc', + u'None\tFoo\tsanta\t\tBar\t\t', + ''] + tsv = content.convert(self.req, 'tab') + self.assertEqual(CRLF.join(expected), tsv.read()) + self.assertEqual(CRLF.join(expected), unicode(tsv)) + self.assertEqual(TEXT_TSV, tsv.type) + self.assertEqual('text/tab-separated-values; charset=utf-8', + tsv.type.mimetype_charset) + + tsv = content.convert(self.req, TEXT_TSV) + self.assertEqual(CRLF.join(expected), tsv.read()) + self.assertEqual(CRLF.join(expected), unicode(tsv)) + self.assertEqual(TEXT_TSV, tsv.type) + self.assertEqual('text/tab-separated-values; charset=utf-8', + tsv.type.mimetype_charset) + def test_rss_conversion(self): - ticket = self._create_a_ticket() - content, mimetype, ext = self.mimeview.convert_content( - self.req, 'trac.ticket.Ticket', ticket, 'rss') - self.assertEqual(('\n\n\n \n ' - 'Ticket \n \n ' - '<p>\nBar\n</p>\n' - '\n en-us\n ' - 'Trac v\n \n\n', - 'application/rss+xml', 'xml'), - (content.replace('\r', ''), mimetype, ext)) + content = self._create_a_ticket_content() + expected = ['', + '', + '', + ' ', + ' Ticket ', + ' ', + ' <p>\r', + 'Bar\r', + '</p>\r', + '', + ' en-us', + ' Trac v', + ' ', + '', + ''] + rss = content.convert(self.req, 'rss') + self.assertEqual('\n'.join(expected), rss.read()) + self.assertEqual('\n'.join(expected), unicode(rss)) + self.assertEqual(APPLICATION_RSS_XML, rss.type) + self.assertEqual('application/rss+xml; charset=utf-8', + rss.type.mimetype_charset) + rss = content.convert(self.req, APPLICATION_RSS_XML) + self.assertEqual('\n'.join(expected), rss.read()) + self.assertEqual('\n'.join(expected), unicode(rss)) + self.assertEqual(APPLICATION_RSS_XML, rss.type) + self.assertEqual('application/rss+xml; charset=utf-8', + rss.type.mimetype_charset) + def suite(): return unittest.makeSuite(TicketConversionTestCase, 'test') Index: trac/ticket/roadmap.py =================================================================== --- trac/ticket/roadmap.py (revision 3605) +++ trac/ticket/roadmap.py (working copy) @@ -184,7 +184,7 @@ # Internal methods - def render_ics(self, req, db, milestones): + def render_ics(self, req, db, milestones): # FIXME: use IContentConverter req.send_response(200) req.send_header('Content-Type', 'text/calendar;charset=utf-8') req.end_headers() Index: trac/ticket/query.py =================================================================== --- trac/ticket/query.py (revision 3605) +++ trac/ticket/query.py (working copy) @@ -21,6 +21,7 @@ from trac.core import * from trac.db import get_column_names +from trac.mimeview.api import * from trac.perm import IPermissionRequestor from trac.ticket import Ticket, TicketSystem from trac.util.datefmt import format_datetime, http_date @@ -31,7 +32,6 @@ INavigationContributor from trac.wiki import wiki_to_html, wiki_to_oneliner, IWikiSyntaxProvider from trac.wiki.macros import WikiMacroBase -from trac.mimeview.api import Mimeview, IContentConverter class QuerySyntaxError(Exception): """Exception raised when a ticket query cannot be parsed from a string.""" @@ -351,21 +351,24 @@ IContentConverter) # IContentConverter methods - def get_supported_conversions(self): - yield ('rss', 'RSS Feed', 'xml', - 'trac.ticket.Query', 'application/rss+xml', 8) - yield ('csv', 'Comma-delimited Text', 'csv', - 'trac.ticket.Query', 'text/csv', 8) - yield ('tab', 'Tab-delimited Text', 'tsv', - 'trac.ticket.Query', 'text/tab-separated-values', 8) + def get_supported_conversions(self, input): + if input.match(Query): + yield Conversion('rss', 8, APPLICATION_RSS_XML) + yield Conversion('csv', 8, TEXT_CSV) + yield Conversion('tab', 8, TEXT_TSV) - def convert_content(self, req, mimetype, query, key): + def convert_content(self, context, conversion, content): + query = content.content + req = context.req + key = conversion.format if key == 'rss': - return self.export_rss(req, query) - elif key == 'csv': - return self.export_csv(req, query, mimetype='text/csv') + data = self.export_rss(req, query) elif key == 'tab': - return self.export_csv(req, query, '\t', 'text/tab-separated-values') + data = self.export_csv(req, query, '\t') + else: # key == 'tab': + data = self.export_csv(req, query) + return StringMimeContent(self.env, data, conversion.output, + filename=content.basename()) # INavigationContributor methods @@ -413,12 +416,11 @@ del req.session[var] req.redirect(query.get_href(req)) - # Add registered converters - for conversion in Mimeview(self.env).get_supported_conversions( - 'trac.ticket.Query'): - add_link(req, 'alternate', - query.get_href(req, format=conversion[0]), - conversion[1], conversion[3]) + # Send as alternate content if required + format = req.args.get('format') + if format: + ObjectContent(self.env, query).convert(req, format).send(req) + # quite Rubyesque, isn't it? ;) constraints = {} for k, v in query.constraints.items(): @@ -435,11 +437,6 @@ constraints[k] = constraint req.hdf['query.constraints'] = constraints - format = req.args.get('format') - if format: - Mimeview(self.env).send_converted(req, 'trac.ticket.Query', query, - format, 'query') - self.display_html(req, query) return 'query.cs', None @@ -615,18 +612,23 @@ self.env.is_component_enabled(ReportModule): req.hdf['query.report_href'] = req.href.report() - def export_csv(self, req, query, sep=',', mimetype='text/plain'): - content = StringIO() + # Add registered converters + for c in Mimeview(self.env).get_conversions(ObjectType(query)): + add_link(req, 'alternate', query.get_href(req, format=c.format), + c.output.name, c.output.mimetype) + + def export_csv(self, req, query, sep=','): + csv = StringIO() cols = query.get_columns() - content.write(sep.join([col for col in cols]) + CRLF) + csv.write(sep.join([col for col in cols]) + CRLF) - results = query.execute(req, self.env.get_db_cnx()) - for result in results: - content.write(sep.join([unicode(result[col]).replace(sep, '_') - .replace('\n', ' ') - .replace('\r', ' ') + lines = query.execute(req, self.env.get_db_cnx()) + for line in lines: + csv.write(sep.join([unicode(line[col]).replace(sep, '_') + .replace('\n', ' ') + .replace('\r', ' ') for col in cols]) + CRLF) - return (content.getvalue(), '%s;charset=utf-8' % mimetype) + return csv.getvalue() def export_rss(self, req, query): query.verbose = True @@ -648,7 +650,7 @@ groupdesc=query.groupdesc and 1 or None, verbose=query.verbose and 1 or None, **query.constraints) - return (req.hdf.render('query_rss.cs'), 'application/rss+xml') + return req.hdf.render('query_rss.cs') # IWikiSyntaxProvider methods Index: trac/versioncontrol/web_ui/util.py =================================================================== --- trac/versioncontrol/web_ui/util.py (revision 3605) +++ trac/versioncontrol/web_ui/util.py (working copy) @@ -16,10 +16,12 @@ # Author: Jonas Borgström # Christian Boos +import posixpath import re import urllib from trac.core import TracError +from trac.mimeview.api import MimeContent from trac.util.datefmt import format_datetime, pretty_timedelta from trac.util.html import escape, html, Markup from trac.util.text import shorten_line @@ -27,8 +29,45 @@ from trac.wiki import wiki_to_html, wiki_to_oneliner __all__ = ['get_changes', 'get_path_links', 'get_path_rev_line', - 'get_existing_node', 'render_node_property'] + 'get_existing_node', 'render_node_property', + 'NodeMimeContent'] + +# MimeContent wrapper for repository files +# +# -- This could be part of trac.versioncontrol.api itself, +# if introducing a dependency on trac.mimeview.api is +# considered to be OK. + +CHUNK_SIZE = 4096 + +class NodeMimeContent(MimeContent): + """MimeContent wrapper for a file Node in the repository.""" + + def __init__(self, env, node, url=None): + MimeContent.__init__(self, env, node, node.content_type, + posixpath.basename(node.path), url) + self.last_modified = node.last_modified + self._stream = None + + def chunks(self): + """Get chunks of the byte content""" + chunk = self.excerpt + self._excerpt = None + while True: + if not chunk: + return + yield chunk + chunk = self._stream.read(CHUNK_SIZE) + + def size(self): + return self.content.get_content_length() + + def _retrieve_excerpt(self): + self._stream = self.content.get_content() + return self._stream.read(CHUNK_SIZE) + + def get_changes(env, repos, revs, full=None, req=None, format=None): db = env.get_db_cnx() changes = {} Index: trac/versioncontrol/web_ui/changeset.py =================================================================== --- trac/versioncontrol/web_ui/changeset.py (revision 3605) +++ trac/versioncontrol/web_ui/changeset.py (working copy) @@ -26,7 +26,6 @@ from trac import util from trac.config import BoolOption, IntOption from trac.core import * -from trac.mimeview import Mimeview, is_binary from trac.perm import IPermissionRequestor from trac.Search import ISearchSource, search_to_sql, shorten_result from trac.Timeline import ITimelineEventProvider @@ -36,7 +35,7 @@ from trac.versioncontrol import Changeset, Node from trac.versioncontrol.diff import get_diff_options, hdf_diff, unified_diff from trac.versioncontrol.svn_authz import SubversionAuthorizer -from trac.versioncontrol.web_ui.util import render_node_property +from trac.versioncontrol.web_ui.util import * from trac.web import IRequestHandler from trac.web.chrome import INavigationContributor, add_link, add_stylesheet from trac.wiki import wiki_to_html, wiki_to_oneliner, IWikiSyntaxProvider, \ @@ -439,17 +438,24 @@ The list is empty when no differences between comparable files are detected, but the return value is None for non-comparable files. """ - old_content = old_node.get_content().read() - if is_binary(old_content): + old_content = NodeMimeContent(self.env, old_node) + if old_content.is_binary: return None - new_content = new_node.get_content().read() - if is_binary(new_content): + new_content = NodeMimeContent(self.env, new_node) + if new_content.is_binary: return None - mview = Mimeview(self.env) - old_content = mview.to_unicode(old_content, old_node.content_type) - new_content = mview.to_unicode(new_content, new_node.content_type) + old_content = unicode(old_content) + new_content = unicode(new_content) + + # We could theoretically avoid converting to unicode() in case + # both files share the same encoding, i.e. + # + # old_content.type.encoding == new_content.type.encoding + # + # however it's not safe to call splitlines on arbitrary str + # see r3236. A workaround could be to use re.split(r'[\r\n]+') ? if old_content != new_content: context = 3 @@ -526,7 +532,6 @@ 'filename=%s.diff' % filename) req.end_headers() - mimeview = Mimeview(self.env) for old_node, new_node, kind, change in repos.get_changes(**diff): # TODO: Property changes @@ -536,23 +541,20 @@ new_content = old_content = '' new_node_info = old_node_info = ('','') - mimeview = Mimeview(self.env) if old_node: - old_content = old_node.get_content().read() - if is_binary(old_content): + old_content = NodeMimeContent(self.env, old_node) + if old_content.is_binary: continue old_node_info = (old_node.path, old_node.rev) - old_content = mimeview.to_unicode(old_content, - old_node.content_type) + old_content = unicode(old_content) if new_node: - new_content = new_node.get_content().read() - if is_binary(new_content): + new_content = NodeMimeContent(self.env, new_node) + if new_content.is_binary: continue new_node_info = (new_node.path, new_node.rev) new_path = new_node.path - new_content = mimeview.to_unicode(new_content, - new_node.content_type) + new_content = unicode(new_content) else: old_node_path = repos.normalize_path(old_node.path) diff_old_path = repos.normalize_path(diff.old_path) Index: trac/versioncontrol/web_ui/browser.py =================================================================== --- trac/versioncontrol/web_ui/browser.py (revision 3605) +++ trac/versioncontrol/web_ui/browser.py (working copy) @@ -23,10 +23,10 @@ from trac import util from trac.config import ListOption, Option from trac.core import * -from trac.mimeview import Mimeview, is_binary, get_mimetype +from trac.mimeview import Mimeview, MimeType, TEXT_PLAIN from trac.perm import IPermissionRequestor from trac.util import sorted, embedded_numbers -from trac.util.datefmt import http_date, format_datetime, pretty_timedelta +from trac.util.datefmt import format_datetime, pretty_timedelta from trac.util.html import escape, html, Markup from trac.util.text import pretty_size from trac.web import IRequestHandler, RequestDone @@ -36,9 +36,6 @@ from trac.versioncontrol.web_ui.util import * -CHUNK_SIZE = 4096 - - class BrowserModule(Component): implements(INavigationContributor, IPermissionRequestor, IRequestHandler, @@ -199,35 +196,20 @@ def _render_file(self, req, repos, node, rev=None): req.perm.assert_permission('FILE_VIEW') - mimeview = Mimeview(self.env) - - # MIME type detection - content = node.get_content() - chunk = content.read(CHUNK_SIZE) - mime_type = node.content_type - if not mime_type or mime_type == 'application/octet-stream': - mime_type = mimeview.get_mimetype(node.name, chunk) or \ - mime_type or 'text/plain' - + content = NodeMimeContent(self.env, node, + req.href.browser(node.path, rev=rev, + format='raw')) # Eventually send the file directly format = req.args.get('format') - if format in ['raw', 'txt']: - req.send_response(200) - req.send_header('Content-Type', - format == 'txt' and 'text/plain' or mime_type) - req.send_header('Content-Length', node.content_length) - req.send_header('Last-Modified', http_date(node.last_modified)) - req.end_headers() - - while 1: - if not chunk: - raise RequestDone - req.write(chunk) - chunk = content.read(CHUNK_SIZE) + if format in ('raw', 'txt'): + if format == 'txt': + content.type = TEXT_PLAIN ### SHOULD BE ENOUGH + content.type = MimeType('text/plain', content.encoding) + content.send(req) else: # The changeset corresponding to the last change on `node` # is more interesting than the `rev` changeset. - changeset = repos.get_changeset(node.rev) + changeset = repos.get_changeset(node.created_rev) message = changeset.message or '--' if self.config['changeset'].getbool('wiki_format_messages'): @@ -238,33 +220,27 @@ req.hdf['file'] = { 'rev': node.rev, - 'changeset_href': req.href.changeset(node.rev), + 'changeset_href': req.href.changeset(node.created_rev), 'date': format_datetime(changeset.date), 'age': pretty_timedelta(changeset.date), 'size': pretty_size(node.content_length), 'author': changeset.author or 'anonymous', 'message': message } - + # add ''Plain Text'' alternate link if needed - if not is_binary(chunk) and mime_type != 'text/plain': + if not content.is_binary and not TEXT_PLAIN.match(content.type): plain_href = req.href.browser(node.path, rev=rev, format='txt') - add_link(req, 'alternate', plain_href, 'Plain Text', - 'text/plain') + add_link(req, 'alternate', plain_href, + TEXT_PLAIN.name, TEXT_PLAIN.mimetype) # add ''Original Format'' alternate link (always) - raw_href = req.href.browser(node.path, rev=rev, format='raw') - add_link(req, 'alternate', raw_href, 'Original Format', mime_type) + add_link(req, 'alternate', content.url, + 'Original Format', content.type.mimetype) - self.log.debug("Rendering preview of node %s@%s with mime-type %s" - % (node.name, str(rev), mime_type)) + req.hdf['file'] = Mimeview(self.env).preview_to_hdf( + req, content, annotations=['lineno']) - del content # the remainder of that content is not needed - - req.hdf['file'] = mimeview.preview_to_hdf( - req, node.get_content(), node.get_content_length(), mime_type, - node.created_path, raw_href, annotations=['lineno']) - add_stylesheet(req, 'common/css/code.css') # IWikiSyntaxProvider methods Index: trac/wiki/web_ui.py =================================================================== --- trac/wiki/web_ui.py (revision 3605) +++ trac/wiki/web_ui.py (working copy) @@ -22,6 +22,7 @@ from trac.attachment import attachments_to_hdf, Attachment, AttachmentModule from trac.core import * +from trac.mimeview.api import * from trac.perm import IPermissionRequestor from trac.Search import ISearchSource, search_to_sql, shorten_result from trac.Timeline import ITimelineEventProvider @@ -34,8 +35,8 @@ from trac.web import HTTPNotFound, IRequestHandler from trac.wiki.api import IWikiPageManipulator, WikiSystem from trac.wiki.model import WikiPage -from trac.wiki.formatter import wiki_to_html, wiki_to_oneliner -from trac.mimeview.api import Mimeview, IContentConverter +from trac.wiki.formatter import wiki_to_html, wiki_to_oneliner, \ + TEXT_X_TRAC_WIKI, APPLICATION_X_TRAC_WIKI class InvalidWikiPage(TracError): @@ -50,12 +51,21 @@ page_manipulators = ExtensionPoint(IWikiPageManipulator) # IContentConverter methods - def get_supported_conversions(self): - yield ('txt', 'Plain Text', 'txt', 'text/x-trac-wiki', 'text/plain', 9) - def convert_content(self, req, mimetype, content, key): - return (content, 'text/plain;charset=utf-8') + def get_supported_conversions(self, input): + if TEXT_X_TRAC_WIKI.match(input) or \ + APPLICATION_X_TRAC_WIKI.match(input): + yield Conversion(format='txt', quality=8, output=TEXT_PLAIN) + yield Conversion(format='html', quality=8, output=TEXT_HTML) + def convert_content(self, context, conversion, content): + if conversion.format == 'txt': + return content # identity transform + elif conversion.format == 'html': + # TODO 0.11: give the context to `wiki_to_html` + html = wiki_to_html(unicode(content), self.env, context.req) + return StringMimeContent(self.env, html, TEXT_HTML) + # INavigationContributor methods def get_active_navigation_item(self, req): @@ -129,8 +139,9 @@ else: format = req.args.get('format') if format: - Mimeview(self.env).send_converted(req, 'text/x-trac-wiki', - page.text, format, page.name) + content = StringMimeContent(self.env, page.text, + TEXT_X_TRAC_WIKI, page.name) + content.convert(req, format).send(req) self._render_view(req, db, page) req.hdf['wiki.action'] = action @@ -430,12 +441,12 @@ req.hdf['html.norobots'] = 1 # Add registered converters - for conversion in Mimeview(self.env).get_supported_conversions( - 'text/x-trac-wiki'): - conversion_href = req.href.wiki(page.name, version=version, - format=conversion[0]) - add_link(req, 'alternate', conversion_href, conversion[1], - conversion[3]) + for c in Mimeview(self.env).get_conversions(TEXT_X_TRAC_WIKI): + if c.format in ('default', 'html'): + continue + add_link(req, 'alternate', + req.href.wiki(page.name, version=version, format=c.format), + c.output.name, c.output.mimetype_charset) latest_page = WikiPage(self.env, page.name) req.hdf['wiki'] = {'exists': page.exists, Index: trac/wiki/formatter.py =================================================================== --- trac/wiki/formatter.py (revision 3605) +++ trac/wiki/formatter.py (working copy) @@ -31,9 +31,17 @@ from trac.util.text import shorten_line, to_unicode __all__ = ['wiki_to_html', 'wiki_to_oneliner', 'wiki_to_outline', - 'wiki_to_link', 'Formatter' ] + 'wiki_to_link', 'Formatter', + 'TEXT_X_TRAC_WIKI', 'APPLICATION_X_TRAC_WIKI'] +TEXT_X_TRAC_WIKI = MimeType('text/x-trac-wiki', + name='Trac Wiki Text', extension='txt') + +APPLICATION_X_TRAC_WIKI = MimeType('application/x-trac-wiki', + name='Trac Wiki Text', extension='txt') + + def system_message(msg, text=None): return html.DIV(html.STRONG(msg), text and html.PRE(text), class_="system-message") @@ -43,12 +51,13 @@ _code_block_re = re.compile('^(.*)$') - def __init__(self, env, name): - # TODO: transmit `formatter` argument - self.env = env + def __init__(self, formatter, name): + self.formatter = formatter + self.env = formatter.env self.name = name self.error = None self.macro_provider = None + self.mimetype = None builtin_processors = {'html': self._html_processor, 'default': self._default_processor, @@ -59,16 +68,15 @@ # Find a matching wiki macro for macro_provider in WikiSystem(self.env).macro_providers: for macro_name in macro_provider.get_macros(): - if self.name == macro_name: + if name == macro_name: self.processor = self._macro_processor self.macro_provider = macro_provider break if not self.processor: # Find a matching mimeview renderer - from trac.mimeview.api import Mimeview - mimetype = Mimeview(self.env).get_mimetype(self.name) - if mimetype: - self.name = mimetype + mimetype = Mimeview(self.env).get_mimetype(name) + if mimetype and not mimetype.is_binary: + self.mimetype = mimetype self.processor = self._mimeview_processor else: self.processor = self._default_processor @@ -94,14 +102,15 @@ # generic processors def _macro_processor(self, req, text): - # TODO: macro should take a `formatter` argument + # TODO: macro should take a `formatter` argument (0.11) self.env.log.debug('Executing Wiki macro %s by provider %s' % (self.name, self.macro_provider)) return self.macro_provider.render_macro(req, self.name, text) def _mimeview_processor(self, req, text): - # TODO: transmit context from `formatter` - return Mimeview(self.env).render(req, self.name, text) + content = StringMimeContent(self.env, text, self.mimetype) + result = content.convert(req, TEXT_HTML, context=self.formatter) + return result.markup() or unicode(result) def process(self, req, text, in_paragraph=False): if self.error: @@ -157,7 +166,8 @@ ENDBLOCK_TOKEN = r"\}\}\}" ENDBLOCK = "}}}" - LINK_SCHEME = r"[\w.+-]+" # as per RFC 2396 + LINK_SCHEME = r"[a-zA-Z][\w.+-]*" # RFC 2396, except upper case is also ok + INTERTRAC_SCHEME = r"[a-zA-Z.+-]*?" # no digits (support for shorthand links) QUOTED_STRING = r"'[^']+'|\"[^\"]+\"" @@ -433,7 +443,7 @@ return '
' args = fullmatch.group('macroargs') try: - macro = WikiProcessor(self.env, name) + macro = WikiProcessor(self, name) return macro.process(self.req, args, True) except Exception, e: self.env.log.error('Macro %s(%s) failed' % (name, args), @@ -721,7 +731,7 @@ else: self.code_text += line + os.linesep if not self.code_processor: - self.code_processor = WikiProcessor(self.env, 'default') + self.code_processor = WikiProcessor(self, 'default') elif line.strip() == Formatter.ENDBLOCK: self.in_code_block -= 1 if self.in_code_block == 0 and self.code_processor: @@ -735,10 +745,10 @@ match = Formatter._processor_re.search(line) if match: name = match.group(1) - self.code_processor = WikiProcessor(self.env, name) + self.code_processor = WikiProcessor(self, name) else: self.code_text += line + os.linesep - self.code_processor = WikiProcessor(self.env, 'default') + self.code_processor = WikiProcessor(self, 'default') else: self.code_text += line + os.linesep Index: trac/web/chrome.py =================================================================== --- trac/web/chrome.py (revision 3605) +++ trac/web/chrome.py (working copy) @@ -17,10 +17,10 @@ import os import re -from trac import mimeview from trac.config import * from trac.core import * from trac.env import IEnvironmentSetupParticipant +from trac.mimeview.api import get_mimetype, MimeContent from trac.util.html import html from trac.web.api import IRequestHandler, HTTPNotFound from trac.web.href import Href @@ -228,7 +228,8 @@ path = os.path.normpath(os.path.join(dir, filename)) assert os.path.commonprefix([dir, path]) == dir if os.path.isfile(path): - req.send_file(path, mimeview.get_mimetype(path)) + mtype = MimeContent(self.env, filename=path).type.mimetype + req.send_file(path, mtype) self.log.warning('File %s not found in any of %s', filename, dirs) raise HTTPNotFound('File %s not found', filename) @@ -288,7 +289,7 @@ icon = href.chrome(icon) else: icon = href.chrome('common', icon) - mimetype = mimeview.get_mimetype(icon) + mimetype = get_mimetype(icon) add_link(req, 'icon', icon, mimetype=mimetype) add_link(req, 'shortcut icon', icon, mimetype=mimetype) Index: trac/util/text.py =================================================================== --- trac/util/text.py (revision 3605) +++ trac/util/text.py (working copy) @@ -25,6 +25,7 @@ CRLF = '\r\n' +IDENTITY_CHARSET = 'iso-8859-1' # not iso-8859-15 # -- Unicode @@ -81,7 +82,7 @@ return urlencode([(k, isinstance(v, unicode) and v.encode('utf-8') or v) for k, v in params]) -def to_utf8(text, charset='iso-8859-15'): +def to_utf8(text, charset=IDENTITY_CHARSET): """Convert a string to UTF-8, assuming the encoding is either UTF-8, ISO Latin-1, or as specified by the optional `charset` parameter. @@ -97,7 +98,7 @@ u = unicode(text, charset) except UnicodeError: # This should always work - u = unicode(text, 'iso-8859-15') + u = unicode(text, IDENTITY_CHARSET) return u.encode('utf-8') Index: templates/browser.cs =================================================================== --- templates/browser.cs (revision 3605) +++ templates/browser.cs (working copy) @@ -111,18 +111,8 @@ -
- HTML preview not available, since the file size exceeds - bytes. Try downloading the file instead.HTML preview not available. To view, download the file. -
Index: templates/macros.cs =================================================================== --- templates/macros.cs (revision 3605) +++ templates/macros.cs (working copy) @@ -194,4 +194,19 @@ def:plural(base, count) ?>s +
+ + HTML preview not available, since the file size exceeds + bytes. You may download the file instead. + HTML preview not available. To view the file, + download the file. +
Index: templates/attachment.cs =================================================================== --- templates/attachment.cs (revision 3605) +++ templates/attachment.cs (working copy) @@ -66,19 +66,8 @@ - -
- - HTML preview not available, since the file size exceeds - bytes. You may download the file instead. - HTML preview not available. To view the file, - download the file. -
+