Index: trac/attachment.py =================================================================== --- trac/attachment.py (revision 3507) +++ trac/attachment.py (working copy) @@ -26,7 +26,7 @@ from trac.config import BoolOption, IntOption from trac.core import * from trac.env import IEnvironmentSetupParticipant -from trac.mimeview import * +from trac.mimeview.api import Mimeview, MimeType, FileMimeContent from trac.util import get_reporter_id, create_unique_file from trac.util.datefmt import format_datetime, pretty_timedelta from trac.util.markup import Markup, html @@ -124,7 +124,7 @@ def parent_href(self, req): return req.href(self.parent_type, self.parent_id) - def _get_title(self): + def _get_title(self): # TODO: should extend to other `parent_type`s return '%s%s: %s' % (self.parent_type == 'ticket' and '#' or '', self.parent_id, self.filename) title = property(_get_title) @@ -227,7 +227,7 @@ select = classmethod(select) - def open(self): + def open(self): # deprecate? self.env.log.debug('Trying to open attachment at %s', self.path) try: fd = open(self.path, 'rb') @@ -505,6 +505,7 @@ 'author': get_reporter_id(req)} def _render_view(self, req, attachment): + # FIXME: perm_map should extend to other `parent_type`s perm_map = {'ticket': 'TICKET_VIEW', 'wiki': 'WIKI_VIEW'} req.perm.assert_permission(perm_map[attachment.parent_type]) @@ -522,53 +523,46 @@ if req.perm.has_permission(perm_map[attachment.parent_type]): req.hdf['attachment.can_delete'] = 1 - fd = attachment.open() - try: - mimeview = Mimeview(self.env) + format = req.args.get('format') + raw_href = attachment.href(req, format='raw') - # MIME type detection - str_data = fd.read(1000) - fd.seek(0) - - binary = is_binary(str_data) - mime_type = mimeview.get_mimetype(attachment.filename, str_data) + mimecontent = FileMimeContent(self.env, attachment.path, raw_href, + 'Attachment') + can_render_as_txt = self.render_unsafe_content and \ + not mimecontent.is_binary - # Eventually send the file directly - format = req.args.get('format') - if format in ('raw', 'txt'): - if not self.render_unsafe_content and not binary: - # Force browser to download HTML/SVG/etc pages that may - # contain malicious code enabling XSS attacks - req.send_header('Content-Disposition', 'attachment;' + - 'filename=' + attachment.filename) - if not mime_type or (self.render_unsafe_content and \ - not binary and format == 'txt'): - mime_type = 'text/plain' - if 'charset=' not in mime_type: - charset = mimeview.get_charset(str_data, mime_type) - mime_type = mime_type + '; charset=' + charset - req.send_file(attachment.path, mime_type) + # Eventually send the file directly + if format in ('raw', 'txt'): + if not self.render_unsafe_content and not mimecontent.is_binary: + # Force browser to download HTML/SVG/etc pages that may + # contain malicious code enabling XSS attacks + req.send_header('Content-Disposition', + 'attachment;filename=%s' % attachment.filename) + if not mimecontent.type.is_known or \ + (format == 'txt' and can_render_as_txt): + # Force the content to be displayed as text + type = MimeType('text/plain', mimecontent.encoding) + else: + type = mimecontent.type + req.send_file(attachment.path, type.mimetype_charset) - # add ''Plain Text'' alternate link if needed - if self.render_unsafe_content and not binary and \ - mime_type and not mime_type.startswith('text/plain'): - plaintext_href = attachment.href(req, format='txt') - add_link(req, 'alternate', plaintext_href, 'Plain Text', - mime_type) + mimetype = mimecontent.type.mimetype - # add ''Original Format'' alternate link (always) - raw_href = attachment.href(req, format='raw') - add_link(req, 'alternate', raw_href, 'Original Format', mime_type) + # add ''Plain Text'' alternate link if needed + if can_render_as_txt and mimecontent.type.is_known and \ + mimetype != 'text/plain': + add_link(req, 'alternate', attachment.href(req, format='txt'), + 'Plain Text', 'text/plain') - self.log.debug("Rendering preview of file %s with mime-type %s" - % (attachment.filename, mime_type)) + # add ''Original Format'' alternate link (always) + add_link(req, 'alternate', raw_href, 'Original Format', mimetype) - req.hdf['attachment'] = mimeview.preview_to_hdf( - req, fd, os.fstat(fd.fileno()).st_size, mime_type, - attachment.filename, raw_href, annotations=['lineno']) - finally: - fd.close() + self.log.debug("Rendering preview of file %s with mime-type %s" % \ + (attachment.filename, mimetype)) + req.hdf['attachment'] = Mimeview(self.env).preview_to_hdf( + req, mimecontent, annotations=['lineno']) + def _render_list(self, req, p_type, p_id): self._parent_to_hdf(req, p_type, p_id) req.hdf['attachment'] = { Index: trac/mimeview/rst.py =================================================================== --- trac/mimeview/rst.py (revision 3507) +++ trac/mimeview/rst.py (working copy) @@ -28,7 +28,7 @@ import re from trac.core import * -from trac.mimeview.api import IHTMLPreviewRenderer, content_to_unicode +from trac.mimeview.api import IHTMLPreviewRenderer from trac.web.href import Href from trac.wiki.formatter import WikiProcessor from trac.wiki import WikiSystem @@ -223,7 +223,7 @@ _inliner = rst.states.Inliner() _parser = rst.Parser(inliner=_inliner) - content = content_to_unicode(self.env, content, mimetype) + content = unicode(content) content = content.encode('utf-8') html = publish_string(content, writer_name='html', parser=_parser, settings_overrides={'halt_level': 6}) Index: trac/mimeview/api.py =================================================================== --- trac/mimeview/api.py (revision 3507) +++ trac/mimeview/api.py (working copy) @@ -19,41 +19,53 @@ # Christian Boos """ -The `trac.mimeview` module centralize the intelligence related to -file metadata, principally concerning the `type` (MIME type) of the content -and, if relevant, concerning the text encoding (charset) used by the content. +The `trac.mimeview` module centralizes the intelligence related to file +metadata, principally concerning the `type` (MIME type) and the text +encoding (charset) used by the content, if the latter one is relevant. There are primarily two approaches for getting the MIME type of a given file: * taking advantage of existing conventions for the file name * examining the file content and applying various heuristics -The module also knows how to convert the file content from one type -to another type. +The module also knows about conversions from one data type to another type, +like conversions to text/html (this is no more a special case). -In some cases, only the `url` pointing to the file's content is actually -needed, that's why we avoid to read the file's content when it's not needed. - -The actual `content` to be converted might be a `unicode` object, -but it can also be the raw byte string (`str`) object, or simply -an object that can be `read()`. +In order to keep the API simple, we deal with a few classes, +each encapsulating a part of the knowledge related to content. + * the `MimeType`, for storing the mime type string, the charset, + but also eventually the name and the commonly used file extension + for that type + * the `MimeContent` and `FileMimeContent`, which provide a flexible + API for handling string and file content, respectively. + Those classes inherit from the abstract `MimeContentBase`, which + can be used to make new wrapper classes for any kind of content. + * the `Conversion` class, which is use to specify conversion from + one `MimeType` to another """ +import os import re from StringIO import StringIO from trac.config import IntOption, ListOption, Option from trac.core import * from trac.util import sorted -from trac.util.text import to_utf8, to_unicode +from trac.util.text import to_utf8, to_unicode, IDENTITY_CHARSET from trac.util.markup import escape, Markup, Fragment, html -__all__ = ['get_mimetype', 'is_binary', 'detect_unicode', 'Mimeview', - 'content_to_unicode'] +__all__ = ['get_mimetype', 'is_binary', 'detect_unicode', + 'Mimeview', 'Conversion', 'MimeContentBase', 'MimeType', + 'MimeContent', 'FileMimeContent', + 'TypeWrapper', 'ObjectContent', 'IContentConverter', + 'TEXT_PLAIN', 'TEXT_HTML', + 'APPLICATION_RSS_XML', 'APPLICATION_OCTET_STREAM'] # Some common MIME types and their associated keywords and/or file extensions +APPLICATION_OCTET_STREAM_STR = 'application/octet-stream' + KNOWN_MIME_TYPES = { 'application/pdf': ['pdf'], 'application/postscript': ['ps'], @@ -114,20 +126,18 @@ for e in exts: MIME_MAP[e] = t -# Simple builtin autodetection from the content using a regexp -MODE_RE = re.compile( - r"#!(?:[/\w.-_]+/)?(\w+)|" # look for shebang - r"-\*-\s*(?:mode:\s*)?([\w+-]+)\s*-\*-|" # look for Emacs' -*- mode -*- - r"vim:.*?syntax=(\w+)" # look for VIM's syntax= - ) -def get_mimetype(filename, content=None, mime_map=MIME_MAP): - """Guess the most probable MIME type of a file with the given name. +# -- a few functions for dealing with MIME types / binary / text content +# in a simple way (get_mimetype, is_binary, detect_unicode) +def get_mimetype_from_filename(filename, mime_map=MIME_MAP): + """Guess the most probable MIME type of file with the given `filename`. + `filename` is either a filename (the lookup will then use the suffix) or some arbitrary keyword. - - `content` is either a `str` or an `unicode` string. + `mime_map` maps keywords to MIME types. + + Return the MIME type as a string, or `None` if not detected. """ suffix = filename.split('.')[-1] if suffix in mime_map: @@ -141,20 +151,42 @@ mimetype = mimetypes.guess_type(filename)[0] except: pass - if not mimetype and content: - match = re.search(MODE_RE, content[:1000]) - if match: - mode = match.group(1) or match.group(3) or \ - match.group(2).lower() - if mode in mime_map: - # 3) mimetype from the content, using the `MODE_RE` - return mime_map[mode] - else: - if is_binary(content): - # 4) mimetype from the content, using`is_binary` - return 'application/octet-stream' return mimetype +# Simple builtin autodetection from the content using a regexp +MODE_RE = re.compile( + r"#!(?:[/\w.-_]+/)?(\w+)|" # look for shebang + r"-\*-\s*(?:mode:\s*)?([\w+-]+)\s*-\*-|" # look for Emacs' -*- mode -*- + r"vim:.*?syntax=(\w+)" # look for VIM's syntax= + ) + +def get_mimetype_from_content(content): + """Guess the most probable MIME type of file with the given `filename`. + + `content` is either a `str` or an `unicode` string + + Return the MIME type as a string, or `None` if not detected. + """ + match = re.search(MODE_RE, content[:1000]) + if match: + mode = match.group(1) or match.group(3) or \ + match.group(2).lower() + if mode in mime_map: + # 3) mimetype from the content, using the `MODE_RE` + return mime_map[mode] + else: + if is_binary(content): + # 4) mimetype from the content, using `is_binary` + return APPLICATION_OCTET_STREAM_STR + +def get_mimetype(filename, content=None, mime_map=MIME_MAP): + """Auto-detect MIME type either from the `filename` or from the `content`. + """ + mimetype = get_mimetype_from_filename(filename, mime_map) + if not mimetype and content: + mimetype = get_mimetype_from_content(content) + return mimetype + def is_binary(data): """Detect binary content by checking the first thousand bytes for zeroes. @@ -165,7 +197,7 @@ return '\0' in data[:1000] def detect_unicode(data): - """Detect different unicode charsets by looking for BOMs (Byte Order Marks). + """Detect different unicode charsets by looking for Byte Order Marks. Operate obviously only on `str` objects. """ @@ -178,19 +210,422 @@ else: return None -def content_to_unicode(env, content, mimetype): - """Retrieve an `unicode` object from a `content` to be previewed""" - mimeview = Mimeview(env) - if hasattr(content, 'read'): - content = content.read(mimeview.max_preview_size) - return mimeview.to_unicode(content, mimetype) +# -- Classes for mimetype, content and conversion +class TypeRepr(object): + """Represent some for of typing.""" + def match(self, other, regexp=False): + raise NotImplementedError + + mimetype = property(lambda x: x._mimetype or APPLICATION_OCTET_STREAM_STR, + doc="MIME Type string (without charset information)") + charset = property(lambda x: x._charset, + doc="Eventual charset information") + is_binary = property(lambda x: x._is_binary()) + +class TypeWrapper(TypeRepr): + """Typing using Python types.""" + + def __init__(self, obj): + self.class_ = isinstance(obj, type) and obj or obj.__class__ + self._mimetype = self._charset = None + + def __repr__(self): + return self.class_.__name__ + + def _is_binary(self): + return True + + def match(self, other, regexp=False): + other_class = isinstance(other, type) and other or \ + isinstance(other, TypeWrapper) and other.class_ or \ + other.__class__ + return other_class == self.class_ + +class MimeType(TypeRepr): + """Typing of arbitrary content using MIME types. + + If the MIME type correspond to text content, the object can also + store a `charset` information. + + A MIME type has a `name` and has an `extension` + that can be used for storing the converted data in a file. + + All the properties of this class are read-only. + """ + + def __init__(self, mimetype, charset=None, name=None, extension=None): + """The `mimetype` string can eventually embed the `charset`.""" + self._mimetype = mimetype + # determine charset + self._charset = charset + if not self._charset and self._mimetype: + sep_idx = mimetype.find(';') + if sep_idx >= 0: + self._mimetype = mimetype[:sep_idx].strip() + charset_idx = mimetype.find('charset=', sep_idx) + if charset_idx >= 0: + self._charset = mimetype[charset_idx+8:].strip() + self._extension = extension + self._name = name + + def __repr__(self): + return 'MIME type: ' + self.mimetype_charset + + def _get_extension(self): + if not self._extension: + self._extension = KNOWN_MIME_TYPES.get(self.mimetype) + if not self._extension: + detail = self.mimetype.split('/', 1)[1] + if detail.startswith('x-'): + self._extension = detail[2:] + return self._extension + + def _get_mimetype_charset(self): + """Combine the MIME type and charset information in a single string. + """ + if self._mimetype and self._charset: + return '%s; charset=%s' % (self.mimetype, self.charset) + else: + return self.mimetype + + def _is_binary(self): + return self._mimetype == APPLICATION_OCTET_STREAM_STR + + name = property(lambda x: x._name or x._extension) + extension = property(lambda x: x._get_extension()) + mimetype_charset = property(lambda x: x._get_mimetype_charset()) + + def match(self, other, regexp=False): + """Compare MIME type string only. + + If `regexp` is set, `self.mimetype` is used as a regexp. + """ + if not isinstance(other, MimeType): + return False + if regexp: + return re.match(self.mimetype, other.mimetype) + else: + return self.mimetype == other.mimetype + + +TEXT_PLAIN = MimeType('text/plain', 'utf-8', 'Plain Text', 'txt') +TEXT_HTML = MimeType('text/html', 'utf-8', 'HTML', 'html') + +APPLICATION_RSS_XML = MimeType('application/rss+xml', 'utf-8', + 'RSS Feed', 'xml') +APPLICATION_OCTET_STREAM = MimeType(APPLICATION_OCTET_STREAM_STR, + IDENTITY_CHARSET, + 'Undefined (binary)', 'bin') + + +class MimeContentBase(object): + """An abstract MIME content, with an associated MimeType. + + Such an object has means to auto-detect both the MIME Type and + the `encoding` of its content. + + That `encoding` is more reliable than the `type.charset` information. + There are additional consistency checks that are performed, and it + can be `None` if the content is an `unicode` object. + + The content itself can be accessed in various ways: through + the iterator protocol, the len() and unicode() operators... + """ + + def __init__(self, env, mimetype=None, filename=None, url=None): + """ + `mimetype` can be specified as a `MimeType` object, + or as string, which will then be a hint about the content. + + If the mimetype is not specified or equal to + "application/octet-stream", then it will be auto-detected when needed. + + In case auto-detection fails, APPLICATION_OCTET_STREAM will be the + corresponding MIME type. + + The `filename` is simply a suggested basename for that content. + + The `url` is a link for retrieving the raw content directly + from the server. This can be useful for converters that can + provide links to objects, instead of having to expand the + content inline. + """ + self.env = env + if isinstance(mimetype, basestring): + mimetype = MimeType(mimetype) + self._type = mimetype + self._filename = filename + self._url = url + self._binary = None + self._encoding = False + + def __repr__(self): + return '<%s %s "%s">' % (self.__class__.__name__, self._type, + self._filename or self._url) + + def __unicode__(self): + """Return the `unicode` object corresponding to the content.""" + return to_unicode(self.content, self.encoding) + # Note: this does the right thing if the content is already `unicode` + + def encode(self, charset): + """Return a `str`, corresponding to the `charset` encoded content.""" + if self.encoding == charset: + return self.content + else: + return unicode(self).encode(charset) + + def _is_binary(self): + """An heuristic for guessing whether the content is binary or not. + + This will eventually fetch an `excerpt` of the content. + """ + if self._binary is None: + self._binary = self.type.is_binary + print `self.type`, self._binary + if self._binary is None: + self._binary = is_binary(self.excerpt) or \ + (self.type.mimetype in \ + Mimeview(self.env).treat_as_binary) + return self._binary + + def _get_type(self): + """Get or determine the MimeType corresponding to this content. + + An `excerpt` of the content will be examined if needed. + """ + if self._type is None: # not set + mimetype = None + if self.filename: + mimemap = Mimeview(self.env).mimemap + mimetype = get_mimetype_from_filename(self.filename, mimemap) + if not mimetype: + mimetype = get_mimetype_from_content(self.excerpt) + if not mimetype: + pass # TODO 0.11: go through IMimeTypeDetectors + self._type = MimeType(mimetype) + return self._type + + def _set_type(self, type): + """Simply replace the existing `type` by the given `MimeType` object. + + If `None` is given, this will force auto-detection the next time + `type` will be accessed. + + Can be used for in-place conversion (e.g. ''any'' to text/plain). + """ + self._type = type + + def _get_encoding(self): + """Get or determine the current encoding of that `content`. + + The encoding will be determined using this order: + * from the charset information present in the mimetype information + * auto-detection of the charset from the `content` + * if nothing else worked, use the configured `default_charset` + + If the `content` happens to be a genuine `unicode` object, then + this returns `None`. + If the `content` is binary, then the encoding will be the identity + charset (ISO Latin 1). + """ + if self._encoding is False: + charset = self.type.charset + if charset: + self._encoding = charset + elif isinstance(self.excerpt, str): + utf_encoding = detect_unicode(self.excerpt) + if utf_encoding is not None: + self._encoding = utf_encoding + elif self.is_binary: + self._encoding = IDENTITY_CHARSET + elif isinstance(self.excerpt, unicode): + self._encoding = None + if self._encoding is False: + pass # TODO 0.11: go through ICharsetDetectors here + if self._encoding is False: + self._encoding = Mimeview(self.env).default_charset + return self._encoding + + def _get_content(self): + """Retrieve all the content. + + Default implementation based on iterator. If the iterator itself + is implemented based on the content... reimplement this one! + """ + return "".join(self.__iter__()) + + def read(self): # TODO: remove in 0.11 + return self.content # (compatibility with IHTMLPreviewRenderer) + + # Methods that need to be reimplemented by subclasses: + + def __iter__(self): + """Iterate on chunks of raw content.""" + raise NotImplementedError + + def __len__(self): + """Length of the raw content, in bytes.""" + raise NotImplementedError + + def _get_excerpt(self, len=1000): + """Extracts the first `len` characters from the content.""" + raise NotImplementedError + + type = property(fget=lambda x: x._get_type(), + fset=lambda x, y: x._set_type(y)) + is_binary = property(lambda x: x._is_binary()) + encoding = property(lambda x: x._get_encoding()) + excerpt = property(lambda x: x._get_excerpt()) + content = property(lambda x: x._get_content()) + filename = property(lambda x: x._filename) + url = property(lambda x: x._url) + + +class MimeContent(MimeContentBase): + """MIME-typed content wrapper for a basestring.""" + + def __init__(self, env, content, mimetype, filename='file', url=None): + MimeContentBase.__init__(self, env, mimetype, filename, url) + self._content = content + + # Reimplemented methods + + def _get_content(self): + """Retrieve the wrapped content. + + Note: therefore this *might* be an `unicode` object. + Remember that in this case, `encoding` will be `None`. + """ + return self._content + + def __iter__(self): + """Iterate on chunks of content. + + If the content `is_binary` property is `False`, those chunks will + be lines, with the line endings kept. + """ + if self.is_binary: + buf = StringIO(self.content) + chunk = buf.read(1000) + while chunk: + yield chunk + chunk = buf.read(1000) + else: + for line in self.content.splitlines(True): + yield line + + def __len__(self): + """Length of the content, in characters.""" + return len(self.content) + + def _get_excerpt(self, len=1000): + """Extracts the first `len` characters from the content.""" + return self._content[:len] + + +class FileMimeContent(MimeContentBase): + """MIME-typed content wrapper for a file.""" + + def __init__(self, env, path, url=None, kind='File', mimetype=None): + self._fd = None + self._path = path + self._kind = kind + self._excerpt = None + MimeContentBase.__init__(self, env, mimetype, os.path.basename(path), + url) + def __del__(self): + if self._fd: + self._fd.close() + + def _ensure_open(self): + if not self._fd: + try: + self._fd = open(self._path) + except IOError: + raise TracError('%s "%s" not found' % (self._kind, + self._filename)) + # Reimplemented methods + + def __iter__(self): + """Iterate on chunks of raw content.""" + chunk = self.excerpt + while chunk: + yield chunk + chunk = self._fd.read(1000) + + def __len__(self): + """Length of the raw content, in bytes.""" + if self._fd: + stat = os.fstat(self._fd.fileno()) + else: + stat = os.stat(self._path) + return stat.st_size + + def _get_excerpt(self, len=1000): + """Extracts the `len` first bytes from the content.""" + if self._excerpt is None: + self._ensure_open() + self._excerpt = self._fd.read(1000) + return self._excerpt + +class ObjectContent(MimeContentBase): + """Wraps a Python object into a MimeContentBase. + + Only supports the bare minimum of the MimeContentBase methods. + """ + + def __init__(self, env, obj, filename="obj"): + self._obj = obj + MimeContentBase.__init__(self, env, TypeWrapper(obj), + filename=filename) + + def _get_content(self): + """Retrieve the wrapped content.""" + return self._obj + + +class NoConversion(TracError): + def __init__(self, msg, from_, output, key): + TracError.__init__(self, '%s, from %s to %s' % + (msg, repr(from_), output and repr(output) or key)) + +class Conversion(object): + """A specification for performing a data conversion. + + Each conversion is identified by a `key` and targets an output `mimetype`. + + A conversion also specifies a `quality` ranking, which is a number + in the range 0 to 9, where 0 means no support and 9 means "perfect" + support (try to keep 9 available for user defined conversions, + though nothing will prevent them from using 10 or 100...) + + Finally, `expand_tabs` indicates whether a tab expansion should precede + the conversion attempt. + + e.g. Conversion(key='latex', quality=8, mimetype=MimeType('text/x-tex')) + """ + + def __init__(self, key, quality=1, mimetype=TEXT_HTML, expand_tabs=False): + self.key = key + self.quality = quality + self.mimetype = mimetype + self.expand_tabs = expand_tabs + + def __repr__(self): + return '<%s conversion to %s [qr=%s, et=%s]>' % \ + (self.key, self.mimetype, self.quality, self.expand_tabs) + + +# -- Deprecated (TODO: remove in 0.11) + class IHTMLPreviewRenderer(Interface): """Extension point interface for components that add HTML renderers of specific content types to the `Mimeview` component. - (Deprecated) + Deprecated in 0.10. Implement `IContentConverter` instead. """ # implementing classes should set this property to True if they @@ -198,31 +633,33 @@ expand_tabs = False def get_quality_ratio(mimetype): - """Return the level of support this renderer provides for the `content` - of the specified MIME type. The return value must be a number between - 0 and 9, where 0 means no support and 9 means "perfect" support. - """ + """Return the level of support this renderer provides""" def render(req, mimetype, content, filename=None, url=None): - """Render an XHTML preview of the raw `content`. + """Render an XHTML preview of the raw `content`.""" - The `content` might be: - * a `str` object - * an `unicode` string - * any object with a `read` method, returning one of the above - It is assumed that the content will correspond to the given `mimetype`. +# -- Interfaces for the extension points - Besides the `content` value, the same content may eventually - be available through the `filename` or `url` parameters. - This is useful for renderers that embed objects, using or - instead of including the content inline. - - Can return the generated XHTML text as a single string or as an - iterable that yields strings. In the latter case, the list will - be considered to correspond to lines of text in the original content. +class IContentConverter(Interface): + """An extension point interface for generic content conversion.""" + + def get_supported_conversions(mimetype): + """Check if conversion of `mimetype` is supported by this converter. + + Return an iterable of `Conversion` objects for which this is + the case. """ + def convert_content(context, conversion, content): + """Convert the given `content` using the specified `conversion`. + + The conversion takes place in the given formatting `context`. + A `context` provides at least a `req` property. + + Return the converted content as a new `MimeContent` object. + """ + class IHTMLPreviewAnnotator(Interface): """Extension point interface for components that can annotate an XHTML representation of file contents with additional information.""" @@ -240,28 +677,20 @@ annotation data.""" -class IContentConverter(Interface): - """An extension point interface for generic MIME based content - conversion.""" +# -- The main Mimeview component - def get_supported_conversions(): - """Return an iterable of tuples in the form (key, name, extension, - in_mimetype, out_mimetype, quality) representing the MIME conversions - supported and - the quality ratio of the conversion in the range 0 to 9, where 0 means - no support and 9 means "perfect" support. eg. ('latex', 'LaTeX', 'tex', - 'text/x-trac-wiki', 'text/plain', 8)""" +class ToplevelContext(object): + """A simple wrapper for the Request object. - def convert_content(req, mimetype, content, key): - """Convert the given content from mimetype to the output MIME type - represented by key. Returns a tuple in the form (content, - output_mime_type) or None if conversion is not possible.""" + Use this when no other context information is available. + """ + def __init__(self, req): + self.req = req - class Mimeview(Component): """A generic class to prettify data, typically source code.""" - renderers = ExtensionPoint(IHTMLPreviewRenderer) + renderers = ExtensionPoint(IHTMLPreviewRenderer) # TODO: remove in 0.11 annotators = ExtensionPoint(IHTMLPreviewAnnotator) converters = ExtensionPoint(IContentConverter) @@ -275,143 +704,183 @@ """Maximum file size for HTML preview. (''since 0.9'').""") mime_map = ListOption('mimeviewer', 'mime_map', - 'text/x-dylan:dylan,text/x-idl:ice,text/x-ada:ads:adb', + 'text/x-dylan:dylan,text/x-idl:ice,text/x-ada:ads:adb', doc= """List of additional MIME types and keyword mappings. - Mappings are comma-separated, and for each MIME type, - there's a colon (":") separated list of associated keywords - or file extensions. (''since 0.10'').""") + Mappings are comma-separated. Each mapping starts with the mimetype, + followed by a colon (":") and the (colon separated) list of associated + keywords or file extensions. (''since 0.10'').""") + + treat_as_binary = ListOption('mimeviewer', 'treat_as_binary', + 'application/pdf,application/postscript,application/rtf', doc= + """List of MIME types that should always be treated as binary content. + + Accounts for the fact that our binary detection heuristic can't + always work for some kind of binary data. (''since 0.10'').""") + def __init__(self): self._mime_map = None - - # Public API - def get_supported_conversions(self, mimetype): - """Return a list of target MIME types in same form as - `IContentConverter.get_supported_conversions()`, but with the converter - component appended. Output is ordered from best to worst quality.""" + def _get_mimemap(self): + """Extend default extension to MIME type mappings""" + if not self._mime_map: + self._mime_map = {} + self._mime_map.update(MIME_MAP) + for mapping in self.config['mimeviewer'].getlist('mime_map'): + if ':' in mapping: + assocations = mapping.split(':') + mimetype = assocations[0] + for keyword in assocations: # mimetype->mimetype on purpose + self._mime_map[keyword] = mimetype + return self._mime_map + + mimemap = property(_get_mimemap) + + def lookup(self, keyword, charset=None): + """Lookup for given `keyword`, among known MIME Types. + + Return a `MimeType` object if found, `None` otherwise. + """ + mimetype = self.mimemap.get(keyword, None) + if mimetype: + return MimeType(mimetype, charset, extension=keyword) + + # -- MIME type conversion + + def get_conversions(self, input): + """Return a list of possible conversions for the `input` MimeType. + + The returned list contains pair of `(conversion, converter)` objects, + ordered from best to worst quality. + """ + # Build list of possible conversions converters = [] for converter in self.converters: - for k, n, e, im, om, q in converter.get_supported_conversions(): - if im == mimetype and q > 0: - converters.append((k, n, e, im, om, q, converter)) - converters = sorted(converters, key=lambda i: i[-1], reverse=True) - return converters + for conversion in converter.get_supported_conversions(input): + if conversion.quality > 0: + converters.append((conversion, converter)) - def convert_content(self, req, mimetype, content, key, filename=None, - url=None): - """Convert the given content to the target MIME type represented by - `key`, which can be either a MIME type or a key. Returns a tuple of - (content, output_mime_type, extension).""" - if not content: - return ('', 'text/plain;charset=utf-8') + # ---- Backward compatibility support for IHTMLPreviewRenderer + class IHTMLPreviewRendererWrapper(object): + def __init__(self, renderer): + self.renderer = renderer + def __repr__(self): + return repr(self.renderer) + def convert_content(self, context, conversion, mimecontent): + return self.renderer.render( + context.req, mimecontent.type.mimetype, + mimecontent, # which is read()able + mimecontent.filename, mimecontent.url) + for renderer in self.renderers: + qr = renderer.get_quality_ratio(input.mimetype) + if qr > 0: + expand_tabs = getattr(renderer, 'expand_tabs', False) + converters.append( + (Conversion(key='', quality=qr, mimetype=TEXT_HTML, + expand_tabs=expand_tabs), + IHTMLPreviewRendererWrapper(renderer))) + # ---- (to be removed in 0.11) - # Ensure we have a MIME type for this content - full_mimetype = mimetype - if not full_mimetype: - if hasattr(content, 'read'): - content = content.read(self.max_preview_size) - full_mimetype = self.get_mimetype(filename, content) - if full_mimetype: - mimetype = full_mimetype.split(';')[0].strip() # split off charset + return sorted(converters, key=lambda c: c[0].quality, reverse=True) + + def convert(self, context, selector, mimecontent): + """Convert the `mimecontent` to another MIME type. + + The conversion to be done is determined by `selector`, + which can be either directly the desired output MIME type or + a key identifying the `Conversion` object. + + Returns a new `MimeContent`. + """ + result, mimetype = self._convert(context, selector, mimecontent) + + if isinstance(result, MimeContentBase): + return result + else: + return MimeContent(self.env, result, mimetype, + filename=mimecontent.filename) + + def _convert(self, context, selector, mimecontent): + """Convert the `mimecontent` to another MIME type. + + Doesn't necessarily return a new `MimeContent`: can be a basestring, + a Fragment, an iterable... + + """ + # See whether we've got a type selector + if isinstance(selector, TypeRepr): + selected_output = selector + selected_key = None else: - mimetype = full_mimetype = 'text/plain' # fallback if not binary + selected_output = self.lookup(selector) + if selected_output: + selected_key = None + else: + selected_key = selector - # Choose best converter - candidates = list(self.get_supported_conversions(mimetype)) - candidates = [c for c in candidates if key in (c[0], c[4])] + # Get all available conversions for the input and filter those which + # are matching either the `selected_output` or the `selected_key` + candidates = [] + for cc_pair in self.get_conversions(mimecontent.type): + conversion = cc_pair[0] + if selected_key == conversion.key or \ + conversion.mimetype.match(selected_output): ### TODO: conversion.type + candidates.append(cc_pair) if not candidates: - raise TracError('No available MIME conversions from %s to %s' % - (mimetype, key)) + raise NoConversion('No available MIME conversions', + mimecontent.type, selected_output, selected_key) - # First successful conversion wins - for ck, name, ext, input_mimettype, output_mimetype, quality, \ - converter in candidates: - output = converter.convert_content(req, mimetype, content, ck) - if not output: - continue - return (output[0], output[1], ext) - raise TracError('No available MIME conversions from %s to %s' % - (mimetype, key)) + tab_expanded = None # we don't want to expand tabs more than once. + # First candidate which converts successfully wins. + for conversion, converter in candidates: + self.log.debug(u'Attempting conversion of %s using %s and %s' % + (mimecontent, conversion, converter)) + if conversion.expand_tabs and not tab_expanded: + tab_expanded = unicode(mimecontent).expandtabs(self.tab_width) + mimecontent = MimeContent(self.env, tab_expanded, + mimecontent.type) + self.log.debug('tab expansion performed.') + try: + res = converter.convert_content(context, conversion, + mimecontent) + if res: + return res, conversion.mimetype + except Exception, e: + self.log.warning('MIME conversion using %s failed (%s)' + % (converter, e), exc_info=True) + raise NoConversion('No MIME conversions succeeded', + mimecontent.type, selected_output, selected_key) + + # -- XHTML rendering and annotations (based on the conversion API) + def get_annotation_types(self): """Generator that returns all available annotation types.""" for annotator in self.annotators: yield annotator.get_annotation_type() - def render(self, req, mimetype, content, filename=None, url=None, - annotations=None): - """Render an XHTML preview of the given `content`. + def render(self, req, mimecontent, annotations=None): + """Render an XHTML preview of the given `mimecontent`. - `content` is the same as an `IHTMLPreviewRenderer.render`'s - `content` argument. - - The specified `mimetype` will be used to select the most appropriate - `IHTMLPreviewRenderer` implementation available for this MIME type. - If not given, the MIME type will be infered from the filename or the - content. - - Return a string containing the XHTML text. + Some `annotations` might be requested as well. """ - if not content: - return '' + result, _ = self._convert(ToplevelContext(req), 'text/html', + mimecontent) + if isinstance(result, Fragment): + return result # might be processed further + elif isinstance(result, basestring): + self.log.warning('HTML rendering: got %s' % + result.__class__.__name__) + return Markup(to_unicode(result)) # needed for compatibility - # Ensure we have a MIME type for this content - full_mimetype = mimetype - if not full_mimetype: - if hasattr(content, 'read'): - content = content.read(self.max_preview_size) - full_mimetype = self.get_mimetype(filename, content) - if full_mimetype: - mimetype = full_mimetype.split(';')[0].strip() # split off charset - else: - mimetype = full_mimetype = 'text/plain' # fallback if not binary + # otherwise, it's an iterable yielding lines + if annotations: + return Markup(self._annotate(result, annotations)) - # Determine candidate `IHTMLPreviewRenderer`s - candidates = [] - for renderer in self.renderers: - qr = renderer.get_quality_ratio(mimetype) - if qr > 0: - candidates.append((qr, renderer)) - candidates.sort(lambda x,y: cmp(y[0], x[0])) + return html.DIV(html.PRE(Markup(''.join(result))), class_="code") - # First candidate which renders successfully wins. - # Also, we don't want to expand tabs more than once. - expanded_content = None - for qr, renderer in candidates: - try: - self.log.debug('Trying to render HTML preview using %s' - % renderer.__class__.__name__) - # check if we need to perform a tab expansion - rendered_content = content - if getattr(renderer, 'expand_tabs', False): - if expanded_content is None: - content = content_to_unicode(self.env, content, - full_mimetype) - expanded_content = content.expandtabs(self.tab_width) - rendered_content = expanded_content - result = renderer.render(req, full_mimetype, rendered_content, - filename, url) - if not result: - continue - elif isinstance(result, Fragment): - return result - elif isinstance(result, basestring): - return Markup(to_unicode(result)) - elif annotations: - return Markup(self._annotate(result, annotations)) - else: - buf = StringIO() - buf.write('
')
-                    for line in result:
-                        buf.write(line + '\n')
-                    buf.write('
') - return Markup(buf.getvalue()) - except Exception, e: - self.log.warning('HTML preview using %s failed (%s)' - % (renderer, e), exc_info=True) - def _annotate(self, lines, annotations): + """Add requested `annotations` to the lines' content.""" buf = StringIO() buf.write('') annotators = [] @@ -445,75 +914,34 @@ buf.write('
') return buf.getvalue() + # -- Deprecated API (TODO: remove in 0.11) + def get_max_preview_size(self): """Deprecated: use `max_preview_size` attribute directly.""" return self.max_preview_size - def get_charset(self, content='', mimetype=None): - """Infer the character encoding from the `content` or the `mimetype`. - - `content` is either a `str` or an `unicode` object. - - The charset will be determined using this order: - * from the charset information present in the `mimetype` argument - * auto-detection of the charset from the `content` - * the configured `default_charset` - """ - if mimetype: - ctpos = mimetype.find('charset=') - if ctpos >= 0: - return mimetype[ctpos + 8:].strip() - if isinstance(content, str): - utf = detect_unicode(content) - if utf is not None: - return utf - return self.default_charset - - def get_mimetype(self, filename, content=None): - """Infer the MIME type from the `filename` or the `content`. - - `content` is either a `str` or an `unicode` object. - - Return the detected MIME type, augmented by the - charset information (i.e. "; charset=..."), - or `None` if detection failed. - """ - # Extend default extension to MIME type mappings with configured ones - if not self._mime_map: - self._mime_map = MIME_MAP - for mapping in self.config['mimeviewer'].getlist('mime_map'): - if ':' in mapping: - assocations = mapping.split(':') - for keyword in assocations: # Note: [0] kept on purpose - self._mime_map[keyword] = assocations[0] - - mimetype = get_mimetype(filename, content, self._mime_map) - charset = None - if mimetype: - charset = self.get_charset(content, mimetype) - if mimetype and charset and not 'charset' in mimetype: - mimetype += '; charset=' + charset - return mimetype - def to_utf8(self, content, mimetype=None): """Convert an encoded `content` to utf-8. ''Deprecated in 0.10. You should use `unicode` strings only.'' """ - return to_utf8(content, self.get_charset(content, mimetype)) + return to_utf8(content) - def to_unicode(self, content, mimetype=None, charset=None): - """Convert `content` (an encoded `str` object) to an `unicode` object. + # -- Utilities - This calls `trac.util.to_unicode` with the `charset` provided, - or the one obtained by `Mimeview.get_charset()`. - """ - if not charset: - charset = self.get_charset(content, mimetype) - return to_unicode(content, charset) + def is_binary(self, typerepr): + """Checks whether a given `TypeRepr` is binary or not.""" + return typerepr.is_binary or typerepr.mimetype in self.treat_as_binary def configured_modes_mapping(self, renderer): - """Return a MIME type to `(mode,quality)` mapping for given `option`""" + """Utility for configurable custom converters + + Return a MIME type to `(mode,quality)` mapping for given `option`, + assuming a format of comma-separated :: + associations. + + See EnscriptConverter and SilverCityConverter. + """ types, option = {}, '%s_modes' % renderer for mapping in self.config['mimeviewer'].getlist(option): if not mapping: @@ -526,37 +954,38 @@ "option." % (mapping, option)) return types - def preview_to_hdf(self, req, content, length, mimetype, filename, - url=None, annotations=None): - """Prepares a rendered preview of the given `content`. - - Note: `content` will usually be an object with a `read` method. - """ - if length >= self.max_preview_size: + def preview_to_hdf(self, req, mimecontent, annotations=None): + """Prepares a rendered preview of the given `mimecontent`.""" + if len(mimecontent) >= self.max_preview_size: return {'max_file_size_reached': True, 'max_file_size': self.max_preview_size, - 'raw_href': url} + 'raw_href': mimecontent.url} else: - return {'preview': self.render(req, mimetype, content, filename, - url, annotations), - 'raw_href': url} + try: + preview = self.render(req, mimecontent, annotations) + except NoConversion, e: + preview = None + return {'preview': preview, + 'raw_href': mimecontent.url} - def send_converted(self, req, in_type, content, selector, filename='file'): - """Helper method for converting `content` and sending it directly. + def send_converted(self, req, selector, mimecontent): + """Helper method for converting `mimecontent` and sending it directly. - `selector` can be either a key or a MIME Type.""" + `selector` can be either a key or the expected output MIME Type. + """ from trac.web import RequestDone - content, output_type, ext = self.convert_content(req, in_type, - content, selector) + result = self.convert(ToplevelContext(req), selector, mimecontent) req.send_response(200) - req.send_header('Content-Type', output_type) - req.send_header('Content-Disposition', 'filename=%s.%s' % (filename, - ext)) + req.send_header('Content-Type', result.type.mimetype_charset) + req.send_header('Content-Disposition', 'filename=%s.%s' % + (result.filename, result.type.extension)) req.end_headers() - req.write(content) - raise RequestDone - + req.write(result.encode('utf-8')) + raise RequestDone + + +# utility for Mimeview._annotate def _html_splitlines(lines): """Tracks open and close tags in lines of HTML text and yields lines that have no tags spanning more than one line.""" @@ -604,65 +1033,67 @@ def annotate_line(self, number, content): return '%s' % (number, number, number) +# return html.TH(html.A(number, href="#L%s" % number), id=number) -# -- Default renderers +# -- Default TEXT_HTML converters (previously ''IHTMLPreviewRenderer'') class PlainTextRenderer(Component): - """HTML preview renderer for plain text, and fallback for any kind of text - for which no more specific renderer is available. + """Convert text to HTML-escaped text. + + Will be used as a fallback for any kind of text + for which no more specific HTML converter is available. """ - implements(IHTMLPreviewRenderer) + implements(IContentConverter) - expand_tabs = True - - TREAT_AS_BINARY = [ - 'application/pdf', - 'application/postscript', - 'application/rtf' - ] - - def get_quality_ratio(self, mimetype): - if mimetype in self.TREAT_AS_BINARY: - return 0 - return 1 - - def render(self, req, mimetype, content, filename=None, url=None): - if is_binary(content): + def get_supported_conversions(self, input): + if Mimeview(self.env).is_binary(input): + return + yield Conversion(key='default', + quality=TEXT_PLAIN.match(input) and 8 or 1, + mimetype=TEXT_HTML, expand_tabs=True) + + def convert_content(self, context, conversion, mimecontent): + if mimecontent.is_binary: self.env.log.debug("Binary data; no preview available") - return + else: + if not TEXT_PLAIN.match(mimecontent.type): + self.env.log.debug("Fallback to plain text renderer.") + mimecontent.type = MimeType('text/plain', mimecontent.encoding) + return mimecontent - self.env.log.debug("Using default plain text mimeviewer") - content = content_to_unicode(self.env, content, mimetype) - for line in content.splitlines(): - yield escape(line) - class ImageRenderer(Component): - """Inline image display. Here we don't need the `content` at all.""" - implements(IHTMLPreviewRenderer) + """Inline image display. - def get_quality_ratio(self, mimetype): - if mimetype.startswith('image/'): - return 8 - return 0 + This renderer doesn't need the actual data at all, only the url. + """ + implements(IContentConverter) - def render(self, req, mimetype, content, filename=None, url=None): - if url: - return html.DIV(html.IMG(src=url,alt=filename), + def get_supported_conversions(self, input): + if MimeType('^image/').match(input, regexp=True): + yield Conversion(key='image', quality=8, mimetype=TEXT_HTML) + + def convert_content(self, context, conversion, mimecontent): + if mimecontent.url: + return html.DIV(html.IMG(src=mimecontent.url, + alt=mimecontent.filename), class_="image-file") class WikiTextRenderer(Component): """Render files containing Trac's own Wiki formatting markup.""" - implements(IHTMLPreviewRenderer) - def get_quality_ratio(self, mimetype): - if mimetype in ('text/x-trac-wiki', 'application/x-trac-wiki'): - return 8 - return 0 + implements(IContentConverter) - def render(self, req, mimetype, content, filename=None, url=None): + def get_supported_conversions(self, input): + from trac.wiki import TEXT_X_TRAC_WIKI, APPLICATION_X_TRAC_WIKI + if TEXT_X_TRAC_WIKI.match(input) or \ + APPLICATION_X_TRAC_WIKI.match(input): + yield Conversion(key='wiki', quality=8, mimetype=TEXT_HTML) + + def convert_content(self, context, conversion, mimecontent): from trac.wiki import wiki_to_html - return wiki_to_html(content_to_unicode(self.env, content, mimetype), - self.env, req) + return MimeContent(self.env, wiki_to_html(unicode(mimecontent), + self.env, context.req), + TEXT_HTML) Index: trac/mimeview/silvercity.py =================================================================== --- trac/mimeview/silvercity.py (revision 3507) +++ trac/mimeview/silvercity.py (working copy) @@ -24,7 +24,8 @@ from trac.core import * from trac.config import ListOption -from trac.mimeview.api import IHTMLPreviewRenderer, Mimeview +from trac.mimeview.api import IContentConverter, Mimeview, MimeContent, \ + Conversion, TEXT_HTML __all__ = ['SilverCityRenderer'] @@ -57,14 +58,14 @@ CRLF_RE = re.compile('\r$', re.MULTILINE) -class SilverCityRenderer(Component): +class SilverCityConverter(Component): """Syntax highlighting based on SilverCity.""" - implements(IHTMLPreviewRenderer) + implements(IContentConverter) - enscript_modes = ListOption('mimeviewer', 'silvercity_modes', - '', + silvercity_modes = ListOption('mimeviewer', 'silvercity_modes', doc= """List of additional MIME types known by SilverCity. + For each, a tuple `mimetype:mode:quality` has to be specified, where `mimetype` is the MIME type, `mode` is the corresponding SilverCity mode to be used @@ -79,20 +80,21 @@ def __init__(self): self._types = None - def get_quality_ratio(self, mimetype): + def get_supported_conversions(self, input): # Extend default MIME type to mode mappings with configured ones if not self._types: self._types = {} self._types.update(types) self._types.update( Mimeview(self.env).configured_modes_mapping('silvercity')) - return self._types.get(mimetype, (None, 0))[1] + quality_ratio = self._types.get(input.mimetype, (None, 0))[1] + if quality_ratio: + yield Conversion('silvercity', quality_ratio, TEXT_HTML) - def render(self, req, mimetype, content, filename=None, rev=None): + def convert_content(self, context, conversion, mimecontent): import SilverCity try: - mimetype = mimetype.split(';', 1)[0] - typelang = self._types[mimetype] + typelang = self._types[mimecontent.type.mimetype] lang = typelang[0] module = getattr(SilverCity, lang) generator = getattr(module, lang + "HTMLGenerator") @@ -108,7 +110,7 @@ raise Exception, err # SilverCity does not like unicode strings - content = content.encode('utf-8') + content = mimecontent.encode('utf-8') # SilverCity generates extra empty line against some types of # the line such as comment or #include with CRLF. So we @@ -122,10 +124,7 @@ span_default_re = re.compile(r'(.*?)', re.DOTALL) html = span_default_re.sub(r'\1', br_re.sub('', buf.getvalue())) - - # Convert the output back to a unicode string - html = html.decode('utf-8') # SilverCity generates _way_ too many non-breaking spaces... # We don't need them anyway, so replace them by normal spaces - return html.replace(' ', ' ').splitlines() + return MimeContent(self.env, html.replace(' ', ' '), TEXT_HTML) Index: trac/mimeview/patch.py =================================================================== --- trac/mimeview/patch.py (revision 3507) +++ trac/mimeview/patch.py (working copy) @@ -16,7 +16,7 @@ # Ludvig Strigeus from trac.core import * -from trac.mimeview.api import content_to_unicode, IHTMLPreviewRenderer, Mimeview +from trac.mimeview.api import IHTMLPreviewRenderer, Mimeview from trac.util.markup import escape, Markup from trac.web.chrome import add_stylesheet @@ -65,7 +65,7 @@ def render(self, req, mimetype, content, filename=None, rev=None): from trac.web.clearsilver import HDFWrapper - content = content_to_unicode(self.env, content, mimetype) + content = unicode(content) d = self._diff_to_hdf(content.splitlines(), Mimeview(self.env).tab_width) if not d: Index: trac/mimeview/enscript.py =================================================================== --- trac/mimeview/enscript.py (revision 3507) +++ trac/mimeview/enscript.py (working copy) @@ -135,6 +135,7 @@ cmdline += ' --color -h -q --language=html -p - -E%s' % mode self.env.log.debug("Enscript command line: %s" % cmdline) + content = unicode(content) np = NaivePopen(cmdline, content.encode('utf-8'), capturestderr=1) if np.errorlevel or np.err: err = 'Running (%s) failed: %s, %s.' % (cmdline, np.errorlevel, Index: trac/mimeview/php.py =================================================================== --- trac/mimeview/php.py (revision 3507) +++ trac/mimeview/php.py (working copy) @@ -20,7 +20,7 @@ from trac.core import * from trac.config import Option -from trac.mimeview.api import IHTMLPreviewRenderer, content_to_unicode +from trac.mimeview.api import IHTMLPreviewRenderer from trac.util import NaivePopen from trac.util.markup import Deuglifier @@ -79,7 +79,7 @@ cmdline += ' -sn' self.env.log.debug("PHP command line: %s" % cmdline) - content = content_to_unicode(self.env, content, mimetype) + content = unicode(content) content = content.encode('utf-8') np = NaivePopen(cmdline, content, capturestderr=1) if np.errorlevel or np.err: Index: trac/ticket/web_ui.py =================================================================== --- trac/ticket/web_ui.py (revision 3507) +++ trac/ticket/web_ui.py (working copy) @@ -23,6 +23,7 @@ from trac.config import BoolOption, Option from trac.core import * from trac.env import IEnvironmentSetupParticipant +from trac.mimeview.api import * from trac.ticket import Milestone, Ticket, TicketSystem, ITicketManipulator from trac.ticket.notification import TicketNotifyEmail from trac.Timeline import ITimelineEventProvider @@ -33,13 +34,18 @@ from trac.web import IRequestHandler from trac.web.chrome import add_link, add_stylesheet, INavigationContributor from trac.wiki import wiki_to_html, wiki_to_oneliner -from trac.mimeview.api import Mimeview, IContentConverter class InvalidTicket(TracError): """Exception raised when a ticket fails validation.""" +TEXT_CSV = MimeType('text/csv', 'utf-8', + 'Comma-delimited Text', 'csv') +TEXT_TSV = MimeType('text/tab-separated-values', 'utf-8', + 'Tab-delimited Text', 'tsv') + + class TicketModuleBase(Component): # FIXME: temporary place-holder for unified ticket validation until # ticket controller unification is merged @@ -211,22 +217,20 @@ # IContentConverter methods - def get_supported_conversions(self): - yield ('csv', 'Comma-delimited Text', 'csv', - 'trac.ticket.Ticket', 'text/csv', 8) - yield ('tab', 'Tab-delimited Text', 'tsv', - 'trac.ticket.Ticket', 'text/tab-separated-values', 8) - yield ('rss', 'RSS Feed', 'xml', - 'trac.ticket.Ticket', 'application/rss+xml', 8) + def get_supported_conversions(self, input): + if input.match(Ticket): + yield Conversion('csv', 8, TEXT_CSV) + yield Conversion('tab', 8, TEXT_TSV) + yield Conversion('rss', 8, APPLICATION_RSS_XML) - def convert_content(self, req, mimetype, ticket, key): + def convert_content(self, context, conversion, objcontent): + key = conversion.key if key == 'csv': - return self.export_csv(ticket, mimetype='text/csv') + return self.export_csv(objcontent, TEXT_CSV) elif key == 'tab': - return self.export_csv(ticket, sep='\t', - mimetype='text/tab-separated-values') + return self.export_csv(objcontent, TEXT_TSV, sep='\t') elif key == 'rss': - return self.export_rss(req, ticket) + return self.export_rss(context.req, objcontent) # INavigationContributor methods @@ -281,11 +285,11 @@ self._insert_ticket_data(req, db, ticket, get_reporter_id(req, 'author')) - mime = Mimeview(self.env) format = req.args.get('format') if format: - mime.send_converted(req, 'trac.ticket.Ticket', ticket, format, - 'ticket_%d' % ticket.id) + Mimeview(self.env).send_converted( + req, format, ObjectContent(self.env, ticket, + filename='ticket_%d' % ticket.id)) # If the ticket is being shown in the context of a query, add # links to help navigate in the query result set @@ -308,10 +312,10 @@ add_stylesheet(req, 'common/css/ticket.css') # Add registered converters - for conversion in mime.get_supported_conversions('trac.ticket.Ticket'): - conversion_href = req.href.ticket(ticket.id, format=conversion[0]) - add_link(req, 'alternate', conversion_href, conversion[1], - conversion[3]) + for conv, _ in Mimeview(self.env).get_conversions(TypeWrapper(Ticket)): + conversion_href = req.href.ticket(ticket.id, format=conv.key) + add_link(req, 'alternate', conversion_href, + conv.mimetype.name, conv.mimetype.mimetype_charset) return 'ticket.cs', None @@ -429,7 +433,8 @@ # Internal methods - def export_csv(self, ticket, sep=',', mimetype='text/plain'): + def export_csv(self, objcontent, mimetype, sep=','): + ticket = objcontent.content content = StringIO() content.write(sep.join(['id'] + [f['name'] for f in ticket.fields]) + CRLF) @@ -438,9 +443,11 @@ .replace(sep, '_').replace('\\', '\\\\') .replace('\n', '\\n').replace('\r', '\\r') for f in ticket.fields]) + CRLF) - return (content.getvalue(), '%s;charset=utf-8' % mimetype) + return MimeContent(self.env, content.getvalue(), mimetype, + filename=objcontent.filename) - def export_rss(self, req, ticket): + def export_rss(self, req, objcontent): + ticket = objcontent.content db = self.env.get_db_cnx() changes = [] change_summary = {} @@ -471,9 +478,9 @@ change['title'] = '; '.join(['%s %s' % (', '.join(v), k) for k, v \ in change_summary.iteritems()]) req.hdf['ticket.changes'] = changes - return (req.hdf.render('ticket_rss.cs'), 'application/rss+xml') + return MimeContent(self.env, req.hdf.render('ticket_rss.cs'), + APPLICATION_RSS_XML, filename=objcontent.filename) - def _do_save(self, req, db, ticket): if req.perm.has_permission('TICKET_CHGPROP'): # TICKET_CHGPROP gives permission to edit the ticket Index: trac/ticket/tests/conversion.py =================================================================== --- trac/ticket/tests/conversion.py (revision 3507) +++ trac/ticket/tests/conversion.py (working copy) @@ -2,7 +2,7 @@ from trac.util import sorted from trac.ticket.model import Ticket from trac.ticket.web_ui import TicketModule -from trac.mimeview.api import Mimeview +from trac.mimeview.api import Mimeview, Conversion from trac.web.clearsilver import HDFWrapper from trac.web.href import Href @@ -17,7 +17,8 @@ self.mimeview = Mimeview(self.env) self.req = Mock(hdf=HDFWrapper(['./templates']), base_path='/trac.cgi', path_info='', - href=Href('/trac.cgi')) + href=Href('/trac.cgi'), + abs_href=Href('http://example.org/trac.cgi')) def _create_a_ticket(self): # 1. Creating ticket @@ -29,24 +30,32 @@ return ticket def test_conversions(self): - conversions = self.mimeview.get_supported_conversions( - 'trac.ticket.Ticket') - expected = sorted([('csv', 'Comma-delimited Text', 'csv', - 'trac.ticket.Ticket', 'text/csv', 8, - self.ticket_module), - ('tab', 'Tab-delimited Text', 'tsv', - 'trac.ticket.Ticket', 'text/tab-separated-values', 8, - self.ticket_module), - ('rss', 'RSS Feed', 'xml', - 'trac.ticket.Ticket', 'application/rss+xml', 8, + conversions = self.mimeview\ + .get_supported_conversions('trac.ticket.Ticket') + expected = sorted([(Conversion('csv', 'Comma-delimited Text', 'csv', + 'trac.ticket.Ticket', 'text/csv', 8), + self.ticket_module), + (Conversion('tab', 'Tab-delimited Text', 'tsv', + 'trac.ticket.Ticket', + 'text/tab-separated-values', 8), + self.ticket_module), + (Conversion('rss', 'RSS Feed', 'xml', + 'trac.ticket.Ticket', + 'application/rss+xml', 8), self.ticket_module)], key=lambda i: i[-1], reverse=True) - self.assertEqual(expected, conversions) + for expected, actual in zip(expected, conversions): + self.assertEqual(expected[1], actual[1]) + for attr in ('key', 'name', 'extension', 'in_type', 'out_type', + 'quality', 'expand_tabs'): + self.assertEqual(getattr(expected[0], attr), + getattr(actual[0], attr)) def test_csv_conversion(self): ticket = self._create_a_ticket() - csv = self.mimeview.convert_content(self.req, 'trac.ticket.Ticket', - ticket, 'csv') + csv = self.mimeview.convert_content(self.req, + ticket, 'trac.ticket.Ticket', + 'csv') self.assertEqual((u'id,summary,reporter,owner,description,keywords,cc' '\r\nNone,Foo,santa,,Bar,,\r\n', 'text/csv;charset=utf-8', 'csv'), csv) @@ -54,8 +63,9 @@ def test_tab_conversion(self): ticket = self._create_a_ticket() - csv = self.mimeview.convert_content(self.req, 'trac.ticket.Ticket', - ticket, 'tab') + csv = self.mimeview.convert_content(self.req, + ticket, 'trac.ticket.Ticket', + 'tab') self.assertEqual((u'id\tsummary\treporter\towner\tdescription\tkeywords' '\tcc\r\nNone\tFoo\tsanta\t\tBar\t\t\r\n', 'text/tab-separated-values;charset=utf-8', 'tsv'), @@ -64,7 +74,7 @@ def test_rss_conversion(self): ticket = self._create_a_ticket() content, mimetype, ext = self.mimeview.convert_content( - self.req, 'trac.ticket.Ticket', ticket, 'rss') + self.req, ticket, 'trac.ticket.Ticket', 'rss') self.assertEqual(('\n\n\n \n ' 'Ticket \n \n ' Index: trac/versioncontrol/web_ui/util.py =================================================================== --- trac/versioncontrol/web_ui/util.py (revision 3507) +++ trac/versioncontrol/web_ui/util.py (working copy) @@ -16,10 +16,12 @@ # Author: Jonas Borgström # Christian Boos +import posixpath import re import urllib from trac.core import TracError +from trac.mimeview.api import MimeContentBase from trac.util.datefmt import format_datetime, pretty_timedelta from trac.util.text import shorten_line from trac.util.markup import escape, html, Markup @@ -27,8 +29,47 @@ from trac.wiki import wiki_to_html, wiki_to_oneliner __all__ = ['get_changes', 'get_path_links', 'get_path_rev_line', - 'get_existing_node', 'render_node_property'] + 'get_existing_node', 'render_node_property', + 'NodeMimeContent'] + +# MimeContent wrapper for repository files +# +# -- This could be part of trac.versioncontrol.api itself, +# if introducing a dependency on trac.mimeview.api is +# considered to be OK. + +CHUNK_SIZE = 4096 + +class NodeMimeContent(MimeContentBase): + """Encapsulation of the content of a file in the repository.""" + + def __init__(self, env, node, url=None): + self.node = node + MimeContentBase.__init__(self, env, node.content_type, + posixpath.basename(node.path), url) + self._excerpt = None + + def __iter__(self): + """Get chunks of the byte content""" + chunk = self.excerpt + self._excerpt = None + while True: + if not chunk: + return + yield chunk + chunk = self._content.read(CHUNK_SIZE) + + def __len__(self): + return self.node.get_content_length() + + def _get_excerpt(self): + if not self._excerpt: + self._content = self.node.get_content() + self._excerpt = self._content.read(CHUNK_SIZE) + return self._excerpt + + def get_changes(env, repos, revs, full=None, req=None, format=None): db = env.get_db_cnx() changes = {} Index: trac/versioncontrol/web_ui/changeset.py =================================================================== --- trac/versioncontrol/web_ui/changeset.py (revision 3507) +++ trac/versioncontrol/web_ui/changeset.py (working copy) @@ -26,7 +26,6 @@ from trac import util from trac.config import BoolOption, IntOption from trac.core import * -from trac.mimeview import Mimeview, is_binary from trac.perm import IPermissionRequestor from trac.Search import ISearchSource, search_to_sql, shorten_result from trac.Timeline import ITimelineEventProvider @@ -36,7 +35,7 @@ from trac.versioncontrol import Changeset, Node from trac.versioncontrol.diff import get_diff_options, hdf_diff, unified_diff from trac.versioncontrol.svn_authz import SubversionAuthorizer -from trac.versioncontrol.web_ui.util import render_node_property +from trac.versioncontrol.web_ui.util import * from trac.web import IRequestHandler from trac.web.chrome import INavigationContributor, add_link, add_stylesheet from trac.wiki import wiki_to_html, wiki_to_oneliner, IWikiSyntaxProvider, \ @@ -439,17 +438,16 @@ The list is empty when no differences between comparable files are detected, but the return value is None for non-comparable files. """ - old_content = old_node.get_content().read() - if is_binary(old_content): + old_content = NodeMimeContent(self.env, old_node) + if old_content.is_binary: return None - new_content = new_node.get_content().read() - if is_binary(new_content): + new_content = NodeMimeContent(self.env, new_content) + if new_content.is_binary: return None - mview = Mimeview(self.env) - old_content = mview.to_unicode(old_content, old_node.content_type) - new_content = mview.to_unicode(new_content, new_node.content_type) + old_content = unicode(old_content) + new_content = unicode(new_content) if old_content != new_content: context = 3 @@ -526,7 +524,6 @@ 'filename=%s.diff' % filename) req.end_headers() - mimeview = Mimeview(self.env) for old_node, new_node, kind, change in repos.get_changes(**diff): # TODO: Property changes @@ -536,23 +533,20 @@ new_content = old_content = '' new_node_info = old_node_info = ('','') - mimeview = Mimeview(self.env) if old_node: - old_content = old_node.get_content().read() - if is_binary(old_content): + old_content = NodeMimeContent(self.env, old_node) + if old_content.is_binary: continue old_node_info = (old_node.path, old_node.rev) - old_content = mimeview.to_unicode(old_content, - old_node.content_type) + old_content = unicode(old_content) if new_node: - new_content = new_node.get_content().read() - if is_binary(new_content): + new_content = NodeMimeContent(self.env, new_node) + if new_content.is_binary: continue new_node_info = (new_node.path, new_node.rev) new_path = new_node.path - new_content = mimeview.to_unicode(new_content, - new_node.content_type) + new_content = unicode(new_content) else: old_node_path = repos.normalize_path(old_node.path) diff_old_path = repos.normalize_path(diff.old_path) Index: trac/versioncontrol/web_ui/browser.py =================================================================== --- trac/versioncontrol/web_ui/browser.py (revision 3507) +++ trac/versioncontrol/web_ui/browser.py (working copy) @@ -23,7 +23,7 @@ from trac import util from trac.config import ListOption, Option from trac.core import * -from trac.mimeview import Mimeview, is_binary, get_mimetype +from trac.mimeview import Mimeview, MimeType from trac.perm import IPermissionRequestor from trac.util import sorted, embedded_numbers from trac.util.datefmt import http_date, format_datetime, pretty_timedelta @@ -36,9 +36,6 @@ from trac.versioncontrol.web_ui.util import * -CHUNK_SIZE = 4096 - - class BrowserModule(Component): implements(INavigationContributor, IPermissionRequestor, IRequestHandler, @@ -199,35 +196,31 @@ def _render_file(self, req, repos, node, rev=None): req.perm.assert_permission('FILE_VIEW') - mimeview = Mimeview(self.env) + format = req.args.get('format') - # MIME type detection - content = node.get_content() - chunk = content.read(CHUNK_SIZE) - mime_type = node.content_type - if not mime_type or mime_type == 'application/octet-stream': - mime_type = mimeview.get_mimetype(node.name, chunk) or \ - mime_type or 'text/plain' + raw_href = req.href.browser(node.path, rev=rev, format='raw') + mimecontent = NodeMimeContent(self.env, node, raw_href) + # Eventually send the file directly - format = req.args.get('format') - if format in ['raw', 'txt']: + if format in ('raw', 'txt'): + if format == 'txt': + type = MimeType('text/plain', mimecontent.encoding) + else: + type = mimecontent.type req.send_response(200) - req.send_header('Content-Type', - format == 'txt' and 'text/plain' or mime_type) - req.send_header('Content-Length', node.content_length) + req.send_header('Content-Type', type.mimetype_charset) + req.send_header('Content-Length', len(mimecontent)) req.send_header('Last-Modified', http_date(node.last_modified)) req.end_headers() - while 1: - if not chunk: - raise RequestDone + for chunk in mimecontent: req.write(chunk) - chunk = content.read(CHUNK_SIZE) + raise RequestDone else: # The changeset corresponding to the last change on `node` # is more interesting than the `rev` changeset. - changeset = repos.get_changeset(node.rev) + changeset = repos.get_changeset(node.created_rev) message = changeset.message or '--' if self.config['changeset'].getbool('wiki_format_messages'): @@ -238,7 +231,7 @@ req.hdf['file'] = { 'rev': node.rev, - 'changeset_href': req.href.changeset(node.rev), + 'changeset_href': req.href.changeset(node.created_rev), 'date': format_datetime(changeset.date), 'age': pretty_timedelta(changeset.date), 'size': pretty_size(node.content_length), @@ -246,25 +239,23 @@ 'message': message } + mimetype = mimecontent.type.mimetype + # add ''Plain Text'' alternate link if needed - if not is_binary(chunk) and mime_type != 'text/plain': + if not mimecontent.is_binary and mimetype != 'text/plain': plain_href = req.href.browser(node.path, rev=rev, format='txt') add_link(req, 'alternate', plain_href, 'Plain Text', 'text/plain') # add ''Original Format'' alternate link (always) - raw_href = req.href.browser(node.path, rev=rev, format='raw') - add_link(req, 'alternate', raw_href, 'Original Format', mime_type) + add_link(req, 'alternate', raw_href, 'Original Format', mimetype) self.log.debug("Rendering preview of node %s@%s with mime-type %s" - % (node.name, str(rev), mime_type)) + % (node.name, str(rev), mimetype)) - del content # the remainder of that content is not needed + req.hdf['file'] = Mimeview(self.env).preview_to_hdf( + req, mimecontent, annotations=['lineno']) - req.hdf['file'] = mimeview.preview_to_hdf( - req, node.get_content(), node.get_content_length(), mime_type, - node.created_path, raw_href, annotations=['lineno']) - add_stylesheet(req, 'common/css/code.css') # IWikiSyntaxProvider methods Index: trac/wiki/web_ui.py =================================================================== --- trac/wiki/web_ui.py (revision 3507) +++ trac/wiki/web_ui.py (working copy) @@ -34,8 +34,10 @@ from trac.web import HTTPNotFound, IRequestHandler from trac.wiki.api import IWikiPageManipulator, WikiSystem from trac.wiki.model import WikiPage -from trac.wiki.formatter import wiki_to_html, wiki_to_oneliner -from trac.mimeview.api import Mimeview, IContentConverter +from trac.wiki.formatter import wiki_to_html, wiki_to_oneliner, \ + TEXT_X_TRAC_WIKI, APPLICATION_X_TRAC_WIKI +from trac.mimeview.api import Mimeview, IContentConverter, Conversion, \ + MimeContent, TEXT_PLAIN class InvalidWikiPage(TracError): @@ -50,12 +52,15 @@ page_manipulators = ExtensionPoint(IWikiPageManipulator) # IContentConverter methods - def get_supported_conversions(self): - yield ('txt', 'Plain Text', 'txt', 'text/x-trac-wiki', 'text/plain', 9) - def convert_content(self, req, mimetype, content, key): - return (content, 'text/plain;charset=utf-8') + def get_supported_conversions(self, input): + if TEXT_X_TRAC_WIKI.match(input) or \ + APPLICATION_X_TRAC_WIKI.match(input): + yield Conversion('txt', quality=8, mimetype=TEXT_PLAIN) + def convert_content(self, context, conversion, content): + return content # identity transform + # INavigationContributor methods def get_active_navigation_item(self, req): @@ -128,8 +133,10 @@ else: format = req.args.get('format') if format: - Mimeview(self.env).send_converted(req, 'text/x-trac-wiki', - page.text, format, page.name) + wiki_content = MimeContent(self.env, page.text, + TEXT_X_TRAC_WIKI, + filename=page.name) + Mimeview(self.env).send_converted(req, format, wiki_content) self._render_view(req, db, page) req.hdf['wiki.action'] = action @@ -429,12 +436,13 @@ req.hdf['html.norobots'] = 1 # Add registered converters - for conversion in Mimeview(self.env).get_supported_conversions( - 'text/x-trac-wiki'): + for conv, _ in Mimeview(self.env).get_conversions(TEXT_X_TRAC_WIKI): + if conv.key in ('wiki', 'default'): + continue conversion_href = req.href.wiki(page.name, version=version, - format=conversion[0]) - add_link(req, 'alternate', conversion_href, conversion[1], - conversion[3]) + format=conv.key) + add_link(req, 'alternate', conversion_href, + conv.mimetype.name, conv.mimetype.mimetype_charset) latest_page = WikiPage(self.env, page.name) req.hdf['wiki'] = {'exists': page.exists, Index: trac/wiki/formatter.py =================================================================== --- trac/wiki/formatter.py (revision 3507) +++ trac/wiki/formatter.py (working copy) @@ -31,9 +31,17 @@ from trac.util.markup import escape, Markup, Element, html __all__ = ['wiki_to_html', 'wiki_to_oneliner', 'wiki_to_outline', - 'wiki_to_link', 'Formatter' ] + 'wiki_to_link', 'Formatter', + 'TEXT_X_TRAC_WIKI', 'APPLICATION_X_TRAC_WIKI'] +TEXT_X_TRAC_WIKI = MimeType('text/x-trac-wiki', + name='Trac Wiki Text', extension='txt') + +APPLICATION_X_TRAC_WIKI = MimeType('application/x-trac-wiki', + name='Trac Wiki Text', extension='txt') + + def system_message(msg, text=None): return html.DIV(html.STRONG(msg), text and html.PRE(text), class_="system-message") @@ -43,12 +51,13 @@ _code_block_re = re.compile('^(.*)$') - def __init__(self, env, name): - # TODO: transmit `formatter` argument - self.env = env + def __init__(self, formatter, name): + self.formatter = formatter + self.env = formatter.env self.name = name self.error = None self.macro_provider = None + self.mimetype = None builtin_processors = {'html': self._html_processor, 'default': self._default_processor, @@ -65,10 +74,9 @@ break if not self.processor: # Find a matching mimeview renderer - from trac.mimeview.api import Mimeview - mimetype = Mimeview(self.env).get_mimetype(self.name) - if mimetype: - self.name = mimetype + mimetype = Mimeview(self.env).lookup(self.name) + if not APPLICATION_OCTET_STREAM.match(mimetype): + self.mimetype = mimetype self.processor = self._mimeview_processor else: self.processor = self._default_processor @@ -94,14 +102,19 @@ # generic processors def _macro_processor(self, req, text): - # TODO: macro should take a `formatter` argument + # TODO: macro should take a `formatter` argument (0.11) self.env.log.debug('Executing Wiki macro %s by provider %s' % (self.name, self.macro_provider)) return self.macro_provider.render_macro(req, self.name, text) def _mimeview_processor(self, req, text): - # TODO: transmit context from `formatter` - return Mimeview(self.env).render(req, self.name, text) + class FormatterContext(object): + def __init__(self, req): + self.req = req + blockcontent = MimeContent(self.env, text, self.mimetype) + return Mimeview(self.env).convert(self.formatter, 'text/html', + blockcontent) + def process(self, req, text, in_paragraph=False): if self.error: @@ -429,7 +442,7 @@ return '
' args = fullmatch.group('macroargs') try: - macro = WikiProcessor(self.env, name) + macro = WikiProcessor(self, name) return macro.process(self.req, args, True) except Exception, e: self.env.log.error('Macro %s(%s) failed' % (name, args), @@ -714,7 +727,7 @@ else: self.code_text += line + os.linesep if not self.code_processor: - self.code_processor = WikiProcessor(self.env, 'default') + self.code_processor = WikiProcessor(self, 'default') elif line.strip() == Formatter.ENDBLOCK: self.in_code_block -= 1 if self.in_code_block == 0 and self.code_processor: @@ -728,10 +741,10 @@ match = Formatter._processor_re.search(line) if match: name = match.group(1) - self.code_processor = WikiProcessor(self.env, name) + self.code_processor = WikiProcessor(self, name) else: self.code_text += line + os.linesep - self.code_processor = WikiProcessor(self.env, 'default') + self.code_processor = WikiProcessor(self, 'default') else: self.code_text += line + os.linesep Index: trac/web/chrome.py =================================================================== --- trac/web/chrome.py (revision 3507) +++ trac/web/chrome.py (working copy) @@ -17,10 +17,10 @@ import os import re -from trac import mimeview from trac.config import * from trac.core import * from trac.env import IEnvironmentSetupParticipant +from trac.mimeview import get_mimetype from trac.util.markup import html from trac.web.api import IRequestHandler, HTTPNotFound from trac.web.href import Href @@ -288,7 +288,7 @@ icon = href.chrome(icon) else: icon = href.chrome('common', icon) - mimetype = mimeview.get_mimetype(icon) + mimetype = get_mimetype(icon) add_link(req, 'icon', icon, mimetype=mimetype) add_link(req, 'shortcut icon', icon, mimetype=mimetype) Index: trac/util/text.py =================================================================== --- trac/util/text.py (revision 3507) +++ trac/util/text.py (working copy) @@ -25,6 +25,7 @@ CRLF = '\r\n' +IDENTITY_CHARSET = 'iso-8859-1' # not iso-8859-15 # -- Unicode @@ -40,8 +41,8 @@ If the `lossy` argument is `True`, which is the default, then we use the 'replace' mode: - If the `lossy` argument is `False`, we fallback to the 'iso-8859-15' - charset in case of an error (encoding a `str` using 'iso-8859-15' + If the `lossy` argument is `False`, we fallback to the 'iso-8859-1' + charset in case of an error (encoding a `str` using 'iso-8859-1' will always work, as there's one Unicode character for each byte of the input). """ @@ -65,7 +66,7 @@ except UnicodeError: return unicode(text, locale.getpreferredencoding(), errors) except UnicodeError: - return unicode(text, 'iso-8859-15') + return unicode(text, IDENTITY_CHARSET) def unicode_quote(value): """A unicode aware version of urllib.quote""" @@ -85,7 +86,7 @@ return urlencode([(k, isinstance(v, unicode) and v.encode('utf-8') or v) for k, v in params]) -def to_utf8(text, charset='iso-8859-15'): +def to_utf8(text, charset=IDENTITY_CHARSET): """Convert a string to UTF-8, assuming the encoding is either UTF-8, ISO Latin-1, or as specified by the optional `charset` parameter. @@ -101,7 +102,7 @@ u = unicode(text, charset) except UnicodeError: # This should always work - u = unicode(text, 'iso-8859-15') + u = unicode(text, IDENTITY_CHARSET) return u.encode('utf-8') Index: templates/browser.cs =================================================================== --- templates/browser.cs (revision 3507) +++ templates/browser.cs (working copy) @@ -111,18 +111,8 @@ -
- HTML preview not available, since the file size exceeds - bytes. Try downloading the file instead.HTML preview not available. To view, download the file. -
Index: templates/macros.cs =================================================================== --- templates/macros.cs (revision 3507) +++ templates/macros.cs (working copy) @@ -194,4 +194,19 @@ def:plural(base, count) ?>s +
+ + HTML preview not available, since the file size exceeds + bytes. You may download the file instead. + HTML preview not available. To view the file, + download the file. +
Index: templates/attachment.cs =================================================================== --- templates/attachment.cs (revision 3507) +++ templates/attachment.cs (working copy) @@ -66,19 +66,8 @@ - -
- - HTML preview not available, since the file size exceeds - bytes. You may download the file instead. - HTML preview not available. To view the file, - download the file. -
+