Index: scripts/trac-admin =================================================================== --- scripts/trac-admin (revision 1343) +++ scripts/trac-admin (working copy) @@ -30,11 +30,13 @@ import sqlite import StringIO +import trac.Environment # has to be imported before Xref from trac import perm from trac import util from trac import sync +from trac import Xref +from trac import Href import trac.siteconfig -import trac.Environment def my_sum(list): """Python2.1 doesn't have sum()""" @@ -272,6 +274,7 @@ docs = (self._help_about + self._help_help + self._help_initenv + self._help_hotcopy + self._help_resync + self._help_upgrade + + self._help_xref + self._help_wiki + # self._help_config + self._help_wiki + self._help_permission + self._help_component + @@ -542,9 +545,13 @@ print ' Installing wiki pages' cursor = cnx.cursor() self._do_wiki_load(trac.siteconfig.__default_wiki_dir__,cursor) + print ' Cross-referencing' + self.__env.href = Href('trac-admin::initenv') + self.__env._wiki_pages = {} + Xref.rebuild_cross_references(self.__env, cnx) print ' Indexing repository' - sync.sync(cnx, rep, fs_ptr, pool) + sync.sync(self.__env, cnx, rep, fs_ptr, pool) except Exception, e: print 'Failed to initialize database.', e sys.exit(2) @@ -592,6 +599,8 @@ pool = core.svn_pool_create(None) self.db_open() # We need to call this function to open the env, really stupid + self.__env.href = Href.Href('trac-admin::resync') # We need this for the WikiFormatter + self.__env._wiki_pages = {} # Remove any trailing slash or else subversion might abort repository_dir = self.__env.get_config('trac', 'repository_dir') @@ -605,9 +614,21 @@ print 'resyncing...' self.db_execsql("DELETE FROM revision") self.db_execsql("DELETE FROM node_change") - sync.sync(cnx, rep, fs_ptr, pool) + sync.sync(self.__env, cnx, rep, fs_ptr, pool) print 'done.' + _help_xref = [('xref', 'Regenerate all the cross-reference information between trac objects')] + + ## XRef + def do_xref(self, line): + self.db_open() # We need to call this function to open the env, really stupid + cnx = self.__env.get_db_cnx() + print 'cross-referencing... (except changesets: use \'resync\' for that)' + self.__env.href = Href.Href('trac-admin::xref') + self.__env._wiki_pages = {} + Xref.rebuild_cross_references(self.__env, cnx, False) + print 'done.' + ## Wiki _help_wiki = [('wiki list', 'List wiki pages'), ('wiki remove ', 'Remove wiki page'), Index: trac/core.py =================================================================== --- trac/core.py (revision 1343) +++ trac/core.py (working copy) @@ -40,6 +40,10 @@ 'search' : ('Search', 'Search', 0), 'report' : ('Report', 'Report', 0), 'ticket' : ('Ticket', 'TicketModule', 0), + 'bug' : ('Ticket', 'TicketModule', 0), + 'issue' : ('Ticket', 'TicketModule', 0), + 'source' : ('Browser', 'Browser', 1), + 'repos' : ('Browser', 'Browser', 1), 'browser' : ('Browser', 'Browser', 1), 'timeline' : ('Timeline', 'Timeline', 1), 'changeset' : ('Changeset', 'Changeset', 1), @@ -48,6 +52,8 @@ 'attachment' : ('File', 'Attachment', 0), 'roadmap' : ('Roadmap', 'Roadmap', 0), 'settings' : ('Settings', 'Settings', 0), + 'xref' : ('Xref', 'XrefModule', 0), + 'orphans' : ('Xref', 'XrefModule', 0), 'milestone' : ('Milestone', 'Milestone', 0) } @@ -74,7 +80,7 @@ pool, rep, fs_ptr = open_svn_repos(repos_dir) module.repos = rep module.fs_ptr = fs_ptr - sync.sync(db, rep, fs_ptr, pool) + sync.sync(env, db, rep, fs_ptr, pool) module.pool = pool return module Index: trac/db_default.py =================================================================== --- trac/db_default.py (revision 1343) +++ trac/db_default.py (working copy) @@ -21,7 +21,7 @@ # Database version identifier. Used for automatic upgrades. -db_version = 9 +db_version = 10 def __mkreports(reports): """Utility function used to create report data in same syntax as the @@ -159,10 +159,23 @@ UNIQUE(sid,var_name) ); +CREATE TABLE xref ( + src_type text, + src_id text, + relation text, + dest_type text, + dest_id text, + facet text, + context text +); + + CREATE INDEX node_change_idx ON node_change(rev); CREATE INDEX ticket_change_idx ON ticket_change(ticket, time); CREATE INDEX wiki_idx ON wiki(name,version); CREATE INDEX session_idx ON session(sid,var_name); +CREATE INDEX xref_src_idx ON xref(src_id,src_type); +CREATE INDEX xref_dest_idx ON xref(dest_id,dest_type); """ ## Index: trac/Milestone.py =================================================================== --- trac/Milestone.py (revision 1343) +++ trac/Milestone.py (working copy) @@ -24,6 +24,7 @@ from trac.Ticket import get_custom_fields, Ticket from trac.WikiFormatter import wiki_to_html from trac.util import * +from trac.Xref import TracRef import time @@ -159,6 +160,7 @@ cursor.execute("INSERT INTO milestone (name,due,completed,description) " "VALUES (%s,%s,%s,%s)", (name, due, completed, description)) + TracRef('milestone', name).replace_xrefs_from_wiki(self.env, self.db, 'description', description) self.db.commit() req.redirect(self.env.href.milestone(name)) @@ -182,6 +184,7 @@ "WHERE milestone=%s", (id,)) self.log.info('Deleting milestone %s' % id) cursor.execute("DELETE FROM milestone WHERE name=%s", (id,)) + TracRef('milestone', id).delete_xrefs(self.db, 'description') self.db.commit() req.redirect(self.env.href.roadmap()) else: @@ -266,6 +269,8 @@ action = req.args.get('action', 'view') id = req.args.get('id') + TracRef('milestone', id).add_backlinks(self.db, req) + if action == 'new': self.perm.assert_permission(perm.MILESTONE_CREATE) self.render_editor(req) Index: trac/tests/href.py =================================================================== --- trac/tests/href.py (revision 1343) +++ trac/tests/href.py (working copy) @@ -94,6 +94,16 @@ self.assertEqual('/attachment/ticket/42/foo.txt?format=raw', self.href.attachment('ticket', '42', 'foo.txt', 'raw')) + def test_xref(self): + """Testing Href.xref""" + self.assertEqual('/xref', self.href.xref()) + self.assertEqual('/xref/wiki', self.href.xref('wiki')) + self.assertEqual('/xref/wiki/WikiStart', self.href.xref('wiki', 'WikiStart')) + + def test_orphans(self): + """Testing Href.orphans""" + self.assertEqual('/orphans', self.href.orphans()) + def suite(): return unittest.makeSuite(HrefTestCase,'test') Index: trac/Report.py =================================================================== --- trac/Report.py (revision 1343) +++ trac/Report.py (working copy) @@ -22,6 +22,7 @@ from trac import perm, util from trac.Module import Module from trac.WikiFormatter import wiki_to_html +from trac.Xref import TracRef import re import time @@ -113,6 +114,7 @@ cursor.execute("INSERT INTO report (title,sql,description) " "VALUES (%s,%s,%s)", (title, sql, description)) id = self.db.get_last_id() + TracRef('report', id).replace_xrefs_from_wiki(self.env, self.db, 'description', description) self.db.commit() req.redirect(self.env.href.report(id)) @@ -122,6 +124,7 @@ if not req.args.has_key('cancel'): cursor = self.db.cursor () cursor.execute("DELETE FROM report WHERE id=%s", (id,)) + TracRef('report', id).delete_xrefs(self.db, 'description') self.db.commit() req.redirect(self.env.href.report()) else: @@ -401,6 +404,9 @@ req.args.get('description', ''), req.args.get('sql', '')) + if id != -1: + TracRef('report', id).add_backlinks(self.db, req) + if id != -1 or action == 'new': self.add_link('up', self.env.href.report(), 'Available Reports') Index: trac/sync.py =================================================================== --- trac/sync.py (revision 1343) +++ trac/sync.py (working copy) @@ -20,11 +20,12 @@ # Author: Jonas Borgström from svn import fs, util, delta, repos, core +from trac.Xref import TracRef import posixpath -def sync(db, repos, fs_ptr, pool): +def sync(env, db, repos, fs_ptr, pool): """ Update the revision and node_change tables to be in sync with the repository. @@ -56,6 +57,7 @@ cursor.execute ('INSERT INTO revision (rev, time, author, message) ' 'VALUES (%s, %s, %s, %s)', rev + offset, date, author, message) + TracRef('changeset', rev + offset).replace_xrefs_from_wiki(env, db, 'content', message) insert_change (subpool, fs_ptr, rev + offset, cursor) core.svn_pool_clear(subpool) @@ -125,7 +127,7 @@ old_path = posixpath.join(old_path, posixpath.split(path)[1]) action = 'A' else: - self._save_change(core.svn_node_file, 'A', path) + self._save_change(core.svn_node_dir, 'A', path) action = None if action: Index: trac/File.py =================================================================== --- trac/File.py (revision 1343) +++ trac/File.py (working copy) @@ -25,6 +25,7 @@ from trac import perm, util from trac.Module import Module from trac.WikiFormatter import wiki_to_html +from trac.Xref import TracRef import svn.core import svn.fs @@ -261,6 +262,9 @@ self.rev = req.args.get('rev', None) self.path = req.args.get('path', '/') + + TracRef('source', self.path).add_backlinks(self.db, req) + if not self.rev: rev_specified = 0 self.rev = svn.fs.youngest_rev(self.fs_ptr, self.pool) Index: trac/Log.py =================================================================== --- trac/Log.py (revision 1343) +++ trac/Log.py (working copy) @@ -22,6 +22,7 @@ from trac import perm, util from trac.Module import Module from trac.WikiFormatter import wiki_to_oneliner +from trac.Xref import TracRef import svn.core import svn.fs @@ -121,6 +122,9 @@ self.path = req.args.get('path', '/') self.authzperm.assert_permission(self.path) + + TracRef('source', self.path).add_backlinks(self.db, req) + if req.args.has_key('rev'): try: rev = int(req.args.get('rev')) Index: trac/upgrades/db10.py =================================================================== --- trac/upgrades/db10.py (revision 0) +++ trac/upgrades/db10.py (revision 0) @@ -0,0 +1,53 @@ +import time + +sql = """ +-- Initial creation of the general cross-reference table +CREATE TABLE xref ( + src_type text, + src_id text, + relation text, + dest_type text, + dest_id text, + facet text, + context text +); + +CREATE INDEX xref_src_idx ON xref(src_id,src_type); +CREATE INDEX xref_dest_idx ON xref(dest_id,dest_type); +""" + +def do_upgrade(env, ver, cursor): + cursor.execute(sql) + +def do_db_upgrade(env, ver, db): + """Renumbering of ticket comments (using the spare 'oldvalue' field)""" + cursor = db.cursor() + update_cursor = db.cursor() + cursor.execute("SELECT ticket, time, author FROM ticket_change " + "WHERE field = 'comment' " + "ORDER BY ticket, time, author ") + previous_ticket = None + for ticket, time, author in cursor: + if ticket != previous_ticket: + previous_ticket = ticket + n = 1 + update_cursor.execute("UPDATE ticket_change SET " + "oldvalue = %s ", + (n)) + n += 1 + + + + + + + + + + + + + + + + Index: trac/Href.py =================================================================== --- trac/Href.py (revision 1343) +++ trac/Href.py (working copy) @@ -56,6 +56,9 @@ else: return href_join(self.base, 'browser', path) + def source(self, path, rev=None): + return self.browser(path, rev) + def login(self): return href_join(self.base, 'login') @@ -176,3 +179,18 @@ if format: href += '?format=%s' % format return href + + def xref(self, module=None, id=None): + if module and id: + href = href_join(self.base, 'xref', str(module), str(id)) + else: + href = href_join(self.base, 'xref') + return href + + def orphans(self, module=None): + if module: + href = href_join(self.base, 'orphans', str(module)) + else: + href = href_join(self.base, 'orphans') + return href + Index: trac/web/main.py =================================================================== --- trac/web/main.py (revision 1343) +++ trac/web/main.py (working copy) @@ -153,13 +153,13 @@ if match: set_if_missing(args, 'mode', match.group(1)) return args - match = re.search('^/(ticket|report)(?:/([0-9]+)/*)?', path_info) + match = re.search('^/(ticket|bug|issue|report)(?:/([0-9]+)/*)?', path_info) if match: set_if_missing(args, 'mode', match.group(1)) if match.group(2): set_if_missing(args, 'id', match.group(2)) return args - match = re.search('^/(browser|log|file)(?:(/.*))?', path_info) + match = re.search('^/(browser|source|repos|log|file)(?:(/.*))?', path_info) if match: set_if_missing(args, 'mode', match.group(1)) if match.group(2): @@ -183,6 +183,14 @@ if match.group(1): set_if_missing(args, 'id', urllib.unquote(match.group(1))) return args + match = re.search('^/(xref|orphans)(?:/([^/]+))?(?:/(.*)/?)?', path_info) + if match: + set_if_missing(args, 'mode', match.group(1)) + set_if_missing(args, 'type', match.group(2)) + id = match.group(3) + if id: + set_if_missing(args, 'id', urllib.unquote(id)) + return args return args def populate_hdf(hdf, env, req=None): @@ -208,6 +216,8 @@ 'login': env.href.login(), 'logout': env.href.logout(), 'settings': env.href.settings(), + 'xref': env.href.xref(), + 'orphans': env.href.orphans(), 'homepage': 'http://trac.edgewall.com/' } Index: trac/Changeset.py =================================================================== --- trac/Changeset.py (revision 1343) +++ trac/Changeset.py (working copy) @@ -24,6 +24,7 @@ from trac.Diff import get_diff_options, hdf_diff, unified_diff from trac.Module import Module from trac.WikiFormatter import wiki_to_html +from trac.Xref import TracRef from trac import authzperm, perm import svn.core @@ -435,6 +436,8 @@ else: self.rev = youngest_rev + TracRef('changeset', self.rev).add_backlinks(self.db, req) + self.diff_options = get_diff_options(req) if req.args.has_key('update'): req.redirect(self.env.href.changeset(self.rev)) @@ -462,7 +465,7 @@ req.hdf['changeset.revision'] = self.rev req.hdf['changeset.changes'] = change_info req.hdf['changeset.href'] = self.env.href.changeset(self.rev) - + if len(change_info) == 0: raise authzperm.AuthzPermissionError() Index: trac/Wiki.py =================================================================== --- trac/Wiki.py (revision 1343) +++ trac/Wiki.py (working copy) @@ -24,6 +24,7 @@ from trac.Module import Module from trac.util import escape, TracError, get_reporter_id from trac.WikiFormatter import * +from trac.Xref import TracRef import os import time @@ -51,8 +52,9 @@ Represents a wiki page (new or existing). """ - def __init__(self, name, version, perm_, db): - self.db = db + def __init__(self, name, version, perm_, env, db): + self.env = env + self.db = db self.name = name self.perm = perm_ cursor = self.db.cursor () @@ -108,6 +110,7 @@ "%s,%s)", (self.name, self.version + 1, int(time.time()), author, remote_addr, self.text, comment, self.readonly)) + TracRef('wiki', self.name).replace_xrefs_from_wiki(self.env, self.db, 'content', self.text) self.db.commit() self.version += 1 self.old_readonly = self.readonly @@ -126,6 +129,9 @@ req.hdf['wiki.page_name'] = escape(pagename) req.hdf['wiki.current_href'] = escape(self.env.href.wiki(pagename)) + self.ref = TracRef('wiki', pagename) + self.ref.add_backlinks(self.db, req) + if action == 'diff': version = int(req.args.get('version', 0)) self._render_diff(req, pagename, version) @@ -136,7 +142,7 @@ elif action == 'delete': version = None if req.args.has_key('delete_version'): - version = int(req.args['version']) + version = int(req.args.get('version')) self._delete_page(req, pagename, version) elif action == 'save': if req.args.has_key('cancel'): @@ -164,10 +170,17 @@ cursor = self.db.cursor() cursor.execute("DELETE FROM wiki WHERE name=%s and version=%s", (pagename, version)) + self.ref.delete_xrefs(self.db, 'comment:%s' % version) self.log.info('Deleted version %d of page %s' % (version, pagename)) cursor.execute("SELECT COUNT(*) FROM wiki WHERE name=%s", (pagename,)) - if not cursor.fetchone(): + last_version = cursor.fetchone()[0] + if last_version == 0: page_deleted = 1 + elif version > last_version: # resurrect the previous 'content' + cursor.execute("SELECT text FROM wiki WHERE name=%s ORDER BY version DESC", + (pagename)) + text = cursor.fetchone()[0] + self.ref.replace_xrefs_from_wiki(self.env, self.db, 'content', text) else: # Delete a wiki page completely cursor.execute("DELETE FROM wiki WHERE name=%s", (pagename,)) page_deleted = 1 @@ -175,6 +188,7 @@ self.db.commit() if page_deleted: + self.ref.delete_xrefs(self.db) # Delete orphaned attachments for attachment in self.env.get_attachments(self.db, 'wiki', pagename): self.env.delete_attachment(self.db, 'wiki', pagename, @@ -233,7 +247,7 @@ def _render_editor(self, req, pagename, preview=0): self.perm.assert_permission(perm.WIKI_MODIFY) - page = WikiPage(pagename, None, self.perm, self.db) + page = WikiPage(pagename, None, self.perm, self.env, self.db) if req.args.has_key('text'): page.set_content(req.args.get('text')) if preview: @@ -311,7 +325,7 @@ self.add_link('alternate', '?format=txt', 'Plain Text', 'text/plain') - page = WikiPage(pagename, version, self.perm, self.db) + page = WikiPage(pagename, version, self.perm, self.env, self.db) info = { 'version': page.version, @@ -331,7 +345,7 @@ def _save_page(self, req, pagename): self.perm.assert_permission(perm.WIKI_MODIFY) - page = WikiPage(pagename, None, self.perm, self.db) + page = WikiPage(pagename, None, self.perm, self.env, self.db) if req.args.has_key('text'): page.set_content(req.args.get('text')) Index: trac/Environment.py =================================================================== --- trac/Environment.py (revision 1343) +++ trac/Environment.py (working copy) @@ -23,7 +23,9 @@ # from trac import db, db_default, Logging, Mimeview, util +from trac.Xref import TracRef + import ConfigParser import os import shutil @@ -253,6 +255,7 @@ cursor.execute('INSERT INTO attachment VALUES(%s,%s,%s,%s,%s,%s,%s,%s)', (type, id, filename, length, int(time.time()), description, author, ipnr)) + TracRef(type, id).replace_xrefs_from_wiki(self, cnx, 'attachment:%s' % filename, description) shutil.copyfileobj(attachment.file, fd) self.log.info('New attachment: %s/%s/%s by %s', type, id, filename, author) cnx.commit() @@ -265,6 +268,7 @@ cursor = cnx.cursor() cursor.execute('DELETE FROM attachment WHERE type=%s AND id=%s AND ' 'filename=%s', (type, id, filename)) + TracRef(type, id).delete_xrefs(cnx, 'attachment:%s' % filename) os.unlink(path) self.log.info('Attachment removed: %s/%s/%s', type, id, filename) cnx.commit() @@ -324,6 +328,8 @@ err = 'No upgrade module for version %i (%s.py)' % (i, upg) raise EnvironmentError, err d.do_upgrade(self, i, cursor) + if hasattr(d, 'do_db_upgrade'): # for more complex upgrades, as in 'db9.py' + d.do_db_upgrade(self, i, cnx) cursor.execute("UPDATE system SET value=%s WHERE " "name='database_version'", (db_default.db_version)) self.log.info('Upgraded db version from %d to %d', Index: trac/Xref.py =================================================================== --- trac/Xref.py (revision 0) +++ trac/Xref.py (revision 0) @@ -0,0 +1,407 @@ +# -*- coding: iso8859-1 -*- +# +# Copyright (C) 2005 Edgewall Software +# Copyright (C) 2005 Christian Boos +# +# Trac is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# Trac is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +# +# Author: Christian Boos + +""" + + This module implements a general cross-reference facility for Trac, + as suggested in #1242. + + Currently, cross-references between the following Trac objects are supported: + * Wiki page + * Ticket + * Changeset + * Report + * Milestone + * Source (only as targets for now) + + Basically, two kinds of references are supported: + * ''implicit references between objects'' + Implicit references are created for every TracLinks that can be found + in (any of) the wiki text of a Trac object. + Indeed, some objects may have separately editable wiki texts, + each of them being a ''facet'' of this object. + (TODO: generic fine grained anchoring: # should go to the facet) + * ''explicit relation between objects'' + An explicit relation must be created explicitely as such, + by some programmatic mean. + Currently, some of the ticket fields are setting up explicit relationships. + + Note that implicit references always have an empty 'relation', + whereas explicit references may use the 'facet' for informative purpose. + +""" + + +from trac.Module import Module +from trac.util import escape +from trac.WikiFormatter import XRefFormatter + +__all__ = ['TracRef', 'rebuild_cross_references'] + + +how_much_context = 40 + + +class TracRef: + """ + A TracRef encapsulate the identity of a Trac Object (changeset, + wiki page, ticket, ...) and can be used to manage the relationships + to other Trac Objects. + + The cross-reference information is stored in the XREF table + (see trac/db_default.py). + + Besides the obvious 'type' and 'id' information for the source and the + destination objects, there is also: + * facet: the location of the cross-reference within the source object. + * context: the wikitext surrounding the cross-reference, if any + * relation: the explicit nature of the relationship. + """ + + def __init__(self, type, id): + self.type = type + self.id = id + if type == 'source': # TODO: oo-ify + self.id = id.strip('/') + + def name(self): + if self.type == 'wiki': # TODO: oo-ify + # TODO: use canonical name if it doesn't follow the WikiPageNames conventions + return escape(self.id) + elif self.type == 'ticket': + return 'Ticket #%s' % self.id + elif self.type == 'changeset': + return 'Changeset [%s]' % self.id + elif self.type == 'report': + return 'Report {%s}' % self.id + else: + return self.type + ':' + escape(self.id) + + def icon(self): + if self.type == 'ticket': # TODO: oo-ify + return 'newticket' + else: + return self.type + + def href(self, env): + m = getattr(env.href, self.type) + if m: + return m(self.id) + else: + return env.href.wiki() + + # -- used by other Modules + + def add_backlinks(self, db, req): + req.hdf['xref_count'] = self.count_sources(db) + + def replace_relation(self, db, relation, dest, facet='', context=''): + """ + Replace the related object for the given 'relation'. + The 'facet' and 'context' are only used to recreate the new relation. + Best suited for 1 to 1 relationships. + """ + print "- %s:%s --[%s]--> *:*" % (self.type, self.id, relation) + cursor = db.cursor() + cursor.execute("DELETE FROM xref " + "WHERE src_id = %s AND src_type = %s " + "AND relation = %s ", + (self.id, self.type, relation)) + self.insert_xref(db, relation, dest, facet, context) + + def replace_xrefs_from_wiki(self, env, db, facet, wikitext): + """ + Remove then re-create the cross-references for the given facet. + """ + self.delete_xrefs(db, facet) + self._create_xrefs_from_wiki(env, db, facet, wikitext) + + + def _create_xrefs_from_wiki(self, env, db, facet, wikitext): + """ + Parse the given 'wikitext' in order to generate all the cross-reference. + It is assumed that self is the source object for these references. + """ + XRefFormatter(env, db, False).format(wikitext, self, facet) + + def delete_xrefs(self, db, facet=None): + """ + Remove all the cross-references having this reference as a source. + Only delete the cross-references for the 'facet', if given. + """ + cursor = db.cursor() + if facet: + facet_clause = "AND facet = %s" + tuple = (self.id, self.type, facet) + print "- %s:%s --[*]--> *:* (in %s)" % (self.type, self.id, facet) + else: + facet_clause = "" + tuple = (self.id, self.type) + print "- %s:%s --[*]--> *:*" % (self.type, self.id) + cursor.execute("DELETE FROM xref " + "WHERE src_id = %s AND src_type = %s " + facet_clause, + tuple) + + + def count_sources(self, db): + return self._count(db, 'dest') + + def count_destinations(self, db): + return self._count(db, 'src') + + def _count(self, db, base): + cursor = db.cursor() + cursor.execute("SELECT count(*) FROM xref " + "WHERE _id = %s AND _type = %s ".replace('', base), + (self.id, self.type)) + return cursor.fetchone()[0] + + + def find_sources(self, db, relation=None): + """ + Retrieve all the incoming relationships for this object. + If 'relation' is given, only the sources for the given relation + are retrieved, otherwise all relations are searched, even implicit ones. + """ + return self._find(db, 'dest', 'src', relation) + + def find_destinations(self, db, relation=None): + """ + Retrieve all the outgoing relationships for this object. + If 'relation' is given, only the targets for the given relation + are retrieved, otherwise all relations are searched, even implicit ones. + """ + return self._find(db, 'src', 'dest', relation) + + def _find(self, db, base, other, relation): + cursor = db.cursor() + if relation: + relation_clause = "AND relation = %s" + tuple = (self.id, self.type, relation) + else: + relation_clause = "" + tuple = (self.id, self.type) + cursor.execute(("SELECT _type, _id, relation, facet, context " + "FROM xref WHERE _id = %s AND _type = %s " + ).replace('', base).replace('', other) + relation_clause, + tuple) + return cursor + + # -- used by the WikiFormatter.XRefFormatter: + + def insert_xref(self, db, relation, dest, facet, context): + print "+ %s:%s --[%s]--> %s:%s (in %s %s)" % (self.type, self.id, + relation, + dest.type, dest.id, + facet, context) + cursor = db.cursor() + cursor.execute("INSERT INTO xref VALUES (%s,%s,%s,%s,%s,%s,%s)", + (self.type, self.id, relation, dest.type, dest.id, facet, context)) + + def extract_context(self, text, start, end): + start_ellipsis = end_ellipsis = '...' + start = start - how_much_context + if start < 0: + start = 0 + start_ellipsis = '' + end = end + how_much_context + if end > len(text): + end = len(text) + end_ellipsis = '' + return start_ellipsis + text[start:end] + end_ellipsis + + + + + +def rebuild_cross_references(env, db, do_changesets=True): + """ + Rebuild all cross-references in the given environment. + + As an option, the rebuilding of the references found in + changesets can be skipped, as this is done by a 'resync' + operation, which is what is advised to do in trac-admin. + """ + cursor = db.cursor() + xreffmt = XRefFormatter(env, db, False) + + def add_xrefs(src, facet, text): + xreffmt.format(text, src, facet) + + # -- wiki objects + cursor.execute("SELECT name, text, comment, version FROM wiki " + "ORDER BY name, version DESC") + previous_page = None + src = None + for name, text, comment, version in cursor: + if name != previous_page: + previous_page = name + src = TracRef('wiki', name) + src.delete_xrefs(db) + add_xrefs(src, 'content', text) + add_xrefs(src, 'comment:%d' % version, comment) + + # -- ticket objects + # -- -- description + cursor.execute("SELECT id, description FROM ticket") + for id, description in cursor: + src = TracRef('ticket', id) + src.delete_xrefs(db) + add_xrefs(src, 'description', description) + # -- -- comments + comment_cursor = db.cursor() + comment_cursor.execute("SELECT oldvalue, newvalue FROM ticket_change " + "WHERE ticket = %s AND field = 'comment'", + (id)) + for n, value in comment_cursor: + add_xrefs(src, 'comment:%s' % n, value) + # -- -- custom fields + # (uncomment when the custom fields will support wiki formatting) + # ... "SELECT name, value FROM ticket_custom" + + # -- changeset objects + if do_changesets: + cursor.execute("SELECT rev, message FROM revision") + for rev, message in cursor: + src = TracRef('changeset', rev) + src.delete_xrefs(db) + add_xrefs(src, 'content', message) + + # -- report objects + cursor.execute("SELECT id, description FROM report") + for id, description in cursor: + src = TracRef('report', id) + src.delete_xrefs(db) + add_xrefs(src, 'description', description) + + # -- milestone objects + cursor.execute("SELECT name, description FROM milestone") + for name, description in cursor: + src = TracRef('milestone', name) + src.delete_xrefs(db) + add_xrefs(src, 'description', description) + + # -- attachment object + cursor.execute("SELECT type, id, description, filename FROM attachment") + for type, id, description, filename in cursor: + add_xrefs(TracRef(type, id), 'attachment:%s' % filename, description) + + db.commit() + + + + +def find_orphaned_objects(db): + # Most interesting 'orphans' order: wikis, tickets, milestones then reports and changesets + queries = [ + "SELECT DISTINCT(name), 'wiki' FROM wiki " + "WHERE name NOT IN (SELECT DISTINCT(dest_id) FROM xref WHERE dest_type = 'wiki') " + , + "SELECT id, 'ticket' FROM ticket " + "WHERE id NOT IN (SELECT DISTINCT(dest_id) FROM xref WHERE dest_type = 'ticket') " + , + "SELECT name, 'milestone' FROM milestone " + "WHERE name NOT IN (SELECT DISTINCT(dest_id) FROM xref WHERE dest_type = 'milestone') " + , + "SELECT id, 'report' FROM report " + "WHERE id NOT IN (SELECT DISTINCT(dest_id) FROM xref WHERE dest_type = 'report') " + , + "SELECT rev, 'changeset' FROM revision " + "WHERE rev NOT IN (SELECT DISTINCT(dest_id) FROM xref WHERE dest_type = 'changeset') " + , + # Not sure about this one: it works, but produces a huge list (good for testing, though :) + # "SELECT DISTINCT(name), 'source' FROM node_change " + # "WHERE name NOT IN (SELECT DISTINCT(dest_id) FROM xref WHERE dest_type = 'source') " + ] + cursor = db.cursor() + cursor.execute(" UNION ALL ".join(queries)) + return cursor + + + + +class XrefModule(Module): + template_name = 'xref.cs' + + def render(self, req): + mode = req.args.get('mode', 'xref') + req.hdf['xref.mode'] = mode + if mode == 'orphans': + self._orphans(req) + self.template_name = 'orphans.cs' + else: + direction = req.args.get('direction','back') + if direction == 'forward': + req.hdf['xref.direction.name'] = 'Forward Link' + base = self._base(req) + self._references(req, base.find_destinations(self.db)) + else: # direction == back + req.hdf['xref.direction.back'] = 1 + req.hdf['xref.direction.name'] = 'Backlink' + base = self._base(req) + self._references(req, base.find_sources(self.db)) + + def _base(self, req): + type = req.args.get('type', 'wiki') + id = req.args.get('id', 'WikiStart') + base = TracRef(type, id) + req.hdf['title'] = req.hdf['xref.direction.name'] + ' for ' + base.name() + req.hdf['xref.base.type'] = type + req.hdf['xref.base.id'] = escape(id) + req.hdf['xref.base.name'] = base.name() + req.hdf['xref.base.icon'] = base.icon() + req.hdf['xref.base.href'] = base.href(self.env) + req.hdf['xref.current_href'] = escape(self.env.href.xref(type, id)) + return base + + def _references(self, req, refs): + links = [] + relations = [] + for type, id, relation, facet, context in refs: + other_ref = TracRef(type, id) + dict = {'type' : type, + 'id' : id, + 'name' : other_ref.name(), + 'icon' : other_ref.icon(), + 'href' : other_ref.href(self.env), + 'relation' : relation, + 'facet' : facet, + 'context' : context} + if relation: + relations.append(dict) + else: + links.append(dict) + req.hdf['xref.links'] = links + req.hdf['xref.relations'] = relations + + def _orphans(self, req): + req.hdf['title'] = 'Orphaned objects' + orphans = [] + for id, type in find_orphaned_objects(self.db): + ref = TracRef(type, id) + obj = {'type' : type, + 'id' : id, + 'name' : ref.name(), + 'icon' : ref.icon(), + 'href' : ref.href(self.env)} + orphans.append(obj) + req.hdf['orphans'] = orphans + Index: trac/Ticket.py =================================================================== --- trac/Ticket.py (revision 1343) +++ trac/Ticket.py (working copy) @@ -23,6 +23,7 @@ from trac.Module import Module from trac.WikiFormatter import wiki_to_html from trac.Notify import TicketNotifyEmail +from trac.Xref import TracRef import time import string @@ -37,10 +38,15 @@ 'reporter', 'owner', 'cc', 'url', 'version', 'status', 'resolution', 'keywords', 'summary', 'description', 'changetime'] + field_xrefs = { 'description' : ('implicit',), + 'summary' : ('implicit',), + 'milestone' : ('1 to 1', 'milestone'), + 'component' : ('1 to 1', 'wiki') } def __init__(self, *args): UserDict.__init__(self) self._old = {} + self.ref = TracRef('ticket', -1) if len(args) == 2: self._fetch_ticket(*args) @@ -65,7 +71,7 @@ raise util.TracError('Ticket %d does not exist.' % id, 'Invalid Ticket Number') - self['id'] = id + self['id'] = self.ref.id = id for i in range(len(Ticket.std_fields)): self[Ticket.std_fields[i]] = row[i] or '' @@ -91,9 +97,10 @@ if not dict.has_key(name): self[name] = '0' - def insert(self, db): + def insert(self, env, db): """Add ticket to database""" assert not self.has_key('id') + assert self.ref.id == -1 # Add a timestamp now = int(time.time()) @@ -107,18 +114,22 @@ % (','.join(std_fields), ','.join(['%s'] * len(std_fields))), map(lambda n, self=self: self[n], std_fields)) - id = db.get_last_id() + self['id'] = self.ref.id = db.get_last_id() + for name in Ticket.field_xrefs.keys(): + self.xref_field(env, db, name, self[name], + time.strftime('%c', time.localtime(now))) + custom_fields = filter(lambda n: n[:7] == 'custom_', self.keys()) for name in custom_fields: cursor.execute("INSERT INTO ticket_custom(ticket,name,value) " - "VALUES(%s,%s,%s)", (id, name[7:], self[name])) + "VALUES(%s,%s,%s)", (self.ref.id, name[7:], self[name])) + # TODO: support xrefs in custom fields too... db.commit() - self['id'] = id self._forget_changes() - return id + return self.ref.id - def save_changes(self, db, author, comment, when = 0): + def save_changes(self, env, db, author, comment, when = 0): """Store ticket changes in the database. The ticket must already exist in the database.""" assert self.has_key('id') @@ -161,16 +172,23 @@ fname = name cursor.execute("UPDATE ticket SET %s=%s WHERE id=%s", (fname, self[name], id)) + self.xref_field(env, db, fname, self[name], + time.strftime('%c', time.localtime(when))) cursor.execute("INSERT INTO ticket_change " "(ticket,time,author,field,oldvalue,newvalue) " "VALUES (%s, %s, %s, %s, %s, %s)", (id, when, author, fname, self._old[name], self[name])) if comment: + cursor.execute("SELECT count(*) FROM ticket_change " + "WHERE ticket = %s", # AND oldvalue LIKE '%%%s.' parent (threads) + (id)) + n = cursor.fetchone()[0] + 1 cursor.execute("INSERT INTO ticket_change " "(ticket,time,author,field,oldvalue,newvalue) " - "VALUES (%s,%s,%s,'comment','',%s)", - (id, when, author, comment)) + "VALUES (%s,%s,%s,'comment',%s,%s)", + (id, when, author, n, comment)) + self.ref.replace_xrefs_from_wiki(env, db, 'comment:%d' % n, comment) cursor.execute("UPDATE ticket SET changetime=%s WHERE id=%s", (when, id)) @@ -208,7 +226,21 @@ log.append((int(row[0]), row[1], row[2], row[3] or '', row[4] or '')) return log + def xref_field(self, env, db, name, value, context=''): + """ + Create cross-references and relationships for this ticket. + """ + assert self.ref.id != -1 + xref_kind = Ticket.field_xrefs[name] + if xref_kind: + if xref_kind[0] == 'implicit': + self.ref.replace_xrefs_from_wiki(env, db, name, self[name]) + elif xref_kind[0] == '1 to 1': + self.ref.replace_relation(db, 'has-%s' % name, TracRef(xref_kind[1], value), + 'field', context) + + def get_custom_fields(env): cfg = env.get_config_items('ticket-custom') if not cfg: @@ -296,7 +328,7 @@ owner = cursor.fetchone()[0] ticket['owner'] = owner - tktid = ticket.insert(self.db) + tktid = ticket.insert(self.env, self.db) # Notify try: @@ -399,7 +431,8 @@ ticket.populate(req.args) now = int(time.time()) - ticket.save_changes(self.db, req.args.get('author', req.authname), + ticket.save_changes(self.env, self.db, + req.args.get('author', req.authname), req.args.get('comment'), when=now) try: @@ -511,6 +544,8 @@ ticket = Ticket(self.db, id) reporter_id = util.get_reporter_id(req) + TracRef('ticket', id).add_backlinks(self.db, req) + if preview: # Use user supplied values for field in Ticket.std_fields: Index: trac/Browser.py =================================================================== --- trac/Browser.py (revision 1343) +++ trac/Browser.py (working copy) @@ -22,6 +22,7 @@ from trac import perm, util from trac.Module import Module from trac.WikiFormatter import wiki_to_oneliner +from trac.Xref import TracRef import svn.core import svn.fs @@ -165,7 +166,9 @@ desc = req.args.has_key('desc') self.authzperm.assert_permission (path) - + + TracRef('source', path).add_backlinks(self.db, req) + if not rev: rev_specified = 0 rev = svn.fs.youngest_rev(self.fs_ptr, self.pool) Index: trac/WikiFormatter.py =================================================================== --- trac/WikiFormatter.py (revision 1343) +++ trac/WikiFormatter.py (working copy) @@ -27,6 +27,7 @@ import util + __all__ = ['Formatter', 'OneLinerFormatter', 'wiki_to_html', 'wiki_to_oneliner'] @@ -51,6 +52,7 @@ _open_tags = [] env = None + xref = None absurls = 0 def __init__(self, env, db, absurls=0): @@ -65,6 +67,8 @@ # Check for preceding escape character '!' if match[0] == '!': return match[1:] + if self.xref: # remember the context + self.context = self.xref.extract_context(self.text, fullmatch.start(), fullmatch.end()) return getattr(self, '_' + itype + '_formatter')(match, fullmatch) def tag_open_p(self, tag): @@ -162,11 +166,22 @@ else: return '%s' % (url, text) + def make_xref(self,type,id): + """ + Create a new cross-reference for the given destination type and id. + All the source information (TracRef, facet and context) is + already known at this point. + """ + if self.xref: + from trac.Xref import TracRef + self.xref.insert_xref(self.db, '', TracRef(type, id), self.facet, self.context) + def _make_wiki_link(self, page, text): anchor = '' if page.find('#') != -1: anchor = page[page.find('#'):] page = page[:page.find('#')] + self.make_xref('wiki', page) if not self.env._wiki_pages.has_key(page): return '%s?' \ % (self._href.wiki(page) + anchor, text) @@ -175,6 +190,7 @@ % (self._href.wiki(page) + anchor, text) def _make_changeset_link(self, rev, text): + self.make_xref('changeset', rev) cursor = self.db.cursor() cursor.execute('SELECT message FROM revision WHERE rev=%s', (rev,)) row = cursor.fetchone() @@ -187,6 +203,7 @@ % (self._href.changeset(rev), text) def _make_ticket_link(self, id, text): + self.make_xref('ticket', id) cursor = self.db.cursor() cursor.execute("SELECT summary,status FROM ticket WHERE id=%s", (id,)) row = cursor.fetchone() @@ -202,12 +219,15 @@ return '%s' \ % (self._href.ticket(id), text) _make_bug_link = _make_ticket_link # alias + _make_issue_link = _make_ticket_link # alias def _make_milestone_link(self, name, text): + self.make_xref('milestone', name) return '%s' \ % (self._href.milestone(name), text) def _make_report_link(self, id, text): + self.make_xref('report', id) return '%s' \ % (self._href.report(id), text) @@ -221,6 +241,7 @@ if match: path = match.group(1) rev = match.group(2) + self.make_xref('source', path) if rev: return '%s' \ % (self._href.browser(path, rev), text) @@ -231,6 +252,50 @@ _make_repos_link = _make_source_link # alias +class XRefFormatter(CommonFormatter): + """ + A special version of the wiki formatter that only cares about Trac objects. + This version is used for generating cross-references (see Xref.py). + """ + + _rules = CommonFormatter._rules + + _compiled_rules = re.compile('(?:' + string.join(_rules, '|') + ')') + + def format(self, text, xref, facet): + if not text: + return + self.xref = xref # remember the source's TracRef + self.facet = facet # remember the source's facet + self._open_tags = [] + + self.in_code_block = 0 + + rules = self._compiled_rules + + for line in text.splitlines(): + # Handle code block + if self.in_code_block or line.strip() == '{{{': + self.handle_code_block(line) + continue + # Handle Horizontal ruler + elif line[0:4] == '----': + continue + # Handle new paragraph + elif line == '': + continue + + self.text = util.escape(line) + # Throw a bunch of regexps on the problem + result = re.sub(rules, self.replace, self.text) + + def handle_code_block(self, line): + if line.strip() == '{{{': + self.in_code_block += 1 + elif line.strip() == '}}}': + self.in_code_block -= 1 + + class OneLinerFormatter(CommonFormatter): """ A special version of the wiki formatter that only implement a @@ -251,7 +316,8 @@ rules = self._compiled_rules - result = re.sub(rules, self.replace, util.escape(text.strip())) + text = util.escape(text.strip()) + result = re.sub(rules, self.replace, text) # Close all open 'one line'-tags result += self.close_tag(None) out.write(result) @@ -630,7 +696,7 @@ return out.getvalue() -def wiki_to_oneliner(wikitext, env, db,absurls=0): +def wiki_to_oneliner(wikitext, env, db, absurls=0): out = StringIO.StringIO() OneLinerFormatter(env, db, absurls).format(wikitext, out) return out.getvalue() Index: templates/report.cs =================================================================== --- templates/report.cs (revision 1343) +++ templates/report.cs (working copy) @@ -4,21 +4,29 @@ Index: templates/file.cs =================================================================== --- templates/file.cs (revision 1343) +++ templates/file.cs (working copy) @@ -3,9 +3,12 @@
Index: templates/log.cs =================================================================== --- templates/log.cs (revision 1343) +++ templates/log.cs (working copy) @@ -3,7 +3,8 @@