Index: trac/versioncontrol/api.py =================================================================== --- trac/versioncontrol/api.py (revision 4871) +++ trac/versioncontrol/api.py (working copy) @@ -136,8 +136,12 @@ """Close the connection to the repository.""" raise NotImplementedError - def clear(self): - """Clear any data that may have been cached in instance properties.""" + def clear(self, youngest_rev=None): + """Clear any data that may have been cached in instance properties. + + `youngest_rev` can be specified as a way to force the value + of the `youngest_rev` property (''will change in 0.12''). + """ pass def get_quickjump_entries(self, rev): @@ -221,6 +225,8 @@ The way revisions are sequenced is version control specific. By default, one assumes that the revisions are sequenced in time (... which is ''not'' correct for most VCS, including Subversion). + + (Deprecated, will not be used anymore in Trac 0.12) """ cursor = db.cursor() cursor.execute("SELECT rev FROM revision ORDER BY time DESC LIMIT 1") Index: trac/versioncontrol/tests/cache.py =================================================================== --- trac/versioncontrol/tests/cache.py (revision 4871) +++ trac/versioncontrol/tests/cache.py (working copy) @@ -17,7 +17,7 @@ from datetime import datetime from trac.log import logger_factory -from trac.test import Mock, InMemoryDatabase +from trac.test import Mock, InMemoryDatabase, EnvironmentStub from trac.util.datefmt import to_timestamp, utc from trac.versioncontrol import Repository, Changeset, Node from trac.versioncontrol.cache import CachedRepository @@ -29,8 +29,12 @@ class CacheTestCase(unittest.TestCase): def setUp(self): - self.db = InMemoryDatabase() + self.env = EnvironmentStub() + self.db = self.env.get_db_cnx() self.log = logger_factory('test') + cursor = self.db.cursor() + cursor.execute("INSERT INTO system (name, value) VALUES (%s,%s)", + ('youngest_rev', '')) def test_initial_sync_with_empty_repos(self): t = datetime(2001, 1, 1, 1, 1, 1, 0, utc) @@ -42,7 +46,7 @@ get_youngest_rev=lambda: 0, normalize_rev=lambda x: x, next_rev=lambda x: None) - cache = CachedRepository(self.db, repos, None, self.log) + cache = CachedRepository(self.env, repos, None, self.log) cache.sync() cursor = self.db.cursor() @@ -66,7 +70,7 @@ get_youngest_rev=lambda: 1, normalize_rev=lambda x: x, next_rev=lambda x: int(x) == 0 and 1 or None) - cache = CachedRepository(self.db, repos, None, self.log) + cache = CachedRepository(self.env, repos, None, self.log) cache.sync() cursor = self.db.cursor() @@ -96,6 +100,7 @@ "VALUES ('1',%s,%s,%s,%s,%s)", [('trunk', 'D', 'A', None, None), ('trunk/README', 'F', 'A', None, None)]) + cursor.execute("UPDATE system SET value='1' WHERE name='youngest_rev'") changes = [('trunk/README', Node.FILE, Changeset.EDIT, 'trunk/README', 1)] changeset = Mock(Changeset, 2, 'Update', 'joe', t3, @@ -103,8 +108,10 @@ repos = Mock(Repository, 'test-repos', None, self.log, get_changeset=lambda x: changeset, get_youngest_rev=lambda: 2, - next_rev=lambda x: int(x) == 1 and 2 or None) - cache = CachedRepository(self.db, repos, None, self.log) + get_oldest_rev=lambda: 0, + normalize_rev=lambda x: x, + next_rev=lambda x: x and int(x) == 1 and 2 or None) + cache = CachedRepository(self.env, repos, None, self.log) cache.sync() cursor = self.db.cursor() @@ -130,12 +137,15 @@ "VALUES ('1',%s,%s,%s,%s,%s)", [('trunk', 'D', 'A', None, None), ('trunk/README', 'F', 'A', None, None)]) + cursor.execute("UPDATE system SET value='1' WHERE name='youngest_rev'") repos = Mock(Repository, 'test-repos', None, self.log, get_changeset=lambda x: None, get_youngest_rev=lambda: 1, - next_rev=lambda x: None, normalize_rev=lambda rev: rev) - cache = CachedRepository(self.db, repos, None, self.log) + get_oldest_rev=lambda: 0, + next_rev=lambda x: None, + normalize_rev=lambda rev: rev) + cache = CachedRepository(self.env, repos, None, self.log) self.assertEqual('1', cache.youngest_rev) changeset = cache.get_changeset(1) self.assertEqual('joe', changeset.author) Index: trac/versioncontrol/svn_fs.py =================================================================== --- trac/versioncontrol/svn_fs.py (revision 4871) +++ trac/versioncontrol/svn_fs.py (working copy) @@ -276,7 +276,7 @@ repos = SubversionRepository(dir, None, self.log, {'tags': self.tags, 'branches': self.branches}) - crepos = CachedRepository(self.env.get_db_cnx(), repos, None, self.log) + crepos = CachedRepository(self.env, repos, None, self.log) if authname: authz = SubversionAuthorizer(self.env, crepos, authname) repos.authz = crepos.authz = authz @@ -392,8 +392,10 @@ assert self.scope[0] == '/' self.clear() - def clear(self): + def clear(self, youngest_rev=None): self.youngest = None + if youngest_rev is not None: + self.youngest = self.normalize_rev(youngest_rev) self.oldest = None def __del__(self): Index: trac/versioncontrol/cache.py =================================================================== --- trac/versioncontrol/cache.py (revision 4871) +++ trac/versioncontrol/cache.py (working copy) @@ -16,7 +16,8 @@ from datetime import datetime -from trac.core import TracError +from trac.core import * +from trac.env import IEnvironmentSetupParticipant from trac.util.datefmt import utc, to_timestamp from trac.versioncontrol import Changeset, Node, Repository, Authorizer, \ NoSuchChangeset @@ -27,19 +28,33 @@ 'D': Changeset.DELETE, 'E': Changeset.EDIT, 'M': Changeset.MOVE} +CACHE_REPOSITORY_DIR = 'repository_dir' +CACHE_YOUNGEST_REV = 'youngest_rev' +def get_cache_metadata(db): + """Retrieve the repository cache metadata from 'system' table.""" + cursor = db.cursor() + cursor.execute("SELECT name, value FROM system " + "WHERE name IN ('%s', '%s')" % + (CACHE_REPOSITORY_DIR, CACHE_YOUNGEST_REV)) + metadata = {} + for name, value in cursor: + metadata[name] = value + return metadata + + class CachedRepository(Repository): - def __init__(self, db, repos, authz, log): + def __init__(self, env, repos, authz, log): + # Note: don't store the db connection anymore, + # since CachedRepository are themselves cached + self.env = env Repository.__init__(self, repos.name, authz, log) - self.db = db self.repos = repos - try: + if CacheSetup(self.env).upgrade_in_progress: + self.log.info("Skipping sync during upgrade") + else: self.sync() - except TracError: - raise - except Exception, e: # most probably 2 concurrent resync attempts - log.warning('Error during sync(): %s' % e) def close(self): self.repos.close() @@ -50,72 +65,125 @@ def get_changeset(self, rev): return CachedChangeset(self.repos, self.repos.normalize_rev(rev), - self.db, self.authz) + self.env, self.authz) def get_changesets(self, start, stop): - cursor = self.db.cursor() + db = self.env.get_db_cnx() + cursor = db.cursor() cursor.execute("SELECT rev FROM revision " "WHERE time >= %s AND time < %s " - "ORDER BY time", (to_timestamp(start), to_timestamp(stop))) + "ORDER BY time", + (to_timestamp(start), to_timestamp(stop))) for rev, in cursor: if self.authz.has_permission_for_changeset(rev): yield self.get_changeset(rev) def sync(self): - cursor = self.db.cursor() - - # -- repository used for populating the cache - cursor.execute("SELECT value FROM system WHERE name='repository_dir'") - for previous_repository_dir, in cursor: - if previous_repository_dir != self.name: + db = self.env.get_db_cnx() + metadata = get_cache_metadata(db) + cursor = db.cursor() + + # -- check that we're populating the cache for the correct repository + repository_dir = metadata.get(CACHE_REPOSITORY_DIR) + if repository_dir: + if repository_dir != self.name: raise TracError("The 'repository_dir' has changed, " "a 'trac-admin resync' operation is needed.") - break else: # no 'repository_dir' stored yet, assume everything's OK - cursor.execute("INSERT INTO system (name,value) " - "VALUES ('repository_dir',%s)", (self.name,)) + cursor.execute("INSERT INTO system (name,value) VALUES (%s,%s)", + (CACHE_REPOSITORY_DIR, self.name,)) + # -- check the latest version stored against the latest in repository + if CACHE_YOUNGEST_REV not in metadata: + raise TracError("The repository cache metadata has changed, " + " a 'trac-admin upgrade' operation is needed.") + + self.youngest = metadata[CACHE_YOUNGEST_REV] self.repos.clear() - youngest_stored = self.repos.get_youngest_rev_in_cache(self.db) + repos_youngest = self.repos.youngest_rev - if youngest_stored != str(self.repos.youngest_rev): + if self.youngest != str(repos_youngest): # must try to resync + if self.youngest: + next_youngest = self.repos.next_rev(self.youngest) + else: + next_youngest = None + try: + next_youngest = self.repos.oldest_rev + next_youngest = self.repos.normalize_rev(next_youngest) + except TracError: + pass + + if next_youngest is None: # nothing to cache yet + return + + # 0. first check if there's no (obvious) resync in progress + cursor.execute("SELECT rev FROM revision WHERE rev=%s", + (str(next_youngest),)) + for rev, in cursor: + # already there, but in progress, so keep ''previous'' + # notion of 'youngest' + self.repos.clear(youngest_rev=self.youngest) + return + + # 1. prepare for resyncing + # (there still might be a race condition at this point) + authz = self.repos.authz self.repos.authz = Authorizer() # remove permission checking kindmap = dict(zip(_kindmap.values(), _kindmap.keys())) actionmap = dict(zip(_actionmap.values(), _actionmap.keys())) - self.log.info("Syncing with repository (%s to %s)" - % (youngest_stored, self.repos.youngest_rev)) - if youngest_stored: - current_rev = self.repos.next_rev(youngest_stored) - else: - try: - current_rev = self.repos.oldest_rev - current_rev = self.repos.normalize_rev(current_rev) - except TracError: - current_rev = None - while current_rev is not None: - changeset = self.repos.get_changeset(current_rev) - cursor.execute("INSERT INTO revision (rev,time,author,message) " - "VALUES (%s,%s,%s,%s)", (str(current_rev), - to_timestamp(changeset.date), - changeset.author, - changeset.message)) - for path,kind,action,base_path,base_rev in changeset.get_changes(): - self.log.debug("Caching node change in [%s]: %s" - % (current_rev, (path, kind, action, - base_path, base_rev))) - kind = kindmap[kind] - action = actionmap[action] - cursor.execute("INSERT INTO node_change (rev,path," - "node_type,change_type,base_path,base_rev) " - "VALUES (%s,%s,%s,%s,%s,%s)", - (str(current_rev), path, kind, action, - base_path, base_rev)) - current_rev = self.repos.next_rev(current_rev) - self.db.commit() - self.repos.authz = authz # restore permission checking + try: + while next_youngest is not None: + + # 1.1 Attempt to resync the 'revision' table + self.log.info("Trying to sync revision [%s]" % + next_youngest) + cset = self.repos.get_changeset(next_youngest) + try: + cursor.execute("INSERT INTO revision " + " (rev,time,author,message) " + "VALUES (%s,%s,%s,%s)", + (str(next_youngest), + to_timestamp(cset.date), + cset.author, cset.message)) + db.commit() + except Exception, e: # *another* 1.1. resync attempt won + log.warning('Revision %s already cached: %s' % e) + # also potentially in progress, so keep ''previous'' + # notion of 'youngest' + return + + # 1.2. now *only* one process was able to get there + # (i.e. there *shouldn't* be any race condition here) + + self.youngest = str(next_youngest) + + for path,kind,action,bpath,brev in cset.get_changes(): + self.log.debug("Caching node change in [%s]: %s" + % (next_youngest, + (path,kind,action,bpath,brev))) + kind = kindmap[kind] + action = actionmap[action] + cursor.execute("INSERT INTO node_change " + " (rev,path,node_type,change_type, " + " base_path,base_rev) " + "VALUES (%s,%s,%s,%s,%s,%s)", + (self.youngest, + path, kind, action, bpath, brev)) + + # 1.3. iterate (1.1 should always succeed now) + next_youngest = self.repos.next_rev(next_youngest) + + # 2. update 'youngest_rev' metadata (minimize failures at 0.) + cursor.execute("UPDATE system SET value=%s WHERE name=%s", + (self.youngest, CACHE_YOUNGEST_REV)) + db.commit() + finally: + # 3. restore permission checking (after 1.) + self.repos.authz = authz + def get_node(self, path, rev=None): return self.repos.get_node(path, rev) @@ -126,7 +194,7 @@ return self.repos.oldest_rev def get_youngest_rev(self): - return self.repos.get_youngest_rev_in_cache(self.db) + return self.youngest def previous_rev(self, rev): return self.repos.previous_rev(rev) @@ -146,15 +214,17 @@ def normalize_rev(self, rev): return self.repos.normalize_rev(rev) - def get_changes(self, old_path, old_rev, new_path, new_rev, ignore_ancestry=1): - return self.repos.get_changes(old_path, old_rev, new_path, new_rev, ignore_ancestry) + def get_changes(self, old_path, old_rev, new_path, new_rev, + ignore_ancestry=1): + return self.repos.get_changes(old_path, old_rev, new_path, new_rev, + ignore_ancestry) class CachedChangeset(Changeset): - def __init__(self, repos, rev, db, authz): + def __init__(self, repos, rev, env, authz): self.repos = repos - self.db = db + self.db = env.get_db_cnx() self.authz = authz cursor = self.db.cursor() cursor.execute("SELECT time,author,message FROM revision " @@ -182,3 +252,29 @@ def get_properties(self): return self.repos.get_changeset(self.rev).get_properties() + + +class CacheSetup(Component): + implements(IEnvironmentSetupParticipant) + + upgrade_in_progress = False + + # IEnvironmentSetupParticipant methods + + def environment_created(self): + pass + + def environment_needs_upgrade(self, db): + metadata = get_cache_metadata(db) + return CACHE_REPOSITORY_DIR in metadata and \ + CACHE_YOUNGEST_REV not in metadata + + def upgrade_environment(self, db): + self.upgrade_in_progress = True + repos = self.env.get_repository() + self.upgrade_in_progress = False + value = repos.get_youngest_rev_in_cache(db) or '' + cursor = db.cursor() + cursor.execute("INSERT INTO system (name, value) VALUES (%s, %s)", + (CACHE_YOUNGEST_REV, value)) + self.log.info('Upgraded cache metadata (youngest_rev=%s)' % value) Index: trac/admin/console.py =================================================================== --- trac/admin/console.py (revision 4871) +++ trac/admin/console.py (working copy) @@ -600,6 +600,9 @@ cursor.execute("DELETE FROM revision") cursor.execute("DELETE FROM node_change") cursor.execute("DELETE FROM system WHERE name='repository_dir'") + cursor.execute("DELETE FROM system WHERE name='youngest_rev'") + cursor.execute("INSERT INTO system (name, value) " + "VALUES ('youngest_rev', '')") repos = self.__env.get_repository() # this will do the sync() print 'Done.' Index: trac/web/api.py =================================================================== --- trac/web/api.py (revision 4871) +++ trac/web/api.py (working copy) @@ -324,9 +324,15 @@ data = self.hdf.render(template) if template.endswith('.html'): - from trac.web.chrome import Chrome - data = Chrome(env).render_template(self, template, data, - 'text/html') + if env: + from trac.web.chrome import Chrome + data = Chrome(env).render_template(self, template, data, + 'text/html') + else: + content_type = 'text/plain' + data = '%s\n\n%s: %s' % (data.get('title'), + data.get('type'), + data.get('message')) except: # failed to render data = get_last_traceback() content_type = 'text/plain' Index: trac/web/main.py =================================================================== --- trac/web/main.py (revision 4871) +++ trac/web/main.py (working copy) @@ -181,14 +181,17 @@ # Select the component that should handle the request chosen_handler = None - if not req.path_info or req.path_info == '/': - chosen_handler = self.default_handler - else: - for handler in self.handlers: - if handler.match_request(req): - chosen_handler = handler - break - chosen_handler = self._pre_process_request(req, chosen_handler) + try: + if not req.path_info or req.path_info == '/': + chosen_handler = self.default_handler + else: + for handler in self.handlers: + if handler.match_request(req): + chosen_handler = handler + break + chosen_handler = self._pre_process_request(req, chosen_handler) + except TracError, e: + chosen_handler = None if not chosen_handler: raise HTTPNotFound('No handler matched request to %s', req.path_info) @@ -407,13 +410,19 @@ 'missing. Trac requires one of these options ' 'to locate the Trac environment(s).') run_once = environ['wsgi.run_once'] - env = _open_environment(env_path, run_once=run_once) - if env.base_url: - environ['trac.base_url'] = env.base_url + env = env_error = None + try: + env = _open_environment(env_path, run_once=run_once) + if env.base_url: + environ['trac.base_url'] = env.base_url + except TracError, e: + env_error = e req = Request(environ, start_response) try: + if not env and env_error: + raise HTTPInternalError(env_error.message) try: try: dispatcher = RequestDispatcher(env) @@ -426,7 +435,8 @@ env.shutdown(threading._get_ident()) except HTTPException, e: - env.log.warn(e) + if env: + env.log.warn(e) title = e.reason or 'Error' data = {'title': title, 'type': 'TracError', 'message': e.message} try: