Ticket #7723: 7723-multirepos-cache-r7590.patch
| File 7723-multirepos-cache-r7590.patch, 15.9 KB (added by rblank, 4 years ago) |
|---|
-
trac/admin/console.py
diff --git a/trac/admin/console.py b/trac/admin/console.py
a b 709 709 project_dir=os.path.basename(self.envname), 710 710 config_path=os.path.join(self.envname, 'conf', 'trac.ini'))) 711 711 712 _help_resync = [('resync', 'Re-synchronize trac with the repository'), 713 ('resync <rev>', 'Re-synchronize only the given <rev>')] 712 ## Resync 713 _help_resync = [('resync', 'Re-synchronize trac with all repositories'), 714 ('resync <repos>', 715 'Re-synchronize only the given <repos>'), 716 ('resync <repos> <rev>', 717 'Re-synchronize only the given <rev> of <repos>')] 714 718 715 719 def _resync_feedback(self, rev): 716 720 sys.stdout.write(' [%s]\r' % rev) 717 721 sys.stdout.flush() 718 722 719 ## Resync720 723 def do_resync(self, line): 721 724 env = self.env_open() 722 725 argv = self.arg_tokenize(line) 723 if argv: 724 rev = argv[0] 725 if rev: 726 env.get_repository().sync_changeset(rev) 727 printout(_("%(rev)s resynced.", rev=rev)) 728 return 726 if argv[0]: 727 reponame = argv[0] 728 if len(argv) >= 2: 729 rev = argv[1] 730 if rev: 731 env.get_repository(reponame).sync_changeset(rev) 732 printout(_("%(rev)s resynced on %(reponame)s.", rev=rev, 733 reponame=reponame)) 734 return 735 repositories = [(reponame, env.get_repository(reponame))] 736 else: 737 repositories = list(env.get_all_repositories()) 738 729 739 from trac.versioncontrol.cache import CACHE_METADATA_KEYS 730 printout(_("Resyncing repository history... ")) 731 cnx = self.db_open() 732 cursor = cnx.cursor() 733 cursor.execute("DELETE FROM revision") 734 cursor.execute("DELETE FROM node_change") 735 cursor.executemany("DELETE FROM system WHERE name=%s", 736 [(k,) for k in CACHE_METADATA_KEYS]) 737 cursor.executemany("INSERT INTO system (name, value) VALUES (%s, %s)", 738 [(k, '') for k in CACHE_METADATA_KEYS]) 739 cnx.commit() 740 repos = env.get_repository().sync(self._resync_feedback) 741 cursor.execute("SELECT count(rev) FROM revision") 742 for cnt, in cursor: 743 printout(ngettext("%(num)s revision cached.", 744 "%(num)s revisions cached.", num=cnt)) 740 for (reponame, repos) in sorted(repositories): 741 printout(_("Resyncing repository history for %(reponame)s... ", 742 reponame=reponame or '(default)')) 743 cnx = self.db_open() 744 cursor = cnx.cursor() 745 cursor.execute("DELETE FROM revision WHERE id=%s", (reponame,)) 746 cursor.execute("DELETE FROM node_change WHERE id=%s", (reponame,)) 747 cursor.executemany("DELETE FROM repository " 748 "WHERE id=%s AND name=%s", 749 [(reponame, k) for k in CACHE_METADATA_KEYS]) 750 cursor.executemany("INSERT INTO repository (id, name, value) " 751 "VALUES (%s, %s, %s)", [(reponame, k, '') 752 for k in CACHE_METADATA_KEYS]) 753 cnx.commit() 754 repos.sync(self._resync_feedback) 755 cursor.execute("SELECT count(rev) FROM revision WHERE id=%s", 756 (reponame,)) 757 for cnt, in cursor: 758 printout(ngettext("%(num)s revision cached.", 759 "%(num)s revisions cached.", num=cnt)) 745 760 printout(_("Done.")) 746 761 747 762 ## Wiki -
trac/db_default.py
diff --git a/trac/db_default.py b/trac/db_default.py
a b 17 17 from trac.db import Table, Column, Index 18 18 19 19 # Database version identifier. Used for automatic upgrades. 20 db_version = 2 120 db_version = 22 21 21 22 22 def __mkreports(reports): 23 23 """Utility function used to create report data in same syntax as the … … 82 82 Index(['time'])], 83 83 84 84 # Version control cache 85 Table('revision', key='rev')[ 85 Table('repository', key=('id', 'name'))[ 86 Column('id'), 87 Column('name'), 88 Column('value')], 89 Table('revision', key=('id', 'rev'))[ 90 Column('id'), 86 91 Column('rev'), 87 92 Column('time', type='int'), 88 93 Column('author'), 89 94 Column('message'), 90 Index(['time'])], 91 Table('node_change', key=('rev', 'path', 'change_type'))[ 95 Index(['id', 'time'])], 96 Table('node_change', key=('id', 'rev', 'path', 'change_type'))[ 97 Column('id'), 92 98 Column('rev'), 93 99 Column('path'), 94 100 Column('node_type', size=1), 95 101 Column('change_type', size=1), 96 102 Column('base_path'), 97 103 Column('base_rev'), 98 Index([' rev'])],104 Index(['id', 'rev'])], 99 105 100 106 # Ticket system 101 107 Table('ticket', key='id')[ -
new file trac/upgrades/db22.py
diff --git a/trac/upgrades/db22.py b/trac/upgrades/db22.py new file mode 100644
- + 1 from trac.db import Table, Column, Index, DatabaseManager 2 3 def do_upgrade(env, ver, cursor): 4 # Make changeset cache multi-repository aware 5 cursor.execute("CREATE TEMPORARY TABLE rev_old " 6 "AS SELECT * FROM revision") 7 cursor.execute("DROP TABLE revision") 8 cursor.execute("CREATE TEMPORARY TABLE nc_old " 9 "AS SELECT * FROM node_change") 10 cursor.execute("DROP TABLE node_change") 11 12 tables = [Table('repository', key=('id', 'name'))[ 13 Column('id'), 14 Column('name'), 15 Column('value')], 16 Table('revision', key=('id', 'rev'))[ 17 Column('id'), 18 Column('rev'), 19 Column('time', type='int'), 20 Column('author'), 21 Column('message'), 22 Index(['id', 'time'])], 23 Table('node_change', key=('id', 'rev', 'path', 'change_type'))[ 24 Column('id'), 25 Column('rev'), 26 Column('path'), 27 Column('node_type', size=1), 28 Column('change_type', size=1), 29 Column('base_path'), 30 Column('base_rev'), 31 Index(['id', 'rev'])]] 32 33 db_connector, _ = DatabaseManager(env)._get_connector() 34 for table in tables: 35 for stmt in db_connector.to_sql(table): 36 cursor.execute(stmt) 37 38 cursor.execute("INSERT INTO revision (id,rev,time,author,message) " 39 "SELECT '',rev,time,author,message FROM rev_old") 40 cursor.execute("DROP TABLE rev_old") 41 cursor.execute("INSERT INTO node_change (id,rev,path,node_type," 42 "change_type,base_path,base_rev) " 43 "SELECT '',rev,path,node_type,change_type,base_path," 44 "base_rev FROM nc_old") 45 cursor.execute("DROP TABLE nc_old") 46 47 cursor.execute("INSERT INTO repository (id,name,value) " 48 "SELECT '',name,value FROM system " 49 "WHERE name IN ('repository_dir', 'youngest_rev')") 50 cursor.execute("DELETE FROM system " 51 "WHERE name IN ('repository_dir', 'youngest_rev')") -
trac/versioncontrol/cache.py
diff --git a/trac/versioncontrol/cache.py b/trac/versioncontrol/cache.py
a b 47 47 self.getdb = lambda: getdb 48 48 self.repos = repos 49 49 50 def _set_reponame(self, value): 51 try: 52 self.repos.reponame = value 53 except AttributeError: 54 pass 55 56 reponame = property(fget=lambda self: self.repos.reponame, 57 fset=_set_reponame) 58 50 59 def close(self): 51 60 self.repos.close() 52 61 … … 62 71 db = self.getdb() 63 72 cursor = db.cursor() 64 73 cursor.execute("SELECT rev FROM revision " 65 "WHERE time >= %s AND time < %s"74 "WHERE id=%s AND time >= %s AND time < %s" 66 75 "ORDER BY time DESC, rev DESC", 67 (to_timestamp(start), to_timestamp(stop))) 76 (self.reponame, to_timestamp(start), 77 to_timestamp(stop))) 68 78 for rev, in cursor: 69 79 try: 70 80 if self.authz.has_permission_for_changeset(rev): … … 77 87 db = self.getdb() 78 88 cursor = db.cursor() 79 89 cursor.execute("UPDATE revision SET time=%s, author=%s, message=%s " 80 "WHERE rev=%s", (to_timestamp(cset.date),81 82 (str(cset.rev))))90 "WHERE id=%s AND rev=%s", 91 (to_timestamp(cset.date), cset.author, cset.message, 92 self.reponame, str(cset.rev))) 83 93 db.commit() 84 94 85 95 def sync(self, feedback=None): 86 96 db = self.getdb() 87 97 cursor = db.cursor() 88 cursor.execute("SELECT name, value FROM system WHERE name IN (%s)" % 89 ','.join(["'%s'" % key for key in CACHE_METADATA_KEYS])) 98 cursor.execute("SELECT name, value FROM repository " 99 "WHERE id=%%s AND name IN (%s)" % 100 ','.join(['%s'] * len(CACHE_METADATA_KEYS)), 101 (self.reponame,) + CACHE_METADATA_KEYS) 90 102 metadata = {} 91 103 for name, value in cursor: 92 104 metadata[name] = value … … 101 113 "'trac-admin resync' operation is needed.")) 102 114 elif repository_dir is None: # 103 115 self.log.info('Storing initial "repository_dir": %s' % self.name) 104 cursor.execute("INSERT INTO system (name,value) VALUES (%s,%s)", 105 (CACHE_REPOSITORY_DIR, self.name,)) 116 cursor.execute("INSERT INTO repository (id,name,value) " 117 "VALUES (%s,%s,%s)", 118 (self.reponame, CACHE_REPOSITORY_DIR, self.name)) 106 119 else: # 'repository_dir' cleared by a resync 107 120 self.log.info('Resetting "repository_dir": %s' % self.name) 108 cursor.execute("UPDATE system SET value=%s WHERE name=%s", 109 (self.name, CACHE_REPOSITORY_DIR)) 121 cursor.execute("UPDATE repository SET value=%s " 122 "WHERE id=%s AND name=%s", 123 (self.name, self.reponame, CACHE_REPOSITORY_DIR)) 110 124 111 125 db.commit() # save metadata changes made up to now 112 126 … … 154 168 return 155 169 156 170 # 0. first check if there's no (obvious) resync in progress 157 cursor.execute("SELECT rev FROM revision WHERE rev=%s",158 (s tr(next_youngest),))171 cursor.execute("SELECT rev FROM revision WHERE id=%s AND rev=%s", 172 (self.reponame, str(next_youngest))) 159 173 for rev, in cursor: 160 174 # already there, but in progress, so keep ''previous'' 161 175 # notion of 'youngest' … … 180 194 cset = self.repos.get_changeset(next_youngest) 181 195 try: 182 196 cursor.execute("INSERT INTO revision " 183 " ( rev,time,author,message) "184 "VALUES (%s,%s,%s,%s )",185 (s tr(next_youngest),197 " (id,rev,time,author,message) " 198 "VALUES (%s,%s,%s,%s,%s)", 199 (self.reponame, str(next_youngest), 186 200 to_timestamp(cset.date), 187 201 cset.author, cset.message)) 188 202 except Exception, e: # *another* 1.1. resync attempt won … … 204 218 kind = kindmap[kind] 205 219 action = actionmap[action] 206 220 cursor.execute("INSERT INTO node_change " 207 " ( rev,path,node_type,change_type, "221 " (id,rev,path,node_type,change_type, " 208 222 " base_path,base_rev) " 209 "VALUES (%s,%s,%s,%s,%s,%s )",210 (s tr(next_youngest),223 "VALUES (%s,%s,%s,%s,%s,%s,%s)", 224 (self.reponame, str(next_youngest), 211 225 path, kind, action, bpath, brev)) 212 226 213 227 # 1.3. iterate (1.1 should always succeed now) … … 216 230 217 231 # 1.4. update 'youngest_rev' metadata 218 232 # (minimize possibility of failures at point 0.) 219 cursor.execute("UPDATE system SET value=%s WHERE name=%s", 220 (str(self.youngest), CACHE_YOUNGEST_REV)) 233 cursor.execute("UPDATE repository SET value=%s " 234 "WHERE id=%s AND name=%s", 235 (str(self.youngest), self.reponame, 236 CACHE_YOUNGEST_REV)) 221 237 db.commit() 222 238 223 239 # 1.5. provide some feedback … … 256 272 def _next_prev_rev(self, direction, rev, path=''): 257 273 db = self.getdb() 258 274 # the changeset revs are sequence of ints: 259 sql = "SELECT rev FROM node_change WHERE " + \275 sql = "SELECT rev FROM node_change WHERE id=%s AND " + \ 260 276 db.cast('rev', 'int') + " " + direction + " %s" 261 args = [ rev]277 args = [self.reponame, rev] 262 278 263 279 if path: 264 280 path = path.lstrip('/') … … 314 330 db = self.getdb() 315 331 cursor = db.cursor() 316 332 cursor.execute("SELECT time,author,message FROM revision " 317 "WHERE rev=%s", (str(rev),)) 333 "WHERE id=%s AND rev=%s", 334 (self.repos.reponame, str(rev))) 318 335 row = cursor.fetchone() 319 336 if row: 320 337 _date, author, message = row … … 328 345 db = self.getdb() 329 346 cursor = db.cursor() 330 347 cursor.execute("SELECT path,node_type,change_type,base_path,base_rev " 331 "FROM node_change WHERE rev=%s "332 "ORDER BY path", (s tr(self.rev),))348 "FROM node_change WHERE id=%s AND rev=%s " 349 "ORDER BY path", (self.repos.reponame, str(self.rev))) 333 350 for path, kind, change, base_path, base_rev in cursor: 334 351 if not self.authz.has_permission(posixpath.join(self.scope, 335 352 path.strip('/'))): -
trac/versioncontrol/web_ui/changeset.py
diff --git a/trac/versioncontrol/web_ui/changeset.py b/trac/versioncontrol/web_ui/changeset.py
a b 906 906 907 907 single = rev_a == rev_b 908 908 if reponame: 909 title = ngettext('Changeset in %(repo)s', 'Changesets in %(repo)s', 909 title = ngettext('Changeset in %(repo)s ', 910 'Changesets in %(repo)s ', 910 911 single and 1 or 2, repo=reponame) 911 912 else: 912 title = ngettext('Changeset ', 'Changesets', single and 1 or 2)913 title = ngettext('Changeset ', 'Changesets ', single and 1 or 2) 913 914 if single: 914 title = tag(title, tag.em(' 915 title = tag(title, tag.em('[%s]' % rev_a)) 915 916 else: 916 title = tag(title, tag.em(' 917 title = tag(title, tag.em('[%s-%s]' % (rev_a, rev_b))) 917 918 if field == 'title': 918 919 return title 919 920 elif field == 'summary':
