1 | # -*- coding: utf-8 -*-
|
---|
2 | #
|
---|
3 | # Copyright (C) 2005-2009 Edgewall Software
|
---|
4 | # Copyright (C) 2005 Christopher Lenz <cmlenz@gmx.de>
|
---|
5 | # All rights reserved.
|
---|
6 | #
|
---|
7 | # This software is licensed as described in the file COPYING, which
|
---|
8 | # you should have received as part of this distribution. The terms
|
---|
9 | # are also available at http://trac.edgewall.org/wiki/TracLicense.
|
---|
10 | #
|
---|
11 | # This software consists of voluntary contributions made by many
|
---|
12 | # individuals. For the exact contribution history, see the revision
|
---|
13 | # history and logs, available at http://trac.edgewall.org/log/.
|
---|
14 | #
|
---|
15 | # Author: Christopher Lenz <cmlenz@gmx.de>
|
---|
16 |
|
---|
17 | import os
|
---|
18 | import posixpath
|
---|
19 | from datetime import datetime
|
---|
20 |
|
---|
21 | from trac.core import TracError
|
---|
22 | from trac.util.datefmt import utc, to_timestamp
|
---|
23 | from trac.util.translation import _
|
---|
24 | from trac.versioncontrol import Changeset, Node, Repository, Authorizer, \
|
---|
25 | NoSuchChangeset
|
---|
26 |
|
---|
27 |
|
---|
28 | _kindmap = {'D': Node.DIRECTORY, 'F': Node.FILE}
|
---|
29 | _actionmap = {'A': Changeset.ADD, 'C': Changeset.COPY,
|
---|
30 | 'D': Changeset.DELETE, 'E': Changeset.EDIT,
|
---|
31 | 'M': Changeset.MOVE}
|
---|
32 |
|
---|
33 | CACHE_REPOSITORY_DIR = 'repository_dir'
|
---|
34 | CACHE_YOUNGEST_REV = 'youngest_rev'
|
---|
35 |
|
---|
36 | CACHE_METADATA_KEYS = (CACHE_REPOSITORY_DIR, CACHE_YOUNGEST_REV)
|
---|
37 |
|
---|
38 |
|
---|
39 | class CachedRepository(Repository):
|
---|
40 |
|
---|
41 | has_linear_changesets = False
|
---|
42 |
|
---|
43 | def __init__(self, getdb, repos, authz, log):
|
---|
44 | Repository.__init__(self, repos.name, authz, log)
|
---|
45 | if callable(getdb):
|
---|
46 | self.getdb = getdb
|
---|
47 | else:
|
---|
48 | self.getdb = lambda: getdb
|
---|
49 | self.repos = repos
|
---|
50 |
|
---|
51 | def close(self):
|
---|
52 | self.repos.close()
|
---|
53 |
|
---|
54 | def get_quickjump_entries(self, rev):
|
---|
55 | for category, name, path, rev in self.repos.get_quickjump_entries(rev):
|
---|
56 | yield category, name, path, rev
|
---|
57 |
|
---|
58 | def get_changeset(self, rev):
|
---|
59 | return CachedChangeset(self.repos, self.repos.normalize_rev(rev),
|
---|
60 | self.getdb, self.authz)
|
---|
61 |
|
---|
62 | def get_changesets(self, start, stop):
|
---|
63 | db = self.getdb()
|
---|
64 | cursor = db.cursor()
|
---|
65 | cursor.execute("SELECT rev FROM revision "
|
---|
66 | "WHERE time >= %s AND time < %s "
|
---|
67 | "ORDER BY time DESC, rev DESC",
|
---|
68 | (to_timestamp(start), to_timestamp(stop)))
|
---|
69 | for rev, in cursor:
|
---|
70 | try:
|
---|
71 | if self.authz.has_permission_for_changeset(rev):
|
---|
72 | yield self.get_changeset(rev)
|
---|
73 | except NoSuchChangeset:
|
---|
74 | pass # skip changesets currently being resync'ed
|
---|
75 |
|
---|
76 | def sync_changeset(self, rev):
|
---|
77 | cset = self.repos.get_changeset(rev)
|
---|
78 | db = self.getdb()
|
---|
79 | cursor = db.cursor()
|
---|
80 | cursor.execute("UPDATE revision SET time=%s, author=%s, message=%s "
|
---|
81 | "WHERE rev=%s", (to_timestamp(cset.date),
|
---|
82 | cset.author, cset.message,
|
---|
83 | (str(cset.rev))))
|
---|
84 | db.commit()
|
---|
85 |
|
---|
86 | def sync(self, feedback=None):
|
---|
87 | db = self.getdb()
|
---|
88 | cursor = db.cursor()
|
---|
89 | cursor.execute("SELECT name, value FROM system WHERE name IN (%s)" %
|
---|
90 | ','.join(["'%s'" % key for key in CACHE_METADATA_KEYS]))
|
---|
91 | metadata = {}
|
---|
92 | for name, value in cursor:
|
---|
93 | metadata[name] = value
|
---|
94 |
|
---|
95 | # -- check that we're populating the cache for the correct repository
|
---|
96 | repository_dir = metadata.get(CACHE_REPOSITORY_DIR)
|
---|
97 | if repository_dir:
|
---|
98 | # directory part of the repo name can vary on case insensitive fs
|
---|
99 | if os.path.normcase(repository_dir) != os.path.normcase(self.name):
|
---|
100 | self.log.info("'repository_dir' has changed from %r to %r"
|
---|
101 | % (repository_dir, self.name))
|
---|
102 | raise TracError(_("The 'repository_dir' has changed, a "
|
---|
103 | "'trac-admin resync' operation is needed."))
|
---|
104 | elif repository_dir is None: #
|
---|
105 | self.log.info('Storing initial "repository_dir": %s' % self.name)
|
---|
106 | cursor.execute("INSERT INTO system (name,value) VALUES (%s,%s)",
|
---|
107 | (CACHE_REPOSITORY_DIR, self.name,))
|
---|
108 | else: # 'repository_dir' cleared by a resync
|
---|
109 | self.log.info('Resetting "repository_dir": %s' % self.name)
|
---|
110 | cursor.execute("UPDATE system SET value=%s WHERE name=%s",
|
---|
111 | (self.name, CACHE_REPOSITORY_DIR))
|
---|
112 |
|
---|
113 | db.commit() # save metadata changes made up to now
|
---|
114 |
|
---|
115 | # -- retrieve the youngest revision in the repository
|
---|
116 | self.repos.clear()
|
---|
117 | repos_youngest = self.repos.youngest_rev
|
---|
118 |
|
---|
119 | # -- retrieve the youngest revision cached so far
|
---|
120 | if CACHE_YOUNGEST_REV not in metadata:
|
---|
121 | raise TracError(_('Missing "youngest_rev" in cache metadata'))
|
---|
122 |
|
---|
123 | self.youngest = metadata[CACHE_YOUNGEST_REV]
|
---|
124 |
|
---|
125 | if self.youngest:
|
---|
126 | self.youngest = self.repos.normalize_rev(self.youngest)
|
---|
127 | if not self.youngest:
|
---|
128 | self.log.debug('normalize_rev failed (youngest_rev=%r)' %
|
---|
129 | self.youngest_rev)
|
---|
130 | else:
|
---|
131 | self.log.debug('cache metadata undefined (youngest_rev=%r)' %
|
---|
132 | self.youngest_rev)
|
---|
133 | self.youngest = None
|
---|
134 |
|
---|
135 | # -- compare them and try to resync if different
|
---|
136 | if self.youngest != repos_youngest:
|
---|
137 | self.log.info("repos rev [%s] != cached rev [%s]" %
|
---|
138 | (repos_youngest, self.youngest))
|
---|
139 | if self.youngest:
|
---|
140 | next_youngest = self.repos.next_rev(self.youngest)
|
---|
141 | else:
|
---|
142 | next_youngest = None
|
---|
143 | try:
|
---|
144 | next_youngest = self.repos.oldest_rev
|
---|
145 | # Ugly hack needed because doing that everytime in
|
---|
146 | # oldest_rev suffers from horrendeous performance (#5213)
|
---|
147 | if hasattr(self.repos, 'scope'):
|
---|
148 | if self.repos.scope != '/':
|
---|
149 | next_youngest = self.repos.next_rev(next_youngest,
|
---|
150 | find_initial_rev=True)
|
---|
151 | next_youngest = self.repos.normalize_rev(next_youngest)
|
---|
152 | except TracError:
|
---|
153 | return # can't normalize oldest_rev: repository was empty
|
---|
154 |
|
---|
155 | if next_youngest is None: # nothing to cache yet
|
---|
156 | return
|
---|
157 |
|
---|
158 | # 0. first check if there's no (obvious) resync in progress
|
---|
159 | cursor.execute("SELECT rev FROM revision WHERE rev=%s",
|
---|
160 | (str(next_youngest),))
|
---|
161 | for rev, in cursor:
|
---|
162 | # already there, but in progress, so keep ''previous''
|
---|
163 | # notion of 'youngest'
|
---|
164 | self.repos.clear(youngest_rev=self.youngest)
|
---|
165 | return
|
---|
166 |
|
---|
167 | # 1. prepare for resyncing
|
---|
168 | # (there still might be a race condition at this point)
|
---|
169 |
|
---|
170 | authz = self.repos.authz
|
---|
171 | self.repos.authz = Authorizer() # remove permission checking
|
---|
172 |
|
---|
173 | kindmap = dict(zip(_kindmap.values(), _kindmap.keys()))
|
---|
174 | actionmap = dict(zip(_actionmap.values(), _actionmap.keys()))
|
---|
175 |
|
---|
176 | try:
|
---|
177 | while next_youngest is not None:
|
---|
178 |
|
---|
179 | # 1.1 Attempt to resync the 'revision' table
|
---|
180 | self.log.info("Trying to sync revision [%s]" %
|
---|
181 | next_youngest)
|
---|
182 | cset = self.repos.get_changeset(next_youngest)
|
---|
183 | try:
|
---|
184 | cursor.execute("INSERT INTO revision "
|
---|
185 | " (rev,time,author,message) "
|
---|
186 | "VALUES (%s,%s,%s,%s)",
|
---|
187 | (str(next_youngest),
|
---|
188 | to_timestamp(cset.date),
|
---|
189 | cset.author, cset.message))
|
---|
190 | except Exception, e: # *another* 1.1. resync attempt won
|
---|
191 | self.log.warning('Revision %s already cached: %s' %
|
---|
192 | (next_youngest, e))
|
---|
193 | # also potentially in progress, so keep ''previous''
|
---|
194 | # notion of 'youngest'
|
---|
195 | self.repos.clear(youngest_rev=self.youngest)
|
---|
196 | db.rollback()
|
---|
197 | return
|
---|
198 |
|
---|
199 | # 1.2. now *only* one process was able to get there
|
---|
200 | # (i.e. there *shouldn't* be any race condition here)
|
---|
201 |
|
---|
202 | for path,kind,action,bpath,brev in cset.get_changes():
|
---|
203 | self.log.debug("Caching node change in [%s]: %s"
|
---|
204 | % (next_youngest,
|
---|
205 | (path,kind,action,bpath,brev)))
|
---|
206 | kind = kindmap[kind]
|
---|
207 | action = actionmap[action]
|
---|
208 | cursor.execute("INSERT INTO node_change "
|
---|
209 | " (rev,path,node_type,change_type, "
|
---|
210 | " base_path,base_rev) "
|
---|
211 | "VALUES (%s,%s,%s,%s,%s,%s)",
|
---|
212 | (str(next_youngest),
|
---|
213 | path, kind, action, bpath, brev))
|
---|
214 |
|
---|
215 | # 1.3. iterate (1.1 should always succeed now)
|
---|
216 | self.youngest = next_youngest
|
---|
217 | next_youngest = self.repos.next_rev(next_youngest)
|
---|
218 |
|
---|
219 | # 1.4. update 'youngest_rev' metadata
|
---|
220 | # (minimize possibility of failures at point 0.)
|
---|
221 | cursor.execute("UPDATE system SET value=%s WHERE name=%s",
|
---|
222 | (str(self.youngest), CACHE_YOUNGEST_REV))
|
---|
223 | db.commit()
|
---|
224 |
|
---|
225 | # 1.5. provide some feedback
|
---|
226 | if feedback:
|
---|
227 | feedback(self.youngest)
|
---|
228 | finally:
|
---|
229 | # 3. restore permission checking (after 1.)
|
---|
230 | self.repos.authz = authz
|
---|
231 |
|
---|
232 | def get_node(self, path, rev=None):
|
---|
233 | return self.repos.get_node(path, rev)
|
---|
234 |
|
---|
235 | def has_node(self, path, rev=None):
|
---|
236 | return self.repos.has_node(path, rev)
|
---|
237 |
|
---|
238 | def get_oldest_rev(self):
|
---|
239 | return self.repos.oldest_rev
|
---|
240 |
|
---|
241 | def get_youngest_rev(self):
|
---|
242 | if not hasattr(self, 'youngest'):
|
---|
243 | self.sync()
|
---|
244 | return self.youngest
|
---|
245 |
|
---|
246 | def previous_rev(self, rev, path=''):
|
---|
247 | if self.has_linear_changesets:
|
---|
248 | return self._next_prev_rev('<', rev, path)
|
---|
249 | else:
|
---|
250 | return self.repos.previous_rev(rev, path)
|
---|
251 |
|
---|
252 | def next_rev(self, rev, path=''):
|
---|
253 | if self.has_linear_changesets:
|
---|
254 | return self._next_prev_rev('>', rev, path)
|
---|
255 | else:
|
---|
256 | return self.repos.next_rev(rev, path)
|
---|
257 |
|
---|
258 | def _next_prev_rev(self, direction, rev, path=''):
|
---|
259 | db = self.getdb()
|
---|
260 | # the changeset revs are sequence of ints:
|
---|
261 | sql = "SELECT rev FROM node_change WHERE " + \
|
---|
262 | db.cast('rev', 'int') + " " + direction + " %s"
|
---|
263 | args = [rev]
|
---|
264 |
|
---|
265 | if path:
|
---|
266 | path = path.lstrip('/')
|
---|
267 | sql += " AND ("
|
---|
268 | # changes on path itself
|
---|
269 | sql += "path=%s "
|
---|
270 | args.append(path)
|
---|
271 | sql += " OR "
|
---|
272 | # changes on path children
|
---|
273 | sql += "path "+db.like()
|
---|
274 | args.append(db.like_escape(path+'/') + '%')
|
---|
275 | sql += " OR "
|
---|
276 | # deletion of path ancestors
|
---|
277 | components = path.lstrip('/').split('/')
|
---|
278 | for i in range(1, len(components)+1):
|
---|
279 | args.append('/'.join(components[:i]))
|
---|
280 | parent_insert = ','.join(('%s',) * len(components))
|
---|
281 | sql += " (path in (" + parent_insert + ") and change_type='D')"
|
---|
282 | sql += ")"
|
---|
283 |
|
---|
284 | sql += " ORDER BY " + db.cast('rev', 'int') + \
|
---|
285 | (direction == '<' and " DESC" or "") + " LIMIT 1"
|
---|
286 |
|
---|
287 | cursor = db.cursor()
|
---|
288 | cursor.execute(sql, args)
|
---|
289 | for rev, in cursor:
|
---|
290 | return rev
|
---|
291 |
|
---|
292 | def rev_older_than(self, rev1, rev2):
|
---|
293 | return self.repos.rev_older_than(rev1, rev2)
|
---|
294 |
|
---|
295 | def get_path_history(self, path, rev=None, limit=None):
|
---|
296 | return self.repos.get_path_history(path, rev, limit)
|
---|
297 |
|
---|
298 | def normalize_path(self, path):
|
---|
299 | return self.repos.normalize_path(path)
|
---|
300 |
|
---|
301 | def normalize_rev(self, rev):
|
---|
302 | return self.repos.normalize_rev(rev)
|
---|
303 |
|
---|
304 | def get_changes(self, old_path, old_rev, new_path, new_rev,
|
---|
305 | ignore_ancestry=1):
|
---|
306 | return self.repos.get_changes(old_path, old_rev, new_path, new_rev,
|
---|
307 | ignore_ancestry)
|
---|
308 |
|
---|
309 |
|
---|
310 | class CachedChangeset(Changeset):
|
---|
311 |
|
---|
312 | def __init__(self, repos, rev, getdb, authz):
|
---|
313 | self.repos = repos
|
---|
314 | self.getdb = getdb
|
---|
315 | self.authz = authz
|
---|
316 | db = self.getdb()
|
---|
317 | cursor = db.cursor()
|
---|
318 | cursor.execute("SELECT time,author,message FROM revision "
|
---|
319 | "WHERE rev=%s", (str(rev),))
|
---|
320 | row = cursor.fetchone()
|
---|
321 | if row:
|
---|
322 | _date, author, message = row
|
---|
323 | date = datetime.fromtimestamp(_date, utc)
|
---|
324 | Changeset.__init__(self, rev, message, author, date)
|
---|
325 | else:
|
---|
326 | raise NoSuchChangeset(rev)
|
---|
327 | self.scope = getattr(repos, 'scope', '')
|
---|
328 |
|
---|
329 | def get_changes(self):
|
---|
330 | db = self.getdb()
|
---|
331 | cursor = db.cursor()
|
---|
332 | cursor.execute("SELECT path,node_type,change_type,base_path,base_rev "
|
---|
333 | "FROM node_change WHERE rev=%s "
|
---|
334 | "ORDER BY path", (str(self.rev),))
|
---|
335 | for path, kind, change, base_path, base_rev in cursor:
|
---|
336 | if not self.authz.has_permission(posixpath.join(self.scope,
|
---|
337 | path.strip('/'))):
|
---|
338 | # FIXME: what about the base_path?
|
---|
339 | continue
|
---|
340 | kind = _kindmap[kind]
|
---|
341 | change = _actionmap[change]
|
---|
342 | yield path, kind, change, base_path, base_rev
|
---|
343 |
|
---|
344 | def get_properties(self):
|
---|
345 | return self.repos.get_changeset(self.rev).get_properties()
|
---|