Trees Indices Help
Trac
Package trac :: Package versioncontrol :: Module cache

Source Code for Module trac.versioncontrol.cache 

 1 # -*- coding: utf-8 -*- 
 2 # 
 3 # Copyright (C) 2005-2009 Edgewall Software 
 4 # Copyright (C) 2005 Christopher Lenz <cmlenz@gmx.de> 
 5 # All rights reserved. 
 6 # 
 7 # This software is licensed as described in the file COPYING, which 
 8 # you should have received as part of this distribution. The terms 
 9 # are also available at http://trac.edgewall.org/wiki/TracLicense. 
 10 # 
 11 # This software consists of voluntary contributions made by many 
 12 # individuals. For the exact contribution history, see the revision 
 13 # history and logs, available at http://trac.edgewall.org/log/. 
 14 # 
 15 # Author: Christopher Lenz <cmlenz@gmx.de> 
 16 
 17 import os 
 18 
 19 from trac .cache  import cached  
 20 from trac .core  import TracError  
 21 from trac .util .datefmt  import from_utimestamp , to_utimestamp  
 22 from trac .util .translation  import _  
 23 from trac .versioncontrol  import Changeset , Node , Repository , NoSuchChangeset  
 24 
 25 
 26 _kindmap  = {'D': Node .DIRECTORY , 'F': Node .FILE } 
 27 _actionmap  = {'A': Changeset .ADD , 'C': Changeset .COPY , 
 28  'D': Changeset .DELETE , 'E': Changeset .EDIT , 
 29  'M': Changeset .MOVE } 

 30 
 31 - def _invert_dict (d): 

 32  return dict(zip(d .values(), d .keys ())) 

 33 
 34 _inverted_kindmap  = _invert_dict(_kindmap ) 
 35 _inverted_actionmap  = _invert_dict(_actionmap ) 
 36 
 37 CACHE_REPOSITORY_DIR  = 'repository_dir' 
 38 CACHE_YOUNGEST_REV  = 'youngest_rev' 
 39 
 40 CACHE_METADATA_KEYS  = (CACHE_REPOSITORY_DIR , CACHE_YOUNGEST_REV ) 

 41 
 42 
 43 - class CachedRepository (Repository): 

 44 
 45  has_linear_changesets  = False 
 46 
 47  scope  = property(lambda self: self.repos.scope ) 
 48 

 49 -  def __init__ (self, env, repos, log): 

 50  self.env  = env  
 51  self.repos = repos 
 52  self._metadata_id = (CachedRepository .__module__ + '.' 
 53  + CachedRepository .__name__ + '.metadata:' 
 54  + str(self.repos.id )) 
 55  Repository .__init__ (self, repos.name , repos.params , log ) 

 56 

 57 -  def close (self): 

 58  self.repos.close () 

 59 

 60 -  def get_base (self): 

 61  return self.repos.get_base () 

 62 

 63 -  def get_quickjump_entries (self, rev): 

 64  return self.repos.get_quickjump_entries (self.normalize_rev (rev)) 

 65 

 66 -  def get_path_url (self, path, rev): 

 67  return self.repos.get_path_url (path , rev) 

 68 

 69 -  def get_changeset (self, rev): 

 70  return CachedChangeset (self, self.normalize_rev (rev), self.env ) 

 71 

 72 -  def get_changeset_uid (self, rev): 

 73  return self.repos.get_changeset_uid (rev) 

 74 

 75 -  def get_changesets (self, start, stop): 

 76  db  = self.env .get_db_cnx () 
 77  cursor  = db .cursor () 
 78  cursor .execute ("SELECT rev FROM revision " 
 79  "WHERE repos=%s AND time >= %s AND time < %s " 
 80  "ORDER BY time DESC, rev DESC", 
 81  (self.id , to_utimestamp (start ), to_utimestamp (stop ))) 
 82  for rev, in cursor : 
 83  try: 
 84  yield self.get_changeset (rev) 
 85  except NoSuchChangeset : 
 86  pass # skip changesets currently being resync'ed 

 87 

 88 -  def sync_changeset (self, rev): 

 89  cset = self.repos.get_changeset (rev) 
 90  srev = self.db_rev (cset.rev) 
 91  old_cset = [None] 
 92 
 93  @self.env .with_transaction () 
 94  def do_sync(db): 
 95  cursor  = db .cursor () 
 96  cursor .execute (""" 
 97  SELECT time,author,message FROM revision 
 98  WHERE repos=%s AND rev=%s 
 99  """, (self.id , srev)) 
100  for time, author, message  in cursor : 
101  old_cset[0] = Changeset (self.repos, cset.rev, message , author, 
102  from_utimestamp (time)) 
103  if old_cset[0]: 
104  cursor .execute (""" 
105  UPDATE revision SET time=%s, author=%s, message=%s 
106  WHERE repos=%s AND rev=%s 
107  """, (to_utimestamp (cset.date), cset.author, cset.message , 
108  self.id , srev)) 
109  else: 
110  self._insert_changeset(cursor , cset.rev, cset) 

111  return old_cset[0] 

112 
113  @cached('_metadata_id') 

114 -  def metadata (self, db): 

115  """Retrieve data for the cached `metadata` attribute.""" 
116  cursor  = db .cursor () 
117  cursor .execute ("SELECT name, value FROM repository " 
118  "WHERE id=%%s AND name IN (%s)" % 
119  ','.join (['%s'] * len(CACHE_METADATA_KEYS )), 
120  (self.id ,) + CACHE_METADATA_KEYS ) 
121  return dict(cursor ) 

122 

123 -  def sync (self, feedback=None, clean=False): 

124  if clean: 
125  self.log .info('Cleaning cache') 
126  @self.env .with_transaction () 
127  def do_clean(db): 
128  cursor  = db .cursor () 
129  cursor .execute ("DELETE FROM revision WHERE repos=%s", 
130  (self.id ,)) 
131  cursor .execute ("DELETE FROM node_change WHERE repos=%s", 
132  (self.id ,)) 
133  cursor .executemany (""" 
134  DELETE FROM repository WHERE id=%s AND name=%s 
135  """, [(self.id , k) for k in CACHE_METADATA_KEYS ]) 
136  cursor .executemany (""" 
137  INSERT INTO repository (id,name,value) VALUES (%s,%s,%s) 
138  """, [(self.id , k, '') for k in CACHE_METADATA_KEYS ]) 
139  del self.metadata  

140 
141  metadata  = self.metadata  
142 
143  @self.env .with_transaction () 
144  def do_transaction(db): 
145  cursor  = db .cursor () 
146  invalidate  = False 
147 
148  # -- check that we're populating the cache for the correct 
149  # repository 
150  repository_dir  = metadata .get (CACHE_REPOSITORY_DIR ) 
151  if repository_dir : 
152  # directory part of the repo name can vary on case insensitive 
153  # fs 
154  if os.path .normcase(repository_dir ) \ 
155  != os.path .normcase(self.name ): 
156  self.log .info("'repository_dir' has changed from %r to %r", 
157  repository_dir , self.name ) 
158  raise TracError (_ ("The repository directory has changed, " 
159  "you should resynchronize the " 
160  "repository with: trac-admin $ENV " 
161  "repository resync '%(reponame)s'", 
162  reponame=self.reponame or '(default)')) 
163  elif repository_dir  is None: #  
164  self.log .info('Storing initial "repository_dir": %s', 
165  self.name ) 
166  cursor .execute (""" 
167  INSERT INTO repository (id,name,value) VALUES (%s,%s,%s) 
168  """, (self.id , CACHE_REPOSITORY_DIR , self.name )) 
169  invalidate  = True 
170  else: # 'repository_dir' cleared by a resync 
171  self.log .info('Resetting "repository_dir": %s', self.name ) 
172  cursor .execute (""" 
173  UPDATE repository SET value=%s WHERE id=%s AND name=%s 
174  """, (self.name , self.id , CACHE_REPOSITORY_DIR )) 
175  invalidate  = True 
176 
177  # -- insert a 'youngeset_rev' for the repository if necessary 
178  if metadata .get (CACHE_YOUNGEST_REV ) is None: 
179  cursor .execute (""" 
180  INSERT INTO repository (id,name,value) VALUES (%s,%s,%s) 
181  """, (self.id , CACHE_YOUNGEST_REV , '')) 
182  invalidate  = True 
183 
184  if invalidate : 
185  del self.metadata
186 187 # -- retrieve the youngest revision in the repository and the youngest 188 # revision cached so far 189 self.repos.clear () 190 repos_youngest = self.repos.youngest_rev 191 youngest = metadata .get (CACHE_YOUNGEST_REV ) 192 193 # -- verify and normalize youngest revision 194 if youngest: 195 youngest = self.repos.normalize_rev (youngest) 196 if not youngest: 197 self.log .debug('normalize_rev failed (youngest_rev=%r)', 198 self.youngest_rev ) 199 else: 200 self.log .debug('cache metadata undefined (youngest_rev=%r)', 201 self.youngest_rev ) 202 youngest = None 203 204 # -- compare them and try to resync if different 205 next_youngest = None 206 if youngest != repos_youngest: 207 self.log .info("repos rev [%s] != cached rev [%s]", 208 repos_youngest, youngest) 209 if youngest: 210 next_youngest = self.repos.next_rev (youngest) 211 else: 212 try: 213 next_youngest = self.repos.oldest_rev 214 # Ugly hack needed because doing that everytime in 215 # oldest_rev suffers from horrendeous performance (#5213) 216 if self.repos.scope != '/' and not \ 217 self.repos.has_node ('/', next_youngest): 218 next_youngest = self.repos.next_rev (next_youngest, 219 find_initial_rev=True) 220 next_youngest = self.repos.normalize_rev (next_youngest) 221 except TracError : 222 # can't normalize oldest_rev: repository was empty 223 return 224 225 if next_youngest is None: # nothing to cache yet 226 return 227 srev = self.db_rev (next_youngest) 228 229 # 0. first check if there's no (obvious) resync in progress 230 db = self.env .get_read_db () 231 cursor = db .cursor () 232 cursor .execute (""" 233 SELECT rev FROM revision WHERE repos=%s AND rev=%s 234 """, (self.id , srev)) 235 for rev, in cursor : 236 # already there, but in progress, so keep ''previous'' 237 # notion of 'youngest' 238 self.repos.clear (youngest_rev =youngest) 239 return 240 241 # prepare for resyncing (there might still be a race 242 # condition at this point) 243 while next_youngest is not None: 244 srev = self.db_rev (next_youngest) 245 exit = [False] 246 247 @self.env .with_transaction () 248 def do_transaction(db): 249 cursor = db .cursor () 250 251 self.log .info("Trying to sync revision [%s]", 252 next_youngest) 253 cset = self.repos.get_changeset (next_youngest) 254 try: 255 # steps 1. and 2. 256 self._insert_changeset(cursor , next_youngest, cset) 257 except Exception, e : # *another* 1.1. resync attempt won 258 self.log .warning('Revision %s already cached: %r', 259 next_youngest, e ) 260 # the other resync attempts is also 261 # potentially still in progress, so for our 262 # process/thread, keep ''previous'' notion of 263 # 'youngest' 264 self.repos.clear (youngest_rev =youngest) 265 # FIXME: This aborts a containing transaction 266 db .rollback () 267 exit[0] = True 268 return 269 270 # 3. update 'youngest_rev' metadata (minimize 271 # possibility of failures at point 0.) 272 cursor .execute (""" 273 UPDATE repository SET value=%s WHERE id=%s AND name=%s 274 """, (str(next_youngest), self.id , CACHE_YOUNGEST_REV )) 275 del self.metadata 276 277 if exit[0]: 278 return 279 280 # 4. iterate (1. should always succeed now) 281 youngest = next_youngest 282 next_youngest = self.repos.next_rev (next_youngest) 283 284 # 5. provide some feedback 285 if feedback: 286 feedback(youngest) 287
288 - def _insert_changeset (self, cursor, rev, cset):
289 srev = self.db_rev (rev) 290 # 1. Attempt to resync the 'revision' table. In case of 291 # concurrent syncs, only such insert into the `revision` table 292 # will succeed, the others will fail and raise an exception. 293 cursor .execute (""" 294 INSERT INTO revision (repos,rev,time,author,message) 295 VALUES (%s,%s,%s,%s,%s) 296 """, (self.id , srev, to_utimestamp (cset.date), 297 cset.author, cset.message )) 298 # 2. now *only* one process was able to get there (i.e. there 299 # *shouldn't* be any race condition here) 300 for path , kind, action, bpath, brev in cset.get_changes (): 301 self.log .debug("Caching node change in [%s]: %r", rev, 302 (path , kind, action, bpath, brev)) 303 kind = _inverted_kindmap [kind] 304 action = _inverted_actionmap [action] 305 cursor .execute (""" 306 INSERT INTO node_change 307 (repos,rev,path,node_type,change_type,base_path, 308 base_rev) 309 VALUES (%s,%s,%s,%s,%s,%s,%s) 310 """, (self.id , srev, path , kind, action, bpath, brev))
311
312 - def get_node (self, path, rev=None):
313 return self.repos.get_node (path , self.normalize_rev (rev))
314
315 - def _get_node_revs (self, path, last=None, first=None):
316 """Return the revisions affecting `path` between `first` and `last` 317 revisions. 318 """ 319 last = self.normalize_rev (last) 320 slast = self.db_rev (last) 321 node = self.get_node (path , last) # Check node existence 322 db = self.env .get_db_cnx () 323 cursor = db .cursor () 324 if first is None: 325 cursor .execute ("SELECT rev FROM node_change " 326 "WHERE repos=%s AND rev<=%s " 327 " AND path=%s " 328 " AND change_type IN ('A', 'C', 'M') " 329 "ORDER BY rev DESC LIMIT 1", 330 (self.id , slast, path )) 331 first = 0 332 for row in cursor : 333 first = int(row[0]) 334 sfirst = self.db_rev (first) 335 cursor .execute ("SELECT DISTINCT rev FROM node_change " 336 "WHERE repos=%%s AND rev>=%%s AND rev<=%%s " 337 " AND (path=%%s OR path %s)" % db .prefix_match (), 338 (self.id , sfirst, slast, path , 339 db .prefix_match_value (path + '/'))) 340 return [int(row[0]) for row in cursor ]
341
342 - def _get_changed_revs (self, node_infos):
343 if not node_infos: 344 return {} 345 346 node_infos = [(node, self.normalize_rev (first)) for node, first 347 in node_infos] 348 sfirst = self.db_rev (min (first for node, first in node_infos)) 349 slast = self.db_rev (max(node.rev for node, first in node_infos)) 350 path_infos = dict((node.path , (node, first)) for node, first 351 in node_infos) 352 path_revs = dict((node.path , []) for node, first in node_infos) 353 354 db = self.env .get_read_db () 355 cursor = db .cursor () 356 prefix_match = db .prefix_match () 357 358 # Prevent "too many SQL variables" since max number of parameters is 359 # 999 on SQLite. No limitation on PostgreSQL and MySQL. 360 idx = 0 361 delta = (999 - 3) // 5 362 while idx < len(node_infos): 363 subset = node_infos[idx:idx + delta] 364 idx += delta 365 count = len(subset) 366 367 holders = ','.join (('%s',) * count ) 368 query = """\ 369 SELECT DISTINCT 370 rev, (CASE WHEN path IN (%s) THEN path %s END) AS path 371 FROM node_change 372 WHERE repos=%%s AND rev>=%%s AND rev<=%%s AND (path IN (%s) %s) 373 """ % \ 374 (holders, 375 ' '.join (('WHEN path ' + prefix_match + ' THEN %s',) * count ), 376 holders, 377 ' '.join (('OR path ' + prefix_match ,) * count )) 378 args = [] 379 args .extend(node.path for node, first in subset) 380 for node, first in subset: 381 args .append(db .prefix_match_value (node.path + '/')) 382 args .append(node.path ) 383 args .extend((self.id , sfirst, slast)) 384 args .extend(node.path for node, first in subset) 385 args .extend(db .prefix_match_value (node.path + '/') 386 for node, first in subset) 387 cursor .execute (query , args ) 388 389 for srev, path in cursor : 390 rev = self.rev_db (srev) 391 node, first = path_infos[path ] 392 if first <= rev <= node.rev: 393 path_revs[path ].append(rev) 394 395 return path_revs
396
397 - def has_node (self, path, rev=None):
398 return self.repos.has_node (path , self.normalize_rev (rev))
399
400 - def get_oldest_rev (self):
401 return self.repos.oldest_rev
402
403 - def get_youngest_rev (self):
404 return self.rev_db (self.metadata .get (CACHE_YOUNGEST_REV ))
405
406 - def previous_rev (self, rev, path=''):
407 # Hitting the repository directly is faster than searching the 408 # database. When there is a long stretch of inactivity on a file (in 409 # particular, when a file is added late in the history) the database 410 # query can take a very long time to determine that there is no 411 # previous revision in the node_changes table. However, the repository 412 # will have a datastructure that will allow it to find the previous 413 # version of a node fairly directly. 414 #if self.has_linear_changesets: 415 # return self._next_prev_rev('<', rev, path) 416 return self.repos.previous_rev (self.normalize_rev (rev), path )
417
418 - def next_rev (self, rev, path=''):
419 if self.has_linear_changesets : 420 return self._next_prev_rev('>', rev, path ) 421 else: 422 return self.repos.next_rev (self.normalize_rev (rev), path )
423
424 - def _next_prev_rev (self, direction, rev, path=''):
425 srev = self.db_rev (rev) 426 db = self.env .get_db_cnx () 427 # the changeset revs are sequence of ints: 428 sql = "SELECT rev FROM node_change WHERE repos=%s AND " + \ 429 "rev" + direction + "%s" 430 args = [self.id , srev] 431 432 if path : 433 path = path .lstrip('/') 434 # changes on path itself or its children 435 sql += " AND (path=%s OR path " + db .prefix_match () 436 args .extend((path , db .prefix_match_value (path + '/'))) 437 # deletion of path ancestors 438 components = path .lstrip('/').split('/') 439 parents = ','.join (('%s',) * len(components)) 440 sql += " OR (path IN (" + parents + ") AND change_type='D'))" 441 for i in range(1, len(components) + 1): 442 args .append('/'.join (components[:i ])) 443 444 sql += " ORDER BY rev" + (direction == '<' and " DESC" or "") \ 445 + " LIMIT 1" 446 447 cursor = db .cursor () 448 cursor .execute (sql , args ) 449 for rev, in cursor : 450 return int(rev)
451
452 - def rev_older_than (self, rev1, rev2):
453 return self.repos.rev_older_than (self.normalize_rev (rev1), 454 self.normalize_rev (rev2))
455
456 - def get_path_history (self, path, rev=None, limit=None):
457 return self.repos.get_path_history (path , self.normalize_rev (rev), 458 limit)
459
460 - def normalize_path (self, path):
461 return self.repos.normalize_path (path )
462
463 - def normalize_rev (self, rev):
464 if rev is None or isinstance(rev, basestring) and \ 465 rev.lower() in ('', 'head', 'latest', 'youngest'): 466 return self.rev_db (self.youngest_rev or 0) 467 else: 468 try: 469 rev = int(rev) 470 if rev <= self.youngest_rev : 471 return rev 472 except (ValueError, TypeError): 473 pass 474 raise NoSuchChangeset (rev)
475
476 - def db_rev (self, rev):
477 """Convert a revision to its representation in the database.""" 478 return str(rev)
479
480 - def rev_db (self, rev):
481 """Convert a revision from its representation in the database.""" 482 return rev
483
484 - def get_changes (self, old_path, old_rev, new_path, new_rev, 485 ignore_ancestry=1):
486 return self.repos.get_changes (old_path, self.normalize_rev (old_rev), 487 new_path, self.normalize_rev (new_rev), 488 ignore_ancestry)
489
490 491 - class CachedChangeset (Changeset):
492
493 - def __init__ (self, repos, rev, env):
494 self.env = env 495 db = self.env .get_db_cnx () 496 cursor = db .cursor () 497 cursor .execute ("SELECT time,author,message FROM revision " 498 "WHERE repos=%s AND rev=%s", 499 (repos.id , repos.db_rev (rev))) 500 row = cursor .fetchone () 501 if row: 502 _date, author, message = row 503 date = from_utimestamp (_date) 504 Changeset .__init__ (self, repos, repos.rev_db (rev), message , author, 505 date) 506 else: 507 raise NoSuchChangeset (rev)
508
509 - def get_changes (self):
510 db = self.env .get_db_cnx () 511 cursor = db .cursor () 512 cursor .execute ("SELECT path,node_type,change_type,base_path,base_rev " 513 "FROM node_change WHERE repos=%s AND rev=%s " 514 "ORDER BY path", 515 (self.repos.id , self.repos.db_rev (self.rev))) 516 for path , kind, change, base_path , base_rev in sorted(cursor ): 517 kind = _kindmap [kind] 518 change = _actionmap [change] 519 yield path , kind, change, base_path , self.repos.rev_db (base_rev)
520
521 - def get_properties (self):
522 return self.repos.repos.get_changeset (self.rev).get_properties ()
523
Trees Indices Help
Trac
Generated by Epydoc 3.0.1 on Mon Feb 13 23:37:32 2023 http://epydoc.sourceforge.net

AltStyle によって変換されたページ (->オリジナル) /