[Python-checkins] python/dist/src/Lib/bsddb init.py,1.11,1.12

Sun Nov 2 20:04:43 EST 2003

Update of /cvsroot/python/python/dist/src/Lib/bsddb
In directory sc8-pr-cvs1:/tmp/cvs-serv10424/Lib/bsddb
Modified Files:
	__init__.py 
Log Message:
* Use weakref's of DBCursor objects for the iterator cursors to avoid a
 memory leak that would've occurred for all iterators that were
 destroyed before having iterated until they raised StopIteration.
* Simplify some code.
* Add new test cases to check for the memleak and ensure that mixing
 iteration with modification of the values for existing keys works.
Index: __init__.py
===================================================================
RCS file: /cvsroot/python/python/dist/src/Lib/bsddb/__init__.py,v
retrieving revision 1.11
retrieving revision 1.12
diff -C2 -d -r1.11 -r1.12
*** __init__.py	2 Nov 2003 09:10:16 -0000	1.11
--- __init__.py	3 Nov 2003 01:04:41 -0000	1.12
***************
*** 68,115 ****
 exec """
 import UserDict
 class _iter_mixin(UserDict.DictMixin):
 def __iter__(self):
 try:
! cur = self.db.cursor()
! self._iter_cursors[str(cur)] = cur

 # since we're only returning keys, we call the cursor
 # methods with flags=0, dlen=0, dofs=0
! curkey = cur.first(0,0,0)[0]
! yield curkey

 next = cur.next
 while 1:
 try:
! curkey = next(0,0,0)[0]
! yield curkey
 except _bsddb.DBCursorClosedError:
! # our cursor object was closed since we last yielded
! # create a new one and attempt to reposition to the
! # right place
! cur = self.db.cursor()
! self._iter_cursors[str(cur)] = cur
 # FIXME-20031101-greg: race condition. cursor could
! # be closed by another thread before this set call.
! try:
! cur.set(curkey,0,0,0)
! except _bsddb.DBCursorClosedError:
! # halt iteration on race condition...
! raise _bsddb.DBNotFoundError
 next = cur.next
 except _bsddb.DBNotFoundError:
! try:
! del self._iter_cursors[str(cur)]
! except KeyError:
! pass
 return

 def iteritems(self):
 try:
! cur = self.db.cursor()
! self._iter_cursors[str(cur)] = cur

 kv = cur.first()
! curkey = kv[0]
 yield kv

--- 68,123 ----
 exec """
 import UserDict
+ from weakref import ref
 class _iter_mixin(UserDict.DictMixin):
+ def _make_iter_cursor(self):
+ cur = self.db.cursor()
+ key = id(cur)
+ self._cursor_refs[key] = ref(cur, self._gen_cref_cleaner(key))
+ return cur
+ 
+ def _gen_cref_cleaner(self, key):
+ # use generate the function for the weakref callback here
+ # to ensure that we do not hold a strict reference to cur
+ # in the callback.
+ return lambda ref: self._cursor_refs.pop(key, None)
+ 
 def __iter__(self):
 try:
! cur = self._make_iter_cursor()
! 
! # FIXME-20031102-greg: race condition. cursor could
! # be closed by another thread before this call.

 # since we're only returning keys, we call the cursor
 # methods with flags=0, dlen=0, dofs=0
! key = cur.first(0,0,0)[0]
! yield key

 next = cur.next
 while 1:
 try:
! key = next(0,0,0)[0]
! yield key
 except _bsddb.DBCursorClosedError:
! cur = self._make_iter_cursor()
 # FIXME-20031101-greg: race condition. cursor could
! # be closed by another thread before this call.
! cur.set(key,0,0,0)
 next = cur.next
 except _bsddb.DBNotFoundError:
! return
! except _bsddb.DBCursorClosedError:
! # the database was modified during iteration. abort.
 return

 def iteritems(self):
 try:
! cur = self._make_iter_cursor()
! 
! # FIXME-20031102-greg: race condition. cursor could
! # be closed by another thread before this call.

 kv = cur.first()
! key = kv[0]
 yield kv

***************
*** 118,142 ****
 try:
 kv = next()
! curkey = kv[0]
 yield kv
 except _bsddb.DBCursorClosedError:
! # our cursor object was closed since we last yielded
! # create a new one and attempt to reposition to the
! # right place
! cur = self.db.cursor()
! self._iter_cursors[str(cur)] = cur
 # FIXME-20031101-greg: race condition. cursor could
! # be closed by another thread before this set call.
! try:
! cur.set(curkey,0,0,0)
! except _bsddb.DBCursorClosedError:
! # halt iteration on race condition...
! raise _bsddb.DBNotFoundError
 next = cur.next
 except _bsddb.DBNotFoundError:
! try:
! del self._iter_cursors[str(cur)]
! except KeyError:
! pass
 return
 """
--- 126,141 ----
 try:
 kv = next()
! key = kv[0]
 yield kv
 except _bsddb.DBCursorClosedError:
! cur = self._make_iter_cursor()
 # FIXME-20031101-greg: race condition. cursor could
! # be closed by another thread before this call.
! cur.set(key,0,0,0)
 next = cur.next
 except _bsddb.DBNotFoundError:
! return
! except _bsddb.DBCursorClosedError:
! # the database was modified during iteration. abort.
 return
 """
***************
*** 160,164 ****
 # reason is that _checkCursor and _closeCursors are not atomic
 # operations. Doing our own locking around self.dbc,
! # self.saved_dbc_key and self._iter_cursors could prevent this.
 # TODO: A test case demonstrating the problem needs to be written.

--- 159,163 ----
 # reason is that _checkCursor and _closeCursors are not atomic
 # operations. Doing our own locking around self.dbc,
! # self.saved_dbc_key and self._cursor_refs could prevent this.
 # TODO: A test case demonstrating the problem needs to be written.

***************
*** 170,182 ****
 # a collection of all DBCursor objects currently allocated
 # by the _iter_mixin interface.
! self._iter_cursors = {}
! 

 def __del__(self):
 self.close()

- def _get_dbc(self):
- return self.dbc
- 
 def _checkCursor(self):
 if self.dbc is None:
--- 169,177 ----
 # a collection of all DBCursor objects currently allocated
 # by the _iter_mixin interface.
! self._cursor_refs = {}

 def __del__(self):
 self.close()

 def _checkCursor(self):
 if self.dbc is None:
***************
*** 198,202 ****
 c.close()
 del c
! map(lambda c: c.close(), self._iter_cursors.values())

 def _checkOpen(self):
--- 193,200 ----
 c.close()
 del c
! for cref in self._cursor_refs.values():
! c = cref()
! if c is not None:
! c.close()

 def _checkOpen(self):

[Python-checkins] python/dist/src/Lib/bsddb __init__.py,1.11,1.12

[Python-checkins] python/dist/src/Lib/bsddb init.py,1.11,1.12