Merge with bsddb3 2002.11.23.10.42.36
diff --git a/Lib/bsddb/dbtables.py b/Lib/bsddb/dbtables.py
index f1e88f2..2c65198 100644
--- a/Lib/bsddb/dbtables.py
+++ b/Lib/bsddb/dbtables.py
@@ -1,6 +1,7 @@
#-----------------------------------------------------------------------
#
# Copyright (C) 2000, 2001 by Autonomous Zone Industries
+# Copyright (C) 2002 Gregory P. Smith
#
# License: This is free software. You may use this software for any
# purpose including modification/redistribution, so long as
@@ -54,6 +55,13 @@
def __call__(self, s):
return s[:len(self.prefix)] == self.prefix
+class PostfixCond(Cond):
+ """Acts as a condition function for matching a string postfix"""
+ def __init__(self, postfix):
+ self.postfix = postfix
+ def __call__(self, s):
+ return s[-len(self.postfix):] == self.postfix
+
class LikeCond(Cond):
"""
Acts as a function that will match using an SQL 'LIKE' style
@@ -523,17 +531,10 @@
# if no condition was specified or the condition
# succeeds, add row to our match list.
if not condition or condition(data) :
- # only create new entries in matcing_rowids on
- # the first pass, otherwise reject the
- # rowid as it must not have matched
- # the previous passes
- if column_num == 0 :
- if not matching_rowids.has_key(rowid) :
- matching_rowids[rowid] = {}
- if savethiscolumndata :
- matching_rowids[rowid][column] = data
- else :
- rejected_rowids[rowid] = rowid
+ if not matching_rowids.has_key(rowid) :
+ matching_rowids[rowid] = {}
+ if savethiscolumndata :
+ matching_rowids[rowid][column] = data
else :
if matching_rowids.has_key(rowid) :
del matching_rowids[rowid]
diff --git a/Lib/bsddb/dbutils.py b/Lib/bsddb/dbutils.py
index fe08407..d21069f 100644
--- a/Lib/bsddb/dbutils.py
+++ b/Lib/bsddb/dbutils.py
@@ -1,11 +1,5 @@
#------------------------------------------------------------------------
#
-# In my performance tests, using this (as in dbtest.py test4) is
-# slightly slower than simply compiling _db.c with MYDB_THREAD
-# undefined to prevent multithreading support in the C module.
-# Using NoDeadlockDb also prevent deadlocks from mutliple processes
-# accessing the same database.
-#
# Copyright (C) 2000 Autonomous Zone Industries
#
# License: This is free software. You may use this software for any
@@ -18,7 +12,7 @@
# Author: Gregory P. Smith <greg@electricrain.com>
#
# Note: I don't know how useful this is in reality since when a
-# DBDeadlockError happens the current transaction is supposed to be
+# DBLockDeadlockError happens the current transaction is supposed to be
# aborted. If it doesn't then when the operation is attempted again
# the deadlock is still happening...
# --Robin
@@ -34,35 +28,47 @@
_sleep = sleep
del sleep
-import _db
+import _bsddb
-_deadlock_MinSleepTime = 1.0/64 # always sleep at least N seconds between retrys
-_deadlock_MaxSleepTime = 1.0 # never sleep more than N seconds between retrys
+_deadlock_MinSleepTime = 1.0/64 # always sleep at least N seconds between retrys
+_deadlock_MaxSleepTime = 3.14159 # never sleep more than N seconds between retrys
+_deadlock_VerboseFile = None # Assign a file object to this for a "sleeping"
+ # message to be written to it each retry
def DeadlockWrap(function, *_args, **_kwargs):
"""DeadlockWrap(function, *_args, **_kwargs) - automatically retries
function in case of a database deadlock.
- This is a DeadlockWrapper method which DB calls can be made using to
- preform infinite retrys with sleeps in between when a DBLockDeadlockError
- exception is raised in a database call:
+ This is a function intended to be used to wrap database calls such
+ that they perform retrys with exponentially backing off sleeps in
+ between when a DBLockDeadlockError exception is raised.
+
+ A 'max_retries' parameter may optionally be passed to prevent it
+ from retrying forever (in which case the exception will be reraised).
d = DB(...)
d.open(...)
DeadlockWrap(d.put, "foo", data="bar") # set key "foo" to "bar"
"""
sleeptime = _deadlock_MinSleepTime
- while (1) :
+ max_retries = _kwargs.get('max_retries', -1)
+ if _kwargs.has_key('max_retries'):
+ del _kwargs['max_retries']
+ while 1:
try:
return apply(function, _args, _kwargs)
except _db.DBLockDeadlockError:
- print 'DeadlockWrap sleeping ', sleeptime
+ if _deadlock_VerboseFile:
+ _deadlock_VerboseFile.write('dbutils.DeadlockWrap: sleeping %1.3f\n' % sleeptime)
_sleep(sleeptime)
# exponential backoff in the sleep time
sleeptime = sleeptime * 2
if sleeptime > _deadlock_MaxSleepTime :
sleeptime = _deadlock_MaxSleepTime
+ max_retries = max_retries - 1
+ if max_retries == -1:
+ raise
#------------------------------------------------------------------------
diff --git a/Lib/bsddb/test/test_dbshelve.py b/Lib/bsddb/test/test_dbshelve.py
index bed7212..580c75f 100644
--- a/Lib/bsddb/test/test_dbshelve.py
+++ b/Lib/bsddb/test/test_dbshelve.py
@@ -210,7 +210,7 @@
class HashShelveTestCase(BasicShelveTestCase):
- dbtype = db.DB_BTREE
+ dbtype = db.DB_HASH
dbflags = db.DB_CREATE
@@ -220,7 +220,7 @@
class ThreadHashShelveTestCase(BasicShelveTestCase):
- dbtype = db.DB_BTREE
+ dbtype = db.DB_HASH
dbflags = db.DB_CREATE | db.DB_THREAD
@@ -261,7 +261,7 @@
class EnvHashShelveTestCase(BasicEnvShelveTestCase):
envflags = 0
- dbtype = db.DB_BTREE
+ dbtype = db.DB_HASH
dbflags = db.DB_CREATE
@@ -273,7 +273,7 @@
class EnvThreadHashShelveTestCase(BasicEnvShelveTestCase):
envflags = db.DB_THREAD
- dbtype = db.DB_BTREE
+ dbtype = db.DB_HASH
dbflags = db.DB_CREATE | db.DB_THREAD
diff --git a/Lib/bsddb/test/test_dbtables.py b/Lib/bsddb/test/test_dbtables.py
index 9809945..a346585 100644
--- a/Lib/bsddb/test/test_dbtables.py
+++ b/Lib/bsddb/test/test_dbtables.py
@@ -5,6 +5,7 @@
#-----------------------------------------------------------------------
#
# Copyright (C) 2000, 2001 by Autonomous Zone Industries
+# Copyright (C) 2002 Gregory P. Smith
#
# March 20, 2000
#
@@ -159,6 +160,40 @@
assert values[0]['b'] == "bad"
+ def test04_MultiCondSelect(self):
+ tabname = "test04_MultiCondSelect"
+ try:
+ self.tdb.Drop(tabname)
+ except dbtables.TableDBError:
+ pass
+ self.tdb.CreateTable(tabname, ['a', 'b', 'c', 'd', 'e'])
+
+ try:
+ self.tdb.Insert(tabname, {'a': "", 'e': pickle.dumps([{4:5, 6:7}, 'foo'], 1), 'f': "Zero"})
+ assert 0
+ except dbtables.TableDBError:
+ pass
+
+ self.tdb.Insert(tabname, {'a': "A", 'b': "B", 'c': "C", 'd': "D", 'e': "E"})
+ self.tdb.Insert(tabname, {'a': "-A", 'b': "-B", 'c': "-C", 'd': "-D", 'e': "-E"})
+ self.tdb.Insert(tabname, {'a': "A-", 'b': "B-", 'c': "C-", 'd': "D-", 'e': "E-"})
+
+ if verbose:
+ self.tdb._db_print()
+
+ # This select should return 0 rows. it is designed to test
+ # the bug identified and fixed in sourceforge bug # 590449
+ # (Big Thanks to "Rob Tillotson (n9mtb)" for tracking this down
+ # and supplying a fix!! This one caused many headaches to say
+ # the least...)
+ values = self.tdb.Select(tabname, ['b', 'a', 'd'],
+ conditions={'e': dbtables.ExactCond('E'),
+ 'a': dbtables.ExactCond('A'),
+ 'd': dbtables.PrefixCond('-')
+ } )
+ assert len(values) == 0, values
+
+
def test_CreateOrExtend(self):
tabname = "test_CreateOrExtend"
diff --git a/Lib/bsddb/test/test_thread.py b/Lib/bsddb/test/test_thread.py
index 60d6dc5..6231f7d 100644
--- a/Lib/bsddb/test/test_thread.py
+++ b/Lib/bsddb/test/test_thread.py
@@ -18,7 +18,7 @@
import unittest
from test.test_support import verbose
-from bsddb import db
+from bsddb import db, dbutils
#----------------------------------------------------------------------
@@ -31,6 +31,9 @@
def setUp(self):
+ if verbose:
+ dbutils._deadlock_VerboseFile = sys.stdout
+
homeDir = os.path.join(os.path.dirname(sys.argv[0]), 'db_home')
self.homeDir = homeDir
try: os.mkdir(homeDir)
@@ -109,7 +112,7 @@
for x in range(start, stop):
key = '%04d' % x
- d.put(key, self.makeData(key))
+ dbutils.DeadlockWrap(d.put, key, self.makeData(key), max_retries=12)
if verbose and x % 100 == 0:
print "%s: records %d - %d finished" % (name, start, x)
@@ -212,7 +215,7 @@
# create a bunch of records
for x in xrange(start, stop):
key = '%04d' % x
- d.put(key, self.makeData(key))
+ dbutils.DeadlockWrap(d.put, key, self.makeData(key), max_retries=12)
if verbose and x % 100 == 0:
print "%s: records %d - %d finished" % (name, start, x)
@@ -221,12 +224,12 @@
if random() <= 0.05:
for y in xrange(start, x):
key = '%04d' % x
- data = d.get(key)
+ data = dbutils.DeadlockWrap(d.get, key, max_retries=12)
assert data == self.makeData(key)
# flush them
try:
- d.sync()
+ dbutils.DeadlockWrap(d.sync, max_retries=12)
except db.DBIncompleteError, val:
if verbose:
print "could not complete sync()..."
@@ -234,12 +237,12 @@
# read them back, deleting a few
for x in xrange(start, stop):
key = '%04d' % x
- data = d.get(key)
+ data = dbutils.DeadlockWrap(d.get, key, max_retries=12)
if verbose and x % 100 == 0:
print "%s: fetched record (%s, %s)" % (name, key, data)
- assert data == self.makeData(key)
+ assert data == self.makeData(key), (key, data, self.makeData(key))
if random() <= 0.10:
- d.delete(key)
+ dbutils.DeadlockWrap(d.delete, key, max_retries=12)
if verbose:
print "%s: deleted record %s" % (name, key)
@@ -273,7 +276,7 @@
class HashSimpleThreaded(SimpleThreadedBase):
- dbtype = db.DB_BTREE
+ dbtype = db.DB_HASH
#----------------------------------------------------------------------