bsddb 4.1.6:
* Extended DB & DBEnv set_get_returns_none functionality to take a
"level" instead of a boolean flag. The boolean 0 and 1 values still
have the same effect. A value of 2 extends the "return None instead
of raising an exception" behaviour to the DBCursor set methods.
This will become the default behaviour in pybsddb 4.2.
* Fixed a typo in DBCursor.join_item method that made it crash instead
of returning a value. Obviously nobody uses it. Wrote a test case
for join and join_item.
diff --git a/Lib/bsddb/dbobj.py b/Lib/bsddb/dbobj.py
index b2632a1..d23f533 100644
--- a/Lib/bsddb/dbobj.py
+++ b/Lib/bsddb/dbobj.py
@@ -15,6 +15,12 @@
# implied.
#
+#
+# TODO it would be *really nice* to have an automatic shadow class populator
+# so that new methods don't need to be added here manually after being
+# added to _bsddb.c.
+#
+
import db
try:
@@ -57,6 +63,8 @@
return apply(self._cobj.set_lk_max_objects, args, kwargs)
def set_mp_mmapsize(self, *args, **kwargs):
return apply(self._cobj.set_mp_mmapsize, args, kwargs)
+ def set_timeout(self, *args, **kwargs):
+ return apply(self._cobj.set_timeout, args, kwargs)
def set_tmp_dir(self, *args, **kwargs):
return apply(self._cobj.set_tmp_dir, args, kwargs)
def txn_begin(self, *args, **kwargs):
diff --git a/Lib/bsddb/dbtables.py b/Lib/bsddb/dbtables.py
index 85dbb4a..d052ca5 100644
--- a/Lib/bsddb/dbtables.py
+++ b/Lib/bsddb/dbtables.py
@@ -155,6 +155,9 @@
if truncate:
myflags |= DB_TRUNCATE
self.db = DB(self.env)
+ # this code relies on DBCursor.set* methods to raise exceptions
+ # rather than returning None
+ self.db.set_get_returns_none(1)
# allow duplicate entries [warning: be careful w/ metadata]
self.db.set_flags(DB_DUP)
self.db.open(filename, DB_BTREE, dbflags | myflags, mode)
diff --git a/Lib/bsddb/test/test_associate.py b/Lib/bsddb/test/test_associate.py
index 3061e45..5fe41a9 100644
--- a/Lib/bsddb/test/test_associate.py
+++ b/Lib/bsddb/test/test_associate.py
@@ -1,5 +1,5 @@
"""
-TestCases for multi-threaded access to a DB.
+TestCases for DB.associate.
"""
import sys, os, string
diff --git a/Lib/bsddb/test/test_basics.py b/Lib/bsddb/test/test_basics.py
index 224aef4..dbab231 100644
--- a/Lib/bsddb/test/test_basics.py
+++ b/Lib/bsddb/test/test_basics.py
@@ -282,11 +282,11 @@
#----------------------------------------
- def test03_SimpleCursorStuff(self):
+ def test03_SimpleCursorStuff(self, get_raises_error=0, set_raises_error=1):
if verbose:
print '\n', '-=' * 30
- print "Running %s.test03_SimpleCursorStuff..." % \
- self.__class__.__name__
+ print "Running %s.test03_SimpleCursorStuff (get_error %s, set_error %s)..." % \
+ (self.__class__.__name__, get_raises_error, set_raises_error)
if self.env and self.dbopenflags & db.DB_AUTO_COMMIT:
txn = self.env.txn_begin()
@@ -300,7 +300,15 @@
count = count + 1
if verbose and count % 100 == 0:
print rec
- rec = c.next()
+ try:
+ rec = c.next()
+ except db.DBNotFoundError, val:
+ if get_raises_error:
+ assert val[0] == db.DB_NOTFOUND
+ if verbose: print val
+ rec = None
+ else:
+ self.fail("unexpected DBNotFoundError")
assert count == 1000
@@ -311,7 +319,15 @@
count = count + 1
if verbose and count % 100 == 0:
print rec
- rec = c.prev()
+ try:
+ rec = c.prev()
+ except db.DBNotFoundError, val:
+ if get_raises_error:
+ assert val[0] == db.DB_NOTFOUND
+ if verbose: print val
+ rec = None
+ else:
+ self.fail("unexpected DBNotFoundError")
assert count == 1000
@@ -322,23 +338,29 @@
assert rec[1] == self.makeData('0505')
try:
- c.set('bad key')
+ n = c.set('bad key')
except db.DBNotFoundError, val:
assert val[0] == db.DB_NOTFOUND
if verbose: print val
else:
- self.fail("expected exception")
+ if set_raises_error:
+ self.fail("expected exception")
+ if n != None:
+ self.fail("expected None: "+`n`)
rec = c.get_both('0404', self.makeData('0404'))
assert rec == ('0404', self.makeData('0404'))
try:
- c.get_both('0404', 'bad data')
+ n = c.get_both('0404', 'bad data')
except db.DBNotFoundError, val:
assert val[0] == db.DB_NOTFOUND
if verbose: print val
else:
- self.fail("expected exception")
+ if get_raises_error:
+ self.fail("expected exception")
+ if n != None:
+ self.fail("expected None: "+`n`)
if self.d.get_type() == db.DB_BTREE:
rec = c.set_range('011')
@@ -414,6 +436,29 @@
# SF pybsddb bug id 667343
del oldcursor
+ def test03b_SimpleCursorWithoutGetReturnsNone0(self):
+ # same test but raise exceptions instead of returning None
+ if verbose:
+ print '\n', '-=' * 30
+ print "Running %s.test03b_SimpleCursorStuffWithoutGetReturnsNone..." % \
+ self.__class__.__name__
+
+ old = self.d.set_get_returns_none(0)
+ assert old == 1
+ self.test03_SimpleCursorStuff(get_raises_error=1, set_raises_error=1)
+
+ def test03c_SimpleCursorGetReturnsNone2(self):
+ # same test but raise exceptions instead of returning None
+ if verbose:
+ print '\n', '-=' * 30
+ print "Running %s.test03c_SimpleCursorStuffWithoutSetReturnsNone..." % \
+ self.__class__.__name__
+
+ old = self.d.set_get_returns_none(2)
+ assert old == 1
+ old = self.d.set_get_returns_none(2)
+ assert old == 2
+ self.test03_SimpleCursorStuff(get_raises_error=0, set_raises_error=0)
#----------------------------------------
diff --git a/Lib/bsddb/test/test_join.py b/Lib/bsddb/test/test_join.py
index ab75ba1..08784ac 100644
--- a/Lib/bsddb/test/test_join.py
+++ b/Lib/bsddb/test/test_join.py
@@ -1,9 +1,117 @@
"""TestCases for using the DB.join and DBCursor.join_item methods.
"""
+import sys, os, string
+import tempfile
+import time
+from pprint import pprint
+
+try:
+ from threading import Thread, currentThread
+ have_threads = 1
+except ImportError:
+ have_threads = 0
+
import unittest
+from test_all import verbose
+
+try:
+ # For Python 2.3
+ from bsddb import db, dbshelve
+except ImportError:
+ # For earlier Pythons w/distutils pybsddb
+ from bsddb3 import db, dbshelve
+
+
+#----------------------------------------------------------------------
+
+ProductIndex = [
+ ('apple', "Convenience Store"),
+ ('blueberry', "Farmer's Market"),
+ ('shotgun', "S-Mart"), # Aisle 12
+ ('pear', "Farmer's Market"),
+ ('chainsaw', "S-Mart"), # "Shop smart. Shop S-Mart!"
+ ('strawberry', "Farmer's Market"),
+]
+
+ColorIndex = [
+ ('blue', "blueberry"),
+ ('red', "apple"),
+ ('red', "chainsaw"),
+ ('red', "strawberry"),
+ ('yellow', "peach"),
+ ('yellow', "pear"),
+ ('black', "shotgun"),
+]
+
+class JoinTestCase(unittest.TestCase):
+ keytype = ''
+
+ def setUp(self):
+ self.filename = self.__class__.__name__ + '.db'
+ homeDir = os.path.join(os.path.dirname(sys.argv[0]), 'db_home')
+ self.homeDir = homeDir
+ try: os.mkdir(homeDir)
+ except os.error: pass
+ self.env = db.DBEnv()
+ self.env.open(homeDir, db.DB_CREATE | db.DB_INIT_MPOOL | db.DB_INIT_LOCK )
+
+ def tearDown(self):
+ self.env.close()
+ import glob
+ files = glob.glob(os.path.join(self.homeDir, '*'))
+ for file in files:
+ os.remove(file)
+
+ def test01_join(self):
+ if verbose:
+ print '\n', '-=' * 30
+ print "Running %s.test01_join..." % \
+ self.__class__.__name__
+
+ # create and populate primary index
+ priDB = db.DB(self.env)
+ priDB.open(self.filename, "primary", db.DB_BTREE, db.DB_CREATE)
+ map(lambda t: apply(priDB.put, t), ProductIndex)
+
+ # create and populate secondary index
+ secDB = db.DB(self.env)
+ secDB.set_flags(db.DB_DUP | db.DB_DUPSORT)
+ secDB.open(self.filename, "secondary", db.DB_BTREE, db.DB_CREATE)
+ map(lambda t: apply(secDB.put, t), ColorIndex)
+
+ sCursor = None
+ jCursor = None
+ try:
+ # lets look up all of the red Products
+ sCursor = secDB.cursor()
+ assert sCursor.set('red')
+
+ # FIXME: jCursor doesn't properly hold a reference to its
+ # cursors, if they are closed before jcursor is used it
+ # can cause a crash.
+ jCursor = priDB.join([sCursor])
+
+ if jCursor.get(0) != ('apple', "Convenience Store"):
+ self.fail("join cursor positioned wrong")
+ if jCursor.join_item() != 'chainsaw':
+ self.fail("DBCursor.join_item returned wrong item")
+ if jCursor.get(0)[0] != 'strawberry':
+ self.fail("join cursor returned wrong thing")
+ if jCursor.get(0): # there were only three red items to return
+ self.fail("join cursor returned too many items")
+ finally:
+ if jCursor:
+ jCursor.close()
+ if sCursor:
+ sCursor.close()
+ priDB.close()
+ secDB.close()
def test_suite():
suite = unittest.TestSuite()
+
+ suite.addTest(unittest.makeSuite(JoinTestCase))
+
return suite
diff --git a/Modules/_bsddb.c b/Modules/_bsddb.c
index b74491c..227fe7b 100644
--- a/Modules/_bsddb.c
+++ b/Modules/_bsddb.c
@@ -93,7 +93,7 @@
/* 40 = 4.0, 33 = 3.3; this will break if the second number is > 9 */
#define DBVER (DB_VERSION_MAJOR * 10 + DB_VERSION_MINOR)
-#define PY_BSDDB_VERSION "4.1.5"
+#define PY_BSDDB_VERSION "4.1.6"
static char *rcs_id = "$Id$";
@@ -141,12 +141,6 @@
#endif
-
-/* What is the default behaviour when DB->get or DBCursor->get returns a
- DB_NOTFOUND error? Return None or raise an exception? */
-#define GET_RETURNS_NONE_DEFAULT 1
-
-
/* Should DB_INCOMPLETE be turned into a warning or an exception? */
#define INCOMPLETE_IS_WARNING 1
@@ -189,12 +183,24 @@
/* --------------------------------------------------------------------- */
/* Structure definitions */
+struct behaviourFlags {
+ /* What is the default behaviour when DB->get or DBCursor->get returns a
+ DB_NOTFOUND error? Return None or raise an exception? */
+ unsigned int getReturnsNone : 1;
+ /* What is the default behaviour for DBCursor.set* methods when DBCursor->get
+ * returns a DB_NOTFOUND error? Return None or raise an exception? */
+ unsigned int cursorSetReturnsNone : 1;
+};
+
+#define DEFAULT_GET_RETURNS_NONE 1
+#define DEFAULT_CURSOR_SET_RETURNS_NONE 0 /* 0 in pybsddb < 4.2, python < 2.4 */
+
typedef struct {
PyObject_HEAD
DB_ENV* db_env;
u_int32_t flags; /* saved flags from open() */
int closed;
- int getReturnsNone;
+ struct behaviourFlags moduleFlags;
} DBEnvObject;
@@ -205,7 +211,7 @@
u_int32_t flags; /* saved flags from open() */
u_int32_t setflags; /* saved flags from set_flags() */
int haveStat;
- int getReturnsNone;
+ struct behaviourFlags moduleFlags;
#if (DBVER >= 33)
PyObject* associateCallback;
int primaryDBType;
@@ -595,7 +601,7 @@
err = self->dbc->c_get(self->dbc, &key, &data, flags);
MYDB_END_ALLOW_THREADS;
- if ((err == DB_NOTFOUND) && self->mydb->getReturnsNone) {
+ if ((err == DB_NOTFOUND) && self->mydb->moduleFlags.getReturnsNone) {
Py_INCREF(Py_None);
retval = Py_None;
}
@@ -681,9 +687,10 @@
}
if (self->myenvobj)
- self->getReturnsNone = self->myenvobj->getReturnsNone;
+ self->moduleFlags = self->myenvobj->moduleFlags;
else
- self->getReturnsNone = GET_RETURNS_NONE_DEFAULT;
+ self->moduleFlags.getReturnsNone = DEFAULT_GET_RETURNS_NONE;
+ self->moduleFlags.cursorSetReturnsNone = DEFAULT_CURSOR_SET_RETURNS_NONE;
MYDB_BEGIN_ALLOW_THREADS;
err = db_create(&self->db, db_env, flags);
@@ -797,7 +804,8 @@
self->closed = 1;
self->flags = flags;
- self->getReturnsNone = GET_RETURNS_NONE_DEFAULT;
+ self->moduleFlags.getReturnsNone = DEFAULT_GET_RETURNS_NONE;
+ self->moduleFlags.cursorSetReturnsNone = DEFAULT_CURSOR_SET_RETURNS_NONE;
MYDB_BEGIN_ALLOW_THREADS;
err = db_env_create(&self->db_env, flags);
@@ -1182,7 +1190,7 @@
err = self->db->get(self->db, txn, &key, &data, flags|consume_flag);
MYDB_END_ALLOW_THREADS;
- if ((err == DB_NOTFOUND) && self->getReturnsNone) {
+ if ((err == DB_NOTFOUND) && self->moduleFlags.getReturnsNone) {
err = 0;
Py_INCREF(Py_None);
retval = Py_None;
@@ -1324,7 +1332,7 @@
Py_INCREF(dfltobj);
retval = dfltobj;
}
- else if ((err == DB_NOTFOUND) && self->getReturnsNone) {
+ else if ((err == DB_NOTFOUND) && self->moduleFlags.getReturnsNone) {
err = 0;
Py_INCREF(Py_None);
retval = Py_None;
@@ -1424,7 +1432,7 @@
err = self->db->get(self->db, txn, &key, &data, flags);
MYDB_END_ALLOW_THREADS;
- if ((err == DB_NOTFOUND) && self->getReturnsNone) {
+ if ((err == DB_NOTFOUND) && self->moduleFlags.getReturnsNone) {
err = 0;
Py_INCREF(Py_None);
retval = Py_None;
@@ -1525,6 +1533,11 @@
free(cursors);
RETURN_IF_ERR();
+ // FIXME: this is a buggy interface. The returned cursor
+ // contains internal references to the passed in cursors
+ // but does not hold python references to them or prevent
+ // them from being closed prematurely. This can cause
+ // python to crash when things are done in the wrong order.
return (PyObject*) newDBCursorObject(dbc, self);
}
@@ -2156,14 +2169,18 @@
DB_set_get_returns_none(DBObject* self, PyObject* args)
{
int flags=0;
- int oldValue;
+ int oldValue=0;
if (!PyArg_ParseTuple(args,"i:set_get_returns_none", &flags))
return NULL;
CHECK_DB_NOT_CLOSED(self);
- oldValue = self->getReturnsNone;
- self->getReturnsNone = flags;
+ if (self->moduleFlags.getReturnsNone)
+ ++oldValue;
+ if (self->moduleFlags.cursorSetReturnsNone)
+ ++oldValue;
+ self->moduleFlags.getReturnsNone = (flags >= 1);
+ self->moduleFlags.cursorSetReturnsNone = (flags >= 2);
return PyInt_FromLong(oldValue);
}
@@ -2643,7 +2660,7 @@
MYDB_END_ALLOW_THREADS;
- if ((err == DB_NOTFOUND) && self->mydb->getReturnsNone) {
+ if ((err == DB_NOTFOUND) && self->mydb->moduleFlags.getReturnsNone) {
Py_INCREF(Py_None);
retval = Py_None;
}
@@ -2790,7 +2807,11 @@
MYDB_BEGIN_ALLOW_THREADS;
err = self->dbc->c_get(self->dbc, &key, &data, flags|DB_SET);
MYDB_END_ALLOW_THREADS;
- if (makeDBError(err)) {
+ if ((err == DB_NOTFOUND) && self->mydb->moduleFlags.cursorSetReturnsNone) {
+ Py_INCREF(Py_None);
+ retval = Py_None;
+ }
+ else if (makeDBError(err)) {
retval = NULL;
}
else {
@@ -2848,7 +2869,11 @@
MYDB_BEGIN_ALLOW_THREADS;
err = self->dbc->c_get(self->dbc, &key, &data, flags|DB_SET_RANGE);
MYDB_END_ALLOW_THREADS;
- if (makeDBError(err)) {
+ if ((err == DB_NOTFOUND) && self->mydb->moduleFlags.cursorSetReturnsNone) {
+ Py_INCREF(Py_None);
+ retval = Py_None;
+ }
+ else if (makeDBError(err)) {
retval = NULL;
}
else {
@@ -2875,19 +2900,15 @@
return retval;
}
-
static PyObject*
-DBC_get_both(DBCursorObject* self, PyObject* args)
+_DBC_get_set_both(DBCursorObject* self, PyObject* keyobj, PyObject* dataobj,
+ int flags, unsigned int returnsNone)
{
- int err, flags=0;
+ int err;
DBT key, data;
- PyObject* retval, *keyobj, *dataobj;
+ PyObject* retval;
- if (!PyArg_ParseTuple(args, "OO|i:get_both", &keyobj, &dataobj, &flags))
- return NULL;
-
- CHECK_CURSOR_NOT_CLOSED(self);
-
+ // the caller did this: CHECK_CURSOR_NOT_CLOSED(self);
if (!make_key_dbt(self->mydb, keyobj, &key, NULL))
return NULL;
if (!make_dbt(dataobj, &data))
@@ -2896,7 +2917,11 @@
MYDB_BEGIN_ALLOW_THREADS;
err = self->dbc->c_get(self->dbc, &key, &data, flags|DB_GET_BOTH);
MYDB_END_ALLOW_THREADS;
- if (makeDBError(err)) {
+ if ((err == DB_NOTFOUND) && returnsNone) {
+ Py_INCREF(Py_None);
+ retval = Py_None;
+ }
+ else if (makeDBError(err)) {
retval = NULL;
}
else {
@@ -2922,6 +2947,38 @@
return retval;
}
+static PyObject*
+DBC_get_both(DBCursorObject* self, PyObject* args)
+{
+ int flags=0;
+ PyObject *keyobj, *dataobj;
+
+ if (!PyArg_ParseTuple(args, "OO|i:get_both", &keyobj, &dataobj, &flags))
+ return NULL;
+
+ // if the cursor is closed, self->mydb may be invalid
+ CHECK_CURSOR_NOT_CLOSED(self);
+
+ return _DBC_get_set_both(self, keyobj, dataobj, flags,
+ self->mydb->moduleFlags.getReturnsNone);
+}
+
+static PyObject*
+DBC_set_both(DBCursorObject* self, PyObject* args)
+{
+ int flags=0;
+ PyObject *keyobj, *dataobj;
+
+ if (!PyArg_ParseTuple(args, "OO|i:set_both", &keyobj, &dataobj, &flags))
+ return NULL;
+
+ // if the cursor is closed, self->mydb may be invalid
+ CHECK_CURSOR_NOT_CLOSED(self);
+
+ return _DBC_get_set_both(self, keyobj, dataobj, flags,
+ self->mydb->moduleFlags.cursorSetReturnsNone);
+}
+
static PyObject*
DBC_set_recno(DBCursorObject* self, PyObject* args, PyObject *kwargs)
@@ -2965,7 +3022,11 @@
MYDB_BEGIN_ALLOW_THREADS;
err = self->dbc->c_get(self->dbc, &key, &data, flags|DB_SET_RECNO);
MYDB_END_ALLOW_THREADS;
- if (makeDBError(err)) {
+ if ((err == DB_NOTFOUND) && self->mydb->moduleFlags.cursorSetReturnsNone) {
+ Py_INCREF(Py_None);
+ retval = Py_None;
+ }
+ else if (makeDBError(err)) {
retval = NULL;
}
else { /* Can only be used for BTrees, so no need to return int key */
@@ -3010,11 +3071,11 @@
static PyObject*
DBC_join_item(DBCursorObject* self, PyObject* args)
{
- int err;
+ int err, flags=0;
DBT key, data;
PyObject* retval;
- if (!PyArg_ParseTuple(args, ":join_item"))
+ if (!PyArg_ParseTuple(args, "|i:join_item", &flags))
return NULL;
CHECK_CURSOR_NOT_CLOSED(self);
@@ -3027,9 +3088,13 @@
}
MYDB_BEGIN_ALLOW_THREADS;
- err = self->dbc->c_get(self->dbc, &key, &data, DB_JOIN_ITEM);
+ err = self->dbc->c_get(self->dbc, &key, &data, flags | DB_JOIN_ITEM);
MYDB_END_ALLOW_THREADS;
- if (makeDBError(err)) {
+ if ((err == DB_NOTFOUND) && self->mydb->moduleFlags.getReturnsNone) {
+ Py_INCREF(Py_None);
+ retval = Py_None;
+ }
+ else if (makeDBError(err)) {
retval = NULL;
}
else {
@@ -3748,14 +3813,18 @@
DBEnv_set_get_returns_none(DBEnvObject* self, PyObject* args)
{
int flags=0;
- int oldValue;
+ int oldValue=0;
if (!PyArg_ParseTuple(args,"i:set_get_returns_none", &flags))
return NULL;
CHECK_ENV_NOT_CLOSED(self);
- oldValue = self->getReturnsNone;
- self->getReturnsNone = flags;
+ if (self->moduleFlags.getReturnsNone)
+ ++oldValue;
+ if (self->moduleFlags.cursorSetReturnsNone)
+ ++oldValue;
+ self->moduleFlags.getReturnsNone = (flags >= 1);
+ self->moduleFlags.cursorSetReturnsNone = (flags >= 2);
return PyInt_FromLong(oldValue);
}
@@ -3977,7 +4046,7 @@
{"set", (PyCFunction)DBC_set, METH_VARARGS|METH_KEYWORDS},
{"set_range", (PyCFunction)DBC_set_range, METH_VARARGS|METH_KEYWORDS},
{"get_both", (PyCFunction)DBC_get_both, METH_VARARGS},
- {"set_both", (PyCFunction)DBC_get_both, METH_VARARGS},
+ {"set_both", (PyCFunction)DBC_set_both, METH_VARARGS},
{"set_recno", (PyCFunction)DBC_set_recno, METH_VARARGS|METH_KEYWORDS},
{"consume", (PyCFunction)DBC_consume, METH_VARARGS|METH_KEYWORDS},
{"next_dup", (PyCFunction)DBC_next_dup, METH_VARARGS|METH_KEYWORDS},