blob: 84a766589082a7f055b6e126080bfdbb0a5fe74e [file] [log] [blame]
Guido van Rossum9f824a71995-08-10 19:29:28 +00001"""A dumb and slow but simple dbm clone.
2
3For database spam, spam.dir contains the index (a text file),
4spam.bak *may* contain a backup of the index (also a text file),
5while spam.dat contains the data (a binary file).
6
7XXX TO DO:
8
9- seems to contain a bug when updating...
10
11- reclaim free space (currently, space once occupied by deleted or expanded
12items is never reused)
13
14- support concurrent access (currently, if two processes take turns making
15updates, they can mess up the index)
16
17- support efficient access to large databases (currently, the whole index
18is read when the database is opened, and some updates rewrite the whole index)
19
20- support opening for read-only (flag = 'm')
21
22"""
23
Martin v. Löwisd0cd95c2001-07-19 10:06:39 +000024import os as _os
Guido van Rossum9f824a71995-08-10 19:29:28 +000025import __builtin__
Raymond Hettinger2dd1ed62003-06-28 07:08:39 +000026import UserDict
Guido van Rossum9f824a71995-08-10 19:29:28 +000027
28_open = __builtin__.open
29
30_BLOCKSIZE = 512
31
Tim Peters88869f92001-01-14 23:36:06 +000032error = IOError # For anydbm
Guido van Rossum22a18901996-05-28 22:58:40 +000033
Raymond Hettinger2dd1ed62003-06-28 07:08:39 +000034class _Database(UserDict.DictMixin):
Guido van Rossum9f824a71995-08-10 19:29:28 +000035
Tim Petersd7472ec2003-07-13 02:22:03 +000036 # The on-disk directory and data files can remain in mutually
37 # inconsistent states for an arbitrarily long time (see comments
38 # at the end of __setitem__). This is only repaired when _commit()
39 # gets called. One place _commit() gets called is from __del__(),
40 # and if that occurs at program shutdown time, module globals may
41 # already have gotten rebound to None. Since it's crucial that
Tim Peters03204642003-07-13 02:37:05 +000042 # _commit() finish successfully, we can't ignore shutdown races
Tim Petersd7472ec2003-07-13 02:22:03 +000043 # here, and _commit() must not reference any globals.
44 _os = _os # for _commit()
45 _open = _open # for _commit()
46
Tim Peters7dfd5702003-07-12 20:11:25 +000047 def __init__(self, filebasename, mode):
Fred Drake2c8373b2001-12-07 21:54:46 +000048 self._mode = mode
Tim Peters7dfd5702003-07-12 20:11:25 +000049
50 # The directory file is a text file. Each line looks like
51 # "%r, (%d, %d)\n" % (key, pos, siz)
52 # where key is the string key, pos is the offset into the dat
53 # file of the associated value's first byte, and siz is the number
54 # of bytes in the associated value.
55 self._dirfile = filebasename + _os.extsep + 'dir'
56
57 # The data file is a binary file pointed into by the directory
58 # file, and holds the values associated with keys. Each value
59 # begins at a _BLOCKSIZE-aligned byte offset, and is a raw
60 # binary 8-bit string value.
61 self._datfile = filebasename + _os.extsep + 'dat'
62 self._bakfile = filebasename + _os.extsep + 'bak'
63
64 # The index is an in-memory dict, mirroring the directory file.
65 self._index = None # maps keys to (pos, siz) pairs
66
Tim Peters88869f92001-01-14 23:36:06 +000067 # Mod by Jack: create data file if needed
68 try:
69 f = _open(self._datfile, 'r')
70 except IOError:
Fred Drake2c8373b2001-12-07 21:54:46 +000071 f = _open(self._datfile, 'w', self._mode)
Tim Peters88869f92001-01-14 23:36:06 +000072 f.close()
73 self._update()
Guido van Rossum9f824a71995-08-10 19:29:28 +000074
Tim Peters7dfd5702003-07-12 20:11:25 +000075 # Read directory file into the in-memory index dict.
Tim Peters88869f92001-01-14 23:36:06 +000076 def _update(self):
77 self._index = {}
78 try:
79 f = _open(self._dirfile)
80 except IOError:
81 pass
82 else:
Tim Peters7dfd5702003-07-12 20:11:25 +000083 for line in f:
Andrew M. Kuchlingecdad852005-06-07 19:36:10 +000084 line = line.rstrip()
Tim Peters7dfd5702003-07-12 20:11:25 +000085 key, pos_and_siz_pair = eval(line)
86 self._index[key] = pos_and_siz_pair
Tim Peters88869f92001-01-14 23:36:06 +000087 f.close()
88
Tim Peters7dfd5702003-07-12 20:11:25 +000089 # Write the index dict to the directory file. The original directory
90 # file (if any) is renamed with a .bak extension first. If a .bak
91 # file currently exists, it's deleted.
Tim Peters88869f92001-01-14 23:36:06 +000092 def _commit(self):
Tim Petersd7472ec2003-07-13 02:22:03 +000093 # CAUTION: It's vital that _commit() succeed, and _commit() can
94 # be called from __del__(). Therefore we must never reference a
95 # global in this routine.
Tim Peters7a6c7332003-07-13 17:21:10 +000096 if self._index is None:
97 return # nothing to do
98
Tim Peters7dfd5702003-07-12 20:11:25 +000099 try:
Tim Petersd7472ec2003-07-13 02:22:03 +0000100 self._os.unlink(self._bakfile)
101 except self._os.error:
Tim Peters7dfd5702003-07-12 20:11:25 +0000102 pass
103
104 try:
Tim Petersd7472ec2003-07-13 02:22:03 +0000105 self._os.rename(self._dirfile, self._bakfile)
106 except self._os.error:
Tim Peters7dfd5702003-07-12 20:11:25 +0000107 pass
108
Tim Petersd7472ec2003-07-13 02:22:03 +0000109 f = self._open(self._dirfile, 'w', self._mode)
Tim Peters3898a702003-07-12 20:23:09 +0000110 for key, pos_and_siz_pair in self._index.iteritems():
111 f.write("%r, %r\n" % (key, pos_and_siz_pair))
Tim Peters88869f92001-01-14 23:36:06 +0000112 f.close()
113
Skip Montanaro6d068152003-07-14 12:15:15 +0000114 sync = _commit
115
Tim Peters88869f92001-01-14 23:36:06 +0000116 def __getitem__(self, key):
117 pos, siz = self._index[key] # may raise KeyError
118 f = _open(self._datfile, 'rb')
119 f.seek(pos)
120 dat = f.read(siz)
121 f.close()
122 return dat
123
Tim Peters7dfd5702003-07-12 20:11:25 +0000124 # Append val to the data file, starting at a _BLOCKSIZE-aligned
125 # offset. The data file is first padded with NUL bytes (if needed)
126 # to get to an aligned offset. Return pair
127 # (starting offset of val, len(val))
Tim Peters88869f92001-01-14 23:36:06 +0000128 def _addval(self, val):
129 f = _open(self._datfile, 'rb+')
130 f.seek(0, 2)
131 pos = int(f.tell())
Guido van Rossum54e54c62001-09-04 19:14:14 +0000132 npos = ((pos + _BLOCKSIZE - 1) // _BLOCKSIZE) * _BLOCKSIZE
Tim Peters88869f92001-01-14 23:36:06 +0000133 f.write('\0'*(npos-pos))
134 pos = npos
Tim Peters88869f92001-01-14 23:36:06 +0000135 f.write(val)
136 f.close()
137 return (pos, len(val))
138
Tim Peters7dfd5702003-07-12 20:11:25 +0000139 # Write val to the data file, starting at offset pos. The caller
140 # is responsible for ensuring that there's enough room starting at
141 # pos to hold val, without overwriting some other value. Return
142 # pair (pos, len(val)).
Tim Peters88869f92001-01-14 23:36:06 +0000143 def _setval(self, pos, val):
144 f = _open(self._datfile, 'rb+')
145 f.seek(pos)
146 f.write(val)
147 f.close()
148 return (pos, len(val))
149
Tim Peters7dfd5702003-07-12 20:11:25 +0000150 # key is a new key whose associated value starts in the data file
Tim Peters1d8d7292003-07-13 02:05:47 +0000151 # at offset pos and with length siz. Add an index record to
152 # the in-memory index dict, and append one to the directory file.
Tim Peters7dfd5702003-07-12 20:11:25 +0000153 def _addkey(self, key, pos_and_siz_pair):
154 self._index[key] = pos_and_siz_pair
Fred Drake2c8373b2001-12-07 21:54:46 +0000155 f = _open(self._dirfile, 'a', self._mode)
Tim Peters7dfd5702003-07-12 20:11:25 +0000156 f.write("%r, %r\n" % (key, pos_and_siz_pair))
Tim Peters88869f92001-01-14 23:36:06 +0000157 f.close()
158
159 def __setitem__(self, key, val):
160 if not type(key) == type('') == type(val):
161 raise TypeError, "keys and values must be strings"
Tim Peters7dfd5702003-07-12 20:11:25 +0000162 if key not in self._index:
163 self._addkey(key, self._addval(val))
Tim Peters88869f92001-01-14 23:36:06 +0000164 else:
Tim Peters7dfd5702003-07-12 20:11:25 +0000165 # See whether the new value is small enough to fit in the
166 # (padded) space currently occupied by the old value.
Tim Peters88869f92001-01-14 23:36:06 +0000167 pos, siz = self._index[key]
Tim Petersef6573e2003-07-11 04:09:55 +0000168 oldblocks = (siz + _BLOCKSIZE - 1) // _BLOCKSIZE
169 newblocks = (len(val) + _BLOCKSIZE - 1) // _BLOCKSIZE
Tim Peters88869f92001-01-14 23:36:06 +0000170 if newblocks <= oldblocks:
Tim Peters7dfd5702003-07-12 20:11:25 +0000171 self._index[key] = self._setval(pos, val)
Tim Peters88869f92001-01-14 23:36:06 +0000172 else:
Tim Peters7dfd5702003-07-12 20:11:25 +0000173 # The new value doesn't fit in the (padded) space used
174 # by the old value. The blocks used by the old value are
175 # forever lost.
176 self._index[key] = self._addval(val)
177
178 # Note that _index may be out of synch with the directory
179 # file now: _setval() and _addval() don't update the directory
Tim Peters1d8d7292003-07-13 02:05:47 +0000180 # file. This also means that the on-disk directory and data
181 # files are in a mutually inconsistent state, and they'll
182 # remain that way until _commit() is called. Note that this
183 # is a disaster (for the database) if the program crashes
184 # (so that _commit() never gets called).
Tim Peters88869f92001-01-14 23:36:06 +0000185
186 def __delitem__(self, key):
Tim Peters7dfd5702003-07-12 20:11:25 +0000187 # The blocks used by the associated value are lost.
Tim Peters88869f92001-01-14 23:36:06 +0000188 del self._index[key]
Tim Peters7dfd5702003-07-12 20:11:25 +0000189 # XXX It's unclear why we do a _commit() here (the code always
190 # XXX has, so I'm not changing it). _setitem__ doesn't try to
191 # XXX keep the directory file in synch. Why should we? Or
192 # XXX why shouldn't __setitem__?
Tim Peters88869f92001-01-14 23:36:06 +0000193 self._commit()
194
195 def keys(self):
196 return self._index.keys()
197
198 def has_key(self, key):
Raymond Hettinger793d4b42002-06-01 14:25:41 +0000199 return key in self._index
Tim Peters88869f92001-01-14 23:36:06 +0000200
Fred Drakea7cc69e2001-05-03 04:55:47 +0000201 def __contains__(self, key):
Raymond Hettinger793d4b42002-06-01 14:25:41 +0000202 return key in self._index
Fred Drakea7cc69e2001-05-03 04:55:47 +0000203
204 def iterkeys(self):
205 return self._index.iterkeys()
206 __iter__ = iterkeys
207
Tim Peters88869f92001-01-14 23:36:06 +0000208 def __len__(self):
209 return len(self._index)
210
211 def close(self):
Anthony Baxtered905702001-12-21 05:13:37 +0000212 self._commit()
Tim Peters7a6c7332003-07-13 17:21:10 +0000213 self._index = self._datfile = self._dirfile = self._bakfile = None
Guido van Rossum9f824a71995-08-10 19:29:28 +0000214
Tim Peters7a6c7332003-07-13 17:21:10 +0000215 __del__ = close
Tim Peterse4418602002-02-16 07:34:19 +0000216
Anthony Baxtered905702001-12-21 05:13:37 +0000217
Guido van Rossum9f824a71995-08-10 19:29:28 +0000218
Fred Drake2c8373b2001-12-07 21:54:46 +0000219def open(file, flag=None, mode=0666):
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000220 """Open the database file, filename, and return corresponding object.
221
222 The flag argument, used to control how the database is opened in the
223 other DBM implementations, is ignored in the dumbdbm module; the
224 database is always opened for update, and will be created if it does
225 not exist.
226
227 The optional mode argument is the UNIX mode of the file, used only when
228 the database has to be created. It defaults to octal code 0666 (and
229 will be modified by the prevailing umask).
230
231 """
Neal Norwitzf421e812003-03-01 22:58:00 +0000232 # flag argument is currently ignored
Fred Drake2c8373b2001-12-07 21:54:46 +0000233 return _Database(file, mode)