blob: 8b00460ec6db8b0945606e36b18a85913f93c653 [file] [log] [blame]
Benjamin Peterson90f5ba52010-03-11 22:53:45 +00001#! /usr/bin/env python3
Guido van Rossum62448671996-09-17 21:33:15 +00002
Thomas Wouters477c8d52006-05-27 19:21:47 +00003"""Read/write support for Maildir, mbox, MH, Babyl, and MMDF mailboxes."""
Guido van Rossum62448671996-09-17 21:33:15 +00004
Thomas Wouters89f507f2006-12-13 04:49:30 +00005# Notes for authors of new mailbox subclasses:
6#
7# Remember to fsync() changes to disk before closing a modified file
8# or returning from a flush() method. See functions _sync_flush() and
9# _sync_close().
10
Thomas Wouters477c8d52006-05-27 19:21:47 +000011import sys
Jack Jansen97157791995-10-23 13:59:53 +000012import os
Thomas Wouters477c8d52006-05-27 19:21:47 +000013import time
14import calendar
15import socket
16import errno
17import copy
R. David Murrayb7deff12011-01-30 06:21:28 +000018import warnings
Thomas Wouters477c8d52006-05-27 19:21:47 +000019import email
Thomas Woutersb2137042007-02-01 18:02:27 +000020import email.message
21import email.generator
Guido van Rossum34d19282007-08-09 01:03:29 +000022import io
R David Murray05ff9902011-06-17 12:54:56 -040023import contextlib
Thomas Wouters477c8d52006-05-27 19:21:47 +000024try:
Thomas Wouters0e3f5912006-08-11 14:57:12 +000025 if sys.platform == 'os2emx':
26 # OS/2 EMX fcntl() not adequate
27 raise ImportError
28 import fcntl
Thomas Wouters477c8d52006-05-27 19:21:47 +000029except ImportError:
30 fcntl = None
Guido van Rossumc7b68821994-04-28 09:53:33 +000031
Thomas Wouters477c8d52006-05-27 19:21:47 +000032__all__ = [ 'Mailbox', 'Maildir', 'mbox', 'MH', 'Babyl', 'MMDF',
33 'Message', 'MaildirMessage', 'mboxMessage', 'MHMessage',
Benjamin Peterson8041d472008-05-11 16:17:24 +000034 'BabylMessage', 'MMDFMessage']
Thomas Wouters477c8d52006-05-27 19:21:47 +000035
R. David Murrayb7deff12011-01-30 06:21:28 +000036linesep = os.linesep.encode('ascii')
37
Thomas Wouters477c8d52006-05-27 19:21:47 +000038class Mailbox:
39 """A group of messages in a particular place."""
40
41 def __init__(self, path, factory=None, create=True):
42 """Initialize a Mailbox instance."""
43 self._path = os.path.abspath(os.path.expanduser(path))
44 self._factory = factory
45
46 def add(self, message):
47 """Add message and return assigned key."""
48 raise NotImplementedError('Method must be implemented by subclass')
49
50 def remove(self, key):
51 """Remove the keyed message; raise KeyError if it doesn't exist."""
52 raise NotImplementedError('Method must be implemented by subclass')
53
54 def __delitem__(self, key):
55 self.remove(key)
56
57 def discard(self, key):
58 """If the keyed message exists, remove it."""
59 try:
60 self.remove(key)
61 except KeyError:
62 pass
63
64 def __setitem__(self, key, message):
65 """Replace the keyed message; raise KeyError if it doesn't exist."""
66 raise NotImplementedError('Method must be implemented by subclass')
67
68 def get(self, key, default=None):
69 """Return the keyed message, or default if it doesn't exist."""
70 try:
71 return self.__getitem__(key)
72 except KeyError:
73 return default
74
75 def __getitem__(self, key):
76 """Return the keyed message; raise KeyError if it doesn't exist."""
77 if not self._factory:
78 return self.get_message(key)
79 else:
R David Murray05ff9902011-06-17 12:54:56 -040080 with contextlib.closing(self.get_file(key)) as file:
81 return self._factory(file)
Thomas Wouters477c8d52006-05-27 19:21:47 +000082
83 def get_message(self, key):
84 """Return a Message representation or raise a KeyError."""
85 raise NotImplementedError('Method must be implemented by subclass')
86
87 def get_string(self, key):
R. David Murrayb7deff12011-01-30 06:21:28 +000088 """Return a string representation or raise a KeyError.
89
90 Uses email.message.Message to create a 7bit clean string
91 representation of the message."""
92 return email.message_from_bytes(self.get_bytes(key)).as_string()
93
94 def get_bytes(self, key):
95 """Return a byte string representation or raise a KeyError."""
Thomas Wouters477c8d52006-05-27 19:21:47 +000096 raise NotImplementedError('Method must be implemented by subclass')
97
98 def get_file(self, key):
99 """Return a file-like representation or raise a KeyError."""
100 raise NotImplementedError('Method must be implemented by subclass')
101
102 def iterkeys(self):
103 """Return an iterator over keys."""
104 raise NotImplementedError('Method must be implemented by subclass')
105
106 def keys(self):
107 """Return a list of keys."""
Guido van Rossumdde4c2c2007-02-12 00:22:55 +0000108 return list(self.iterkeys())
Thomas Wouters477c8d52006-05-27 19:21:47 +0000109
110 def itervalues(self):
111 """Return an iterator over all messages."""
Guido van Rossumcc2b0162007-02-11 06:12:03 +0000112 for key in self.keys():
Thomas Wouters477c8d52006-05-27 19:21:47 +0000113 try:
114 value = self[key]
115 except KeyError:
116 continue
117 yield value
118
119 def __iter__(self):
Guido van Rossumb5a755e2007-07-18 18:15:48 +0000120 return self.itervalues()
Thomas Wouters477c8d52006-05-27 19:21:47 +0000121
122 def values(self):
123 """Return a list of messages. Memory intensive."""
Guido van Rossumdde4c2c2007-02-12 00:22:55 +0000124 return list(self.itervalues())
Thomas Wouters477c8d52006-05-27 19:21:47 +0000125
126 def iteritems(self):
127 """Return an iterator over (key, message) tuples."""
Guido van Rossumcc2b0162007-02-11 06:12:03 +0000128 for key in self.keys():
Thomas Wouters477c8d52006-05-27 19:21:47 +0000129 try:
130 value = self[key]
131 except KeyError:
132 continue
133 yield (key, value)
134
135 def items(self):
136 """Return a list of (key, message) tuples. Memory intensive."""
Guido van Rossumdde4c2c2007-02-12 00:22:55 +0000137 return list(self.iteritems())
Thomas Wouters477c8d52006-05-27 19:21:47 +0000138
Guido van Rossume2b70bc2006-08-18 22:13:04 +0000139 def __contains__(self, key):
Thomas Wouters477c8d52006-05-27 19:21:47 +0000140 """Return True if the keyed message exists, False otherwise."""
141 raise NotImplementedError('Method must be implemented by subclass')
142
Thomas Wouters477c8d52006-05-27 19:21:47 +0000143 def __len__(self):
144 """Return a count of messages in the mailbox."""
145 raise NotImplementedError('Method must be implemented by subclass')
146
147 def clear(self):
148 """Delete all messages."""
Guido van Rossumcc2b0162007-02-11 06:12:03 +0000149 for key in self.keys():
Thomas Wouters477c8d52006-05-27 19:21:47 +0000150 self.discard(key)
151
152 def pop(self, key, default=None):
153 """Delete the keyed message and return it, or default."""
154 try:
155 result = self[key]
156 except KeyError:
157 return default
158 self.discard(key)
159 return result
160
161 def popitem(self):
162 """Delete an arbitrary (key, message) pair and return it."""
Guido van Rossumcc2b0162007-02-11 06:12:03 +0000163 for key in self.keys():
Thomas Wouters477c8d52006-05-27 19:21:47 +0000164 return (key, self.pop(key)) # This is only run once.
165 else:
166 raise KeyError('No messages in mailbox')
167
168 def update(self, arg=None):
169 """Change the messages that correspond to certain keys."""
170 if hasattr(arg, 'iteritems'):
Guido van Rossumcc2b0162007-02-11 06:12:03 +0000171 source = arg.items()
Thomas Wouters477c8d52006-05-27 19:21:47 +0000172 elif hasattr(arg, 'items'):
173 source = arg.items()
174 else:
175 source = arg
176 bad_key = False
177 for key, message in source:
178 try:
179 self[key] = message
180 except KeyError:
181 bad_key = True
182 if bad_key:
183 raise KeyError('No message with key(s)')
184
185 def flush(self):
186 """Write any pending changes to the disk."""
187 raise NotImplementedError('Method must be implemented by subclass')
188
189 def lock(self):
190 """Lock the mailbox."""
191 raise NotImplementedError('Method must be implemented by subclass')
192
193 def unlock(self):
194 """Unlock the mailbox if it is locked."""
195 raise NotImplementedError('Method must be implemented by subclass')
196
197 def close(self):
198 """Flush and close the mailbox."""
199 raise NotImplementedError('Method must be implemented by subclass')
200
R. David Murrayb7deff12011-01-30 06:21:28 +0000201 def _string_to_bytes(self, message):
202 # If a message is not 7bit clean, we refuse to handle it since it
203 # likely came from reading invalid messages in text mode, and that way
204 # lies mojibake.
205 try:
206 return message.encode('ascii')
207 except UnicodeError:
208 raise ValueError("String input must be ASCII-only; "
209 "use bytes or a Message instead")
210
Thomas Wouters477c8d52006-05-27 19:21:47 +0000211 def _dump_message(self, message, target, mangle_from_=False):
R. David Murrayb7deff12011-01-30 06:21:28 +0000212 # This assumes the target file is open in binary mode.
Thomas Wouters477c8d52006-05-27 19:21:47 +0000213 """Dump message contents to target file."""
Thomas Woutersb2137042007-02-01 18:02:27 +0000214 if isinstance(message, email.message.Message):
R. David Murrayb7deff12011-01-30 06:21:28 +0000215 buffer = io.BytesIO()
216 gen = email.generator.BytesGenerator(buffer, mangle_from_, 0)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000217 gen.flatten(message)
218 buffer.seek(0)
Guido van Rossum671117a2007-08-31 04:25:05 +0000219 data = buffer.read()
R. David Murrayb7deff12011-01-30 06:21:28 +0000220 data = data.replace(b'\n', linesep)
Guido van Rossum671117a2007-08-31 04:25:05 +0000221 target.write(data)
R. David Murrayb7deff12011-01-30 06:21:28 +0000222 elif isinstance(message, (str, bytes, io.StringIO)):
223 if isinstance(message, io.StringIO):
224 warnings.warn("Use of StringIO input is deprecated, "
225 "use BytesIO instead", DeprecationWarning, 3)
226 message = message.getvalue()
227 if isinstance(message, str):
228 message = self._string_to_bytes(message)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000229 if mangle_from_:
R. David Murrayb7deff12011-01-30 06:21:28 +0000230 message = message.replace(b'\nFrom ', b'\n>From ')
231 message = message.replace(b'\n', linesep)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000232 target.write(message)
233 elif hasattr(message, 'read'):
R. David Murrayb7deff12011-01-30 06:21:28 +0000234 if hasattr(message, 'buffer'):
235 warnings.warn("Use of text mode files is deprecated, "
236 "use a binary mode file instead", DeprecationWarning, 3)
237 message = message.buffer
Thomas Wouters477c8d52006-05-27 19:21:47 +0000238 while True:
239 line = message.readline()
R. David Murrayb7deff12011-01-30 06:21:28 +0000240 # Universal newline support.
241 if line.endswith(b'\r\n'):
242 line = line[:-2] + b'\n'
243 elif line.endswith(b'\r'):
244 line = line[:-1] + b'\n'
Guido van Rossum7cad4f32007-06-07 23:58:54 +0000245 if not line:
Thomas Wouters477c8d52006-05-27 19:21:47 +0000246 break
R. David Murrayb7deff12011-01-30 06:21:28 +0000247 if mangle_from_ and line.startswith(b'From '):
248 line = b'>From ' + line[5:]
249 line = line.replace(b'\n', linesep)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000250 target.write(line)
251 else:
252 raise TypeError('Invalid message type: %s' % type(message))
253
254
255class Maildir(Mailbox):
256 """A qmail-style Maildir mailbox."""
257
258 colon = ':'
259
Georg Brandlaa5b4112008-05-11 20:51:18 +0000260 def __init__(self, dirname, factory=None, create=True):
Thomas Wouters477c8d52006-05-27 19:21:47 +0000261 """Initialize a Maildir instance."""
262 Mailbox.__init__(self, dirname, factory, create)
R David Murraycaed7fe2011-05-06 22:07:19 -0400263 self._paths = {
264 'tmp': os.path.join(self._path, 'tmp'),
265 'new': os.path.join(self._path, 'new'),
266 'cur': os.path.join(self._path, 'cur'),
267 }
Thomas Wouters477c8d52006-05-27 19:21:47 +0000268 if not os.path.exists(self._path):
269 if create:
Guido van Rossumcd16bf62007-06-13 18:07:49 +0000270 os.mkdir(self._path, 0o700)
R David Murraycaed7fe2011-05-06 22:07:19 -0400271 for path in self._paths.values():
272 os.mkdir(path, 0o700)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000273 else:
274 raise NoSuchMailboxError(self._path)
275 self._toc = {}
Petri Lehtinen8c482ee2011-11-05 09:44:59 +0200276 self._toc_mtimes = {'cur': 0, 'new': 0}
277 self._last_read = 0 # Records last time we read cur/new
278 self._skewfactor = 0.1 # Adjust if os/fs clocks are skewing
Thomas Wouters477c8d52006-05-27 19:21:47 +0000279
280 def add(self, message):
281 """Add message and return assigned key."""
282 tmp_file = self._create_tmp()
283 try:
284 self._dump_message(message, tmp_file)
R. David Murrayfc141142011-02-11 22:47:17 +0000285 except BaseException:
286 tmp_file.close()
287 os.remove(tmp_file.name)
288 raise
289 _sync_close(tmp_file)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000290 if isinstance(message, MaildirMessage):
291 subdir = message.get_subdir()
292 suffix = self.colon + message.get_info()
293 if suffix == self.colon:
294 suffix = ''
295 else:
296 subdir = 'new'
297 suffix = ''
298 uniq = os.path.basename(tmp_file.name).split(self.colon)[0]
299 dest = os.path.join(self._path, subdir, uniq + suffix)
Thomas Wouters89f507f2006-12-13 04:49:30 +0000300 try:
301 if hasattr(os, 'link'):
302 os.link(tmp_file.name, dest)
303 os.remove(tmp_file.name)
304 else:
305 os.rename(tmp_file.name, dest)
Guido van Rossumb940e112007-01-10 16:19:56 +0000306 except OSError as e:
Thomas Wouters89f507f2006-12-13 04:49:30 +0000307 os.remove(tmp_file.name)
308 if e.errno == errno.EEXIST:
309 raise ExternalClashError('Name clash with existing message: %s'
310 % dest)
311 else:
312 raise
Thomas Wouters477c8d52006-05-27 19:21:47 +0000313 if isinstance(message, MaildirMessage):
314 os.utime(dest, (os.path.getatime(dest), message.get_date()))
315 return uniq
316
317 def remove(self, key):
318 """Remove the keyed message; raise KeyError if it doesn't exist."""
319 os.remove(os.path.join(self._path, self._lookup(key)))
320
321 def discard(self, key):
322 """If the keyed message exists, remove it."""
323 # This overrides an inapplicable implementation in the superclass.
324 try:
325 self.remove(key)
326 except KeyError:
327 pass
Guido van Rossumb940e112007-01-10 16:19:56 +0000328 except OSError as e:
Thomas Wouters477c8d52006-05-27 19:21:47 +0000329 if e.errno != errno.ENOENT:
330 raise
331
332 def __setitem__(self, key, message):
333 """Replace the keyed message; raise KeyError if it doesn't exist."""
334 old_subpath = self._lookup(key)
335 temp_key = self.add(message)
336 temp_subpath = self._lookup(temp_key)
337 if isinstance(message, MaildirMessage):
338 # temp's subdir and suffix were specified by message.
339 dominant_subpath = temp_subpath
340 else:
341 # temp's subdir and suffix were defaults from add().
342 dominant_subpath = old_subpath
343 subdir = os.path.dirname(dominant_subpath)
344 if self.colon in dominant_subpath:
345 suffix = self.colon + dominant_subpath.split(self.colon)[-1]
346 else:
347 suffix = ''
348 self.discard(key)
349 new_path = os.path.join(self._path, subdir, key + suffix)
350 os.rename(os.path.join(self._path, temp_subpath), new_path)
351 if isinstance(message, MaildirMessage):
352 os.utime(new_path, (os.path.getatime(new_path),
353 message.get_date()))
354
355 def get_message(self, key):
356 """Return a Message representation or raise a KeyError."""
357 subpath = self._lookup(key)
R. David Murrayb7deff12011-01-30 06:21:28 +0000358 f = open(os.path.join(self._path, subpath), 'rb')
Thomas Wouters477c8d52006-05-27 19:21:47 +0000359 try:
Christian Heimesfdab48e2008-01-20 09:06:41 +0000360 if self._factory:
361 msg = self._factory(f)
362 else:
363 msg = MaildirMessage(f)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000364 finally:
365 f.close()
366 subdir, name = os.path.split(subpath)
367 msg.set_subdir(subdir)
368 if self.colon in name:
369 msg.set_info(name.split(self.colon)[-1])
370 msg.set_date(os.path.getmtime(os.path.join(self._path, subpath)))
371 return msg
372
R. David Murrayb7deff12011-01-30 06:21:28 +0000373 def get_bytes(self, key):
374 """Return a bytes representation or raise a KeyError."""
375 f = open(os.path.join(self._path, self._lookup(key)), 'rb')
Thomas Wouters477c8d52006-05-27 19:21:47 +0000376 try:
R. David Murrayb7deff12011-01-30 06:21:28 +0000377 return f.read().replace(linesep, b'\n')
Thomas Wouters477c8d52006-05-27 19:21:47 +0000378 finally:
379 f.close()
380
381 def get_file(self, key):
382 """Return a file-like representation or raise a KeyError."""
R. David Murrayb7deff12011-01-30 06:21:28 +0000383 f = open(os.path.join(self._path, self._lookup(key)), 'rb')
Thomas Wouters477c8d52006-05-27 19:21:47 +0000384 return _ProxyFile(f)
385
386 def iterkeys(self):
387 """Return an iterator over keys."""
388 self._refresh()
389 for key in self._toc:
390 try:
391 self._lookup(key)
392 except KeyError:
393 continue
394 yield key
395
Guido van Rossume2b70bc2006-08-18 22:13:04 +0000396 def __contains__(self, key):
Thomas Wouters477c8d52006-05-27 19:21:47 +0000397 """Return True if the keyed message exists, False otherwise."""
398 self._refresh()
399 return key in self._toc
400
401 def __len__(self):
402 """Return a count of messages in the mailbox."""
403 self._refresh()
404 return len(self._toc)
405
406 def flush(self):
407 """Write any pending changes to disk."""
Antoine Pitrou03f13442009-11-02 11:36:51 +0000408 # Maildir changes are always written immediately, so there's nothing
R David Murraycaed7fe2011-05-06 22:07:19 -0400409 # to do.
410 pass
Thomas Wouters477c8d52006-05-27 19:21:47 +0000411
412 def lock(self):
413 """Lock the mailbox."""
414 return
415
416 def unlock(self):
417 """Unlock the mailbox if it is locked."""
418 return
419
420 def close(self):
421 """Flush and close the mailbox."""
422 return
423
424 def list_folders(self):
425 """Return a list of folder names."""
426 result = []
427 for entry in os.listdir(self._path):
428 if len(entry) > 1 and entry[0] == '.' and \
429 os.path.isdir(os.path.join(self._path, entry)):
430 result.append(entry[1:])
431 return result
432
433 def get_folder(self, folder):
434 """Return a Maildir instance for the named folder."""
Thomas Wouters89f507f2006-12-13 04:49:30 +0000435 return Maildir(os.path.join(self._path, '.' + folder),
436 factory=self._factory,
437 create=False)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000438
439 def add_folder(self, folder):
440 """Create a folder and return a Maildir instance representing it."""
441 path = os.path.join(self._path, '.' + folder)
Thomas Wouters89f507f2006-12-13 04:49:30 +0000442 result = Maildir(path, factory=self._factory)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000443 maildirfolder_path = os.path.join(path, 'maildirfolder')
444 if not os.path.exists(maildirfolder_path):
Georg Brandl6aa2d1f2008-08-12 08:35:52 +0000445 os.close(os.open(maildirfolder_path, os.O_CREAT | os.O_WRONLY,
446 0o666))
Thomas Wouters477c8d52006-05-27 19:21:47 +0000447 return result
448
449 def remove_folder(self, folder):
450 """Delete the named folder, which must be empty."""
451 path = os.path.join(self._path, '.' + folder)
452 for entry in os.listdir(os.path.join(path, 'new')) + \
453 os.listdir(os.path.join(path, 'cur')):
454 if len(entry) < 1 or entry[0] != '.':
455 raise NotEmptyError('Folder contains message(s): %s' % folder)
456 for entry in os.listdir(path):
457 if entry != 'new' and entry != 'cur' and entry != 'tmp' and \
458 os.path.isdir(os.path.join(path, entry)):
459 raise NotEmptyError("Folder contains subdirectory '%s': %s" %
460 (folder, entry))
461 for root, dirs, files in os.walk(path, topdown=False):
462 for entry in files:
463 os.remove(os.path.join(root, entry))
464 for entry in dirs:
465 os.rmdir(os.path.join(root, entry))
466 os.rmdir(path)
467
468 def clean(self):
469 """Delete old files in "tmp"."""
470 now = time.time()
471 for entry in os.listdir(os.path.join(self._path, 'tmp')):
472 path = os.path.join(self._path, 'tmp', entry)
473 if now - os.path.getatime(path) > 129600: # 60 * 60 * 36
474 os.remove(path)
475
476 _count = 1 # This is used to generate unique file names.
477
478 def _create_tmp(self):
479 """Create a file in the tmp subdirectory and open and return it."""
480 now = time.time()
481 hostname = socket.gethostname()
482 if '/' in hostname:
483 hostname = hostname.replace('/', r'\057')
484 if ':' in hostname:
485 hostname = hostname.replace(':', r'\072')
486 uniq = "%s.M%sP%sQ%s.%s" % (int(now), int(now % 1 * 1e6), os.getpid(),
487 Maildir._count, hostname)
488 path = os.path.join(self._path, 'tmp', uniq)
489 try:
490 os.stat(path)
Guido van Rossumb940e112007-01-10 16:19:56 +0000491 except OSError as e:
Thomas Wouters477c8d52006-05-27 19:21:47 +0000492 if e.errno == errno.ENOENT:
493 Maildir._count += 1
Thomas Wouters89f507f2006-12-13 04:49:30 +0000494 try:
495 return _create_carefully(path)
Guido van Rossumb940e112007-01-10 16:19:56 +0000496 except OSError as e:
Thomas Wouters89f507f2006-12-13 04:49:30 +0000497 if e.errno != errno.EEXIST:
498 raise
Thomas Wouters477c8d52006-05-27 19:21:47 +0000499 else:
500 raise
Thomas Wouters89f507f2006-12-13 04:49:30 +0000501
502 # Fall through to here if stat succeeded or open raised EEXIST.
503 raise ExternalClashError('Name clash prevented file creation: %s' %
504 path)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000505
506 def _refresh(self):
507 """Update table of contents mapping."""
R David Murraycaed7fe2011-05-06 22:07:19 -0400508 # If it has been less than two seconds since the last _refresh() call,
509 # we have to unconditionally re-read the mailbox just in case it has
510 # been modified, because os.path.mtime() has a 2 sec resolution in the
511 # most common worst case (FAT) and a 1 sec resolution typically. This
512 # results in a few unnecessary re-reads when _refresh() is called
513 # multiple times in that interval, but once the clock ticks over, we
514 # will only re-read as needed. Because the filesystem might be being
515 # served by an independent system with its own clock, we record and
516 # compare with the mtimes from the filesystem. Because the other
517 # system's clock might be skewing relative to our clock, we add an
518 # extra delta to our wait. The default is one tenth second, but is an
519 # instance variable and so can be adjusted if dealing with a
520 # particularly skewed or irregular system.
521 if time.time() - self._last_read > 2 + self._skewfactor:
522 refresh = False
523 for subdir in self._toc_mtimes:
524 mtime = os.path.getmtime(self._paths[subdir])
525 if mtime > self._toc_mtimes[subdir]:
526 refresh = True
527 self._toc_mtimes[subdir] = mtime
528 if not refresh:
Benjamin Petersonf6489f92009-11-25 17:46:26 +0000529 return
R David Murraycaed7fe2011-05-06 22:07:19 -0400530 # Refresh toc
Thomas Wouters477c8d52006-05-27 19:21:47 +0000531 self._toc = {}
R David Murraycaed7fe2011-05-06 22:07:19 -0400532 for subdir in self._toc_mtimes:
533 path = self._paths[subdir]
Andrew M. Kuchlingfa815342009-05-03 02:52:20 +0000534 for entry in os.listdir(path):
535 p = os.path.join(path, entry)
Guido van Rossumb5a755e2007-07-18 18:15:48 +0000536 if os.path.isdir(p):
537 continue
Thomas Wouters477c8d52006-05-27 19:21:47 +0000538 uniq = entry.split(self.colon)[0]
539 self._toc[uniq] = os.path.join(subdir, entry)
R David Murraycaed7fe2011-05-06 22:07:19 -0400540 self._last_read = time.time()
Andrew M. Kuchlingfa815342009-05-03 02:52:20 +0000541
Thomas Wouters477c8d52006-05-27 19:21:47 +0000542 def _lookup(self, key):
543 """Use TOC to return subpath for given key, or raise a KeyError."""
544 try:
545 if os.path.exists(os.path.join(self._path, self._toc[key])):
546 return self._toc[key]
547 except KeyError:
548 pass
549 self._refresh()
550 try:
551 return self._toc[key]
552 except KeyError:
553 raise KeyError('No message with key: %s' % key)
554
555 # This method is for backward compatibility only.
556 def next(self):
557 """Return the next message in a one-time iteration."""
558 if not hasattr(self, '_onetime_keys'):
Guido van Rossumdde4c2c2007-02-12 00:22:55 +0000559 self._onetime_keys = iter(self.keys())
Thomas Wouters477c8d52006-05-27 19:21:47 +0000560 while True:
561 try:
Georg Brandla18af4e2007-04-21 15:47:16 +0000562 return self[next(self._onetime_keys)]
Thomas Wouters477c8d52006-05-27 19:21:47 +0000563 except StopIteration:
564 return None
565 except KeyError:
566 continue
567
568
569class _singlefileMailbox(Mailbox):
570 """A single-file mailbox."""
571
572 def __init__(self, path, factory=None, create=True):
573 """Initialize a single-file mailbox."""
574 Mailbox.__init__(self, path, factory, create)
575 try:
R. David Murrayb7deff12011-01-30 06:21:28 +0000576 f = open(self._path, 'rb+')
Guido van Rossumb940e112007-01-10 16:19:56 +0000577 except IOError as e:
Thomas Wouters477c8d52006-05-27 19:21:47 +0000578 if e.errno == errno.ENOENT:
579 if create:
R. David Murrayb7deff12011-01-30 06:21:28 +0000580 f = open(self._path, 'wb+')
Thomas Wouters477c8d52006-05-27 19:21:47 +0000581 else:
582 raise NoSuchMailboxError(self._path)
R. David Murray752b9502011-03-03 18:12:34 +0000583 elif e.errno in (errno.EACCES, errno.EROFS):
R. David Murrayb7deff12011-01-30 06:21:28 +0000584 f = open(self._path, 'rb')
Thomas Wouters477c8d52006-05-27 19:21:47 +0000585 else:
586 raise
587 self._file = f
588 self._toc = None
589 self._next_key = 0
Petri Lehtinenf29435f2012-06-28 13:48:17 +0300590 self._pending = False # No changes require rewriting the file.
591 self._pending_sync = False # No need to sync the file
Thomas Wouters477c8d52006-05-27 19:21:47 +0000592 self._locked = False
Petri Lehtinenf29435f2012-06-28 13:48:17 +0300593 self._file_length = None # Used to record mailbox size
Thomas Wouters477c8d52006-05-27 19:21:47 +0000594
595 def add(self, message):
596 """Add message and return assigned key."""
597 self._lookup()
598 self._toc[self._next_key] = self._append_message(message)
599 self._next_key += 1
Petri Lehtinenf29435f2012-06-28 13:48:17 +0300600 # _append_message appends the message to the mailbox file. We
601 # don't need a full rewrite + rename, sync is enough.
602 self._pending_sync = True
Thomas Wouters477c8d52006-05-27 19:21:47 +0000603 return self._next_key - 1
604
605 def remove(self, key):
606 """Remove the keyed message; raise KeyError if it doesn't exist."""
607 self._lookup(key)
608 del self._toc[key]
609 self._pending = True
610
611 def __setitem__(self, key, message):
612 """Replace the keyed message; raise KeyError if it doesn't exist."""
613 self._lookup(key)
614 self._toc[key] = self._append_message(message)
615 self._pending = True
616
617 def iterkeys(self):
618 """Return an iterator over keys."""
619 self._lookup()
620 for key in self._toc.keys():
621 yield key
622
Guido van Rossume2b70bc2006-08-18 22:13:04 +0000623 def __contains__(self, key):
Thomas Wouters477c8d52006-05-27 19:21:47 +0000624 """Return True if the keyed message exists, False otherwise."""
625 self._lookup()
626 return key in self._toc
627
628 def __len__(self):
629 """Return a count of messages in the mailbox."""
630 self._lookup()
631 return len(self._toc)
632
633 def lock(self):
634 """Lock the mailbox."""
635 if not self._locked:
636 _lock_file(self._file)
637 self._locked = True
638
639 def unlock(self):
640 """Unlock the mailbox if it is locked."""
641 if self._locked:
642 _unlock_file(self._file)
643 self._locked = False
644
645 def flush(self):
646 """Write any pending changes to disk."""
647 if not self._pending:
Petri Lehtinenf29435f2012-06-28 13:48:17 +0300648 if self._pending_sync:
649 # Messages have only been added, so syncing the file
650 # is enough.
651 _sync_flush(self._file)
652 self._pending_sync = False
Thomas Wouters477c8d52006-05-27 19:21:47 +0000653 return
Thomas Wouters902d6eb2007-01-09 23:18:33 +0000654
655 # In order to be writing anything out at all, self._toc must
656 # already have been generated (and presumably has been modified
657 # by adding or deleting an item).
658 assert self._toc is not None
Thomas Wouters9fe394c2007-02-05 01:24:16 +0000659
Thomas Wouters902d6eb2007-01-09 23:18:33 +0000660 # Check length of self._file; if it's changed, some other process
661 # has modified the mailbox since we scanned it.
662 self._file.seek(0, 2)
663 cur_len = self._file.tell()
664 if cur_len != self._file_length:
665 raise ExternalClashError('Size of mailbox file changed '
666 '(expected %i, found %i)' %
667 (self._file_length, cur_len))
Thomas Wouters9fe394c2007-02-05 01:24:16 +0000668
Thomas Wouters477c8d52006-05-27 19:21:47 +0000669 new_file = _create_temporary(self._path)
670 try:
671 new_toc = {}
672 self._pre_mailbox_hook(new_file)
673 for key in sorted(self._toc.keys()):
674 start, stop = self._toc[key]
675 self._file.seek(start)
676 self._pre_message_hook(new_file)
677 new_start = new_file.tell()
678 while True:
679 buffer = self._file.read(min(4096,
680 stop - self._file.tell()))
Guido van Rossum7cad4f32007-06-07 23:58:54 +0000681 if not buffer:
Thomas Wouters477c8d52006-05-27 19:21:47 +0000682 break
683 new_file.write(buffer)
684 new_toc[key] = (new_start, new_file.tell())
685 self._post_message_hook(new_file)
Petri Lehtinen02653f12012-06-15 20:50:51 +0300686 self._file_length = new_file.tell()
Thomas Wouters477c8d52006-05-27 19:21:47 +0000687 except:
688 new_file.close()
689 os.remove(new_file.name)
690 raise
Thomas Wouters89f507f2006-12-13 04:49:30 +0000691 _sync_close(new_file)
692 # self._file is about to get replaced, so no need to sync.
Thomas Wouters477c8d52006-05-27 19:21:47 +0000693 self._file.close()
Petri Lehtinen5b5619f2012-06-29 15:09:12 +0300694 # Make sure the new file's mode is the same as the old file's
695 mode = os.stat(self._path).st_mode
696 os.chmod(new_file.name, mode)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000697 try:
698 os.rename(new_file.name, self._path)
Guido van Rossumb940e112007-01-10 16:19:56 +0000699 except OSError as e:
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000700 if e.errno == errno.EEXIST or \
701 (os.name == 'os2' and e.errno == errno.EACCES):
Thomas Wouters477c8d52006-05-27 19:21:47 +0000702 os.remove(self._path)
703 os.rename(new_file.name, self._path)
704 else:
705 raise
706 self._file = open(self._path, 'rb+')
707 self._toc = new_toc
708 self._pending = False
Petri Lehtinenf29435f2012-06-28 13:48:17 +0300709 self._pending_sync = False
Thomas Wouters477c8d52006-05-27 19:21:47 +0000710 if self._locked:
Thomas Wouters89f507f2006-12-13 04:49:30 +0000711 _lock_file(self._file, dotlock=False)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000712
713 def _pre_mailbox_hook(self, f):
714 """Called before writing the mailbox to file f."""
715 return
716
717 def _pre_message_hook(self, f):
718 """Called before writing each message to file f."""
719 return
720
721 def _post_message_hook(self, f):
722 """Called after writing each message to file f."""
723 return
724
725 def close(self):
726 """Flush and close the mailbox."""
727 self.flush()
728 if self._locked:
729 self.unlock()
Thomas Wouters89f507f2006-12-13 04:49:30 +0000730 self._file.close() # Sync has been done by self.flush() above.
Thomas Wouters477c8d52006-05-27 19:21:47 +0000731
732 def _lookup(self, key=None):
733 """Return (start, stop) or raise KeyError."""
734 if self._toc is None:
735 self._generate_toc()
736 if key is not None:
737 try:
738 return self._toc[key]
739 except KeyError:
740 raise KeyError('No message with key: %s' % key)
741
742 def _append_message(self, message):
743 """Append message to mailbox and return (start, stop) offsets."""
744 self._file.seek(0, 2)
R. David Murrayfc141142011-02-11 22:47:17 +0000745 before = self._file.tell()
Petri Lehtinenb6ee3d62012-06-29 13:43:37 +0300746 if len(self._toc) == 0 and not self._pending:
747 # This is the first message, and the _pre_mailbox_hook
748 # hasn't yet been called. If self._pending is True,
749 # messages have been removed, so _pre_mailbox_hook must
750 # have been called already.
Petri Lehtinenf29435f2012-06-28 13:48:17 +0300751 self._pre_mailbox_hook(self._file)
R. David Murrayfc141142011-02-11 22:47:17 +0000752 try:
753 self._pre_message_hook(self._file)
754 offsets = self._install_message(message)
755 self._post_message_hook(self._file)
756 except BaseException:
757 self._file.truncate(before)
758 raise
Thomas Wouters477c8d52006-05-27 19:21:47 +0000759 self._file.flush()
Thomas Wouters902d6eb2007-01-09 23:18:33 +0000760 self._file_length = self._file.tell() # Record current length of mailbox
Thomas Wouters477c8d52006-05-27 19:21:47 +0000761 return offsets
762
763
764
765class _mboxMMDF(_singlefileMailbox):
766 """An mbox or MMDF mailbox."""
767
768 _mangle_from_ = True
769
770 def get_message(self, key):
771 """Return a Message representation or raise a KeyError."""
772 start, stop = self._lookup(key)
773 self._file.seek(start)
R. David Murrayb7deff12011-01-30 06:21:28 +0000774 from_line = self._file.readline().replace(linesep, b'')
Thomas Wouters477c8d52006-05-27 19:21:47 +0000775 string = self._file.read(stop - self._file.tell())
R. David Murrayb7deff12011-01-30 06:21:28 +0000776 msg = self._message_factory(string.replace(linesep, b'\n'))
777 msg.set_from(from_line[5:].decode('ascii'))
Thomas Wouters477c8d52006-05-27 19:21:47 +0000778 return msg
779
780 def get_string(self, key, from_=False):
781 """Return a string representation or raise a KeyError."""
R. David Murrayb7deff12011-01-30 06:21:28 +0000782 return email.message_from_bytes(
783 self.get_bytes(key)).as_string(unixfrom=from_)
784
785 def get_bytes(self, key, from_=False):
786 """Return a string representation or raise a KeyError."""
Thomas Wouters477c8d52006-05-27 19:21:47 +0000787 start, stop = self._lookup(key)
788 self._file.seek(start)
789 if not from_:
790 self._file.readline()
791 string = self._file.read(stop - self._file.tell())
R. David Murrayb7deff12011-01-30 06:21:28 +0000792 return string.replace(linesep, b'\n')
Thomas Wouters477c8d52006-05-27 19:21:47 +0000793
794 def get_file(self, key, from_=False):
795 """Return a file-like representation or raise a KeyError."""
796 start, stop = self._lookup(key)
797 self._file.seek(start)
798 if not from_:
799 self._file.readline()
800 return _PartialFile(self._file, self._file.tell(), stop)
801
802 def _install_message(self, message):
803 """Format a message and blindly write to self._file."""
804 from_line = None
R. David Murrayb7deff12011-01-30 06:21:28 +0000805 if isinstance(message, str):
806 message = self._string_to_bytes(message)
807 if isinstance(message, bytes) and message.startswith(b'From '):
808 newline = message.find(b'\n')
Thomas Wouters477c8d52006-05-27 19:21:47 +0000809 if newline != -1:
810 from_line = message[:newline]
811 message = message[newline + 1:]
812 else:
813 from_line = message
R. David Murrayb7deff12011-01-30 06:21:28 +0000814 message = b''
Thomas Wouters477c8d52006-05-27 19:21:47 +0000815 elif isinstance(message, _mboxMMDFMessage):
R. David Murrayb7deff12011-01-30 06:21:28 +0000816 author = message.get_from().encode('ascii')
817 from_line = b'From ' + author
Thomas Woutersb2137042007-02-01 18:02:27 +0000818 elif isinstance(message, email.message.Message):
Thomas Wouters477c8d52006-05-27 19:21:47 +0000819 from_line = message.get_unixfrom() # May be None.
R. David Murrayb7deff12011-01-30 06:21:28 +0000820 if from_line is not None:
821 from_line = from_line.encode('ascii')
Thomas Wouters477c8d52006-05-27 19:21:47 +0000822 if from_line is None:
R. David Murrayb7deff12011-01-30 06:21:28 +0000823 from_line = b'From MAILER-DAEMON ' + time.asctime(time.gmtime()).encode()
Thomas Wouters477c8d52006-05-27 19:21:47 +0000824 start = self._file.tell()
R. David Murrayb7deff12011-01-30 06:21:28 +0000825 self._file.write(from_line + linesep)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000826 self._dump_message(message, self._file, self._mangle_from_)
827 stop = self._file.tell()
828 return (start, stop)
829
830
831class mbox(_mboxMMDF):
832 """A classic mbox mailbox."""
833
834 _mangle_from_ = True
835
836 def __init__(self, path, factory=None, create=True):
837 """Initialize an mbox mailbox."""
838 self._message_factory = mboxMessage
839 _mboxMMDF.__init__(self, path, factory, create)
840
841 def _pre_message_hook(self, f):
842 """Called before writing each message to file f."""
843 if f.tell() != 0:
R. David Murrayb7deff12011-01-30 06:21:28 +0000844 f.write(linesep)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000845
846 def _generate_toc(self):
847 """Generate key-to-(start, stop) table of contents."""
848 starts, stops = [], []
849 self._file.seek(0)
850 while True:
851 line_pos = self._file.tell()
852 line = self._file.readline()
R. David Murrayb7deff12011-01-30 06:21:28 +0000853 if line.startswith(b'From '):
Thomas Wouters477c8d52006-05-27 19:21:47 +0000854 if len(stops) < len(starts):
R. David Murrayb7deff12011-01-30 06:21:28 +0000855 stops.append(line_pos - len(linesep))
Thomas Wouters477c8d52006-05-27 19:21:47 +0000856 starts.append(line_pos)
Guido van Rossum7cad4f32007-06-07 23:58:54 +0000857 elif not line:
Thomas Wouters477c8d52006-05-27 19:21:47 +0000858 stops.append(line_pos)
859 break
860 self._toc = dict(enumerate(zip(starts, stops)))
861 self._next_key = len(self._toc)
Thomas Wouters902d6eb2007-01-09 23:18:33 +0000862 self._file_length = self._file.tell()
Thomas Wouters477c8d52006-05-27 19:21:47 +0000863
864
865class MMDF(_mboxMMDF):
866 """An MMDF mailbox."""
867
868 def __init__(self, path, factory=None, create=True):
869 """Initialize an MMDF mailbox."""
870 self._message_factory = MMDFMessage
871 _mboxMMDF.__init__(self, path, factory, create)
872
873 def _pre_message_hook(self, f):
874 """Called before writing each message to file f."""
R. David Murrayb7deff12011-01-30 06:21:28 +0000875 f.write(b'\001\001\001\001' + linesep)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000876
877 def _post_message_hook(self, f):
878 """Called after writing each message to file f."""
R. David Murrayb7deff12011-01-30 06:21:28 +0000879 f.write(linesep + b'\001\001\001\001' + linesep)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000880
881 def _generate_toc(self):
882 """Generate key-to-(start, stop) table of contents."""
883 starts, stops = [], []
884 self._file.seek(0)
885 next_pos = 0
886 while True:
887 line_pos = next_pos
888 line = self._file.readline()
889 next_pos = self._file.tell()
R. David Murrayb7deff12011-01-30 06:21:28 +0000890 if line.startswith(b'\001\001\001\001' + linesep):
Thomas Wouters477c8d52006-05-27 19:21:47 +0000891 starts.append(next_pos)
892 while True:
893 line_pos = next_pos
894 line = self._file.readline()
895 next_pos = self._file.tell()
R. David Murrayb7deff12011-01-30 06:21:28 +0000896 if line == b'\001\001\001\001' + linesep:
897 stops.append(line_pos - len(linesep))
Thomas Wouters477c8d52006-05-27 19:21:47 +0000898 break
Guido van Rossum7cad4f32007-06-07 23:58:54 +0000899 elif not line:
Thomas Wouters477c8d52006-05-27 19:21:47 +0000900 stops.append(line_pos)
901 break
Guido van Rossum7cad4f32007-06-07 23:58:54 +0000902 elif not line:
Thomas Wouters477c8d52006-05-27 19:21:47 +0000903 break
904 self._toc = dict(enumerate(zip(starts, stops)))
905 self._next_key = len(self._toc)
Thomas Wouters902d6eb2007-01-09 23:18:33 +0000906 self._file.seek(0, 2)
907 self._file_length = self._file.tell()
Thomas Wouters477c8d52006-05-27 19:21:47 +0000908
909
910class MH(Mailbox):
911 """An MH mailbox."""
912
913 def __init__(self, path, factory=None, create=True):
914 """Initialize an MH instance."""
915 Mailbox.__init__(self, path, factory, create)
916 if not os.path.exists(self._path):
917 if create:
Guido van Rossumcd16bf62007-06-13 18:07:49 +0000918 os.mkdir(self._path, 0o700)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000919 os.close(os.open(os.path.join(self._path, '.mh_sequences'),
Guido van Rossumcd16bf62007-06-13 18:07:49 +0000920 os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600))
Thomas Wouters477c8d52006-05-27 19:21:47 +0000921 else:
922 raise NoSuchMailboxError(self._path)
923 self._locked = False
924
925 def add(self, message):
926 """Add message and return assigned key."""
927 keys = self.keys()
928 if len(keys) == 0:
929 new_key = 1
930 else:
931 new_key = max(keys) + 1
932 new_path = os.path.join(self._path, str(new_key))
933 f = _create_carefully(new_path)
R. David Murray774a66d2011-02-12 00:03:31 +0000934 closed = False
Thomas Wouters477c8d52006-05-27 19:21:47 +0000935 try:
936 if self._locked:
937 _lock_file(f)
938 try:
R. David Murrayfc141142011-02-11 22:47:17 +0000939 try:
940 self._dump_message(message, f)
941 except BaseException:
R. David Murray774a66d2011-02-12 00:03:31 +0000942 # Unlock and close so it can be deleted on Windows
943 if self._locked:
944 _unlock_file(f)
945 _sync_close(f)
946 closed = True
R. David Murrayfc141142011-02-11 22:47:17 +0000947 os.remove(new_path)
948 raise
Thomas Wouters477c8d52006-05-27 19:21:47 +0000949 if isinstance(message, MHMessage):
950 self._dump_sequences(message, new_key)
951 finally:
952 if self._locked:
953 _unlock_file(f)
954 finally:
R. David Murray774a66d2011-02-12 00:03:31 +0000955 if not closed:
956 _sync_close(f)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000957 return new_key
958
959 def remove(self, key):
960 """Remove the keyed message; raise KeyError if it doesn't exist."""
961 path = os.path.join(self._path, str(key))
962 try:
963 f = open(path, 'rb+')
Guido van Rossumb940e112007-01-10 16:19:56 +0000964 except IOError as e:
Thomas Wouters477c8d52006-05-27 19:21:47 +0000965 if e.errno == errno.ENOENT:
966 raise KeyError('No message with key: %s' % key)
967 else:
968 raise
Benjamin Peterson21896a32010-03-21 22:03:03 +0000969 else:
Thomas Wouters477c8d52006-05-27 19:21:47 +0000970 f.close()
Benjamin Peterson21896a32010-03-21 22:03:03 +0000971 os.remove(path)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000972
973 def __setitem__(self, key, message):
974 """Replace the keyed message; raise KeyError if it doesn't exist."""
975 path = os.path.join(self._path, str(key))
976 try:
R. David Murrayb7deff12011-01-30 06:21:28 +0000977 f = open(path, 'rb+')
Guido van Rossumb940e112007-01-10 16:19:56 +0000978 except IOError as e:
Thomas Wouters477c8d52006-05-27 19:21:47 +0000979 if e.errno == errno.ENOENT:
980 raise KeyError('No message with key: %s' % key)
981 else:
982 raise
983 try:
984 if self._locked:
985 _lock_file(f)
986 try:
987 os.close(os.open(path, os.O_WRONLY | os.O_TRUNC))
988 self._dump_message(message, f)
989 if isinstance(message, MHMessage):
990 self._dump_sequences(message, key)
991 finally:
992 if self._locked:
993 _unlock_file(f)
994 finally:
Thomas Wouters89f507f2006-12-13 04:49:30 +0000995 _sync_close(f)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000996
997 def get_message(self, key):
998 """Return a Message representation or raise a KeyError."""
999 try:
1000 if self._locked:
R. David Murrayb7deff12011-01-30 06:21:28 +00001001 f = open(os.path.join(self._path, str(key)), 'rb+')
Thomas Wouters477c8d52006-05-27 19:21:47 +00001002 else:
R. David Murrayb7deff12011-01-30 06:21:28 +00001003 f = open(os.path.join(self._path, str(key)), 'rb')
Guido van Rossumb940e112007-01-10 16:19:56 +00001004 except IOError as e:
Thomas Wouters477c8d52006-05-27 19:21:47 +00001005 if e.errno == errno.ENOENT:
1006 raise KeyError('No message with key: %s' % key)
1007 else:
1008 raise
1009 try:
1010 if self._locked:
1011 _lock_file(f)
1012 try:
1013 msg = MHMessage(f)
1014 finally:
1015 if self._locked:
1016 _unlock_file(f)
1017 finally:
1018 f.close()
R. David Murray548ac412009-04-02 19:44:43 +00001019 for name, key_list in self.get_sequences().items():
Thomas Wouters477c8d52006-05-27 19:21:47 +00001020 if key in key_list:
1021 msg.add_sequence(name)
1022 return msg
1023
R. David Murrayb7deff12011-01-30 06:21:28 +00001024 def get_bytes(self, key):
1025 """Return a bytes representation or raise a KeyError."""
Thomas Wouters477c8d52006-05-27 19:21:47 +00001026 try:
1027 if self._locked:
R. David Murrayb7deff12011-01-30 06:21:28 +00001028 f = open(os.path.join(self._path, str(key)), 'rb+')
Thomas Wouters477c8d52006-05-27 19:21:47 +00001029 else:
R. David Murrayb7deff12011-01-30 06:21:28 +00001030 f = open(os.path.join(self._path, str(key)), 'rb')
Guido van Rossumb940e112007-01-10 16:19:56 +00001031 except IOError as e:
Thomas Wouters477c8d52006-05-27 19:21:47 +00001032 if e.errno == errno.ENOENT:
1033 raise KeyError('No message with key: %s' % key)
1034 else:
1035 raise
1036 try:
1037 if self._locked:
1038 _lock_file(f)
1039 try:
R. David Murrayb7deff12011-01-30 06:21:28 +00001040 return f.read().replace(linesep, b'\n')
Thomas Wouters477c8d52006-05-27 19:21:47 +00001041 finally:
1042 if self._locked:
1043 _unlock_file(f)
1044 finally:
1045 f.close()
1046
1047 def get_file(self, key):
1048 """Return a file-like representation or raise a KeyError."""
1049 try:
R. David Murrayb7deff12011-01-30 06:21:28 +00001050 f = open(os.path.join(self._path, str(key)), 'rb')
Guido van Rossumb940e112007-01-10 16:19:56 +00001051 except IOError as e:
Thomas Wouters477c8d52006-05-27 19:21:47 +00001052 if e.errno == errno.ENOENT:
1053 raise KeyError('No message with key: %s' % key)
1054 else:
1055 raise
1056 return _ProxyFile(f)
1057
1058 def iterkeys(self):
1059 """Return an iterator over keys."""
1060 return iter(sorted(int(entry) for entry in os.listdir(self._path)
1061 if entry.isdigit()))
1062
Guido van Rossume2b70bc2006-08-18 22:13:04 +00001063 def __contains__(self, key):
Thomas Wouters477c8d52006-05-27 19:21:47 +00001064 """Return True if the keyed message exists, False otherwise."""
1065 return os.path.exists(os.path.join(self._path, str(key)))
1066
1067 def __len__(self):
1068 """Return a count of messages in the mailbox."""
Guido van Rossumcc2b0162007-02-11 06:12:03 +00001069 return len(list(self.keys()))
Thomas Wouters477c8d52006-05-27 19:21:47 +00001070
1071 def lock(self):
1072 """Lock the mailbox."""
1073 if not self._locked:
1074 self._file = open(os.path.join(self._path, '.mh_sequences'), 'rb+')
1075 _lock_file(self._file)
1076 self._locked = True
1077
1078 def unlock(self):
1079 """Unlock the mailbox if it is locked."""
1080 if self._locked:
1081 _unlock_file(self._file)
Thomas Wouters89f507f2006-12-13 04:49:30 +00001082 _sync_close(self._file)
Thomas Wouters477c8d52006-05-27 19:21:47 +00001083 del self._file
1084 self._locked = False
1085
1086 def flush(self):
1087 """Write any pending changes to the disk."""
1088 return
1089
1090 def close(self):
1091 """Flush and close the mailbox."""
1092 if self._locked:
1093 self.unlock()
1094
1095 def list_folders(self):
1096 """Return a list of folder names."""
1097 result = []
1098 for entry in os.listdir(self._path):
1099 if os.path.isdir(os.path.join(self._path, entry)):
1100 result.append(entry)
1101 return result
1102
1103 def get_folder(self, folder):
1104 """Return an MH instance for the named folder."""
Thomas Wouters89f507f2006-12-13 04:49:30 +00001105 return MH(os.path.join(self._path, folder),
1106 factory=self._factory, create=False)
Thomas Wouters477c8d52006-05-27 19:21:47 +00001107
1108 def add_folder(self, folder):
1109 """Create a folder and return an MH instance representing it."""
Thomas Wouters89f507f2006-12-13 04:49:30 +00001110 return MH(os.path.join(self._path, folder),
1111 factory=self._factory)
Thomas Wouters477c8d52006-05-27 19:21:47 +00001112
1113 def remove_folder(self, folder):
1114 """Delete the named folder, which must be empty."""
1115 path = os.path.join(self._path, folder)
1116 entries = os.listdir(path)
1117 if entries == ['.mh_sequences']:
1118 os.remove(os.path.join(path, '.mh_sequences'))
1119 elif entries == []:
1120 pass
1121 else:
1122 raise NotEmptyError('Folder not empty: %s' % self._path)
1123 os.rmdir(path)
1124
1125 def get_sequences(self):
1126 """Return a name-to-key-list dictionary to define each sequence."""
1127 results = {}
R. David Murrayb7deff12011-01-30 06:21:28 +00001128 f = open(os.path.join(self._path, '.mh_sequences'), 'r')
Thomas Wouters477c8d52006-05-27 19:21:47 +00001129 try:
1130 all_keys = set(self.keys())
1131 for line in f:
1132 try:
1133 name, contents = line.split(':')
1134 keys = set()
1135 for spec in contents.split():
1136 if spec.isdigit():
1137 keys.add(int(spec))
1138 else:
1139 start, stop = (int(x) for x in spec.split('-'))
1140 keys.update(range(start, stop + 1))
1141 results[name] = [key for key in sorted(keys) \
1142 if key in all_keys]
1143 if len(results[name]) == 0:
1144 del results[name]
1145 except ValueError:
1146 raise FormatError('Invalid sequence specification: %s' %
1147 line.rstrip())
1148 finally:
1149 f.close()
1150 return results
1151
1152 def set_sequences(self, sequences):
1153 """Set sequences using the given name-to-key-list dictionary."""
R. David Murrayb7deff12011-01-30 06:21:28 +00001154 f = open(os.path.join(self._path, '.mh_sequences'), 'r+')
Thomas Wouters477c8d52006-05-27 19:21:47 +00001155 try:
1156 os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC))
Guido van Rossumcc2b0162007-02-11 06:12:03 +00001157 for name, keys in sequences.items():
Thomas Wouters477c8d52006-05-27 19:21:47 +00001158 if len(keys) == 0:
1159 continue
R. David Murrayb7deff12011-01-30 06:21:28 +00001160 f.write(name + ':')
Thomas Wouters477c8d52006-05-27 19:21:47 +00001161 prev = None
1162 completing = False
1163 for key in sorted(set(keys)):
1164 if key - 1 == prev:
1165 if not completing:
1166 completing = True
1167 f.write('-')
1168 elif completing:
1169 completing = False
1170 f.write('%s %s' % (prev, key))
1171 else:
1172 f.write(' %s' % key)
1173 prev = key
1174 if completing:
1175 f.write(str(prev) + '\n')
1176 else:
1177 f.write('\n')
1178 finally:
Thomas Wouters89f507f2006-12-13 04:49:30 +00001179 _sync_close(f)
Thomas Wouters477c8d52006-05-27 19:21:47 +00001180
1181 def pack(self):
1182 """Re-name messages to eliminate numbering gaps. Invalidates keys."""
1183 sequences = self.get_sequences()
1184 prev = 0
1185 changes = []
Guido van Rossumcc2b0162007-02-11 06:12:03 +00001186 for key in self.keys():
Thomas Wouters477c8d52006-05-27 19:21:47 +00001187 if key - 1 != prev:
1188 changes.append((key, prev + 1))
Thomas Wouters89f507f2006-12-13 04:49:30 +00001189 if hasattr(os, 'link'):
1190 os.link(os.path.join(self._path, str(key)),
1191 os.path.join(self._path, str(prev + 1)))
1192 os.unlink(os.path.join(self._path, str(key)))
1193 else:
1194 os.rename(os.path.join(self._path, str(key)),
1195 os.path.join(self._path, str(prev + 1)))
Thomas Wouters477c8d52006-05-27 19:21:47 +00001196 prev += 1
1197 self._next_key = prev + 1
1198 if len(changes) == 0:
1199 return
1200 for name, key_list in sequences.items():
1201 for old, new in changes:
1202 if old in key_list:
1203 key_list[key_list.index(old)] = new
1204 self.set_sequences(sequences)
1205
1206 def _dump_sequences(self, message, key):
1207 """Inspect a new MHMessage and update sequences appropriately."""
1208 pending_sequences = message.get_sequences()
1209 all_sequences = self.get_sequences()
Guido van Rossumcc2b0162007-02-11 06:12:03 +00001210 for name, key_list in all_sequences.items():
Thomas Wouters477c8d52006-05-27 19:21:47 +00001211 if name in pending_sequences:
1212 key_list.append(key)
1213 elif key in key_list:
1214 del key_list[key_list.index(key)]
1215 for sequence in pending_sequences:
1216 if sequence not in all_sequences:
1217 all_sequences[sequence] = [key]
1218 self.set_sequences(all_sequences)
1219
1220
1221class Babyl(_singlefileMailbox):
1222 """An Rmail-style Babyl mailbox."""
1223
1224 _special_labels = frozenset(('unseen', 'deleted', 'filed', 'answered',
1225 'forwarded', 'edited', 'resent'))
1226
1227 def __init__(self, path, factory=None, create=True):
1228 """Initialize a Babyl mailbox."""
1229 _singlefileMailbox.__init__(self, path, factory, create)
1230 self._labels = {}
1231
1232 def add(self, message):
1233 """Add message and return assigned key."""
1234 key = _singlefileMailbox.add(self, message)
1235 if isinstance(message, BabylMessage):
1236 self._labels[key] = message.get_labels()
1237 return key
1238
1239 def remove(self, key):
1240 """Remove the keyed message; raise KeyError if it doesn't exist."""
1241 _singlefileMailbox.remove(self, key)
1242 if key in self._labels:
1243 del self._labels[key]
1244
1245 def __setitem__(self, key, message):
1246 """Replace the keyed message; raise KeyError if it doesn't exist."""
1247 _singlefileMailbox.__setitem__(self, key, message)
1248 if isinstance(message, BabylMessage):
1249 self._labels[key] = message.get_labels()
1250
1251 def get_message(self, key):
1252 """Return a Message representation or raise a KeyError."""
1253 start, stop = self._lookup(key)
1254 self._file.seek(start)
R. David Murrayb7deff12011-01-30 06:21:28 +00001255 self._file.readline() # Skip b'1,' line specifying labels.
1256 original_headers = io.BytesIO()
Thomas Wouters477c8d52006-05-27 19:21:47 +00001257 while True:
1258 line = self._file.readline()
R. David Murrayb7deff12011-01-30 06:21:28 +00001259 if line == b'*** EOOH ***' + linesep or not line:
Thomas Wouters477c8d52006-05-27 19:21:47 +00001260 break
R. David Murrayb7deff12011-01-30 06:21:28 +00001261 original_headers.write(line.replace(linesep, b'\n'))
1262 visible_headers = io.BytesIO()
Thomas Wouters477c8d52006-05-27 19:21:47 +00001263 while True:
1264 line = self._file.readline()
R. David Murrayb7deff12011-01-30 06:21:28 +00001265 if line == linesep or not line:
Thomas Wouters477c8d52006-05-27 19:21:47 +00001266 break
R. David Murrayb7deff12011-01-30 06:21:28 +00001267 visible_headers.write(line.replace(linesep, b'\n'))
1268 # Read up to the stop, or to the end
1269 n = stop - self._file.tell()
1270 assert n >= 0
1271 body = self._file.read(n)
1272 body = body.replace(linesep, b'\n')
Thomas Wouters477c8d52006-05-27 19:21:47 +00001273 msg = BabylMessage(original_headers.getvalue() + body)
1274 msg.set_visible(visible_headers.getvalue())
1275 if key in self._labels:
1276 msg.set_labels(self._labels[key])
1277 return msg
1278
R. David Murrayb7deff12011-01-30 06:21:28 +00001279 def get_bytes(self, key):
Thomas Wouters477c8d52006-05-27 19:21:47 +00001280 """Return a string representation or raise a KeyError."""
1281 start, stop = self._lookup(key)
1282 self._file.seek(start)
R. David Murrayb7deff12011-01-30 06:21:28 +00001283 self._file.readline() # Skip b'1,' line specifying labels.
1284 original_headers = io.BytesIO()
Thomas Wouters477c8d52006-05-27 19:21:47 +00001285 while True:
1286 line = self._file.readline()
R. David Murrayb7deff12011-01-30 06:21:28 +00001287 if line == b'*** EOOH ***' + linesep or not line:
Thomas Wouters477c8d52006-05-27 19:21:47 +00001288 break
R. David Murrayb7deff12011-01-30 06:21:28 +00001289 original_headers.write(line.replace(linesep, b'\n'))
Thomas Wouters477c8d52006-05-27 19:21:47 +00001290 while True:
1291 line = self._file.readline()
R. David Murrayb7deff12011-01-30 06:21:28 +00001292 if line == linesep or not line:
Thomas Wouters477c8d52006-05-27 19:21:47 +00001293 break
R. David Murrayb7deff12011-01-30 06:21:28 +00001294 headers = original_headers.getvalue()
1295 n = stop - self._file.tell()
1296 assert n >= 0
1297 data = self._file.read(n)
1298 data = data.replace(linesep, b'\n')
1299 return headers + data
Thomas Wouters477c8d52006-05-27 19:21:47 +00001300
1301 def get_file(self, key):
1302 """Return a file-like representation or raise a KeyError."""
R. David Murrayb7deff12011-01-30 06:21:28 +00001303 return io.BytesIO(self.get_bytes(key).replace(b'\n', linesep))
Thomas Wouters477c8d52006-05-27 19:21:47 +00001304
1305 def get_labels(self):
1306 """Return a list of user-defined labels in the mailbox."""
1307 self._lookup()
1308 labels = set()
1309 for label_list in self._labels.values():
1310 labels.update(label_list)
1311 labels.difference_update(self._special_labels)
1312 return list(labels)
1313
1314 def _generate_toc(self):
1315 """Generate key-to-(start, stop) table of contents."""
1316 starts, stops = [], []
1317 self._file.seek(0)
1318 next_pos = 0
1319 label_lists = []
1320 while True:
1321 line_pos = next_pos
1322 line = self._file.readline()
1323 next_pos = self._file.tell()
R. David Murrayb7deff12011-01-30 06:21:28 +00001324 if line == b'\037\014' + linesep:
Thomas Wouters477c8d52006-05-27 19:21:47 +00001325 if len(stops) < len(starts):
R. David Murrayb7deff12011-01-30 06:21:28 +00001326 stops.append(line_pos - len(linesep))
Thomas Wouters477c8d52006-05-27 19:21:47 +00001327 starts.append(next_pos)
1328 labels = [label.strip() for label
R. David Murrayb7deff12011-01-30 06:21:28 +00001329 in self._file.readline()[1:].split(b',')
Guido van Rossum7cad4f32007-06-07 23:58:54 +00001330 if label.strip()]
Thomas Wouters477c8d52006-05-27 19:21:47 +00001331 label_lists.append(labels)
R. David Murrayb7deff12011-01-30 06:21:28 +00001332 elif line == b'\037' or line == b'\037' + linesep:
Thomas Wouters477c8d52006-05-27 19:21:47 +00001333 if len(stops) < len(starts):
R. David Murrayb7deff12011-01-30 06:21:28 +00001334 stops.append(line_pos - len(linesep))
Guido van Rossum7cad4f32007-06-07 23:58:54 +00001335 elif not line:
R. David Murrayb7deff12011-01-30 06:21:28 +00001336 stops.append(line_pos - len(linesep))
Thomas Wouters477c8d52006-05-27 19:21:47 +00001337 break
1338 self._toc = dict(enumerate(zip(starts, stops)))
1339 self._labels = dict(enumerate(label_lists))
1340 self._next_key = len(self._toc)
Thomas Wouters902d6eb2007-01-09 23:18:33 +00001341 self._file.seek(0, 2)
1342 self._file_length = self._file.tell()
Thomas Wouters9fe394c2007-02-05 01:24:16 +00001343
Thomas Wouters477c8d52006-05-27 19:21:47 +00001344 def _pre_mailbox_hook(self, f):
1345 """Called before writing the mailbox to file f."""
R. David Murrayb7deff12011-01-30 06:21:28 +00001346 babyl = b'BABYL OPTIONS:' + linesep
1347 babyl += b'Version: 5' + linesep
1348 labels = self.get_labels()
1349 labels = (label.encode() for label in labels)
1350 babyl += b'Labels:' + b','.join(labels) + linesep
1351 babyl += b'\037'
1352 f.write(babyl)
Thomas Wouters477c8d52006-05-27 19:21:47 +00001353
1354 def _pre_message_hook(self, f):
1355 """Called before writing each message to file f."""
R. David Murrayb7deff12011-01-30 06:21:28 +00001356 f.write(b'\014' + linesep)
Thomas Wouters477c8d52006-05-27 19:21:47 +00001357
1358 def _post_message_hook(self, f):
1359 """Called after writing each message to file f."""
R. David Murrayb7deff12011-01-30 06:21:28 +00001360 f.write(linesep + b'\037')
Thomas Wouters477c8d52006-05-27 19:21:47 +00001361
1362 def _install_message(self, message):
1363 """Write message contents and return (start, stop)."""
1364 start = self._file.tell()
1365 if isinstance(message, BabylMessage):
1366 special_labels = []
1367 labels = []
1368 for label in message.get_labels():
1369 if label in self._special_labels:
1370 special_labels.append(label)
1371 else:
1372 labels.append(label)
R. David Murrayb7deff12011-01-30 06:21:28 +00001373 self._file.write(b'1')
Thomas Wouters477c8d52006-05-27 19:21:47 +00001374 for label in special_labels:
R. David Murrayb7deff12011-01-30 06:21:28 +00001375 self._file.write(b', ' + label.encode())
1376 self._file.write(b',,')
Thomas Wouters477c8d52006-05-27 19:21:47 +00001377 for label in labels:
R. David Murrayb7deff12011-01-30 06:21:28 +00001378 self._file.write(b' ' + label.encode() + b',')
1379 self._file.write(linesep)
Thomas Wouters477c8d52006-05-27 19:21:47 +00001380 else:
R. David Murrayb7deff12011-01-30 06:21:28 +00001381 self._file.write(b'1,,' + linesep)
Thomas Woutersb2137042007-02-01 18:02:27 +00001382 if isinstance(message, email.message.Message):
R. David Murrayb7deff12011-01-30 06:21:28 +00001383 orig_buffer = io.BytesIO()
1384 orig_generator = email.generator.BytesGenerator(orig_buffer, False, 0)
Thomas Wouters477c8d52006-05-27 19:21:47 +00001385 orig_generator.flatten(message)
1386 orig_buffer.seek(0)
1387 while True:
1388 line = orig_buffer.readline()
R. David Murrayb7deff12011-01-30 06:21:28 +00001389 self._file.write(line.replace(b'\n', linesep))
1390 if line == b'\n' or not line:
Thomas Wouters477c8d52006-05-27 19:21:47 +00001391 break
R. David Murrayb7deff12011-01-30 06:21:28 +00001392 self._file.write(b'*** EOOH ***' + linesep)
Thomas Wouters477c8d52006-05-27 19:21:47 +00001393 if isinstance(message, BabylMessage):
R. David Murrayb7deff12011-01-30 06:21:28 +00001394 vis_buffer = io.BytesIO()
1395 vis_generator = email.generator.BytesGenerator(vis_buffer, False, 0)
Thomas Wouters477c8d52006-05-27 19:21:47 +00001396 vis_generator.flatten(message.get_visible())
1397 while True:
1398 line = vis_buffer.readline()
R. David Murrayb7deff12011-01-30 06:21:28 +00001399 self._file.write(line.replace(b'\n', linesep))
1400 if line == b'\n' or not line:
Thomas Wouters477c8d52006-05-27 19:21:47 +00001401 break
1402 else:
1403 orig_buffer.seek(0)
1404 while True:
1405 line = orig_buffer.readline()
R. David Murrayb7deff12011-01-30 06:21:28 +00001406 self._file.write(line.replace(b'\n', linesep))
1407 if line == b'\n' or not line:
Thomas Wouters477c8d52006-05-27 19:21:47 +00001408 break
1409 while True:
1410 buffer = orig_buffer.read(4096) # Buffer size is arbitrary.
Guido van Rossum7cad4f32007-06-07 23:58:54 +00001411 if not buffer:
Thomas Wouters477c8d52006-05-27 19:21:47 +00001412 break
R. David Murrayb7deff12011-01-30 06:21:28 +00001413 self._file.write(buffer.replace(b'\n', linesep))
1414 elif isinstance(message, (bytes, str, io.StringIO)):
1415 if isinstance(message, io.StringIO):
1416 warnings.warn("Use of StringIO input is deprecated, "
1417 "use BytesIO instead", DeprecationWarning, 3)
1418 message = message.getvalue()
1419 if isinstance(message, str):
1420 message = self._string_to_bytes(message)
1421 body_start = message.find(b'\n\n') + 2
Thomas Wouters477c8d52006-05-27 19:21:47 +00001422 if body_start - 2 != -1:
R. David Murrayb7deff12011-01-30 06:21:28 +00001423 self._file.write(message[:body_start].replace(b'\n', linesep))
1424 self._file.write(b'*** EOOH ***' + linesep)
1425 self._file.write(message[:body_start].replace(b'\n', linesep))
1426 self._file.write(message[body_start:].replace(b'\n', linesep))
Thomas Wouters477c8d52006-05-27 19:21:47 +00001427 else:
R. David Murrayb7deff12011-01-30 06:21:28 +00001428 self._file.write(b'*** EOOH ***' + linesep + linesep)
1429 self._file.write(message.replace(b'\n', linesep))
Thomas Wouters477c8d52006-05-27 19:21:47 +00001430 elif hasattr(message, 'readline'):
R. David Murrayb7deff12011-01-30 06:21:28 +00001431 if hasattr(message, 'buffer'):
1432 warnings.warn("Use of text mode files is deprecated, "
1433 "use a binary mode file instead", DeprecationWarning, 3)
1434 message = message.buffer
Thomas Wouters477c8d52006-05-27 19:21:47 +00001435 original_pos = message.tell()
1436 first_pass = True
1437 while True:
1438 line = message.readline()
R. David Murrayb7deff12011-01-30 06:21:28 +00001439 # Universal newline support.
1440 if line.endswith(b'\r\n'):
1441 line = line[:-2] + b'\n'
1442 elif line.endswith(b'\r'):
1443 line = line[:-1] + b'\n'
1444 self._file.write(line.replace(b'\n', linesep))
1445 if line == b'\n' or not line:
1446 self._file.write(b'*** EOOH ***' + linesep)
Thomas Wouters477c8d52006-05-27 19:21:47 +00001447 if first_pass:
1448 first_pass = False
1449 message.seek(original_pos)
1450 else:
1451 break
1452 while True:
1453 buffer = message.read(4096) # Buffer size is arbitrary.
Guido van Rossum7cad4f32007-06-07 23:58:54 +00001454 if not buffer:
Thomas Wouters477c8d52006-05-27 19:21:47 +00001455 break
R. David Murrayb7deff12011-01-30 06:21:28 +00001456 self._file.write(buffer.replace(b'\n', linesep))
Thomas Wouters477c8d52006-05-27 19:21:47 +00001457 else:
1458 raise TypeError('Invalid message type: %s' % type(message))
1459 stop = self._file.tell()
1460 return (start, stop)
1461
1462
Thomas Woutersb2137042007-02-01 18:02:27 +00001463class Message(email.message.Message):
Thomas Wouters477c8d52006-05-27 19:21:47 +00001464 """Message with mailbox-format-specific properties."""
1465
1466 def __init__(self, message=None):
1467 """Initialize a Message instance."""
Thomas Woutersb2137042007-02-01 18:02:27 +00001468 if isinstance(message, email.message.Message):
Thomas Wouters477c8d52006-05-27 19:21:47 +00001469 self._become_message(copy.deepcopy(message))
1470 if isinstance(message, Message):
1471 message._explain_to(self)
R. David Murrayb7deff12011-01-30 06:21:28 +00001472 elif isinstance(message, bytes):
1473 self._become_message(email.message_from_bytes(message))
Thomas Wouters477c8d52006-05-27 19:21:47 +00001474 elif isinstance(message, str):
1475 self._become_message(email.message_from_string(message))
R. David Murrayb7deff12011-01-30 06:21:28 +00001476 elif isinstance(message, io.TextIOWrapper):
Thomas Wouters477c8d52006-05-27 19:21:47 +00001477 self._become_message(email.message_from_file(message))
R. David Murrayb7deff12011-01-30 06:21:28 +00001478 elif hasattr(message, "read"):
1479 self._become_message(email.message_from_binary_file(message))
Thomas Wouters477c8d52006-05-27 19:21:47 +00001480 elif message is None:
Thomas Woutersb2137042007-02-01 18:02:27 +00001481 email.message.Message.__init__(self)
Thomas Wouters477c8d52006-05-27 19:21:47 +00001482 else:
1483 raise TypeError('Invalid message type: %s' % type(message))
1484
1485 def _become_message(self, message):
1486 """Assume the non-format-specific state of message."""
1487 for name in ('_headers', '_unixfrom', '_payload', '_charset',
1488 'preamble', 'epilogue', 'defects', '_default_type'):
1489 self.__dict__[name] = message.__dict__[name]
1490
1491 def _explain_to(self, message):
1492 """Copy format-specific state to message insofar as possible."""
1493 if isinstance(message, Message):
1494 return # There's nothing format-specific to explain.
1495 else:
1496 raise TypeError('Cannot convert to specified type')
1497
1498
1499class MaildirMessage(Message):
1500 """Message with Maildir-specific properties."""
1501
1502 def __init__(self, message=None):
1503 """Initialize a MaildirMessage instance."""
1504 self._subdir = 'new'
1505 self._info = ''
1506 self._date = time.time()
1507 Message.__init__(self, message)
1508
1509 def get_subdir(self):
1510 """Return 'new' or 'cur'."""
1511 return self._subdir
1512
1513 def set_subdir(self, subdir):
1514 """Set subdir to 'new' or 'cur'."""
1515 if subdir == 'new' or subdir == 'cur':
1516 self._subdir = subdir
1517 else:
1518 raise ValueError("subdir must be 'new' or 'cur': %s" % subdir)
1519
1520 def get_flags(self):
1521 """Return as a string the flags that are set."""
1522 if self._info.startswith('2,'):
1523 return self._info[2:]
1524 else:
1525 return ''
1526
1527 def set_flags(self, flags):
1528 """Set the given flags and unset all others."""
1529 self._info = '2,' + ''.join(sorted(flags))
1530
1531 def add_flag(self, flag):
1532 """Set the given flag(s) without changing others."""
1533 self.set_flags(''.join(set(self.get_flags()) | set(flag)))
1534
1535 def remove_flag(self, flag):
1536 """Unset the given string flag(s) without changing others."""
Guido van Rossum7cad4f32007-06-07 23:58:54 +00001537 if self.get_flags():
Thomas Wouters477c8d52006-05-27 19:21:47 +00001538 self.set_flags(''.join(set(self.get_flags()) - set(flag)))
1539
1540 def get_date(self):
1541 """Return delivery date of message, in seconds since the epoch."""
1542 return self._date
1543
1544 def set_date(self, date):
1545 """Set delivery date of message, in seconds since the epoch."""
1546 try:
1547 self._date = float(date)
1548 except ValueError:
1549 raise TypeError("can't convert to float: %s" % date)
1550
1551 def get_info(self):
1552 """Get the message's "info" as a string."""
1553 return self._info
1554
1555 def set_info(self, info):
1556 """Set the message's "info" string."""
1557 if isinstance(info, str):
1558 self._info = info
1559 else:
1560 raise TypeError('info must be a string: %s' % type(info))
1561
1562 def _explain_to(self, message):
1563 """Copy Maildir-specific state to message insofar as possible."""
1564 if isinstance(message, MaildirMessage):
1565 message.set_flags(self.get_flags())
1566 message.set_subdir(self.get_subdir())
1567 message.set_date(self.get_date())
1568 elif isinstance(message, _mboxMMDFMessage):
1569 flags = set(self.get_flags())
1570 if 'S' in flags:
1571 message.add_flag('R')
1572 if self.get_subdir() == 'cur':
1573 message.add_flag('O')
1574 if 'T' in flags:
1575 message.add_flag('D')
1576 if 'F' in flags:
1577 message.add_flag('F')
1578 if 'R' in flags:
1579 message.add_flag('A')
1580 message.set_from('MAILER-DAEMON', time.gmtime(self.get_date()))
1581 elif isinstance(message, MHMessage):
1582 flags = set(self.get_flags())
1583 if 'S' not in flags:
1584 message.add_sequence('unseen')
1585 if 'R' in flags:
1586 message.add_sequence('replied')
1587 if 'F' in flags:
1588 message.add_sequence('flagged')
1589 elif isinstance(message, BabylMessage):
1590 flags = set(self.get_flags())
1591 if 'S' not in flags:
1592 message.add_label('unseen')
1593 if 'T' in flags:
1594 message.add_label('deleted')
1595 if 'R' in flags:
1596 message.add_label('answered')
1597 if 'P' in flags:
1598 message.add_label('forwarded')
1599 elif isinstance(message, Message):
1600 pass
1601 else:
1602 raise TypeError('Cannot convert to specified type: %s' %
1603 type(message))
1604
1605
1606class _mboxMMDFMessage(Message):
1607 """Message with mbox- or MMDF-specific properties."""
1608
1609 def __init__(self, message=None):
1610 """Initialize an mboxMMDFMessage instance."""
1611 self.set_from('MAILER-DAEMON', True)
Thomas Woutersb2137042007-02-01 18:02:27 +00001612 if isinstance(message, email.message.Message):
Thomas Wouters477c8d52006-05-27 19:21:47 +00001613 unixfrom = message.get_unixfrom()
1614 if unixfrom is not None and unixfrom.startswith('From '):
1615 self.set_from(unixfrom[5:])
1616 Message.__init__(self, message)
1617
1618 def get_from(self):
1619 """Return contents of "From " line."""
1620 return self._from
1621
1622 def set_from(self, from_, time_=None):
1623 """Set "From " line, formatting and appending time_ if specified."""
1624 if time_ is not None:
1625 if time_ is True:
1626 time_ = time.gmtime()
1627 from_ += ' ' + time.asctime(time_)
1628 self._from = from_
1629
1630 def get_flags(self):
1631 """Return as a string the flags that are set."""
1632 return self.get('Status', '') + self.get('X-Status', '')
1633
1634 def set_flags(self, flags):
1635 """Set the given flags and unset all others."""
1636 flags = set(flags)
1637 status_flags, xstatus_flags = '', ''
1638 for flag in ('R', 'O'):
1639 if flag in flags:
1640 status_flags += flag
1641 flags.remove(flag)
1642 for flag in ('D', 'F', 'A'):
1643 if flag in flags:
1644 xstatus_flags += flag
1645 flags.remove(flag)
1646 xstatus_flags += ''.join(sorted(flags))
1647 try:
1648 self.replace_header('Status', status_flags)
1649 except KeyError:
1650 self.add_header('Status', status_flags)
1651 try:
1652 self.replace_header('X-Status', xstatus_flags)
1653 except KeyError:
1654 self.add_header('X-Status', xstatus_flags)
1655
1656 def add_flag(self, flag):
1657 """Set the given flag(s) without changing others."""
1658 self.set_flags(''.join(set(self.get_flags()) | set(flag)))
1659
1660 def remove_flag(self, flag):
1661 """Unset the given string flag(s) without changing others."""
1662 if 'Status' in self or 'X-Status' in self:
1663 self.set_flags(''.join(set(self.get_flags()) - set(flag)))
1664
1665 def _explain_to(self, message):
1666 """Copy mbox- or MMDF-specific state to message insofar as possible."""
1667 if isinstance(message, MaildirMessage):
1668 flags = set(self.get_flags())
1669 if 'O' in flags:
1670 message.set_subdir('cur')
1671 if 'F' in flags:
1672 message.add_flag('F')
1673 if 'A' in flags:
1674 message.add_flag('R')
1675 if 'R' in flags:
1676 message.add_flag('S')
1677 if 'D' in flags:
1678 message.add_flag('T')
1679 del message['status']
1680 del message['x-status']
1681 maybe_date = ' '.join(self.get_from().split()[-5:])
1682 try:
1683 message.set_date(calendar.timegm(time.strptime(maybe_date,
1684 '%a %b %d %H:%M:%S %Y')))
1685 except (ValueError, OverflowError):
1686 pass
1687 elif isinstance(message, _mboxMMDFMessage):
1688 message.set_flags(self.get_flags())
1689 message.set_from(self.get_from())
1690 elif isinstance(message, MHMessage):
1691 flags = set(self.get_flags())
1692 if 'R' not in flags:
1693 message.add_sequence('unseen')
1694 if 'A' in flags:
1695 message.add_sequence('replied')
1696 if 'F' in flags:
1697 message.add_sequence('flagged')
1698 del message['status']
1699 del message['x-status']
1700 elif isinstance(message, BabylMessage):
1701 flags = set(self.get_flags())
1702 if 'R' not in flags:
1703 message.add_label('unseen')
1704 if 'D' in flags:
1705 message.add_label('deleted')
1706 if 'A' in flags:
1707 message.add_label('answered')
1708 del message['status']
1709 del message['x-status']
1710 elif isinstance(message, Message):
1711 pass
1712 else:
1713 raise TypeError('Cannot convert to specified type: %s' %
1714 type(message))
1715
1716
1717class mboxMessage(_mboxMMDFMessage):
1718 """Message with mbox-specific properties."""
1719
1720
1721class MHMessage(Message):
1722 """Message with MH-specific properties."""
1723
1724 def __init__(self, message=None):
1725 """Initialize an MHMessage instance."""
1726 self._sequences = []
1727 Message.__init__(self, message)
1728
1729 def get_sequences(self):
1730 """Return a list of sequences that include the message."""
1731 return self._sequences[:]
1732
1733 def set_sequences(self, sequences):
1734 """Set the list of sequences that include the message."""
1735 self._sequences = list(sequences)
1736
1737 def add_sequence(self, sequence):
1738 """Add sequence to list of sequences including the message."""
1739 if isinstance(sequence, str):
1740 if not sequence in self._sequences:
1741 self._sequences.append(sequence)
1742 else:
R. David Murrayb7deff12011-01-30 06:21:28 +00001743 raise TypeError('sequence type must be str: %s' % type(sequence))
Thomas Wouters477c8d52006-05-27 19:21:47 +00001744
1745 def remove_sequence(self, sequence):
1746 """Remove sequence from the list of sequences including the message."""
1747 try:
1748 self._sequences.remove(sequence)
1749 except ValueError:
1750 pass
1751
1752 def _explain_to(self, message):
1753 """Copy MH-specific state to message insofar as possible."""
1754 if isinstance(message, MaildirMessage):
1755 sequences = set(self.get_sequences())
1756 if 'unseen' in sequences:
1757 message.set_subdir('cur')
1758 else:
1759 message.set_subdir('cur')
1760 message.add_flag('S')
1761 if 'flagged' in sequences:
1762 message.add_flag('F')
1763 if 'replied' in sequences:
1764 message.add_flag('R')
1765 elif isinstance(message, _mboxMMDFMessage):
1766 sequences = set(self.get_sequences())
1767 if 'unseen' not in sequences:
1768 message.add_flag('RO')
1769 else:
1770 message.add_flag('O')
1771 if 'flagged' in sequences:
1772 message.add_flag('F')
1773 if 'replied' in sequences:
1774 message.add_flag('A')
1775 elif isinstance(message, MHMessage):
1776 for sequence in self.get_sequences():
1777 message.add_sequence(sequence)
1778 elif isinstance(message, BabylMessage):
1779 sequences = set(self.get_sequences())
1780 if 'unseen' in sequences:
1781 message.add_label('unseen')
1782 if 'replied' in sequences:
1783 message.add_label('answered')
1784 elif isinstance(message, Message):
1785 pass
1786 else:
1787 raise TypeError('Cannot convert to specified type: %s' %
1788 type(message))
1789
1790
1791class BabylMessage(Message):
1792 """Message with Babyl-specific properties."""
1793
1794 def __init__(self, message=None):
1795 """Initialize an BabylMessage instance."""
1796 self._labels = []
1797 self._visible = Message()
1798 Message.__init__(self, message)
1799
1800 def get_labels(self):
1801 """Return a list of labels on the message."""
1802 return self._labels[:]
1803
1804 def set_labels(self, labels):
1805 """Set the list of labels on the message."""
1806 self._labels = list(labels)
1807
1808 def add_label(self, label):
1809 """Add label to list of labels on the message."""
1810 if isinstance(label, str):
1811 if label not in self._labels:
1812 self._labels.append(label)
1813 else:
1814 raise TypeError('label must be a string: %s' % type(label))
1815
1816 def remove_label(self, label):
1817 """Remove label from the list of labels on the message."""
1818 try:
1819 self._labels.remove(label)
1820 except ValueError:
1821 pass
1822
1823 def get_visible(self):
1824 """Return a Message representation of visible headers."""
1825 return Message(self._visible)
1826
1827 def set_visible(self, visible):
1828 """Set the Message representation of visible headers."""
1829 self._visible = Message(visible)
1830
1831 def update_visible(self):
1832 """Update and/or sensibly generate a set of visible headers."""
1833 for header in self._visible.keys():
1834 if header in self:
1835 self._visible.replace_header(header, self[header])
1836 else:
1837 del self._visible[header]
1838 for header in ('Date', 'From', 'Reply-To', 'To', 'CC', 'Subject'):
1839 if header in self and header not in self._visible:
1840 self._visible[header] = self[header]
1841
1842 def _explain_to(self, message):
1843 """Copy Babyl-specific state to message insofar as possible."""
1844 if isinstance(message, MaildirMessage):
1845 labels = set(self.get_labels())
1846 if 'unseen' in labels:
1847 message.set_subdir('cur')
1848 else:
1849 message.set_subdir('cur')
1850 message.add_flag('S')
1851 if 'forwarded' in labels or 'resent' in labels:
1852 message.add_flag('P')
1853 if 'answered' in labels:
1854 message.add_flag('R')
1855 if 'deleted' in labels:
1856 message.add_flag('T')
1857 elif isinstance(message, _mboxMMDFMessage):
1858 labels = set(self.get_labels())
1859 if 'unseen' not in labels:
1860 message.add_flag('RO')
1861 else:
1862 message.add_flag('O')
1863 if 'deleted' in labels:
1864 message.add_flag('D')
1865 if 'answered' in labels:
1866 message.add_flag('A')
1867 elif isinstance(message, MHMessage):
1868 labels = set(self.get_labels())
1869 if 'unseen' in labels:
1870 message.add_sequence('unseen')
1871 if 'answered' in labels:
1872 message.add_sequence('replied')
1873 elif isinstance(message, BabylMessage):
1874 message.set_visible(self.get_visible())
1875 for label in self.get_labels():
1876 message.add_label(label)
1877 elif isinstance(message, Message):
1878 pass
1879 else:
1880 raise TypeError('Cannot convert to specified type: %s' %
1881 type(message))
1882
1883
1884class MMDFMessage(_mboxMMDFMessage):
1885 """Message with MMDF-specific properties."""
1886
1887
1888class _ProxyFile:
1889 """A read-only wrapper of a file."""
1890
1891 def __init__(self, f, pos=None):
1892 """Initialize a _ProxyFile."""
1893 self._file = f
1894 if pos is None:
1895 self._pos = f.tell()
1896 else:
1897 self._pos = pos
1898
1899 def read(self, size=None):
1900 """Read bytes."""
Guido van Rossum98297ee2007-11-06 21:34:58 +00001901 return self._read(size, self._file.read)
Thomas Wouters477c8d52006-05-27 19:21:47 +00001902
R. David Murrayb7deff12011-01-30 06:21:28 +00001903 def read1(self, size=None):
1904 """Read bytes."""
1905 return self._read(size, self._file.read1)
1906
Thomas Wouters477c8d52006-05-27 19:21:47 +00001907 def readline(self, size=None):
1908 """Read a line."""
Guido van Rossum98297ee2007-11-06 21:34:58 +00001909 return self._read(size, self._file.readline)
Thomas Wouters477c8d52006-05-27 19:21:47 +00001910
1911 def readlines(self, sizehint=None):
1912 """Read multiple lines."""
1913 result = []
1914 for line in self:
1915 result.append(line)
1916 if sizehint is not None:
1917 sizehint -= len(line)
1918 if sizehint <= 0:
1919 break
1920 return result
1921
1922 def __iter__(self):
1923 """Iterate over lines."""
Guido van Rossum7cad4f32007-06-07 23:58:54 +00001924 while True:
1925 line = self.readline()
1926 if not line:
1927 raise StopIteration
1928 yield line
Thomas Wouters477c8d52006-05-27 19:21:47 +00001929
1930 def tell(self):
1931 """Return the position."""
1932 return self._pos
1933
1934 def seek(self, offset, whence=0):
1935 """Change position."""
1936 if whence == 1:
1937 self._file.seek(self._pos)
1938 self._file.seek(offset, whence)
1939 self._pos = self._file.tell()
1940
1941 def close(self):
1942 """Close the file."""
R David Murrayc88bce12011-06-17 22:24:05 -04001943 if hasattr(self, '_file'):
1944 if hasattr(self._file, 'close'):
1945 self._file.close()
1946 del self._file
Thomas Wouters477c8d52006-05-27 19:21:47 +00001947
1948 def _read(self, size, read_method):
1949 """Read size bytes using read_method."""
1950 if size is None:
1951 size = -1
1952 self._file.seek(self._pos)
1953 result = read_method(size)
1954 self._pos = self._file.tell()
1955 return result
1956
Georg Brandl6ce29fa2010-10-30 14:33:28 +00001957 def __enter__(self):
1958 """Context manager protocol support."""
1959 return self
1960
1961 def __exit__(self, *exc):
1962 self.close()
1963
R. David Murrayb7deff12011-01-30 06:21:28 +00001964 def readable(self):
1965 return self._file.readable()
1966
1967 def writable(self):
1968 return self._file.writable()
1969
1970 def seekable(self):
1971 return self._file.seekable()
1972
1973 def flush(self):
1974 return self._file.flush()
1975
1976 @property
1977 def closed(self):
R David Murrayc88bce12011-06-17 22:24:05 -04001978 if not hasattr(self, '_file'):
1979 return True
1980 if not hasattr(self._file, 'closed'):
1981 return False
R. David Murrayb7deff12011-01-30 06:21:28 +00001982 return self._file.closed
1983
Thomas Wouters477c8d52006-05-27 19:21:47 +00001984
1985class _PartialFile(_ProxyFile):
1986 """A read-only wrapper of part of a file."""
1987
1988 def __init__(self, f, start=None, stop=None):
1989 """Initialize a _PartialFile."""
1990 _ProxyFile.__init__(self, f, start)
1991 self._start = start
1992 self._stop = stop
1993
1994 def tell(self):
1995 """Return the position with respect to start."""
1996 return _ProxyFile.tell(self) - self._start
1997
1998 def seek(self, offset, whence=0):
1999 """Change position, possibly with respect to start or stop."""
2000 if whence == 0:
2001 self._pos = self._start
2002 whence = 1
2003 elif whence == 2:
2004 self._pos = self._stop
2005 whence = 1
2006 _ProxyFile.seek(self, offset, whence)
2007
2008 def _read(self, size, read_method):
2009 """Read size bytes using read_method, honoring start and stop."""
2010 remaining = self._stop - self._pos
2011 if remaining <= 0:
R. David Murrayb7deff12011-01-30 06:21:28 +00002012 return b''
Thomas Wouters477c8d52006-05-27 19:21:47 +00002013 if size is None or size < 0 or size > remaining:
2014 size = remaining
2015 return _ProxyFile._read(self, size, read_method)
2016
Georg Brandl6ce29fa2010-10-30 14:33:28 +00002017 def close(self):
2018 # do *not* close the underlying file object for partial files,
2019 # since it's global to the mailbox object
R David Murrayc88bce12011-06-17 22:24:05 -04002020 if hasattr(self, '_file'):
2021 del self._file
Georg Brandl6ce29fa2010-10-30 14:33:28 +00002022
Thomas Wouters477c8d52006-05-27 19:21:47 +00002023
2024def _lock_file(f, dotlock=True):
Thomas Wouters0e3f5912006-08-11 14:57:12 +00002025 """Lock file f using lockf and dot locking."""
Thomas Wouters477c8d52006-05-27 19:21:47 +00002026 dotlock_done = False
2027 try:
2028 if fcntl:
2029 try:
2030 fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
Guido van Rossumb940e112007-01-10 16:19:56 +00002031 except IOError as e:
R. David Murray752b9502011-03-03 18:12:34 +00002032 if e.errno in (errno.EAGAIN, errno.EACCES, errno.EROFS):
Thomas Wouters477c8d52006-05-27 19:21:47 +00002033 raise ExternalClashError('lockf: lock unavailable: %s' %
2034 f.name)
2035 else:
2036 raise
Thomas Wouters477c8d52006-05-27 19:21:47 +00002037 if dotlock:
2038 try:
2039 pre_lock = _create_temporary(f.name + '.lock')
2040 pre_lock.close()
Guido van Rossumb940e112007-01-10 16:19:56 +00002041 except IOError as e:
R. David Murray752b9502011-03-03 18:12:34 +00002042 if e.errno in (errno.EACCES, errno.EROFS):
Thomas Wouters477c8d52006-05-27 19:21:47 +00002043 return # Without write access, just skip dotlocking.
2044 else:
2045 raise
2046 try:
2047 if hasattr(os, 'link'):
2048 os.link(pre_lock.name, f.name + '.lock')
2049 dotlock_done = True
2050 os.unlink(pre_lock.name)
2051 else:
2052 os.rename(pre_lock.name, f.name + '.lock')
2053 dotlock_done = True
Guido van Rossumb940e112007-01-10 16:19:56 +00002054 except OSError as e:
Thomas Wouters0e3f5912006-08-11 14:57:12 +00002055 if e.errno == errno.EEXIST or \
2056 (os.name == 'os2' and e.errno == errno.EACCES):
Thomas Wouters477c8d52006-05-27 19:21:47 +00002057 os.remove(pre_lock.name)
2058 raise ExternalClashError('dot lock unavailable: %s' %
2059 f.name)
2060 else:
2061 raise
2062 except:
2063 if fcntl:
2064 fcntl.lockf(f, fcntl.LOCK_UN)
Thomas Wouters477c8d52006-05-27 19:21:47 +00002065 if dotlock_done:
2066 os.remove(f.name + '.lock')
2067 raise
2068
2069def _unlock_file(f):
Thomas Wouters0e3f5912006-08-11 14:57:12 +00002070 """Unlock file f using lockf and dot locking."""
Thomas Wouters477c8d52006-05-27 19:21:47 +00002071 if fcntl:
2072 fcntl.lockf(f, fcntl.LOCK_UN)
Thomas Wouters477c8d52006-05-27 19:21:47 +00002073 if os.path.exists(f.name + '.lock'):
2074 os.remove(f.name + '.lock')
2075
2076def _create_carefully(path):
2077 """Create a file if it doesn't exist and open for reading and writing."""
Georg Brandl6aa2d1f2008-08-12 08:35:52 +00002078 fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o666)
Thomas Wouters477c8d52006-05-27 19:21:47 +00002079 try:
R. David Murrayb7deff12011-01-30 06:21:28 +00002080 return open(path, 'rb+')
Thomas Wouters477c8d52006-05-27 19:21:47 +00002081 finally:
2082 os.close(fd)
2083
2084def _create_temporary(path):
2085 """Create a temp file based on path and open for reading and writing."""
2086 return _create_carefully('%s.%s.%s.%s' % (path, int(time.time()),
2087 socket.gethostname(),
2088 os.getpid()))
2089
Thomas Wouters89f507f2006-12-13 04:49:30 +00002090def _sync_flush(f):
2091 """Ensure changes to file f are physically on disk."""
2092 f.flush()
Thomas Wouters902d6eb2007-01-09 23:18:33 +00002093 if hasattr(os, 'fsync'):
2094 os.fsync(f.fileno())
Thomas Wouters89f507f2006-12-13 04:49:30 +00002095
2096def _sync_close(f):
2097 """Close file f, ensuring all changes are physically on disk."""
2098 _sync_flush(f)
2099 f.close()
Thomas Wouters477c8d52006-05-27 19:21:47 +00002100
Guido van Rossum62448671996-09-17 21:33:15 +00002101
Thomas Wouters477c8d52006-05-27 19:21:47 +00002102class Error(Exception):
2103 """Raised for module-specific errors."""
2104
2105class NoSuchMailboxError(Error):
2106 """The specified mailbox does not exist and won't be created."""
2107
2108class NotEmptyError(Error):
2109 """The specified mailbox is not empty and deletion was requested."""
2110
2111class ExternalClashError(Error):
2112 """Another process caused an action to fail."""
2113
2114class FormatError(Error):
2115 """A file appears to have an invalid format."""