darn! I converted half of the files the wrong way.
diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py
index ea89090..7235a41 100644
--- a/Lib/multiprocessing/queues.py
+++ b/Lib/multiprocessing/queues.py
@@ -1,356 +1,356 @@
-#
-# Module implementing queues
-#
-# multiprocessing/queues.py
-#
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
-#
-
-__all__ = ['Queue', 'SimpleQueue']
-
-import sys
-import os
-import threading
-import collections
-import time
-import atexit
-import weakref
-
-from Queue import Empty, Full
-import _multiprocessing
-from multiprocessing import Pipe
-from multiprocessing.synchronize import Lock, BoundedSemaphore, Semaphore, Condition
-from multiprocessing.util import debug, info, Finalize, register_after_fork
-from multiprocessing.forking import assert_spawning
-
-#
-# Queue type using a pipe, buffer and thread
-#
-
-class Queue(object):
-
- def __init__(self, maxsize=0):
- if maxsize <= 0:
- maxsize = _multiprocessing.SemLock.SEM_VALUE_MAX
- self._maxsize = maxsize
- self._reader, self._writer = Pipe(duplex=False)
- self._rlock = Lock()
- self._opid = os.getpid()
- if sys.platform == 'win32':
- self._wlock = None
- else:
- self._wlock = Lock()
- self._sem = BoundedSemaphore(maxsize)
-
- self._after_fork()
-
- if sys.platform != 'win32':
- register_after_fork(self, Queue._after_fork)
-
- def __getstate__(self):
- assert_spawning(self)
- return (self._maxsize, self._reader, self._writer,
- self._rlock, self._wlock, self._sem, self._opid)
-
- def __setstate__(self, state):
- (self._maxsize, self._reader, self._writer,
- self._rlock, self._wlock, self._sem, self._opid) = state
- self._after_fork()
-
- def _after_fork(self):
- debug('Queue._after_fork()')
- self._notempty = threading.Condition(threading.Lock())
- self._buffer = collections.deque()
- self._thread = None
- self._jointhread = None
- self._joincancelled = False
- self._closed = False
- self._close = None
- self._send = self._writer.send
- self._recv = self._reader.recv
- self._poll = self._reader.poll
-
- def put(self, obj, block=True, timeout=None):
- assert not self._closed
- if not self._sem.acquire(block, timeout):
- raise Full
-
- self._notempty.acquire()
- try:
- if self._thread is None:
- self._start_thread()
- self._buffer.append(obj)
- self._notempty.notify()
- finally:
- self._notempty.release()
-
- def get(self, block=True, timeout=None):
- if block and timeout is None:
- self._rlock.acquire()
- try:
- res = self._recv()
- self._sem.release()
- return res
- finally:
- self._rlock.release()
-
- else:
- if block:
- deadline = time.time() + timeout
- if not self._rlock.acquire(block, timeout):
- raise Empty
- try:
- if not self._poll(block and (deadline-time.time()) or 0.0):
- raise Empty
- res = self._recv()
- self._sem.release()
- return res
- finally:
- self._rlock.release()
-
- def qsize(self):
- # Raises NotImplementError on Mac OSX because of broken sem_getvalue()
- return self._maxsize - self._sem._semlock._get_value()
-
- def empty(self):
- return not self._poll()
-
- def full(self):
- return self._sem._semlock._is_zero()
-
- def get_nowait(self):
- return self.get(False)
-
- def put_nowait(self, obj):
- return self.put(obj, False)
-
- def close(self):
- self._closed = True
- self._reader.close()
- if self._close:
- self._close()
-
- def join_thread(self):
- debug('Queue.join_thread()')
- assert self._closed
- if self._jointhread:
- self._jointhread()
-
- def cancel_join_thread(self):
- debug('Queue.cancel_join_thread()')
- self._joincancelled = True
- try:
- self._jointhread.cancel()
- except AttributeError:
- pass
-
- def _start_thread(self):
- debug('Queue._start_thread()')
-
- # Start thread which transfers data from buffer to pipe
- self._buffer.clear()
- self._thread = threading.Thread(
- target=Queue._feed,
- args=(self._buffer, self._notempty, self._send,
- self._wlock, self._writer.close),
- name='QueueFeederThread'
- )
- self._thread.set_daemon(True)
-
- debug('doing self._thread.start()')
- self._thread.start()
- debug('... done self._thread.start()')
-
- # On process exit we will wait for data to be flushed to pipe.
- #
- # However, if this process created the queue then all
- # processes which use the queue will be descendants of this
- # process. Therefore waiting for the queue to be flushed
- # is pointless once all the child processes have been joined.
- created_by_this_process = (self._opid == os.getpid())
- if not self._joincancelled and not created_by_this_process:
- self._jointhread = Finalize(
- self._thread, Queue._finalize_join,
- [weakref.ref(self._thread)],
- exitpriority=-5
- )
-
- # Send sentinel to the thread queue object when garbage collected
- self._close = Finalize(
- self, Queue._finalize_close,
- [self._buffer, self._notempty],
- exitpriority=10
- )
-
- @staticmethod
- def _finalize_join(twr):
- debug('joining queue thread')
- thread = twr()
- if thread is not None:
- thread.join()
- debug('... queue thread joined')
- else:
- debug('... queue thread already dead')
-
- @staticmethod
- def _finalize_close(buffer, notempty):
- debug('telling queue thread to quit')
- notempty.acquire()
- try:
- buffer.append(_sentinel)
- notempty.notify()
- finally:
- notempty.release()
-
- @staticmethod
- def _feed(buffer, notempty, send, writelock, close):
- debug('starting thread to feed data to pipe')
- from .util import is_exiting
-
- nacquire = notempty.acquire
- nrelease = notempty.release
- nwait = notempty.wait
- bpopleft = buffer.popleft
- sentinel = _sentinel
- if sys.platform != 'win32':
- wacquire = writelock.acquire
- wrelease = writelock.release
- else:
- wacquire = None
-
- try:
- while 1:
- nacquire()
- try:
- if not buffer:
- nwait()
- finally:
- nrelease()
- try:
- while 1:
- obj = bpopleft()
- if obj is sentinel:
- debug('feeder thread got sentinel -- exiting')
- close()
- return
-
- if wacquire is None:
- send(obj)
- else:
- wacquire()
- try:
- send(obj)
- finally:
- wrelease()
- except IndexError:
- pass
- except Exception, e:
- # Since this runs in a daemon thread the resources it uses
- # may be become unusable while the process is cleaning up.
- # We ignore errors which happen after the process has
- # started to cleanup.
- try:
- if is_exiting():
- info('error in queue thread: %s', e)
- else:
- import traceback
- traceback.print_exc()
- except Exception:
- pass
-
-_sentinel = object()
-
-#
-# A queue type which also supports join() and task_done() methods
-#
-# Note that if you do not call task_done() for each finished task then
-# eventually the counter's semaphore may overflow causing Bad Things
-# to happen.
-#
-
-class JoinableQueue(Queue):
-
- def __init__(self, maxsize=0):
- Queue.__init__(self, maxsize)
- self._unfinished_tasks = Semaphore(0)
- self._cond = Condition()
-
- def __getstate__(self):
- return Queue.__getstate__(self) + (self._cond, self._unfinished_tasks)
-
- def __setstate__(self, state):
- Queue.__setstate__(self, state[:-2])
- self._cond, self._unfinished_tasks = state[-2:]
-
- def put(self, item, block=True, timeout=None):
- Queue.put(self, item, block, timeout)
- self._unfinished_tasks.release()
-
- def task_done(self):
- self._cond.acquire()
- try:
- if not self._unfinished_tasks.acquire(False):
- raise ValueError('task_done() called too many times')
- if self._unfinished_tasks._semlock._is_zero():
- self._cond.notify_all()
- finally:
- self._cond.release()
-
- def join(self):
- self._cond.acquire()
- try:
- if not self._unfinished_tasks._semlock._is_zero():
- self._cond.wait()
- finally:
- self._cond.release()
-
-#
-# Simplified Queue type -- really just a locked pipe
-#
-
-class SimpleQueue(object):
-
- def __init__(self):
- self._reader, self._writer = Pipe(duplex=False)
- self._rlock = Lock()
- if sys.platform == 'win32':
- self._wlock = None
- else:
- self._wlock = Lock()
- self._make_methods()
-
- def empty(self):
- return not self._reader.poll()
-
- def __getstate__(self):
- assert_spawning(self)
- return (self._reader, self._writer, self._rlock, self._wlock)
-
- def __setstate__(self, state):
- (self._reader, self._writer, self._rlock, self._wlock) = state
- self._make_methods()
-
- def _make_methods(self):
- recv = self._reader.recv
- racquire, rrelease = self._rlock.acquire, self._rlock.release
- def get():
- racquire()
- try:
- return recv()
- finally:
- rrelease()
- self.get = get
-
- if self._wlock is None:
- # writes to a message oriented win32 pipe are atomic
- self.put = self._writer.send
- else:
- send = self._writer.send
- wacquire, wrelease = self._wlock.acquire, self._wlock.release
- def put(obj):
- wacquire()
- try:
- return send(obj)
- finally:
- wrelease()
- self.put = put
+#
+# Module implementing queues
+#
+# multiprocessing/queues.py
+#
+# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+#
+
+__all__ = ['Queue', 'SimpleQueue']
+
+import sys
+import os
+import threading
+import collections
+import time
+import atexit
+import weakref
+
+from Queue import Empty, Full
+import _multiprocessing
+from multiprocessing import Pipe
+from multiprocessing.synchronize import Lock, BoundedSemaphore, Semaphore, Condition
+from multiprocessing.util import debug, info, Finalize, register_after_fork
+from multiprocessing.forking import assert_spawning
+
+#
+# Queue type using a pipe, buffer and thread
+#
+
+class Queue(object):
+
+ def __init__(self, maxsize=0):
+ if maxsize <= 0:
+ maxsize = _multiprocessing.SemLock.SEM_VALUE_MAX
+ self._maxsize = maxsize
+ self._reader, self._writer = Pipe(duplex=False)
+ self._rlock = Lock()
+ self._opid = os.getpid()
+ if sys.platform == 'win32':
+ self._wlock = None
+ else:
+ self._wlock = Lock()
+ self._sem = BoundedSemaphore(maxsize)
+
+ self._after_fork()
+
+ if sys.platform != 'win32':
+ register_after_fork(self, Queue._after_fork)
+
+ def __getstate__(self):
+ assert_spawning(self)
+ return (self._maxsize, self._reader, self._writer,
+ self._rlock, self._wlock, self._sem, self._opid)
+
+ def __setstate__(self, state):
+ (self._maxsize, self._reader, self._writer,
+ self._rlock, self._wlock, self._sem, self._opid) = state
+ self._after_fork()
+
+ def _after_fork(self):
+ debug('Queue._after_fork()')
+ self._notempty = threading.Condition(threading.Lock())
+ self._buffer = collections.deque()
+ self._thread = None
+ self._jointhread = None
+ self._joincancelled = False
+ self._closed = False
+ self._close = None
+ self._send = self._writer.send
+ self._recv = self._reader.recv
+ self._poll = self._reader.poll
+
+ def put(self, obj, block=True, timeout=None):
+ assert not self._closed
+ if not self._sem.acquire(block, timeout):
+ raise Full
+
+ self._notempty.acquire()
+ try:
+ if self._thread is None:
+ self._start_thread()
+ self._buffer.append(obj)
+ self._notempty.notify()
+ finally:
+ self._notempty.release()
+
+ def get(self, block=True, timeout=None):
+ if block and timeout is None:
+ self._rlock.acquire()
+ try:
+ res = self._recv()
+ self._sem.release()
+ return res
+ finally:
+ self._rlock.release()
+
+ else:
+ if block:
+ deadline = time.time() + timeout
+ if not self._rlock.acquire(block, timeout):
+ raise Empty
+ try:
+ if not self._poll(block and (deadline-time.time()) or 0.0):
+ raise Empty
+ res = self._recv()
+ self._sem.release()
+ return res
+ finally:
+ self._rlock.release()
+
+ def qsize(self):
+ # Raises NotImplementError on Mac OSX because of broken sem_getvalue()
+ return self._maxsize - self._sem._semlock._get_value()
+
+ def empty(self):
+ return not self._poll()
+
+ def full(self):
+ return self._sem._semlock._is_zero()
+
+ def get_nowait(self):
+ return self.get(False)
+
+ def put_nowait(self, obj):
+ return self.put(obj, False)
+
+ def close(self):
+ self._closed = True
+ self._reader.close()
+ if self._close:
+ self._close()
+
+ def join_thread(self):
+ debug('Queue.join_thread()')
+ assert self._closed
+ if self._jointhread:
+ self._jointhread()
+
+ def cancel_join_thread(self):
+ debug('Queue.cancel_join_thread()')
+ self._joincancelled = True
+ try:
+ self._jointhread.cancel()
+ except AttributeError:
+ pass
+
+ def _start_thread(self):
+ debug('Queue._start_thread()')
+
+ # Start thread which transfers data from buffer to pipe
+ self._buffer.clear()
+ self._thread = threading.Thread(
+ target=Queue._feed,
+ args=(self._buffer, self._notempty, self._send,
+ self._wlock, self._writer.close),
+ name='QueueFeederThread'
+ )
+ self._thread.set_daemon(True)
+
+ debug('doing self._thread.start()')
+ self._thread.start()
+ debug('... done self._thread.start()')
+
+ # On process exit we will wait for data to be flushed to pipe.
+ #
+ # However, if this process created the queue then all
+ # processes which use the queue will be descendants of this
+ # process. Therefore waiting for the queue to be flushed
+ # is pointless once all the child processes have been joined.
+ created_by_this_process = (self._opid == os.getpid())
+ if not self._joincancelled and not created_by_this_process:
+ self._jointhread = Finalize(
+ self._thread, Queue._finalize_join,
+ [weakref.ref(self._thread)],
+ exitpriority=-5
+ )
+
+ # Send sentinel to the thread queue object when garbage collected
+ self._close = Finalize(
+ self, Queue._finalize_close,
+ [self._buffer, self._notempty],
+ exitpriority=10
+ )
+
+ @staticmethod
+ def _finalize_join(twr):
+ debug('joining queue thread')
+ thread = twr()
+ if thread is not None:
+ thread.join()
+ debug('... queue thread joined')
+ else:
+ debug('... queue thread already dead')
+
+ @staticmethod
+ def _finalize_close(buffer, notempty):
+ debug('telling queue thread to quit')
+ notempty.acquire()
+ try:
+ buffer.append(_sentinel)
+ notempty.notify()
+ finally:
+ notempty.release()
+
+ @staticmethod
+ def _feed(buffer, notempty, send, writelock, close):
+ debug('starting thread to feed data to pipe')
+ from .util import is_exiting
+
+ nacquire = notempty.acquire
+ nrelease = notempty.release
+ nwait = notempty.wait
+ bpopleft = buffer.popleft
+ sentinel = _sentinel
+ if sys.platform != 'win32':
+ wacquire = writelock.acquire
+ wrelease = writelock.release
+ else:
+ wacquire = None
+
+ try:
+ while 1:
+ nacquire()
+ try:
+ if not buffer:
+ nwait()
+ finally:
+ nrelease()
+ try:
+ while 1:
+ obj = bpopleft()
+ if obj is sentinel:
+ debug('feeder thread got sentinel -- exiting')
+ close()
+ return
+
+ if wacquire is None:
+ send(obj)
+ else:
+ wacquire()
+ try:
+ send(obj)
+ finally:
+ wrelease()
+ except IndexError:
+ pass
+ except Exception, e:
+ # Since this runs in a daemon thread the resources it uses
+ # may be become unusable while the process is cleaning up.
+ # We ignore errors which happen after the process has
+ # started to cleanup.
+ try:
+ if is_exiting():
+ info('error in queue thread: %s', e)
+ else:
+ import traceback
+ traceback.print_exc()
+ except Exception:
+ pass
+
+_sentinel = object()
+
+#
+# A queue type which also supports join() and task_done() methods
+#
+# Note that if you do not call task_done() for each finished task then
+# eventually the counter's semaphore may overflow causing Bad Things
+# to happen.
+#
+
+class JoinableQueue(Queue):
+
+ def __init__(self, maxsize=0):
+ Queue.__init__(self, maxsize)
+ self._unfinished_tasks = Semaphore(0)
+ self._cond = Condition()
+
+ def __getstate__(self):
+ return Queue.__getstate__(self) + (self._cond, self._unfinished_tasks)
+
+ def __setstate__(self, state):
+ Queue.__setstate__(self, state[:-2])
+ self._cond, self._unfinished_tasks = state[-2:]
+
+ def put(self, item, block=True, timeout=None):
+ Queue.put(self, item, block, timeout)
+ self._unfinished_tasks.release()
+
+ def task_done(self):
+ self._cond.acquire()
+ try:
+ if not self._unfinished_tasks.acquire(False):
+ raise ValueError('task_done() called too many times')
+ if self._unfinished_tasks._semlock._is_zero():
+ self._cond.notify_all()
+ finally:
+ self._cond.release()
+
+ def join(self):
+ self._cond.acquire()
+ try:
+ if not self._unfinished_tasks._semlock._is_zero():
+ self._cond.wait()
+ finally:
+ self._cond.release()
+
+#
+# Simplified Queue type -- really just a locked pipe
+#
+
+class SimpleQueue(object):
+
+ def __init__(self):
+ self._reader, self._writer = Pipe(duplex=False)
+ self._rlock = Lock()
+ if sys.platform == 'win32':
+ self._wlock = None
+ else:
+ self._wlock = Lock()
+ self._make_methods()
+
+ def empty(self):
+ return not self._reader.poll()
+
+ def __getstate__(self):
+ assert_spawning(self)
+ return (self._reader, self._writer, self._rlock, self._wlock)
+
+ def __setstate__(self, state):
+ (self._reader, self._writer, self._rlock, self._wlock) = state
+ self._make_methods()
+
+ def _make_methods(self):
+ recv = self._reader.recv
+ racquire, rrelease = self._rlock.acquire, self._rlock.release
+ def get():
+ racquire()
+ try:
+ return recv()
+ finally:
+ rrelease()
+ self.get = get
+
+ if self._wlock is None:
+ # writes to a message oriented win32 pipe are atomic
+ self.put = self._writer.send
+ else:
+ send = self._writer.send
+ wacquire, wrelease = self._wlock.acquire, self._wlock.release
+ def put(obj):
+ wacquire()
+ try:
+ return send(obj)
+ finally:
+ wrelease()
+ self.put = put