blob: 4dd6502c87d073d84dac9b61d7de49289115e51a [file] [log] [blame]
Benjamin Petersone711caf2008-06-11 16:44:04 +00001#
2# A higher level module for using sockets (or Windows named pipes)
3#
4# multiprocessing/connection.py
5#
R. David Murray3fc969a2010-12-14 01:38:16 +00006# Copyright (c) 2006-2008, R Oudkerk
Richard Oudkerk3e268aa2012-04-30 12:13:55 +01007# Licensed to PSF under a Contributor Agreement.
Benjamin Petersone711caf2008-06-11 16:44:04 +00008#
9
Antoine Pitroubdb1cf12012-03-05 19:28:37 +010010__all__ = [ 'Client', 'Listener', 'Pipe', 'wait' ]
Benjamin Petersone711caf2008-06-11 16:44:04 +000011
Antoine Pitrou87cf2202011-05-09 17:04:27 +020012import io
Benjamin Petersone711caf2008-06-11 16:44:04 +000013import os
14import sys
Antoine Pitrou87cf2202011-05-09 17:04:27 +020015import select
Benjamin Petersone711caf2008-06-11 16:44:04 +000016import socket
Antoine Pitrou87cf2202011-05-09 17:04:27 +020017import struct
Georg Brandl6aa2d1f2008-08-12 08:35:52 +000018import errno
Benjamin Petersone711caf2008-06-11 16:44:04 +000019import time
20import tempfile
21import itertools
22
23import _multiprocessing
Antoine Pitrou87cf2202011-05-09 17:04:27 +020024from multiprocessing import current_process, AuthenticationError, BufferTooShort
Richard Oudkerk59d54042012-05-10 16:11:12 +010025from multiprocessing.util import get_temp_dir, Finalize, sub_debug, debug
Antoine Pitrou5438ed12012-04-24 22:56:57 +020026from multiprocessing.forking import ForkingPickler
Antoine Pitrou87cf2202011-05-09 17:04:27 +020027try:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +020028 import _winapi
29 from _winapi import WAIT_OBJECT_0, WAIT_TIMEOUT, INFINITE
Antoine Pitrou87cf2202011-05-09 17:04:27 +020030except ImportError:
31 if sys.platform == 'win32':
32 raise
Antoine Pitrou23bba4c2012-04-18 20:51:15 +020033 _winapi = None
Benjamin Petersone711caf2008-06-11 16:44:04 +000034
35#
36#
37#
38
39BUFSIZE = 8192
Antoine Pitrou45d61a32009-11-13 22:35:18 +000040# A very generous timeout when it comes to local connections...
41CONNECTION_TIMEOUT = 20.
Benjamin Petersone711caf2008-06-11 16:44:04 +000042
43_mmap_counter = itertools.count()
44
45default_family = 'AF_INET'
46families = ['AF_INET']
47
48if hasattr(socket, 'AF_UNIX'):
49 default_family = 'AF_UNIX'
50 families += ['AF_UNIX']
51
52if sys.platform == 'win32':
53 default_family = 'AF_PIPE'
54 families += ['AF_PIPE']
55
Antoine Pitrou45d61a32009-11-13 22:35:18 +000056
57def _init_timeout(timeout=CONNECTION_TIMEOUT):
58 return time.time() + timeout
59
60def _check_timeout(t):
61 return time.time() > t
62
Benjamin Petersone711caf2008-06-11 16:44:04 +000063#
64#
65#
66
67def arbitrary_address(family):
68 '''
69 Return an arbitrary free address for the given family
70 '''
71 if family == 'AF_INET':
72 return ('localhost', 0)
73 elif family == 'AF_UNIX':
74 return tempfile.mktemp(prefix='listener-', dir=get_temp_dir())
75 elif family == 'AF_PIPE':
76 return tempfile.mktemp(prefix=r'\\.\pipe\pyc-%d-%d-' %
77 (os.getpid(), next(_mmap_counter)))
78 else:
79 raise ValueError('unrecognized family')
80
Antoine Pitrou709176f2012-04-01 17:19:09 +020081def _validate_family(family):
82 '''
83 Checks if the family is valid for the current environment.
84 '''
85 if sys.platform != 'win32' and family == 'AF_PIPE':
86 raise ValueError('Family %s is not recognized.' % family)
87
Antoine Pitrou6d20cba2012-04-03 20:12:23 +020088 if sys.platform == 'win32' and family == 'AF_UNIX':
89 # double check
90 if not hasattr(socket, family):
91 raise ValueError('Family %s is not recognized.' % family)
Benjamin Petersone711caf2008-06-11 16:44:04 +000092
93def address_type(address):
94 '''
95 Return the types of the address
96
97 This can be 'AF_INET', 'AF_UNIX', or 'AF_PIPE'
98 '''
99 if type(address) == tuple:
100 return 'AF_INET'
101 elif type(address) is str and address.startswith('\\\\'):
102 return 'AF_PIPE'
103 elif type(address) is str:
104 return 'AF_UNIX'
105 else:
106 raise ValueError('address type of %r unrecognized' % address)
107
108#
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200109# Connection classes
110#
111
112class _ConnectionBase:
113 _handle = None
114
115 def __init__(self, handle, readable=True, writable=True):
116 handle = handle.__index__()
117 if handle < 0:
118 raise ValueError("invalid handle")
119 if not readable and not writable:
120 raise ValueError(
121 "at least one of `readable` and `writable` must be True")
122 self._handle = handle
123 self._readable = readable
124 self._writable = writable
125
Antoine Pitrou60001202011-07-09 01:03:46 +0200126 # XXX should we use util.Finalize instead of a __del__?
127
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200128 def __del__(self):
129 if self._handle is not None:
130 self._close()
131
132 def _check_closed(self):
133 if self._handle is None:
Andrew Svetlovf7a17b42012-12-25 16:47:37 +0200134 raise OSError("handle is closed")
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200135
136 def _check_readable(self):
137 if not self._readable:
Andrew Svetlovf7a17b42012-12-25 16:47:37 +0200138 raise OSError("connection is write-only")
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200139
140 def _check_writable(self):
141 if not self._writable:
Andrew Svetlovf7a17b42012-12-25 16:47:37 +0200142 raise OSError("connection is read-only")
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200143
144 def _bad_message_length(self):
145 if self._writable:
146 self._readable = False
147 else:
148 self.close()
Andrew Svetlovf7a17b42012-12-25 16:47:37 +0200149 raise OSError("bad message length")
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200150
151 @property
152 def closed(self):
153 """True if the connection is closed"""
154 return self._handle is None
155
156 @property
157 def readable(self):
158 """True if the connection is readable"""
159 return self._readable
160
161 @property
162 def writable(self):
163 """True if the connection is writable"""
164 return self._writable
165
166 def fileno(self):
167 """File descriptor or handle of the connection"""
168 self._check_closed()
169 return self._handle
170
171 def close(self):
172 """Close the connection"""
173 if self._handle is not None:
174 try:
175 self._close()
176 finally:
177 self._handle = None
178
179 def send_bytes(self, buf, offset=0, size=None):
180 """Send the bytes data from a bytes-like object"""
181 self._check_closed()
182 self._check_writable()
183 m = memoryview(buf)
184 # HACK for byte-indexing of non-bytewise buffers (e.g. array.array)
185 if m.itemsize > 1:
186 m = memoryview(bytes(m))
187 n = len(m)
188 if offset < 0:
189 raise ValueError("offset is negative")
190 if n < offset:
191 raise ValueError("buffer length < offset")
192 if size is None:
193 size = n - offset
194 elif size < 0:
195 raise ValueError("size is negative")
196 elif offset + size > n:
197 raise ValueError("buffer length < offset + size")
198 self._send_bytes(m[offset:offset + size])
199
200 def send(self, obj):
201 """Send a (picklable) object"""
202 self._check_closed()
203 self._check_writable()
Charles-François Natalia6550752013-03-24 15:21:49 +0100204 self._send_bytes(ForkingPickler.dumps(obj))
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200205
206 def recv_bytes(self, maxlength=None):
207 """
208 Receive bytes data as a bytes object.
209 """
210 self._check_closed()
211 self._check_readable()
212 if maxlength is not None and maxlength < 0:
213 raise ValueError("negative maxlength")
214 buf = self._recv_bytes(maxlength)
215 if buf is None:
216 self._bad_message_length()
217 return buf.getvalue()
218
219 def recv_bytes_into(self, buf, offset=0):
220 """
221 Receive bytes data into a writeable buffer-like object.
222 Return the number of bytes read.
223 """
224 self._check_closed()
225 self._check_readable()
226 with memoryview(buf) as m:
227 # Get bytesize of arbitrary buffer
228 itemsize = m.itemsize
229 bytesize = itemsize * len(m)
230 if offset < 0:
231 raise ValueError("negative offset")
232 elif offset > bytesize:
233 raise ValueError("offset too large")
234 result = self._recv_bytes()
235 size = result.tell()
236 if bytesize < offset + size:
237 raise BufferTooShort(result.getvalue())
238 # Message can fit in dest
239 result.seek(0)
240 result.readinto(m[offset // itemsize :
241 (offset + size) // itemsize])
242 return size
243
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100244 def recv(self):
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200245 """Receive a (picklable) object"""
246 self._check_closed()
247 self._check_readable()
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100248 buf = self._recv_bytes()
Charles-François Natalia6550752013-03-24 15:21:49 +0100249 return ForkingPickler.loads(buf.getbuffer())
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200250
251 def poll(self, timeout=0.0):
252 """Whether there is any input available to be read"""
253 self._check_closed()
254 self._check_readable()
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200255 return self._poll(timeout)
256
Richard Oudkerkd69cfe82012-06-18 17:47:52 +0100257 def __enter__(self):
258 return self
259
260 def __exit__(self, exc_type, exc_value, exc_tb):
261 self.close()
262
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200263
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200264if _winapi:
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200265
266 class PipeConnection(_ConnectionBase):
267 """
268 Connection class based on a Windows named pipe.
Antoine Pitroudd696492011-06-08 17:21:55 +0200269 Overlapped I/O is used, so the handles must have been created
270 with FILE_FLAG_OVERLAPPED.
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200271 """
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100272 _got_empty_message = False
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200273
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200274 def _close(self, _CloseHandle=_winapi.CloseHandle):
Antoine Pitrou71a28a92011-07-09 01:03:00 +0200275 _CloseHandle(self._handle)
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200276
277 def _send_bytes(self, buf):
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200278 ov, err = _winapi.WriteFile(self._handle, buf, overlapped=True)
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100279 try:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200280 if err == _winapi.ERROR_IO_PENDING:
281 waitres = _winapi.WaitForMultipleObjects(
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100282 [ov.event], False, INFINITE)
283 assert waitres == WAIT_OBJECT_0
284 except:
285 ov.cancel()
286 raise
287 finally:
288 nwritten, err = ov.GetOverlappedResult(True)
289 assert err == 0
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200290 assert nwritten == len(buf)
291
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100292 def _recv_bytes(self, maxsize=None):
293 if self._got_empty_message:
294 self._got_empty_message = False
295 return io.BytesIO()
Antoine Pitroudd696492011-06-08 17:21:55 +0200296 else:
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100297 bsize = 128 if maxsize is None else min(maxsize, 128)
Antoine Pitroudd696492011-06-08 17:21:55 +0200298 try:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200299 ov, err = _winapi.ReadFile(self._handle, bsize,
300 overlapped=True)
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100301 try:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200302 if err == _winapi.ERROR_IO_PENDING:
303 waitres = _winapi.WaitForMultipleObjects(
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100304 [ov.event], False, INFINITE)
305 assert waitres == WAIT_OBJECT_0
306 except:
307 ov.cancel()
308 raise
309 finally:
310 nread, err = ov.GetOverlappedResult(True)
311 if err == 0:
312 f = io.BytesIO()
313 f.write(ov.getbuffer())
314 return f
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200315 elif err == _winapi.ERROR_MORE_DATA:
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100316 return self._get_more_data(ov, maxsize)
Andrew Svetlovf7a17b42012-12-25 16:47:37 +0200317 except OSError as e:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200318 if e.winerror == _winapi.ERROR_BROKEN_PIPE:
Antoine Pitroudd696492011-06-08 17:21:55 +0200319 raise EOFError
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100320 else:
321 raise
322 raise RuntimeError("shouldn't get here; expected KeyboardInterrupt")
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200323
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100324 def _poll(self, timeout):
325 if (self._got_empty_message or
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200326 _winapi.PeekNamedPipe(self._handle)[0] != 0):
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200327 return True
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100328 return bool(wait([self], timeout))
329
330 def _get_more_data(self, ov, maxsize):
331 buf = ov.getbuffer()
332 f = io.BytesIO()
333 f.write(buf)
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200334 left = _winapi.PeekNamedPipe(self._handle)[1]
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100335 assert left > 0
336 if maxsize is not None and len(buf) + left > maxsize:
337 self._bad_message_length()
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200338 ov, err = _winapi.ReadFile(self._handle, left, overlapped=True)
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100339 rbytes, err = ov.GetOverlappedResult(True)
340 assert err == 0
341 assert rbytes == left
342 f.write(ov.getbuffer())
343 return f
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200344
345
346class Connection(_ConnectionBase):
347 """
348 Connection class based on an arbitrary file descriptor (Unix only), or
349 a socket handle (Windows).
350 """
351
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200352 if _winapi:
353 def _close(self, _close=_multiprocessing.closesocket):
Antoine Pitrou71a28a92011-07-09 01:03:00 +0200354 _close(self._handle)
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200355 _write = _multiprocessing.send
356 _read = _multiprocessing.recv
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200357 else:
Antoine Pitrou71a28a92011-07-09 01:03:00 +0200358 def _close(self, _close=os.close):
359 _close(self._handle)
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200360 _write = os.write
361 _read = os.read
362
363 def _send(self, buf, write=_write):
364 remaining = len(buf)
365 while True:
366 n = write(self._handle, buf)
367 remaining -= n
368 if remaining == 0:
369 break
370 buf = buf[n:]
371
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100372 def _recv(self, size, read=_read):
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200373 buf = io.BytesIO()
Antoine Pitroudd696492011-06-08 17:21:55 +0200374 handle = self._handle
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200375 remaining = size
376 while remaining > 0:
Antoine Pitroudd696492011-06-08 17:21:55 +0200377 chunk = read(handle, remaining)
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200378 n = len(chunk)
379 if n == 0:
380 if remaining == size:
381 raise EOFError
382 else:
Andrew Svetlovf7a17b42012-12-25 16:47:37 +0200383 raise OSError("got end of file during message")
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200384 buf.write(chunk)
385 remaining -= n
386 return buf
387
388 def _send_bytes(self, buf):
389 # For wire compatibility with 3.2 and lower
390 n = len(buf)
Charles-François Natali225aa4f2011-09-20 19:27:39 +0200391 self._send(struct.pack("!i", n))
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200392 # The condition is necessary to avoid "broken pipe" errors
393 # when sending a 0-length buffer if the other end closed the pipe.
394 if n > 0:
395 self._send(buf)
396
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100397 def _recv_bytes(self, maxsize=None):
398 buf = self._recv(4)
Charles-François Natali225aa4f2011-09-20 19:27:39 +0200399 size, = struct.unpack("!i", buf.getvalue())
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200400 if maxsize is not None and size > maxsize:
401 return None
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100402 return self._recv(size)
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200403
404 def _poll(self, timeout):
Richard Oudkerked9e06c2013-01-13 22:46:48 +0000405 r = wait([self], timeout)
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200406 return bool(r)
407
408
409#
Benjamin Petersone711caf2008-06-11 16:44:04 +0000410# Public functions
411#
412
413class Listener(object):
414 '''
415 Returns a listener object.
416
417 This is a wrapper for a bound socket which is 'listening' for
418 connections, or for a Windows named pipe.
419 '''
420 def __init__(self, address=None, family=None, backlog=1, authkey=None):
421 family = family or (address and address_type(address)) \
422 or default_family
423 address = address or arbitrary_address(family)
424
Antoine Pitrou709176f2012-04-01 17:19:09 +0200425 _validate_family(family)
Benjamin Petersone711caf2008-06-11 16:44:04 +0000426 if family == 'AF_PIPE':
427 self._listener = PipeListener(address, backlog)
428 else:
429 self._listener = SocketListener(address, family, backlog)
430
431 if authkey is not None and not isinstance(authkey, bytes):
432 raise TypeError('authkey should be a byte string')
433
434 self._authkey = authkey
435
436 def accept(self):
437 '''
438 Accept a connection on the bound socket or named pipe of `self`.
439
440 Returns a `Connection` object.
441 '''
Richard Oudkerkd69cfe82012-06-18 17:47:52 +0100442 if self._listener is None:
Andrew Svetlovf7a17b42012-12-25 16:47:37 +0200443 raise OSError('listener is closed')
Benjamin Petersone711caf2008-06-11 16:44:04 +0000444 c = self._listener.accept()
445 if self._authkey:
446 deliver_challenge(c, self._authkey)
447 answer_challenge(c, self._authkey)
448 return c
449
450 def close(self):
451 '''
452 Close the bound socket or named pipe of `self`.
453 '''
Richard Oudkerkd69cfe82012-06-18 17:47:52 +0100454 if self._listener is not None:
455 self._listener.close()
456 self._listener = None
Benjamin Petersone711caf2008-06-11 16:44:04 +0000457
458 address = property(lambda self: self._listener._address)
459 last_accepted = property(lambda self: self._listener._last_accepted)
460
Richard Oudkerkd69cfe82012-06-18 17:47:52 +0100461 def __enter__(self):
462 return self
463
464 def __exit__(self, exc_type, exc_value, exc_tb):
465 self.close()
466
Benjamin Petersone711caf2008-06-11 16:44:04 +0000467
468def Client(address, family=None, authkey=None):
469 '''
470 Returns a connection to the address of a `Listener`
471 '''
472 family = family or address_type(address)
Antoine Pitrou709176f2012-04-01 17:19:09 +0200473 _validate_family(family)
Benjamin Petersone711caf2008-06-11 16:44:04 +0000474 if family == 'AF_PIPE':
475 c = PipeClient(address)
476 else:
477 c = SocketClient(address)
478
479 if authkey is not None and not isinstance(authkey, bytes):
480 raise TypeError('authkey should be a byte string')
481
482 if authkey is not None:
483 answer_challenge(c, authkey)
484 deliver_challenge(c, authkey)
485
486 return c
487
488
489if sys.platform != 'win32':
490
491 def Pipe(duplex=True):
492 '''
493 Returns pair of connection objects at either end of a pipe
494 '''
495 if duplex:
496 s1, s2 = socket.socketpair()
Richard Oudkerkb15e6222012-07-27 14:19:00 +0100497 s1.setblocking(True)
498 s2.setblocking(True)
Antoine Pitrou5aa878c2011-05-09 21:00:28 +0200499 c1 = Connection(s1.detach())
500 c2 = Connection(s2.detach())
Benjamin Petersone711caf2008-06-11 16:44:04 +0000501 else:
502 fd1, fd2 = os.pipe()
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200503 c1 = Connection(fd1, writable=False)
504 c2 = Connection(fd2, readable=False)
Benjamin Petersone711caf2008-06-11 16:44:04 +0000505
506 return c1, c2
507
508else:
509
Benjamin Petersone711caf2008-06-11 16:44:04 +0000510 def Pipe(duplex=True):
511 '''
512 Returns pair of connection objects at either end of a pipe
513 '''
514 address = arbitrary_address('AF_PIPE')
515 if duplex:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200516 openmode = _winapi.PIPE_ACCESS_DUPLEX
517 access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE
Benjamin Petersone711caf2008-06-11 16:44:04 +0000518 obsize, ibsize = BUFSIZE, BUFSIZE
519 else:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200520 openmode = _winapi.PIPE_ACCESS_INBOUND
521 access = _winapi.GENERIC_WRITE
Benjamin Petersone711caf2008-06-11 16:44:04 +0000522 obsize, ibsize = 0, BUFSIZE
523
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200524 h1 = _winapi.CreateNamedPipe(
525 address, openmode | _winapi.FILE_FLAG_OVERLAPPED |
526 _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE,
527 _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
528 _winapi.PIPE_WAIT,
529 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL
Benjamin Petersone711caf2008-06-11 16:44:04 +0000530 )
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200531 h2 = _winapi.CreateFile(
532 address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING,
533 _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL
Benjamin Petersone711caf2008-06-11 16:44:04 +0000534 )
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200535 _winapi.SetNamedPipeHandleState(
536 h2, _winapi.PIPE_READMODE_MESSAGE, None, None
Benjamin Petersone711caf2008-06-11 16:44:04 +0000537 )
538
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200539 overlapped = _winapi.ConnectNamedPipe(h1, overlapped=True)
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100540 _, err = overlapped.GetOverlappedResult(True)
541 assert err == 0
Benjamin Petersone711caf2008-06-11 16:44:04 +0000542
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200543 c1 = PipeConnection(h1, writable=duplex)
544 c2 = PipeConnection(h2, readable=duplex)
Benjamin Petersone711caf2008-06-11 16:44:04 +0000545
546 return c1, c2
547
548#
549# Definitions for connections based on sockets
550#
551
552class SocketListener(object):
553 '''
Georg Brandl734e2682008-08-12 08:18:18 +0000554 Representation of a socket which is bound to an address and listening
Benjamin Petersone711caf2008-06-11 16:44:04 +0000555 '''
556 def __init__(self, address, family, backlog=1):
557 self._socket = socket.socket(getattr(socket, family))
Charles-François Nataliedc67fe2012-02-04 15:12:08 +0100558 try:
Charles-François Natalied4a8fc2012-02-08 21:15:58 +0100559 # SO_REUSEADDR has different semantics on Windows (issue #2550).
560 if os.name == 'posix':
561 self._socket.setsockopt(socket.SOL_SOCKET,
562 socket.SO_REUSEADDR, 1)
Richard Oudkerkb15e6222012-07-27 14:19:00 +0100563 self._socket.setblocking(True)
Charles-François Nataliedc67fe2012-02-04 15:12:08 +0100564 self._socket.bind(address)
565 self._socket.listen(backlog)
566 self._address = self._socket.getsockname()
567 except OSError:
568 self._socket.close()
569 raise
Benjamin Petersone711caf2008-06-11 16:44:04 +0000570 self._family = family
571 self._last_accepted = None
572
Benjamin Petersone711caf2008-06-11 16:44:04 +0000573 if family == 'AF_UNIX':
574 self._unlink = Finalize(
Georg Brandl2ee470f2008-07-16 12:55:28 +0000575 self, os.unlink, args=(address,), exitpriority=0
Benjamin Petersone711caf2008-06-11 16:44:04 +0000576 )
577 else:
578 self._unlink = None
579
580 def accept(self):
581 s, self._last_accepted = self._socket.accept()
Richard Oudkerkb15e6222012-07-27 14:19:00 +0100582 s.setblocking(True)
Antoine Pitroudf97cbe2012-04-07 22:38:52 +0200583 return Connection(s.detach())
Benjamin Petersone711caf2008-06-11 16:44:04 +0000584
585 def close(self):
586 self._socket.close()
587 if self._unlink is not None:
588 self._unlink()
589
590
591def SocketClient(address):
592 '''
593 Return a connection object connected to the socket given by `address`
594 '''
595 family = address_type(address)
Victor Stinner2b695062011-01-03 15:47:59 +0000596 with socket.socket( getattr(socket, family) ) as s:
Richard Oudkerkb15e6222012-07-27 14:19:00 +0100597 s.setblocking(True)
Charles-François Natalie6eabd42011-11-19 09:59:43 +0100598 s.connect(address)
Antoine Pitroudf97cbe2012-04-07 22:38:52 +0200599 return Connection(s.detach())
Benjamin Petersone711caf2008-06-11 16:44:04 +0000600
601#
602# Definitions for connections based on named pipes
603#
604
605if sys.platform == 'win32':
606
607 class PipeListener(object):
608 '''
609 Representation of a named pipe
610 '''
611 def __init__(self, address, backlog=None):
612 self._address = address
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100613 self._handle_queue = [self._new_handle(first=True)]
614
Benjamin Petersone711caf2008-06-11 16:44:04 +0000615 self._last_accepted = None
Benjamin Petersone711caf2008-06-11 16:44:04 +0000616 sub_debug('listener created with address=%r', self._address)
Benjamin Petersone711caf2008-06-11 16:44:04 +0000617 self.close = Finalize(
618 self, PipeListener._finalize_pipe_listener,
619 args=(self._handle_queue, self._address), exitpriority=0
620 )
621
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100622 def _new_handle(self, first=False):
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200623 flags = _winapi.PIPE_ACCESS_DUPLEX | _winapi.FILE_FLAG_OVERLAPPED
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100624 if first:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200625 flags |= _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE
626 return _winapi.CreateNamedPipe(
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100627 self._address, flags,
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200628 _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
629 _winapi.PIPE_WAIT,
630 _winapi.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE,
631 _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL
Benjamin Petersone711caf2008-06-11 16:44:04 +0000632 )
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100633
634 def accept(self):
635 self._handle_queue.append(self._new_handle())
Benjamin Petersone711caf2008-06-11 16:44:04 +0000636 handle = self._handle_queue.pop(0)
637 try:
Richard Oudkerkfdb8dcf2012-05-05 19:45:37 +0100638 ov = _winapi.ConnectNamedPipe(handle, overlapped=True)
639 except OSError as e:
640 if e.winerror != _winapi.ERROR_NO_DATA:
641 raise
642 # ERROR_NO_DATA can occur if a client has already connected,
643 # written data and then disconnected -- see Issue 14725.
644 else:
645 try:
646 res = _winapi.WaitForMultipleObjects(
647 [ov.event], False, INFINITE)
648 except:
649 ov.cancel()
650 _winapi.CloseHandle(handle)
651 raise
652 finally:
653 _, err = ov.GetOverlappedResult(True)
654 assert err == 0
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200655 return PipeConnection(handle)
Benjamin Petersone711caf2008-06-11 16:44:04 +0000656
657 @staticmethod
658 def _finalize_pipe_listener(queue, address):
659 sub_debug('closing listener with address=%r', address)
660 for handle in queue:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200661 _winapi.CloseHandle(handle)
Benjamin Petersone711caf2008-06-11 16:44:04 +0000662
663 def PipeClient(address):
664 '''
665 Return a connection object connected to the pipe given by `address`
666 '''
Antoine Pitrou45d61a32009-11-13 22:35:18 +0000667 t = _init_timeout()
Benjamin Petersone711caf2008-06-11 16:44:04 +0000668 while 1:
669 try:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200670 _winapi.WaitNamedPipe(address, 1000)
671 h = _winapi.CreateFile(
672 address, _winapi.GENERIC_READ | _winapi.GENERIC_WRITE,
673 0, _winapi.NULL, _winapi.OPEN_EXISTING,
674 _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL
Benjamin Petersone711caf2008-06-11 16:44:04 +0000675 )
Andrew Svetlov2606a6f2012-12-19 14:33:35 +0200676 except OSError as e:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200677 if e.winerror not in (_winapi.ERROR_SEM_TIMEOUT,
678 _winapi.ERROR_PIPE_BUSY) or _check_timeout(t):
Benjamin Petersone711caf2008-06-11 16:44:04 +0000679 raise
680 else:
681 break
682 else:
683 raise
684
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200685 _winapi.SetNamedPipeHandleState(
686 h, _winapi.PIPE_READMODE_MESSAGE, None, None
Benjamin Petersone711caf2008-06-11 16:44:04 +0000687 )
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200688 return PipeConnection(h)
Benjamin Petersone711caf2008-06-11 16:44:04 +0000689
690#
691# Authentication stuff
692#
693
694MESSAGE_LENGTH = 20
695
Benjamin Peterson1fcfe212008-06-25 12:54:22 +0000696CHALLENGE = b'#CHALLENGE#'
697WELCOME = b'#WELCOME#'
698FAILURE = b'#FAILURE#'
Benjamin Petersone711caf2008-06-11 16:44:04 +0000699
700def deliver_challenge(connection, authkey):
701 import hmac
702 assert isinstance(authkey, bytes)
703 message = os.urandom(MESSAGE_LENGTH)
704 connection.send_bytes(CHALLENGE + message)
705 digest = hmac.new(authkey, message).digest()
706 response = connection.recv_bytes(256) # reject large message
707 if response == digest:
708 connection.send_bytes(WELCOME)
709 else:
710 connection.send_bytes(FAILURE)
711 raise AuthenticationError('digest received was wrong')
712
713def answer_challenge(connection, authkey):
714 import hmac
715 assert isinstance(authkey, bytes)
716 message = connection.recv_bytes(256) # reject large message
717 assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message
718 message = message[len(CHALLENGE):]
719 digest = hmac.new(authkey, message).digest()
720 connection.send_bytes(digest)
721 response = connection.recv_bytes(256) # reject large message
722 if response != WELCOME:
723 raise AuthenticationError('digest sent was rejected')
724
725#
726# Support for using xmlrpclib for serialization
727#
728
729class ConnectionWrapper(object):
730 def __init__(self, conn, dumps, loads):
731 self._conn = conn
732 self._dumps = dumps
733 self._loads = loads
734 for attr in ('fileno', 'close', 'poll', 'recv_bytes', 'send_bytes'):
735 obj = getattr(conn, attr)
736 setattr(self, attr, obj)
737 def send(self, obj):
738 s = self._dumps(obj)
739 self._conn.send_bytes(s)
740 def recv(self):
741 s = self._conn.recv_bytes()
742 return self._loads(s)
743
744def _xml_dumps(obj):
Marc-André Lemburg8f36af72011-02-25 15:42:01 +0000745 return xmlrpclib.dumps((obj,), None, None, None, 1).encode('utf-8')
Benjamin Petersone711caf2008-06-11 16:44:04 +0000746
747def _xml_loads(s):
Marc-André Lemburg8f36af72011-02-25 15:42:01 +0000748 (obj,), method = xmlrpclib.loads(s.decode('utf-8'))
Benjamin Petersone711caf2008-06-11 16:44:04 +0000749 return obj
750
751class XmlListener(Listener):
752 def accept(self):
753 global xmlrpclib
754 import xmlrpc.client as xmlrpclib
755 obj = Listener.accept(self)
756 return ConnectionWrapper(obj, _xml_dumps, _xml_loads)
757
758def XmlClient(*args, **kwds):
759 global xmlrpclib
760 import xmlrpc.client as xmlrpclib
761 return ConnectionWrapper(Client(*args, **kwds), _xml_dumps, _xml_loads)
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200762
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100763#
764# Wait
765#
766
767if sys.platform == 'win32':
768
769 def _exhaustive_wait(handles, timeout):
770 # Return ALL handles which are currently signalled. (Only
771 # returning the first signalled might create starvation issues.)
772 L = list(handles)
773 ready = []
774 while L:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200775 res = _winapi.WaitForMultipleObjects(L, False, timeout)
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100776 if res == WAIT_TIMEOUT:
777 break
778 elif WAIT_OBJECT_0 <= res < WAIT_OBJECT_0 + len(L):
779 res -= WAIT_OBJECT_0
780 elif WAIT_ABANDONED_0 <= res < WAIT_ABANDONED_0 + len(L):
781 res -= WAIT_ABANDONED_0
782 else:
783 raise RuntimeError('Should not get here')
784 ready.append(L[res])
785 L = L[res+1:]
786 timeout = 0
787 return ready
788
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200789 _ready_errors = {_winapi.ERROR_BROKEN_PIPE, _winapi.ERROR_NETNAME_DELETED}
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100790
791 def wait(object_list, timeout=None):
792 '''
793 Wait till an object in object_list is ready/readable.
794
795 Returns list of those objects in object_list which are ready/readable.
796 '''
797 if timeout is None:
798 timeout = INFINITE
799 elif timeout < 0:
800 timeout = 0
801 else:
802 timeout = int(timeout * 1000 + 0.5)
803
804 object_list = list(object_list)
805 waithandle_to_obj = {}
806 ov_list = []
807 ready_objects = set()
808 ready_handles = set()
809
810 try:
811 for o in object_list:
812 try:
813 fileno = getattr(o, 'fileno')
814 except AttributeError:
815 waithandle_to_obj[o.__index__()] = o
816 else:
817 # start an overlapped read of length zero
818 try:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200819 ov, err = _winapi.ReadFile(fileno(), 0, True)
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100820 except OSError as e:
821 err = e.winerror
822 if err not in _ready_errors:
823 raise
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200824 if err == _winapi.ERROR_IO_PENDING:
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100825 ov_list.append(ov)
826 waithandle_to_obj[ov.event] = o
827 else:
828 # If o.fileno() is an overlapped pipe handle and
829 # err == 0 then there is a zero length message
830 # in the pipe, but it HAS NOT been consumed.
831 ready_objects.add(o)
832 timeout = 0
833
834 ready_handles = _exhaustive_wait(waithandle_to_obj.keys(), timeout)
835 finally:
836 # request that overlapped reads stop
837 for ov in ov_list:
838 ov.cancel()
839
840 # wait for all overlapped reads to stop
841 for ov in ov_list:
842 try:
843 _, err = ov.GetOverlappedResult(True)
844 except OSError as e:
845 err = e.winerror
846 if err not in _ready_errors:
847 raise
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200848 if err != _winapi.ERROR_OPERATION_ABORTED:
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100849 o = waithandle_to_obj[ov.event]
850 ready_objects.add(o)
851 if err == 0:
852 # If o.fileno() is an overlapped pipe handle then
853 # a zero length message HAS been consumed.
854 if hasattr(o, '_got_empty_message'):
855 o._got_empty_message = True
856
857 ready_objects.update(waithandle_to_obj[h] for h in ready_handles)
858 return [o for o in object_list if o in ready_objects]
859
860else:
861
Giampaolo Rodola'0c8ad612013-01-14 02:24:05 +0100862 if hasattr(select, 'poll'):
863 def _poll(fds, timeout):
864 if timeout is not None:
865 timeout = int(timeout) * 1000 # timeout is in milliseconds
866 fd_map = {}
867 pollster = select.poll()
868 for fd in fds:
869 pollster.register(fd, select.POLLIN)
870 if hasattr(fd, 'fileno'):
871 fd_map[fd.fileno()] = fd
872 else:
873 fd_map[fd] = fd
874 ls = []
875 for fd, event in pollster.poll(timeout):
876 if event & select.POLLNVAL:
877 raise ValueError('invalid file descriptor %i' % fd)
878 ls.append(fd_map[fd])
879 return ls
880 else:
881 def _poll(fds, timeout):
882 return select.select(fds, [], [], timeout)[0]
883
884
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100885 def wait(object_list, timeout=None):
886 '''
887 Wait till an object in object_list is ready/readable.
888
889 Returns list of those objects in object_list which are ready/readable.
890 '''
891 if timeout is not None:
892 if timeout <= 0:
Giampaolo Rodola'0c8ad612013-01-14 02:24:05 +0100893 return _poll(object_list, 0)
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100894 else:
895 deadline = time.time() + timeout
896 while True:
897 try:
Giampaolo Rodola'0c8ad612013-01-14 02:24:05 +0100898 return _poll(object_list, timeout)
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100899 except OSError as e:
900 if e.errno != errno.EINTR:
901 raise
902 if timeout is not None:
903 timeout = deadline - time.time()
Antoine Pitrou5438ed12012-04-24 22:56:57 +0200904
905#
906# Make connection and socket objects sharable if possible
907#
908
909if sys.platform == 'win32':
910 from . import reduction
911 ForkingPickler.register(socket.socket, reduction.reduce_socket)
912 ForkingPickler.register(Connection, reduction.reduce_connection)
913 ForkingPickler.register(PipeConnection, reduction.reduce_pipe_connection)
914else:
915 try:
916 from . import reduction
917 except ImportError:
918 pass
919 else:
920 ForkingPickler.register(socket.socket, reduction.reduce_socket)
921 ForkingPickler.register(Connection, reduction.reduce_connection)