blob: fbbd5d91d39047a2a43811f5038a8267c29a066a [file] [log] [blame]
Benjamin Petersone711caf2008-06-11 16:44:04 +00001#
2# A higher level module for using sockets (or Windows named pipes)
3#
4# multiprocessing/connection.py
5#
R. David Murray3fc969a2010-12-14 01:38:16 +00006# Copyright (c) 2006-2008, R Oudkerk
Richard Oudkerk3e268aa2012-04-30 12:13:55 +01007# Licensed to PSF under a Contributor Agreement.
Benjamin Petersone711caf2008-06-11 16:44:04 +00008#
9
Antoine Pitroubdb1cf12012-03-05 19:28:37 +010010__all__ = [ 'Client', 'Listener', 'Pipe', 'wait' ]
Benjamin Petersone711caf2008-06-11 16:44:04 +000011
Antoine Pitrou87cf2202011-05-09 17:04:27 +020012import io
Benjamin Petersone711caf2008-06-11 16:44:04 +000013import os
14import sys
Antoine Pitrou87cf2202011-05-09 17:04:27 +020015import pickle
16import select
Benjamin Petersone711caf2008-06-11 16:44:04 +000017import socket
Antoine Pitrou87cf2202011-05-09 17:04:27 +020018import struct
Georg Brandl6aa2d1f2008-08-12 08:35:52 +000019import errno
Benjamin Petersone711caf2008-06-11 16:44:04 +000020import time
21import tempfile
22import itertools
23
24import _multiprocessing
Antoine Pitrou87cf2202011-05-09 17:04:27 +020025from multiprocessing import current_process, AuthenticationError, BufferTooShort
Richard Oudkerk59d54042012-05-10 16:11:12 +010026from multiprocessing.util import get_temp_dir, Finalize, sub_debug, debug
Antoine Pitrou5438ed12012-04-24 22:56:57 +020027from multiprocessing.forking import ForkingPickler
Antoine Pitrou87cf2202011-05-09 17:04:27 +020028try:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +020029 import _winapi
30 from _winapi import WAIT_OBJECT_0, WAIT_TIMEOUT, INFINITE
Antoine Pitrou87cf2202011-05-09 17:04:27 +020031except ImportError:
32 if sys.platform == 'win32':
33 raise
Antoine Pitrou23bba4c2012-04-18 20:51:15 +020034 _winapi = None
Benjamin Petersone711caf2008-06-11 16:44:04 +000035
36#
37#
38#
39
40BUFSIZE = 8192
Antoine Pitrou45d61a32009-11-13 22:35:18 +000041# A very generous timeout when it comes to local connections...
42CONNECTION_TIMEOUT = 20.
Benjamin Petersone711caf2008-06-11 16:44:04 +000043
44_mmap_counter = itertools.count()
45
46default_family = 'AF_INET'
47families = ['AF_INET']
48
49if hasattr(socket, 'AF_UNIX'):
50 default_family = 'AF_UNIX'
51 families += ['AF_UNIX']
52
53if sys.platform == 'win32':
54 default_family = 'AF_PIPE'
55 families += ['AF_PIPE']
56
Antoine Pitrou45d61a32009-11-13 22:35:18 +000057
58def _init_timeout(timeout=CONNECTION_TIMEOUT):
59 return time.time() + timeout
60
61def _check_timeout(t):
62 return time.time() > t
63
Benjamin Petersone711caf2008-06-11 16:44:04 +000064#
65#
66#
67
68def arbitrary_address(family):
69 '''
70 Return an arbitrary free address for the given family
71 '''
72 if family == 'AF_INET':
73 return ('localhost', 0)
74 elif family == 'AF_UNIX':
75 return tempfile.mktemp(prefix='listener-', dir=get_temp_dir())
76 elif family == 'AF_PIPE':
77 return tempfile.mktemp(prefix=r'\\.\pipe\pyc-%d-%d-' %
78 (os.getpid(), next(_mmap_counter)))
79 else:
80 raise ValueError('unrecognized family')
81
Antoine Pitrou709176f2012-04-01 17:19:09 +020082def _validate_family(family):
83 '''
84 Checks if the family is valid for the current environment.
85 '''
86 if sys.platform != 'win32' and family == 'AF_PIPE':
87 raise ValueError('Family %s is not recognized.' % family)
88
Antoine Pitrou6d20cba2012-04-03 20:12:23 +020089 if sys.platform == 'win32' and family == 'AF_UNIX':
90 # double check
91 if not hasattr(socket, family):
92 raise ValueError('Family %s is not recognized.' % family)
Benjamin Petersone711caf2008-06-11 16:44:04 +000093
94def address_type(address):
95 '''
96 Return the types of the address
97
98 This can be 'AF_INET', 'AF_UNIX', or 'AF_PIPE'
99 '''
100 if type(address) == tuple:
101 return 'AF_INET'
102 elif type(address) is str and address.startswith('\\\\'):
103 return 'AF_PIPE'
104 elif type(address) is str:
105 return 'AF_UNIX'
106 else:
107 raise ValueError('address type of %r unrecognized' % address)
108
109#
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200110# Connection classes
111#
112
113class _ConnectionBase:
114 _handle = None
115
116 def __init__(self, handle, readable=True, writable=True):
117 handle = handle.__index__()
118 if handle < 0:
119 raise ValueError("invalid handle")
120 if not readable and not writable:
121 raise ValueError(
122 "at least one of `readable` and `writable` must be True")
123 self._handle = handle
124 self._readable = readable
125 self._writable = writable
126
Antoine Pitrou60001202011-07-09 01:03:46 +0200127 # XXX should we use util.Finalize instead of a __del__?
128
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200129 def __del__(self):
130 if self._handle is not None:
131 self._close()
132
133 def _check_closed(self):
134 if self._handle is None:
135 raise IOError("handle is closed")
136
137 def _check_readable(self):
138 if not self._readable:
139 raise IOError("connection is write-only")
140
141 def _check_writable(self):
142 if not self._writable:
143 raise IOError("connection is read-only")
144
145 def _bad_message_length(self):
146 if self._writable:
147 self._readable = False
148 else:
149 self.close()
150 raise IOError("bad message length")
151
152 @property
153 def closed(self):
154 """True if the connection is closed"""
155 return self._handle is None
156
157 @property
158 def readable(self):
159 """True if the connection is readable"""
160 return self._readable
161
162 @property
163 def writable(self):
164 """True if the connection is writable"""
165 return self._writable
166
167 def fileno(self):
168 """File descriptor or handle of the connection"""
169 self._check_closed()
170 return self._handle
171
172 def close(self):
173 """Close the connection"""
174 if self._handle is not None:
175 try:
176 self._close()
177 finally:
178 self._handle = None
179
180 def send_bytes(self, buf, offset=0, size=None):
181 """Send the bytes data from a bytes-like object"""
182 self._check_closed()
183 self._check_writable()
184 m = memoryview(buf)
185 # HACK for byte-indexing of non-bytewise buffers (e.g. array.array)
186 if m.itemsize > 1:
187 m = memoryview(bytes(m))
188 n = len(m)
189 if offset < 0:
190 raise ValueError("offset is negative")
191 if n < offset:
192 raise ValueError("buffer length < offset")
193 if size is None:
194 size = n - offset
195 elif size < 0:
196 raise ValueError("size is negative")
197 elif offset + size > n:
198 raise ValueError("buffer length < offset + size")
199 self._send_bytes(m[offset:offset + size])
200
201 def send(self, obj):
202 """Send a (picklable) object"""
203 self._check_closed()
204 self._check_writable()
Antoine Pitrou5438ed12012-04-24 22:56:57 +0200205 buf = io.BytesIO()
206 ForkingPickler(buf, pickle.HIGHEST_PROTOCOL).dump(obj)
207 self._send_bytes(buf.getbuffer())
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200208
209 def recv_bytes(self, maxlength=None):
210 """
211 Receive bytes data as a bytes object.
212 """
213 self._check_closed()
214 self._check_readable()
215 if maxlength is not None and maxlength < 0:
216 raise ValueError("negative maxlength")
217 buf = self._recv_bytes(maxlength)
218 if buf is None:
219 self._bad_message_length()
220 return buf.getvalue()
221
222 def recv_bytes_into(self, buf, offset=0):
223 """
224 Receive bytes data into a writeable buffer-like object.
225 Return the number of bytes read.
226 """
227 self._check_closed()
228 self._check_readable()
229 with memoryview(buf) as m:
230 # Get bytesize of arbitrary buffer
231 itemsize = m.itemsize
232 bytesize = itemsize * len(m)
233 if offset < 0:
234 raise ValueError("negative offset")
235 elif offset > bytesize:
236 raise ValueError("offset too large")
237 result = self._recv_bytes()
238 size = result.tell()
239 if bytesize < offset + size:
240 raise BufferTooShort(result.getvalue())
241 # Message can fit in dest
242 result.seek(0)
243 result.readinto(m[offset // itemsize :
244 (offset + size) // itemsize])
245 return size
246
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100247 def recv(self):
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200248 """Receive a (picklable) object"""
249 self._check_closed()
250 self._check_readable()
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100251 buf = self._recv_bytes()
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200252 return pickle.loads(buf.getbuffer())
253
254 def poll(self, timeout=0.0):
255 """Whether there is any input available to be read"""
256 self._check_closed()
257 self._check_readable()
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200258 return self._poll(timeout)
259
Richard Oudkerkd69cfe82012-06-18 17:47:52 +0100260 def __enter__(self):
261 return self
262
263 def __exit__(self, exc_type, exc_value, exc_tb):
264 self.close()
265
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200266
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200267if _winapi:
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200268
269 class PipeConnection(_ConnectionBase):
270 """
271 Connection class based on a Windows named pipe.
Antoine Pitroudd696492011-06-08 17:21:55 +0200272 Overlapped I/O is used, so the handles must have been created
273 with FILE_FLAG_OVERLAPPED.
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200274 """
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100275 _got_empty_message = False
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200276
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200277 def _close(self, _CloseHandle=_winapi.CloseHandle):
Antoine Pitrou71a28a92011-07-09 01:03:00 +0200278 _CloseHandle(self._handle)
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200279
280 def _send_bytes(self, buf):
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200281 ov, err = _winapi.WriteFile(self._handle, buf, overlapped=True)
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100282 try:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200283 if err == _winapi.ERROR_IO_PENDING:
284 waitres = _winapi.WaitForMultipleObjects(
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100285 [ov.event], False, INFINITE)
286 assert waitres == WAIT_OBJECT_0
287 except:
288 ov.cancel()
289 raise
290 finally:
291 nwritten, err = ov.GetOverlappedResult(True)
292 assert err == 0
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200293 assert nwritten == len(buf)
294
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100295 def _recv_bytes(self, maxsize=None):
296 if self._got_empty_message:
297 self._got_empty_message = False
298 return io.BytesIO()
Antoine Pitroudd696492011-06-08 17:21:55 +0200299 else:
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100300 bsize = 128 if maxsize is None else min(maxsize, 128)
Antoine Pitroudd696492011-06-08 17:21:55 +0200301 try:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200302 ov, err = _winapi.ReadFile(self._handle, bsize,
303 overlapped=True)
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100304 try:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200305 if err == _winapi.ERROR_IO_PENDING:
306 waitres = _winapi.WaitForMultipleObjects(
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100307 [ov.event], False, INFINITE)
308 assert waitres == WAIT_OBJECT_0
309 except:
310 ov.cancel()
311 raise
312 finally:
313 nread, err = ov.GetOverlappedResult(True)
314 if err == 0:
315 f = io.BytesIO()
316 f.write(ov.getbuffer())
317 return f
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200318 elif err == _winapi.ERROR_MORE_DATA:
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100319 return self._get_more_data(ov, maxsize)
Antoine Pitroudd696492011-06-08 17:21:55 +0200320 except IOError as e:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200321 if e.winerror == _winapi.ERROR_BROKEN_PIPE:
Antoine Pitroudd696492011-06-08 17:21:55 +0200322 raise EOFError
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100323 else:
324 raise
325 raise RuntimeError("shouldn't get here; expected KeyboardInterrupt")
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200326
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100327 def _poll(self, timeout):
328 if (self._got_empty_message or
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200329 _winapi.PeekNamedPipe(self._handle)[0] != 0):
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200330 return True
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100331 return bool(wait([self], timeout))
332
333 def _get_more_data(self, ov, maxsize):
334 buf = ov.getbuffer()
335 f = io.BytesIO()
336 f.write(buf)
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200337 left = _winapi.PeekNamedPipe(self._handle)[1]
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100338 assert left > 0
339 if maxsize is not None and len(buf) + left > maxsize:
340 self._bad_message_length()
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200341 ov, err = _winapi.ReadFile(self._handle, left, overlapped=True)
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100342 rbytes, err = ov.GetOverlappedResult(True)
343 assert err == 0
344 assert rbytes == left
345 f.write(ov.getbuffer())
346 return f
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200347
348
349class Connection(_ConnectionBase):
350 """
351 Connection class based on an arbitrary file descriptor (Unix only), or
352 a socket handle (Windows).
353 """
354
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200355 if _winapi:
356 def _close(self, _close=_multiprocessing.closesocket):
Antoine Pitrou71a28a92011-07-09 01:03:00 +0200357 _close(self._handle)
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200358 _write = _multiprocessing.send
359 _read = _multiprocessing.recv
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200360 else:
Antoine Pitrou71a28a92011-07-09 01:03:00 +0200361 def _close(self, _close=os.close):
362 _close(self._handle)
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200363 _write = os.write
364 _read = os.read
365
366 def _send(self, buf, write=_write):
367 remaining = len(buf)
368 while True:
369 n = write(self._handle, buf)
370 remaining -= n
371 if remaining == 0:
372 break
373 buf = buf[n:]
374
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100375 def _recv(self, size, read=_read):
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200376 buf = io.BytesIO()
Antoine Pitroudd696492011-06-08 17:21:55 +0200377 handle = self._handle
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200378 remaining = size
379 while remaining > 0:
Antoine Pitroudd696492011-06-08 17:21:55 +0200380 chunk = read(handle, remaining)
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200381 n = len(chunk)
382 if n == 0:
383 if remaining == size:
384 raise EOFError
385 else:
386 raise IOError("got end of file during message")
387 buf.write(chunk)
388 remaining -= n
389 return buf
390
391 def _send_bytes(self, buf):
392 # For wire compatibility with 3.2 and lower
393 n = len(buf)
Charles-François Natali225aa4f2011-09-20 19:27:39 +0200394 self._send(struct.pack("!i", n))
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200395 # The condition is necessary to avoid "broken pipe" errors
396 # when sending a 0-length buffer if the other end closed the pipe.
397 if n > 0:
398 self._send(buf)
399
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100400 def _recv_bytes(self, maxsize=None):
401 buf = self._recv(4)
Charles-François Natali225aa4f2011-09-20 19:27:39 +0200402 size, = struct.unpack("!i", buf.getvalue())
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200403 if maxsize is not None and size > maxsize:
404 return None
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100405 return self._recv(size)
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200406
407 def _poll(self, timeout):
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100408 r = wait([self._handle], timeout)
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200409 return bool(r)
410
411
412#
Benjamin Petersone711caf2008-06-11 16:44:04 +0000413# Public functions
414#
415
416class Listener(object):
417 '''
418 Returns a listener object.
419
420 This is a wrapper for a bound socket which is 'listening' for
421 connections, or for a Windows named pipe.
422 '''
423 def __init__(self, address=None, family=None, backlog=1, authkey=None):
424 family = family or (address and address_type(address)) \
425 or default_family
426 address = address or arbitrary_address(family)
427
Antoine Pitrou709176f2012-04-01 17:19:09 +0200428 _validate_family(family)
Benjamin Petersone711caf2008-06-11 16:44:04 +0000429 if family == 'AF_PIPE':
430 self._listener = PipeListener(address, backlog)
431 else:
432 self._listener = SocketListener(address, family, backlog)
433
434 if authkey is not None and not isinstance(authkey, bytes):
435 raise TypeError('authkey should be a byte string')
436
437 self._authkey = authkey
438
439 def accept(self):
440 '''
441 Accept a connection on the bound socket or named pipe of `self`.
442
443 Returns a `Connection` object.
444 '''
Richard Oudkerkd69cfe82012-06-18 17:47:52 +0100445 if self._listener is None:
446 raise IOError('listener is closed')
Benjamin Petersone711caf2008-06-11 16:44:04 +0000447 c = self._listener.accept()
448 if self._authkey:
449 deliver_challenge(c, self._authkey)
450 answer_challenge(c, self._authkey)
451 return c
452
453 def close(self):
454 '''
455 Close the bound socket or named pipe of `self`.
456 '''
Richard Oudkerkd69cfe82012-06-18 17:47:52 +0100457 if self._listener is not None:
458 self._listener.close()
459 self._listener = None
Benjamin Petersone711caf2008-06-11 16:44:04 +0000460
461 address = property(lambda self: self._listener._address)
462 last_accepted = property(lambda self: self._listener._last_accepted)
463
Richard Oudkerkd69cfe82012-06-18 17:47:52 +0100464 def __enter__(self):
465 return self
466
467 def __exit__(self, exc_type, exc_value, exc_tb):
468 self.close()
469
Benjamin Petersone711caf2008-06-11 16:44:04 +0000470
471def Client(address, family=None, authkey=None):
472 '''
473 Returns a connection to the address of a `Listener`
474 '''
475 family = family or address_type(address)
Antoine Pitrou709176f2012-04-01 17:19:09 +0200476 _validate_family(family)
Benjamin Petersone711caf2008-06-11 16:44:04 +0000477 if family == 'AF_PIPE':
478 c = PipeClient(address)
479 else:
480 c = SocketClient(address)
481
482 if authkey is not None and not isinstance(authkey, bytes):
483 raise TypeError('authkey should be a byte string')
484
485 if authkey is not None:
486 answer_challenge(c, authkey)
487 deliver_challenge(c, authkey)
488
489 return c
490
491
492if sys.platform != 'win32':
493
494 def Pipe(duplex=True):
495 '''
496 Returns pair of connection objects at either end of a pipe
497 '''
498 if duplex:
499 s1, s2 = socket.socketpair()
Richard Oudkerkb15e6222012-07-27 14:19:00 +0100500 s1.setblocking(True)
501 s2.setblocking(True)
Antoine Pitrou5aa878c2011-05-09 21:00:28 +0200502 c1 = Connection(s1.detach())
503 c2 = Connection(s2.detach())
Benjamin Petersone711caf2008-06-11 16:44:04 +0000504 else:
505 fd1, fd2 = os.pipe()
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200506 c1 = Connection(fd1, writable=False)
507 c2 = Connection(fd2, readable=False)
Benjamin Petersone711caf2008-06-11 16:44:04 +0000508
509 return c1, c2
510
511else:
512
Benjamin Petersone711caf2008-06-11 16:44:04 +0000513 def Pipe(duplex=True):
514 '''
515 Returns pair of connection objects at either end of a pipe
516 '''
517 address = arbitrary_address('AF_PIPE')
518 if duplex:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200519 openmode = _winapi.PIPE_ACCESS_DUPLEX
520 access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE
Benjamin Petersone711caf2008-06-11 16:44:04 +0000521 obsize, ibsize = BUFSIZE, BUFSIZE
522 else:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200523 openmode = _winapi.PIPE_ACCESS_INBOUND
524 access = _winapi.GENERIC_WRITE
Benjamin Petersone711caf2008-06-11 16:44:04 +0000525 obsize, ibsize = 0, BUFSIZE
526
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200527 h1 = _winapi.CreateNamedPipe(
528 address, openmode | _winapi.FILE_FLAG_OVERLAPPED |
529 _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE,
530 _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
531 _winapi.PIPE_WAIT,
532 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL
Benjamin Petersone711caf2008-06-11 16:44:04 +0000533 )
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200534 h2 = _winapi.CreateFile(
535 address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING,
536 _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL
Benjamin Petersone711caf2008-06-11 16:44:04 +0000537 )
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200538 _winapi.SetNamedPipeHandleState(
539 h2, _winapi.PIPE_READMODE_MESSAGE, None, None
Benjamin Petersone711caf2008-06-11 16:44:04 +0000540 )
541
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200542 overlapped = _winapi.ConnectNamedPipe(h1, overlapped=True)
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100543 _, err = overlapped.GetOverlappedResult(True)
544 assert err == 0
Benjamin Petersone711caf2008-06-11 16:44:04 +0000545
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200546 c1 = PipeConnection(h1, writable=duplex)
547 c2 = PipeConnection(h2, readable=duplex)
Benjamin Petersone711caf2008-06-11 16:44:04 +0000548
549 return c1, c2
550
551#
552# Definitions for connections based on sockets
553#
554
555class SocketListener(object):
556 '''
Georg Brandl734e2682008-08-12 08:18:18 +0000557 Representation of a socket which is bound to an address and listening
Benjamin Petersone711caf2008-06-11 16:44:04 +0000558 '''
559 def __init__(self, address, family, backlog=1):
560 self._socket = socket.socket(getattr(socket, family))
Charles-François Nataliedc67fe2012-02-04 15:12:08 +0100561 try:
Charles-François Natalied4a8fc2012-02-08 21:15:58 +0100562 # SO_REUSEADDR has different semantics on Windows (issue #2550).
563 if os.name == 'posix':
564 self._socket.setsockopt(socket.SOL_SOCKET,
565 socket.SO_REUSEADDR, 1)
Richard Oudkerkb15e6222012-07-27 14:19:00 +0100566 self._socket.setblocking(True)
Charles-François Nataliedc67fe2012-02-04 15:12:08 +0100567 self._socket.bind(address)
568 self._socket.listen(backlog)
569 self._address = self._socket.getsockname()
570 except OSError:
571 self._socket.close()
572 raise
Benjamin Petersone711caf2008-06-11 16:44:04 +0000573 self._family = family
574 self._last_accepted = None
575
Benjamin Petersone711caf2008-06-11 16:44:04 +0000576 if family == 'AF_UNIX':
577 self._unlink = Finalize(
Georg Brandl2ee470f2008-07-16 12:55:28 +0000578 self, os.unlink, args=(address,), exitpriority=0
Benjamin Petersone711caf2008-06-11 16:44:04 +0000579 )
580 else:
581 self._unlink = None
582
583 def accept(self):
584 s, self._last_accepted = self._socket.accept()
Richard Oudkerkb15e6222012-07-27 14:19:00 +0100585 s.setblocking(True)
Antoine Pitroudf97cbe2012-04-07 22:38:52 +0200586 return Connection(s.detach())
Benjamin Petersone711caf2008-06-11 16:44:04 +0000587
588 def close(self):
589 self._socket.close()
590 if self._unlink is not None:
591 self._unlink()
592
593
594def SocketClient(address):
595 '''
596 Return a connection object connected to the socket given by `address`
597 '''
598 family = address_type(address)
Victor Stinner2b695062011-01-03 15:47:59 +0000599 with socket.socket( getattr(socket, family) ) as s:
Richard Oudkerkb15e6222012-07-27 14:19:00 +0100600 s.setblocking(True)
Charles-François Natalie6eabd42011-11-19 09:59:43 +0100601 s.connect(address)
Antoine Pitroudf97cbe2012-04-07 22:38:52 +0200602 return Connection(s.detach())
Benjamin Petersone711caf2008-06-11 16:44:04 +0000603
604#
605# Definitions for connections based on named pipes
606#
607
608if sys.platform == 'win32':
609
610 class PipeListener(object):
611 '''
612 Representation of a named pipe
613 '''
614 def __init__(self, address, backlog=None):
615 self._address = address
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100616 self._handle_queue = [self._new_handle(first=True)]
617
Benjamin Petersone711caf2008-06-11 16:44:04 +0000618 self._last_accepted = None
Benjamin Petersone711caf2008-06-11 16:44:04 +0000619 sub_debug('listener created with address=%r', self._address)
Benjamin Petersone711caf2008-06-11 16:44:04 +0000620 self.close = Finalize(
621 self, PipeListener._finalize_pipe_listener,
622 args=(self._handle_queue, self._address), exitpriority=0
623 )
624
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100625 def _new_handle(self, first=False):
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200626 flags = _winapi.PIPE_ACCESS_DUPLEX | _winapi.FILE_FLAG_OVERLAPPED
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100627 if first:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200628 flags |= _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE
629 return _winapi.CreateNamedPipe(
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100630 self._address, flags,
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200631 _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
632 _winapi.PIPE_WAIT,
633 _winapi.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE,
634 _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL
Benjamin Petersone711caf2008-06-11 16:44:04 +0000635 )
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100636
637 def accept(self):
638 self._handle_queue.append(self._new_handle())
Benjamin Petersone711caf2008-06-11 16:44:04 +0000639 handle = self._handle_queue.pop(0)
640 try:
Richard Oudkerkfdb8dcf2012-05-05 19:45:37 +0100641 ov = _winapi.ConnectNamedPipe(handle, overlapped=True)
642 except OSError as e:
643 if e.winerror != _winapi.ERROR_NO_DATA:
644 raise
645 # ERROR_NO_DATA can occur if a client has already connected,
646 # written data and then disconnected -- see Issue 14725.
647 else:
648 try:
649 res = _winapi.WaitForMultipleObjects(
650 [ov.event], False, INFINITE)
651 except:
652 ov.cancel()
653 _winapi.CloseHandle(handle)
654 raise
655 finally:
656 _, err = ov.GetOverlappedResult(True)
657 assert err == 0
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200658 return PipeConnection(handle)
Benjamin Petersone711caf2008-06-11 16:44:04 +0000659
660 @staticmethod
661 def _finalize_pipe_listener(queue, address):
662 sub_debug('closing listener with address=%r', address)
663 for handle in queue:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200664 _winapi.CloseHandle(handle)
Benjamin Petersone711caf2008-06-11 16:44:04 +0000665
666 def PipeClient(address):
667 '''
668 Return a connection object connected to the pipe given by `address`
669 '''
Antoine Pitrou45d61a32009-11-13 22:35:18 +0000670 t = _init_timeout()
Benjamin Petersone711caf2008-06-11 16:44:04 +0000671 while 1:
672 try:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200673 _winapi.WaitNamedPipe(address, 1000)
674 h = _winapi.CreateFile(
675 address, _winapi.GENERIC_READ | _winapi.GENERIC_WRITE,
676 0, _winapi.NULL, _winapi.OPEN_EXISTING,
677 _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL
Benjamin Petersone711caf2008-06-11 16:44:04 +0000678 )
679 except WindowsError as e:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200680 if e.winerror not in (_winapi.ERROR_SEM_TIMEOUT,
681 _winapi.ERROR_PIPE_BUSY) or _check_timeout(t):
Benjamin Petersone711caf2008-06-11 16:44:04 +0000682 raise
683 else:
684 break
685 else:
686 raise
687
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200688 _winapi.SetNamedPipeHandleState(
689 h, _winapi.PIPE_READMODE_MESSAGE, None, None
Benjamin Petersone711caf2008-06-11 16:44:04 +0000690 )
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200691 return PipeConnection(h)
Benjamin Petersone711caf2008-06-11 16:44:04 +0000692
693#
694# Authentication stuff
695#
696
697MESSAGE_LENGTH = 20
698
Benjamin Peterson1fcfe212008-06-25 12:54:22 +0000699CHALLENGE = b'#CHALLENGE#'
700WELCOME = b'#WELCOME#'
701FAILURE = b'#FAILURE#'
Benjamin Petersone711caf2008-06-11 16:44:04 +0000702
703def deliver_challenge(connection, authkey):
704 import hmac
705 assert isinstance(authkey, bytes)
706 message = os.urandom(MESSAGE_LENGTH)
707 connection.send_bytes(CHALLENGE + message)
708 digest = hmac.new(authkey, message).digest()
709 response = connection.recv_bytes(256) # reject large message
710 if response == digest:
711 connection.send_bytes(WELCOME)
712 else:
713 connection.send_bytes(FAILURE)
714 raise AuthenticationError('digest received was wrong')
715
716def answer_challenge(connection, authkey):
717 import hmac
718 assert isinstance(authkey, bytes)
719 message = connection.recv_bytes(256) # reject large message
720 assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message
721 message = message[len(CHALLENGE):]
722 digest = hmac.new(authkey, message).digest()
723 connection.send_bytes(digest)
724 response = connection.recv_bytes(256) # reject large message
725 if response != WELCOME:
726 raise AuthenticationError('digest sent was rejected')
727
728#
729# Support for using xmlrpclib for serialization
730#
731
732class ConnectionWrapper(object):
733 def __init__(self, conn, dumps, loads):
734 self._conn = conn
735 self._dumps = dumps
736 self._loads = loads
737 for attr in ('fileno', 'close', 'poll', 'recv_bytes', 'send_bytes'):
738 obj = getattr(conn, attr)
739 setattr(self, attr, obj)
740 def send(self, obj):
741 s = self._dumps(obj)
742 self._conn.send_bytes(s)
743 def recv(self):
744 s = self._conn.recv_bytes()
745 return self._loads(s)
746
747def _xml_dumps(obj):
Marc-André Lemburg8f36af72011-02-25 15:42:01 +0000748 return xmlrpclib.dumps((obj,), None, None, None, 1).encode('utf-8')
Benjamin Petersone711caf2008-06-11 16:44:04 +0000749
750def _xml_loads(s):
Marc-André Lemburg8f36af72011-02-25 15:42:01 +0000751 (obj,), method = xmlrpclib.loads(s.decode('utf-8'))
Benjamin Petersone711caf2008-06-11 16:44:04 +0000752 return obj
753
754class XmlListener(Listener):
755 def accept(self):
756 global xmlrpclib
757 import xmlrpc.client as xmlrpclib
758 obj = Listener.accept(self)
759 return ConnectionWrapper(obj, _xml_dumps, _xml_loads)
760
761def XmlClient(*args, **kwds):
762 global xmlrpclib
763 import xmlrpc.client as xmlrpclib
764 return ConnectionWrapper(Client(*args, **kwds), _xml_dumps, _xml_loads)
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200765
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100766#
767# Wait
768#
769
770if sys.platform == 'win32':
771
772 def _exhaustive_wait(handles, timeout):
773 # Return ALL handles which are currently signalled. (Only
774 # returning the first signalled might create starvation issues.)
775 L = list(handles)
776 ready = []
777 while L:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200778 res = _winapi.WaitForMultipleObjects(L, False, timeout)
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100779 if res == WAIT_TIMEOUT:
780 break
781 elif WAIT_OBJECT_0 <= res < WAIT_OBJECT_0 + len(L):
782 res -= WAIT_OBJECT_0
783 elif WAIT_ABANDONED_0 <= res < WAIT_ABANDONED_0 + len(L):
784 res -= WAIT_ABANDONED_0
785 else:
786 raise RuntimeError('Should not get here')
787 ready.append(L[res])
788 L = L[res+1:]
789 timeout = 0
790 return ready
791
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200792 _ready_errors = {_winapi.ERROR_BROKEN_PIPE, _winapi.ERROR_NETNAME_DELETED}
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100793
794 def wait(object_list, timeout=None):
795 '''
796 Wait till an object in object_list is ready/readable.
797
798 Returns list of those objects in object_list which are ready/readable.
799 '''
800 if timeout is None:
801 timeout = INFINITE
802 elif timeout < 0:
803 timeout = 0
804 else:
805 timeout = int(timeout * 1000 + 0.5)
806
807 object_list = list(object_list)
808 waithandle_to_obj = {}
809 ov_list = []
810 ready_objects = set()
811 ready_handles = set()
812
813 try:
814 for o in object_list:
815 try:
816 fileno = getattr(o, 'fileno')
817 except AttributeError:
818 waithandle_to_obj[o.__index__()] = o
819 else:
820 # start an overlapped read of length zero
821 try:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200822 ov, err = _winapi.ReadFile(fileno(), 0, True)
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100823 except OSError as e:
824 err = e.winerror
825 if err not in _ready_errors:
826 raise
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200827 if err == _winapi.ERROR_IO_PENDING:
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100828 ov_list.append(ov)
829 waithandle_to_obj[ov.event] = o
830 else:
831 # If o.fileno() is an overlapped pipe handle and
832 # err == 0 then there is a zero length message
833 # in the pipe, but it HAS NOT been consumed.
834 ready_objects.add(o)
835 timeout = 0
836
837 ready_handles = _exhaustive_wait(waithandle_to_obj.keys(), timeout)
838 finally:
839 # request that overlapped reads stop
840 for ov in ov_list:
841 ov.cancel()
842
843 # wait for all overlapped reads to stop
844 for ov in ov_list:
845 try:
846 _, err = ov.GetOverlappedResult(True)
847 except OSError as e:
848 err = e.winerror
849 if err not in _ready_errors:
850 raise
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200851 if err != _winapi.ERROR_OPERATION_ABORTED:
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100852 o = waithandle_to_obj[ov.event]
853 ready_objects.add(o)
854 if err == 0:
855 # If o.fileno() is an overlapped pipe handle then
856 # a zero length message HAS been consumed.
857 if hasattr(o, '_got_empty_message'):
858 o._got_empty_message = True
859
860 ready_objects.update(waithandle_to_obj[h] for h in ready_handles)
861 return [o for o in object_list if o in ready_objects]
862
863else:
864
865 def wait(object_list, timeout=None):
866 '''
867 Wait till an object in object_list is ready/readable.
868
869 Returns list of those objects in object_list which are ready/readable.
870 '''
871 if timeout is not None:
872 if timeout <= 0:
873 return select.select(object_list, [], [], 0)[0]
874 else:
875 deadline = time.time() + timeout
876 while True:
877 try:
878 return select.select(object_list, [], [], timeout)[0]
879 except OSError as e:
880 if e.errno != errno.EINTR:
881 raise
882 if timeout is not None:
883 timeout = deadline - time.time()
Antoine Pitrou5438ed12012-04-24 22:56:57 +0200884
885#
886# Make connection and socket objects sharable if possible
887#
888
889if sys.platform == 'win32':
890 from . import reduction
891 ForkingPickler.register(socket.socket, reduction.reduce_socket)
892 ForkingPickler.register(Connection, reduction.reduce_connection)
893 ForkingPickler.register(PipeConnection, reduction.reduce_pipe_connection)
894else:
895 try:
896 from . import reduction
897 except ImportError:
898 pass
899 else:
900 ForkingPickler.register(socket.socket, reduction.reduce_socket)
901 ForkingPickler.register(Connection, reduction.reduce_connection)