blob: 56f375d237a025c7c3e1ec177679bdc74f19d544 [file] [log] [blame]
Benjamin Petersone711caf2008-06-11 16:44:04 +00001#
2# A higher level module for using sockets (or Windows named pipes)
3#
4# multiprocessing/connection.py
5#
R. David Murray3fc969a2010-12-14 01:38:16 +00006# Copyright (c) 2006-2008, R Oudkerk
Richard Oudkerk3e268aa2012-04-30 12:13:55 +01007# Licensed to PSF under a Contributor Agreement.
Benjamin Petersone711caf2008-06-11 16:44:04 +00008#
9
Antoine Pitroubdb1cf12012-03-05 19:28:37 +010010__all__ = [ 'Client', 'Listener', 'Pipe', 'wait' ]
Benjamin Petersone711caf2008-06-11 16:44:04 +000011
Antoine Pitrou87cf2202011-05-09 17:04:27 +020012import io
Benjamin Petersone711caf2008-06-11 16:44:04 +000013import os
14import sys
Antoine Pitrou87cf2202011-05-09 17:04:27 +020015import pickle
16import select
Benjamin Petersone711caf2008-06-11 16:44:04 +000017import socket
Antoine Pitrou87cf2202011-05-09 17:04:27 +020018import struct
Georg Brandl6aa2d1f2008-08-12 08:35:52 +000019import errno
Benjamin Petersone711caf2008-06-11 16:44:04 +000020import time
21import tempfile
22import itertools
23
24import _multiprocessing
Antoine Pitrou87cf2202011-05-09 17:04:27 +020025from multiprocessing import current_process, AuthenticationError, BufferTooShort
Richard Oudkerk59d54042012-05-10 16:11:12 +010026from multiprocessing.util import get_temp_dir, Finalize, sub_debug, debug
Antoine Pitrou5438ed12012-04-24 22:56:57 +020027from multiprocessing.forking import ForkingPickler
Antoine Pitrou87cf2202011-05-09 17:04:27 +020028try:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +020029 import _winapi
30 from _winapi import WAIT_OBJECT_0, WAIT_TIMEOUT, INFINITE
Antoine Pitrou87cf2202011-05-09 17:04:27 +020031except ImportError:
32 if sys.platform == 'win32':
33 raise
Antoine Pitrou23bba4c2012-04-18 20:51:15 +020034 _winapi = None
Benjamin Petersone711caf2008-06-11 16:44:04 +000035
36#
37#
38#
39
40BUFSIZE = 8192
Antoine Pitrou45d61a32009-11-13 22:35:18 +000041# A very generous timeout when it comes to local connections...
42CONNECTION_TIMEOUT = 20.
Benjamin Petersone711caf2008-06-11 16:44:04 +000043
44_mmap_counter = itertools.count()
45
46default_family = 'AF_INET'
47families = ['AF_INET']
48
49if hasattr(socket, 'AF_UNIX'):
50 default_family = 'AF_UNIX'
51 families += ['AF_UNIX']
52
53if sys.platform == 'win32':
54 default_family = 'AF_PIPE'
55 families += ['AF_PIPE']
56
Antoine Pitrou45d61a32009-11-13 22:35:18 +000057
58def _init_timeout(timeout=CONNECTION_TIMEOUT):
59 return time.time() + timeout
60
61def _check_timeout(t):
62 return time.time() > t
63
Benjamin Petersone711caf2008-06-11 16:44:04 +000064#
65#
66#
67
68def arbitrary_address(family):
69 '''
70 Return an arbitrary free address for the given family
71 '''
72 if family == 'AF_INET':
73 return ('localhost', 0)
74 elif family == 'AF_UNIX':
75 return tempfile.mktemp(prefix='listener-', dir=get_temp_dir())
76 elif family == 'AF_PIPE':
77 return tempfile.mktemp(prefix=r'\\.\pipe\pyc-%d-%d-' %
78 (os.getpid(), next(_mmap_counter)))
79 else:
80 raise ValueError('unrecognized family')
81
Antoine Pitrou709176f2012-04-01 17:19:09 +020082def _validate_family(family):
83 '''
84 Checks if the family is valid for the current environment.
85 '''
86 if sys.platform != 'win32' and family == 'AF_PIPE':
87 raise ValueError('Family %s is not recognized.' % family)
88
Antoine Pitrou6d20cba2012-04-03 20:12:23 +020089 if sys.platform == 'win32' and family == 'AF_UNIX':
90 # double check
91 if not hasattr(socket, family):
92 raise ValueError('Family %s is not recognized.' % family)
Benjamin Petersone711caf2008-06-11 16:44:04 +000093
94def address_type(address):
95 '''
96 Return the types of the address
97
98 This can be 'AF_INET', 'AF_UNIX', or 'AF_PIPE'
99 '''
100 if type(address) == tuple:
101 return 'AF_INET'
102 elif type(address) is str and address.startswith('\\\\'):
103 return 'AF_PIPE'
104 elif type(address) is str:
105 return 'AF_UNIX'
106 else:
107 raise ValueError('address type of %r unrecognized' % address)
108
109#
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200110# Connection classes
111#
112
113class _ConnectionBase:
114 _handle = None
115
116 def __init__(self, handle, readable=True, writable=True):
117 handle = handle.__index__()
118 if handle < 0:
119 raise ValueError("invalid handle")
120 if not readable and not writable:
121 raise ValueError(
122 "at least one of `readable` and `writable` must be True")
123 self._handle = handle
124 self._readable = readable
125 self._writable = writable
126
Antoine Pitrou60001202011-07-09 01:03:46 +0200127 # XXX should we use util.Finalize instead of a __del__?
128
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200129 def __del__(self):
130 if self._handle is not None:
131 self._close()
132
133 def _check_closed(self):
134 if self._handle is None:
135 raise IOError("handle is closed")
136
137 def _check_readable(self):
138 if not self._readable:
139 raise IOError("connection is write-only")
140
141 def _check_writable(self):
142 if not self._writable:
143 raise IOError("connection is read-only")
144
145 def _bad_message_length(self):
146 if self._writable:
147 self._readable = False
148 else:
149 self.close()
150 raise IOError("bad message length")
151
152 @property
153 def closed(self):
154 """True if the connection is closed"""
155 return self._handle is None
156
157 @property
158 def readable(self):
159 """True if the connection is readable"""
160 return self._readable
161
162 @property
163 def writable(self):
164 """True if the connection is writable"""
165 return self._writable
166
167 def fileno(self):
168 """File descriptor or handle of the connection"""
169 self._check_closed()
170 return self._handle
171
172 def close(self):
173 """Close the connection"""
174 if self._handle is not None:
175 try:
176 self._close()
177 finally:
178 self._handle = None
179
180 def send_bytes(self, buf, offset=0, size=None):
181 """Send the bytes data from a bytes-like object"""
182 self._check_closed()
183 self._check_writable()
184 m = memoryview(buf)
185 # HACK for byte-indexing of non-bytewise buffers (e.g. array.array)
186 if m.itemsize > 1:
187 m = memoryview(bytes(m))
188 n = len(m)
189 if offset < 0:
190 raise ValueError("offset is negative")
191 if n < offset:
192 raise ValueError("buffer length < offset")
193 if size is None:
194 size = n - offset
195 elif size < 0:
196 raise ValueError("size is negative")
197 elif offset + size > n:
198 raise ValueError("buffer length < offset + size")
199 self._send_bytes(m[offset:offset + size])
200
201 def send(self, obj):
202 """Send a (picklable) object"""
203 self._check_closed()
204 self._check_writable()
Antoine Pitrou5438ed12012-04-24 22:56:57 +0200205 buf = io.BytesIO()
206 ForkingPickler(buf, pickle.HIGHEST_PROTOCOL).dump(obj)
207 self._send_bytes(buf.getbuffer())
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200208
209 def recv_bytes(self, maxlength=None):
210 """
211 Receive bytes data as a bytes object.
212 """
213 self._check_closed()
214 self._check_readable()
215 if maxlength is not None and maxlength < 0:
216 raise ValueError("negative maxlength")
217 buf = self._recv_bytes(maxlength)
218 if buf is None:
219 self._bad_message_length()
220 return buf.getvalue()
221
222 def recv_bytes_into(self, buf, offset=0):
223 """
224 Receive bytes data into a writeable buffer-like object.
225 Return the number of bytes read.
226 """
227 self._check_closed()
228 self._check_readable()
229 with memoryview(buf) as m:
230 # Get bytesize of arbitrary buffer
231 itemsize = m.itemsize
232 bytesize = itemsize * len(m)
233 if offset < 0:
234 raise ValueError("negative offset")
235 elif offset > bytesize:
236 raise ValueError("offset too large")
237 result = self._recv_bytes()
238 size = result.tell()
239 if bytesize < offset + size:
240 raise BufferTooShort(result.getvalue())
241 # Message can fit in dest
242 result.seek(0)
243 result.readinto(m[offset // itemsize :
244 (offset + size) // itemsize])
245 return size
246
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100247 def recv(self):
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200248 """Receive a (picklable) object"""
249 self._check_closed()
250 self._check_readable()
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100251 buf = self._recv_bytes()
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200252 return pickle.loads(buf.getbuffer())
253
254 def poll(self, timeout=0.0):
255 """Whether there is any input available to be read"""
256 self._check_closed()
257 self._check_readable()
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200258 return self._poll(timeout)
259
260
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200261if _winapi:
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200262
263 class PipeConnection(_ConnectionBase):
264 """
265 Connection class based on a Windows named pipe.
Antoine Pitroudd696492011-06-08 17:21:55 +0200266 Overlapped I/O is used, so the handles must have been created
267 with FILE_FLAG_OVERLAPPED.
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200268 """
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100269 _got_empty_message = False
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200270
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200271 def _close(self, _CloseHandle=_winapi.CloseHandle):
Antoine Pitrou71a28a92011-07-09 01:03:00 +0200272 _CloseHandle(self._handle)
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200273
274 def _send_bytes(self, buf):
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200275 ov, err = _winapi.WriteFile(self._handle, buf, overlapped=True)
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100276 try:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200277 if err == _winapi.ERROR_IO_PENDING:
278 waitres = _winapi.WaitForMultipleObjects(
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100279 [ov.event], False, INFINITE)
280 assert waitres == WAIT_OBJECT_0
281 except:
282 ov.cancel()
283 raise
284 finally:
285 nwritten, err = ov.GetOverlappedResult(True)
286 assert err == 0
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200287 assert nwritten == len(buf)
288
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100289 def _recv_bytes(self, maxsize=None):
290 if self._got_empty_message:
291 self._got_empty_message = False
292 return io.BytesIO()
Antoine Pitroudd696492011-06-08 17:21:55 +0200293 else:
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100294 bsize = 128 if maxsize is None else min(maxsize, 128)
Antoine Pitroudd696492011-06-08 17:21:55 +0200295 try:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200296 ov, err = _winapi.ReadFile(self._handle, bsize,
297 overlapped=True)
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100298 try:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200299 if err == _winapi.ERROR_IO_PENDING:
300 waitres = _winapi.WaitForMultipleObjects(
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100301 [ov.event], False, INFINITE)
302 assert waitres == WAIT_OBJECT_0
303 except:
304 ov.cancel()
305 raise
306 finally:
307 nread, err = ov.GetOverlappedResult(True)
308 if err == 0:
309 f = io.BytesIO()
310 f.write(ov.getbuffer())
311 return f
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200312 elif err == _winapi.ERROR_MORE_DATA:
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100313 return self._get_more_data(ov, maxsize)
Antoine Pitroudd696492011-06-08 17:21:55 +0200314 except IOError as e:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200315 if e.winerror == _winapi.ERROR_BROKEN_PIPE:
Antoine Pitroudd696492011-06-08 17:21:55 +0200316 raise EOFError
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100317 else:
318 raise
319 raise RuntimeError("shouldn't get here; expected KeyboardInterrupt")
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200320
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100321 def _poll(self, timeout):
322 if (self._got_empty_message or
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200323 _winapi.PeekNamedPipe(self._handle)[0] != 0):
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200324 return True
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100325 return bool(wait([self], timeout))
326
327 def _get_more_data(self, ov, maxsize):
328 buf = ov.getbuffer()
329 f = io.BytesIO()
330 f.write(buf)
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200331 left = _winapi.PeekNamedPipe(self._handle)[1]
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100332 assert left > 0
333 if maxsize is not None and len(buf) + left > maxsize:
334 self._bad_message_length()
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200335 ov, err = _winapi.ReadFile(self._handle, left, overlapped=True)
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100336 rbytes, err = ov.GetOverlappedResult(True)
337 assert err == 0
338 assert rbytes == left
339 f.write(ov.getbuffer())
340 return f
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200341
342
343class Connection(_ConnectionBase):
344 """
345 Connection class based on an arbitrary file descriptor (Unix only), or
346 a socket handle (Windows).
347 """
348
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200349 if _winapi:
350 def _close(self, _close=_multiprocessing.closesocket):
Antoine Pitrou71a28a92011-07-09 01:03:00 +0200351 _close(self._handle)
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200352 _write = _multiprocessing.send
353 _read = _multiprocessing.recv
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200354 else:
Antoine Pitrou71a28a92011-07-09 01:03:00 +0200355 def _close(self, _close=os.close):
356 _close(self._handle)
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200357 _write = os.write
358 _read = os.read
359
360 def _send(self, buf, write=_write):
361 remaining = len(buf)
362 while True:
363 n = write(self._handle, buf)
364 remaining -= n
365 if remaining == 0:
366 break
367 buf = buf[n:]
368
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100369 def _recv(self, size, read=_read):
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200370 buf = io.BytesIO()
Antoine Pitroudd696492011-06-08 17:21:55 +0200371 handle = self._handle
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200372 remaining = size
373 while remaining > 0:
Antoine Pitroudd696492011-06-08 17:21:55 +0200374 chunk = read(handle, remaining)
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200375 n = len(chunk)
376 if n == 0:
377 if remaining == size:
378 raise EOFError
379 else:
380 raise IOError("got end of file during message")
381 buf.write(chunk)
382 remaining -= n
383 return buf
384
385 def _send_bytes(self, buf):
386 # For wire compatibility with 3.2 and lower
387 n = len(buf)
Charles-François Natali225aa4f2011-09-20 19:27:39 +0200388 self._send(struct.pack("!i", n))
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200389 # The condition is necessary to avoid "broken pipe" errors
390 # when sending a 0-length buffer if the other end closed the pipe.
391 if n > 0:
392 self._send(buf)
393
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100394 def _recv_bytes(self, maxsize=None):
395 buf = self._recv(4)
Charles-François Natali225aa4f2011-09-20 19:27:39 +0200396 size, = struct.unpack("!i", buf.getvalue())
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200397 if maxsize is not None and size > maxsize:
398 return None
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100399 return self._recv(size)
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200400
401 def _poll(self, timeout):
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100402 r = wait([self._handle], timeout)
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200403 return bool(r)
404
405
406#
Benjamin Petersone711caf2008-06-11 16:44:04 +0000407# Public functions
408#
409
410class Listener(object):
411 '''
412 Returns a listener object.
413
414 This is a wrapper for a bound socket which is 'listening' for
415 connections, or for a Windows named pipe.
416 '''
417 def __init__(self, address=None, family=None, backlog=1, authkey=None):
418 family = family or (address and address_type(address)) \
419 or default_family
420 address = address or arbitrary_address(family)
421
Antoine Pitrou709176f2012-04-01 17:19:09 +0200422 _validate_family(family)
Benjamin Petersone711caf2008-06-11 16:44:04 +0000423 if family == 'AF_PIPE':
424 self._listener = PipeListener(address, backlog)
425 else:
426 self._listener = SocketListener(address, family, backlog)
427
428 if authkey is not None and not isinstance(authkey, bytes):
429 raise TypeError('authkey should be a byte string')
430
431 self._authkey = authkey
432
433 def accept(self):
434 '''
435 Accept a connection on the bound socket or named pipe of `self`.
436
437 Returns a `Connection` object.
438 '''
439 c = self._listener.accept()
440 if self._authkey:
441 deliver_challenge(c, self._authkey)
442 answer_challenge(c, self._authkey)
443 return c
444
445 def close(self):
446 '''
447 Close the bound socket or named pipe of `self`.
448 '''
449 return self._listener.close()
450
451 address = property(lambda self: self._listener._address)
452 last_accepted = property(lambda self: self._listener._last_accepted)
453
454
455def Client(address, family=None, authkey=None):
456 '''
457 Returns a connection to the address of a `Listener`
458 '''
459 family = family or address_type(address)
Antoine Pitrou709176f2012-04-01 17:19:09 +0200460 _validate_family(family)
Benjamin Petersone711caf2008-06-11 16:44:04 +0000461 if family == 'AF_PIPE':
462 c = PipeClient(address)
463 else:
464 c = SocketClient(address)
465
466 if authkey is not None and not isinstance(authkey, bytes):
467 raise TypeError('authkey should be a byte string')
468
469 if authkey is not None:
470 answer_challenge(c, authkey)
471 deliver_challenge(c, authkey)
472
473 return c
474
475
476if sys.platform != 'win32':
477
478 def Pipe(duplex=True):
479 '''
480 Returns pair of connection objects at either end of a pipe
481 '''
482 if duplex:
483 s1, s2 = socket.socketpair()
Antoine Pitrou5aa878c2011-05-09 21:00:28 +0200484 c1 = Connection(s1.detach())
485 c2 = Connection(s2.detach())
Benjamin Petersone711caf2008-06-11 16:44:04 +0000486 else:
487 fd1, fd2 = os.pipe()
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200488 c1 = Connection(fd1, writable=False)
489 c2 = Connection(fd2, readable=False)
Benjamin Petersone711caf2008-06-11 16:44:04 +0000490
491 return c1, c2
492
493else:
494
Benjamin Petersone711caf2008-06-11 16:44:04 +0000495 def Pipe(duplex=True):
496 '''
497 Returns pair of connection objects at either end of a pipe
498 '''
499 address = arbitrary_address('AF_PIPE')
500 if duplex:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200501 openmode = _winapi.PIPE_ACCESS_DUPLEX
502 access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE
Benjamin Petersone711caf2008-06-11 16:44:04 +0000503 obsize, ibsize = BUFSIZE, BUFSIZE
504 else:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200505 openmode = _winapi.PIPE_ACCESS_INBOUND
506 access = _winapi.GENERIC_WRITE
Benjamin Petersone711caf2008-06-11 16:44:04 +0000507 obsize, ibsize = 0, BUFSIZE
508
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200509 h1 = _winapi.CreateNamedPipe(
510 address, openmode | _winapi.FILE_FLAG_OVERLAPPED |
511 _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE,
512 _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
513 _winapi.PIPE_WAIT,
514 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL
Benjamin Petersone711caf2008-06-11 16:44:04 +0000515 )
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200516 h2 = _winapi.CreateFile(
517 address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING,
518 _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL
Benjamin Petersone711caf2008-06-11 16:44:04 +0000519 )
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200520 _winapi.SetNamedPipeHandleState(
521 h2, _winapi.PIPE_READMODE_MESSAGE, None, None
Benjamin Petersone711caf2008-06-11 16:44:04 +0000522 )
523
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200524 overlapped = _winapi.ConnectNamedPipe(h1, overlapped=True)
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100525 _, err = overlapped.GetOverlappedResult(True)
526 assert err == 0
Benjamin Petersone711caf2008-06-11 16:44:04 +0000527
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200528 c1 = PipeConnection(h1, writable=duplex)
529 c2 = PipeConnection(h2, readable=duplex)
Benjamin Petersone711caf2008-06-11 16:44:04 +0000530
531 return c1, c2
532
533#
534# Definitions for connections based on sockets
535#
536
537class SocketListener(object):
538 '''
Georg Brandl734e2682008-08-12 08:18:18 +0000539 Representation of a socket which is bound to an address and listening
Benjamin Petersone711caf2008-06-11 16:44:04 +0000540 '''
541 def __init__(self, address, family, backlog=1):
542 self._socket = socket.socket(getattr(socket, family))
Charles-François Nataliedc67fe2012-02-04 15:12:08 +0100543 try:
Charles-François Natalied4a8fc2012-02-08 21:15:58 +0100544 # SO_REUSEADDR has different semantics on Windows (issue #2550).
545 if os.name == 'posix':
546 self._socket.setsockopt(socket.SOL_SOCKET,
547 socket.SO_REUSEADDR, 1)
Charles-François Nataliedc67fe2012-02-04 15:12:08 +0100548 self._socket.bind(address)
549 self._socket.listen(backlog)
550 self._address = self._socket.getsockname()
551 except OSError:
552 self._socket.close()
553 raise
Benjamin Petersone711caf2008-06-11 16:44:04 +0000554 self._family = family
555 self._last_accepted = None
556
Benjamin Petersone711caf2008-06-11 16:44:04 +0000557 if family == 'AF_UNIX':
558 self._unlink = Finalize(
Georg Brandl2ee470f2008-07-16 12:55:28 +0000559 self, os.unlink, args=(address,), exitpriority=0
Benjamin Petersone711caf2008-06-11 16:44:04 +0000560 )
561 else:
562 self._unlink = None
563
564 def accept(self):
565 s, self._last_accepted = self._socket.accept()
Antoine Pitroudf97cbe2012-04-07 22:38:52 +0200566 return Connection(s.detach())
Benjamin Petersone711caf2008-06-11 16:44:04 +0000567
568 def close(self):
569 self._socket.close()
570 if self._unlink is not None:
571 self._unlink()
572
573
574def SocketClient(address):
575 '''
576 Return a connection object connected to the socket given by `address`
577 '''
578 family = address_type(address)
Victor Stinner2b695062011-01-03 15:47:59 +0000579 with socket.socket( getattr(socket, family) ) as s:
Charles-François Natalie6eabd42011-11-19 09:59:43 +0100580 s.connect(address)
Antoine Pitroudf97cbe2012-04-07 22:38:52 +0200581 return Connection(s.detach())
Benjamin Petersone711caf2008-06-11 16:44:04 +0000582
583#
584# Definitions for connections based on named pipes
585#
586
587if sys.platform == 'win32':
588
589 class PipeListener(object):
590 '''
591 Representation of a named pipe
592 '''
593 def __init__(self, address, backlog=None):
594 self._address = address
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100595 self._handle_queue = [self._new_handle(first=True)]
596
Benjamin Petersone711caf2008-06-11 16:44:04 +0000597 self._last_accepted = None
Benjamin Petersone711caf2008-06-11 16:44:04 +0000598 sub_debug('listener created with address=%r', self._address)
Benjamin Petersone711caf2008-06-11 16:44:04 +0000599 self.close = Finalize(
600 self, PipeListener._finalize_pipe_listener,
601 args=(self._handle_queue, self._address), exitpriority=0
602 )
603
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100604 def _new_handle(self, first=False):
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200605 flags = _winapi.PIPE_ACCESS_DUPLEX | _winapi.FILE_FLAG_OVERLAPPED
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100606 if first:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200607 flags |= _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE
608 return _winapi.CreateNamedPipe(
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100609 self._address, flags,
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200610 _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
611 _winapi.PIPE_WAIT,
612 _winapi.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE,
613 _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL
Benjamin Petersone711caf2008-06-11 16:44:04 +0000614 )
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100615
616 def accept(self):
617 self._handle_queue.append(self._new_handle())
Benjamin Petersone711caf2008-06-11 16:44:04 +0000618 handle = self._handle_queue.pop(0)
619 try:
Richard Oudkerkfdb8dcf2012-05-05 19:45:37 +0100620 ov = _winapi.ConnectNamedPipe(handle, overlapped=True)
621 except OSError as e:
622 if e.winerror != _winapi.ERROR_NO_DATA:
623 raise
624 # ERROR_NO_DATA can occur if a client has already connected,
625 # written data and then disconnected -- see Issue 14725.
626 else:
627 try:
628 res = _winapi.WaitForMultipleObjects(
629 [ov.event], False, INFINITE)
630 except:
631 ov.cancel()
632 _winapi.CloseHandle(handle)
633 raise
634 finally:
635 _, err = ov.GetOverlappedResult(True)
636 assert err == 0
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200637 return PipeConnection(handle)
Benjamin Petersone711caf2008-06-11 16:44:04 +0000638
639 @staticmethod
640 def _finalize_pipe_listener(queue, address):
641 sub_debug('closing listener with address=%r', address)
642 for handle in queue:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200643 _winapi.CloseHandle(handle)
Benjamin Petersone711caf2008-06-11 16:44:04 +0000644
645 def PipeClient(address):
646 '''
647 Return a connection object connected to the pipe given by `address`
648 '''
Antoine Pitrou45d61a32009-11-13 22:35:18 +0000649 t = _init_timeout()
Benjamin Petersone711caf2008-06-11 16:44:04 +0000650 while 1:
651 try:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200652 _winapi.WaitNamedPipe(address, 1000)
653 h = _winapi.CreateFile(
654 address, _winapi.GENERIC_READ | _winapi.GENERIC_WRITE,
655 0, _winapi.NULL, _winapi.OPEN_EXISTING,
656 _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL
Benjamin Petersone711caf2008-06-11 16:44:04 +0000657 )
658 except WindowsError as e:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200659 if e.winerror not in (_winapi.ERROR_SEM_TIMEOUT,
660 _winapi.ERROR_PIPE_BUSY) or _check_timeout(t):
Benjamin Petersone711caf2008-06-11 16:44:04 +0000661 raise
662 else:
663 break
664 else:
665 raise
666
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200667 _winapi.SetNamedPipeHandleState(
668 h, _winapi.PIPE_READMODE_MESSAGE, None, None
Benjamin Petersone711caf2008-06-11 16:44:04 +0000669 )
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200670 return PipeConnection(h)
Benjamin Petersone711caf2008-06-11 16:44:04 +0000671
672#
673# Authentication stuff
674#
675
676MESSAGE_LENGTH = 20
677
Benjamin Peterson1fcfe212008-06-25 12:54:22 +0000678CHALLENGE = b'#CHALLENGE#'
679WELCOME = b'#WELCOME#'
680FAILURE = b'#FAILURE#'
Benjamin Petersone711caf2008-06-11 16:44:04 +0000681
682def deliver_challenge(connection, authkey):
683 import hmac
684 assert isinstance(authkey, bytes)
685 message = os.urandom(MESSAGE_LENGTH)
686 connection.send_bytes(CHALLENGE + message)
687 digest = hmac.new(authkey, message).digest()
688 response = connection.recv_bytes(256) # reject large message
689 if response == digest:
690 connection.send_bytes(WELCOME)
691 else:
692 connection.send_bytes(FAILURE)
693 raise AuthenticationError('digest received was wrong')
694
695def answer_challenge(connection, authkey):
696 import hmac
697 assert isinstance(authkey, bytes)
698 message = connection.recv_bytes(256) # reject large message
699 assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message
700 message = message[len(CHALLENGE):]
701 digest = hmac.new(authkey, message).digest()
702 connection.send_bytes(digest)
703 response = connection.recv_bytes(256) # reject large message
704 if response != WELCOME:
705 raise AuthenticationError('digest sent was rejected')
706
707#
708# Support for using xmlrpclib for serialization
709#
710
711class ConnectionWrapper(object):
712 def __init__(self, conn, dumps, loads):
713 self._conn = conn
714 self._dumps = dumps
715 self._loads = loads
716 for attr in ('fileno', 'close', 'poll', 'recv_bytes', 'send_bytes'):
717 obj = getattr(conn, attr)
718 setattr(self, attr, obj)
719 def send(self, obj):
720 s = self._dumps(obj)
721 self._conn.send_bytes(s)
722 def recv(self):
723 s = self._conn.recv_bytes()
724 return self._loads(s)
725
726def _xml_dumps(obj):
Marc-André Lemburg8f36af72011-02-25 15:42:01 +0000727 return xmlrpclib.dumps((obj,), None, None, None, 1).encode('utf-8')
Benjamin Petersone711caf2008-06-11 16:44:04 +0000728
729def _xml_loads(s):
Marc-André Lemburg8f36af72011-02-25 15:42:01 +0000730 (obj,), method = xmlrpclib.loads(s.decode('utf-8'))
Benjamin Petersone711caf2008-06-11 16:44:04 +0000731 return obj
732
733class XmlListener(Listener):
734 def accept(self):
735 global xmlrpclib
736 import xmlrpc.client as xmlrpclib
737 obj = Listener.accept(self)
738 return ConnectionWrapper(obj, _xml_dumps, _xml_loads)
739
740def XmlClient(*args, **kwds):
741 global xmlrpclib
742 import xmlrpc.client as xmlrpclib
743 return ConnectionWrapper(Client(*args, **kwds), _xml_dumps, _xml_loads)
Antoine Pitrou87cf2202011-05-09 17:04:27 +0200744
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100745#
746# Wait
747#
748
749if sys.platform == 'win32':
750
751 def _exhaustive_wait(handles, timeout):
752 # Return ALL handles which are currently signalled. (Only
753 # returning the first signalled might create starvation issues.)
754 L = list(handles)
755 ready = []
756 while L:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200757 res = _winapi.WaitForMultipleObjects(L, False, timeout)
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100758 if res == WAIT_TIMEOUT:
759 break
760 elif WAIT_OBJECT_0 <= res < WAIT_OBJECT_0 + len(L):
761 res -= WAIT_OBJECT_0
762 elif WAIT_ABANDONED_0 <= res < WAIT_ABANDONED_0 + len(L):
763 res -= WAIT_ABANDONED_0
764 else:
765 raise RuntimeError('Should not get here')
766 ready.append(L[res])
767 L = L[res+1:]
768 timeout = 0
769 return ready
770
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200771 _ready_errors = {_winapi.ERROR_BROKEN_PIPE, _winapi.ERROR_NETNAME_DELETED}
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100772
773 def wait(object_list, timeout=None):
774 '''
775 Wait till an object in object_list is ready/readable.
776
777 Returns list of those objects in object_list which are ready/readable.
778 '''
779 if timeout is None:
780 timeout = INFINITE
781 elif timeout < 0:
782 timeout = 0
783 else:
784 timeout = int(timeout * 1000 + 0.5)
785
786 object_list = list(object_list)
787 waithandle_to_obj = {}
788 ov_list = []
789 ready_objects = set()
790 ready_handles = set()
791
792 try:
793 for o in object_list:
794 try:
795 fileno = getattr(o, 'fileno')
796 except AttributeError:
797 waithandle_to_obj[o.__index__()] = o
798 else:
799 # start an overlapped read of length zero
800 try:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200801 ov, err = _winapi.ReadFile(fileno(), 0, True)
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100802 except OSError as e:
803 err = e.winerror
804 if err not in _ready_errors:
805 raise
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200806 if err == _winapi.ERROR_IO_PENDING:
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100807 ov_list.append(ov)
808 waithandle_to_obj[ov.event] = o
809 else:
810 # If o.fileno() is an overlapped pipe handle and
811 # err == 0 then there is a zero length message
812 # in the pipe, but it HAS NOT been consumed.
813 ready_objects.add(o)
814 timeout = 0
815
816 ready_handles = _exhaustive_wait(waithandle_to_obj.keys(), timeout)
817 finally:
818 # request that overlapped reads stop
819 for ov in ov_list:
820 ov.cancel()
821
822 # wait for all overlapped reads to stop
823 for ov in ov_list:
824 try:
825 _, err = ov.GetOverlappedResult(True)
826 except OSError as e:
827 err = e.winerror
828 if err not in _ready_errors:
829 raise
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200830 if err != _winapi.ERROR_OPERATION_ABORTED:
Antoine Pitroubdb1cf12012-03-05 19:28:37 +0100831 o = waithandle_to_obj[ov.event]
832 ready_objects.add(o)
833 if err == 0:
834 # If o.fileno() is an overlapped pipe handle then
835 # a zero length message HAS been consumed.
836 if hasattr(o, '_got_empty_message'):
837 o._got_empty_message = True
838
839 ready_objects.update(waithandle_to_obj[h] for h in ready_handles)
840 return [o for o in object_list if o in ready_objects]
841
842else:
843
844 def wait(object_list, timeout=None):
845 '''
846 Wait till an object in object_list is ready/readable.
847
848 Returns list of those objects in object_list which are ready/readable.
849 '''
850 if timeout is not None:
851 if timeout <= 0:
852 return select.select(object_list, [], [], 0)[0]
853 else:
854 deadline = time.time() + timeout
855 while True:
856 try:
857 return select.select(object_list, [], [], timeout)[0]
858 except OSError as e:
859 if e.errno != errno.EINTR:
860 raise
861 if timeout is not None:
862 timeout = deadline - time.time()
Antoine Pitrou5438ed12012-04-24 22:56:57 +0200863
864#
865# Make connection and socket objects sharable if possible
866#
867
868if sys.platform == 'win32':
869 from . import reduction
870 ForkingPickler.register(socket.socket, reduction.reduce_socket)
871 ForkingPickler.register(Connection, reduction.reduce_connection)
872 ForkingPickler.register(PipeConnection, reduction.reduce_pipe_connection)
873else:
874 try:
875 from . import reduction
876 except ImportError:
877 pass
878 else:
879 ForkingPickler.register(socket.socket, reduction.reduce_socket)
880 ForkingPickler.register(Connection, reduction.reduce_connection)