Guido van Rossum | 27b7c7e | 2013-10-17 13:40:50 -0700 | [diff] [blame] | 1 | """Event loop using a proactor and related classes. |
| 2 | |
| 3 | A proactor is a "notify-on-completion" multiplexer. Currently a |
| 4 | proactor is only implemented on Windows with IOCP. |
| 5 | """ |
| 6 | |
Victor Stinner | 8dffc45 | 2014-01-25 15:32:06 +0100 | [diff] [blame] | 7 | __all__ = ['BaseProactorEventLoop'] |
| 8 | |
Guido van Rossum | 27b7c7e | 2013-10-17 13:40:50 -0700 | [diff] [blame] | 9 | import socket |
| 10 | |
| 11 | from . import base_events |
| 12 | from . import constants |
| 13 | from . import futures |
| 14 | from . import transports |
Guido van Rossum | fc29e0f | 2013-10-17 15:39:45 -0700 | [diff] [blame] | 15 | from .log import logger |
Guido van Rossum | 27b7c7e | 2013-10-17 13:40:50 -0700 | [diff] [blame] | 16 | |
| 17 | |
| 18 | class _ProactorBasePipeTransport(transports.BaseTransport): |
| 19 | """Base class for pipe and socket transports.""" |
| 20 | |
| 21 | def __init__(self, loop, sock, protocol, waiter=None, |
| 22 | extra=None, server=None): |
| 23 | super().__init__(extra) |
| 24 | self._set_extra(sock) |
| 25 | self._loop = loop |
| 26 | self._sock = sock |
| 27 | self._protocol = protocol |
| 28 | self._server = server |
Guido van Rossum | ebb8e58 | 2013-12-04 12:12:07 -0800 | [diff] [blame] | 29 | self._buffer = None # None or bytearray. |
Guido van Rossum | 27b7c7e | 2013-10-17 13:40:50 -0700 | [diff] [blame] | 30 | self._read_fut = None |
| 31 | self._write_fut = None |
| 32 | self._conn_lost = 0 |
| 33 | self._closing = False # Set when close() called. |
| 34 | self._eof_written = False |
Guido van Rossum | ebb8e58 | 2013-12-04 12:12:07 -0800 | [diff] [blame] | 35 | self._protocol_paused = False |
| 36 | self.set_write_buffer_limits() |
Guido van Rossum | 27b7c7e | 2013-10-17 13:40:50 -0700 | [diff] [blame] | 37 | if self._server is not None: |
| 38 | self._server.attach(self) |
| 39 | self._loop.call_soon(self._protocol.connection_made, self) |
| 40 | if waiter is not None: |
| 41 | self._loop.call_soon(waiter.set_result, None) |
| 42 | |
| 43 | def _set_extra(self, sock): |
| 44 | self._extra['pipe'] = sock |
| 45 | |
| 46 | def close(self): |
| 47 | if self._closing: |
| 48 | return |
| 49 | self._closing = True |
| 50 | self._conn_lost += 1 |
| 51 | if not self._buffer and self._write_fut is None: |
| 52 | self._loop.call_soon(self._call_connection_lost, None) |
| 53 | if self._read_fut is not None: |
| 54 | self._read_fut.cancel() |
| 55 | |
| 56 | def _fatal_error(self, exc): |
Guido van Rossum | fc29e0f | 2013-10-17 15:39:45 -0700 | [diff] [blame] | 57 | logger.exception('Fatal error for %s', self) |
Guido van Rossum | 27b7c7e | 2013-10-17 13:40:50 -0700 | [diff] [blame] | 58 | self._force_close(exc) |
| 59 | |
| 60 | def _force_close(self, exc): |
| 61 | if self._closing: |
| 62 | return |
| 63 | self._closing = True |
| 64 | self._conn_lost += 1 |
| 65 | if self._write_fut: |
| 66 | self._write_fut.cancel() |
| 67 | if self._read_fut: |
| 68 | self._read_fut.cancel() |
| 69 | self._write_fut = self._read_fut = None |
Guido van Rossum | ebb8e58 | 2013-12-04 12:12:07 -0800 | [diff] [blame] | 70 | self._buffer = None |
Guido van Rossum | 27b7c7e | 2013-10-17 13:40:50 -0700 | [diff] [blame] | 71 | self._loop.call_soon(self._call_connection_lost, exc) |
| 72 | |
| 73 | def _call_connection_lost(self, exc): |
| 74 | try: |
| 75 | self._protocol.connection_lost(exc) |
| 76 | finally: |
| 77 | # XXX If there is a pending overlapped read on the other |
| 78 | # end then it may fail with ERROR_NETNAME_DELETED if we |
| 79 | # just close our end. First calling shutdown() seems to |
| 80 | # cure it, but maybe using DisconnectEx() would be better. |
| 81 | if hasattr(self._sock, 'shutdown'): |
| 82 | self._sock.shutdown(socket.SHUT_RDWR) |
| 83 | self._sock.close() |
| 84 | server = self._server |
| 85 | if server is not None: |
| 86 | server.detach(self) |
| 87 | self._server = None |
| 88 | |
Guido van Rossum | ebb8e58 | 2013-12-04 12:12:07 -0800 | [diff] [blame] | 89 | # XXX The next four methods are nearly identical to corresponding |
| 90 | # ones in _SelectorTransport. Maybe refactor buffer management to |
| 91 | # share the implementations? (Also these are really only needed |
| 92 | # by _ProactorWritePipeTransport but since _buffer is defined on |
| 93 | # the base class I am putting it here for now.) |
| 94 | |
| 95 | def _maybe_pause_protocol(self): |
| 96 | size = self.get_write_buffer_size() |
| 97 | if size <= self._high_water: |
| 98 | return |
| 99 | if not self._protocol_paused: |
| 100 | self._protocol_paused = True |
| 101 | try: |
| 102 | self._protocol.pause_writing() |
| 103 | except Exception: |
| 104 | logger.exception('pause_writing() failed') |
| 105 | |
| 106 | def _maybe_resume_protocol(self): |
| 107 | if (self._protocol_paused and |
| 108 | self.get_write_buffer_size() <= self._low_water): |
| 109 | self._protocol_paused = False |
| 110 | try: |
| 111 | self._protocol.resume_writing() |
| 112 | except Exception: |
| 113 | logger.exception('resume_writing() failed') |
| 114 | |
| 115 | def set_write_buffer_limits(self, high=None, low=None): |
| 116 | if high is None: |
| 117 | if low is None: |
| 118 | high = 64*1024 |
| 119 | else: |
| 120 | high = 4*low |
| 121 | if low is None: |
| 122 | low = high // 4 |
| 123 | if not high >= low >= 0: |
| 124 | raise ValueError('high (%r) must be >= low (%r) must be >= 0' % |
| 125 | (high, low)) |
| 126 | self._high_water = high |
| 127 | self._low_water = low |
| 128 | |
| 129 | def get_write_buffer_size(self): |
| 130 | # NOTE: This doesn't take into account data already passed to |
| 131 | # send() even if send() hasn't finished yet. |
| 132 | if not self._buffer: |
| 133 | return 0 |
| 134 | return len(self._buffer) |
| 135 | |
Guido van Rossum | 27b7c7e | 2013-10-17 13:40:50 -0700 | [diff] [blame] | 136 | |
| 137 | class _ProactorReadPipeTransport(_ProactorBasePipeTransport, |
| 138 | transports.ReadTransport): |
| 139 | """Transport for read pipes.""" |
| 140 | |
| 141 | def __init__(self, loop, sock, protocol, waiter=None, |
| 142 | extra=None, server=None): |
| 143 | super().__init__(loop, sock, protocol, waiter, extra, server) |
| 144 | self._read_fut = None |
| 145 | self._paused = False |
| 146 | self._loop.call_soon(self._loop_reading) |
| 147 | |
Guido van Rossum | 57497ad | 2013-10-18 07:58:20 -0700 | [diff] [blame] | 148 | def pause_reading(self): |
Guido van Rossum | ebb8e58 | 2013-12-04 12:12:07 -0800 | [diff] [blame] | 149 | if self._closing: |
| 150 | raise RuntimeError('Cannot pause_reading() when closing') |
| 151 | if self._paused: |
| 152 | raise RuntimeError('Already paused') |
Guido van Rossum | 27b7c7e | 2013-10-17 13:40:50 -0700 | [diff] [blame] | 153 | self._paused = True |
| 154 | |
Guido van Rossum | 57497ad | 2013-10-18 07:58:20 -0700 | [diff] [blame] | 155 | def resume_reading(self): |
Guido van Rossum | ebb8e58 | 2013-12-04 12:12:07 -0800 | [diff] [blame] | 156 | if not self._paused: |
| 157 | raise RuntimeError('Not paused') |
Guido van Rossum | 27b7c7e | 2013-10-17 13:40:50 -0700 | [diff] [blame] | 158 | self._paused = False |
| 159 | if self._closing: |
| 160 | return |
| 161 | self._loop.call_soon(self._loop_reading, self._read_fut) |
| 162 | |
| 163 | def _loop_reading(self, fut=None): |
| 164 | if self._paused: |
| 165 | return |
| 166 | data = None |
| 167 | |
| 168 | try: |
| 169 | if fut is not None: |
| 170 | assert self._read_fut is fut or (self._read_fut is None and |
| 171 | self._closing) |
| 172 | self._read_fut = None |
| 173 | data = fut.result() # deliver data later in "finally" clause |
| 174 | |
| 175 | if self._closing: |
| 176 | # since close() has been called we ignore any read data |
| 177 | data = None |
| 178 | return |
| 179 | |
| 180 | if data == b'': |
| 181 | # we got end-of-file so no need to reschedule a new read |
| 182 | return |
| 183 | |
| 184 | # reschedule a new read |
| 185 | self._read_fut = self._loop._proactor.recv(self._sock, 4096) |
| 186 | except ConnectionAbortedError as exc: |
| 187 | if not self._closing: |
| 188 | self._fatal_error(exc) |
| 189 | except ConnectionResetError as exc: |
| 190 | self._force_close(exc) |
| 191 | except OSError as exc: |
| 192 | self._fatal_error(exc) |
| 193 | except futures.CancelledError: |
| 194 | if not self._closing: |
| 195 | raise |
| 196 | else: |
| 197 | self._read_fut.add_done_callback(self._loop_reading) |
| 198 | finally: |
| 199 | if data: |
| 200 | self._protocol.data_received(data) |
| 201 | elif data is not None: |
| 202 | keep_open = self._protocol.eof_received() |
| 203 | if not keep_open: |
| 204 | self.close() |
| 205 | |
| 206 | |
| 207 | class _ProactorWritePipeTransport(_ProactorBasePipeTransport, |
| 208 | transports.WriteTransport): |
| 209 | """Transport for write pipes.""" |
| 210 | |
| 211 | def write(self, data): |
Guido van Rossum | ebb8e58 | 2013-12-04 12:12:07 -0800 | [diff] [blame] | 212 | if not isinstance(data, (bytes, bytearray, memoryview)): |
| 213 | raise TypeError('data argument must be byte-ish (%r)', |
| 214 | type(data)) |
Guido van Rossum | 27b7c7e | 2013-10-17 13:40:50 -0700 | [diff] [blame] | 215 | if self._eof_written: |
Guido van Rossum | ebb8e58 | 2013-12-04 12:12:07 -0800 | [diff] [blame] | 216 | raise RuntimeError('write_eof() already called') |
Guido van Rossum | 27b7c7e | 2013-10-17 13:40:50 -0700 | [diff] [blame] | 217 | |
| 218 | if not data: |
| 219 | return |
| 220 | |
| 221 | if self._conn_lost: |
| 222 | if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES: |
Guido van Rossum | fc29e0f | 2013-10-17 15:39:45 -0700 | [diff] [blame] | 223 | logger.warning('socket.send() raised exception.') |
Guido van Rossum | 27b7c7e | 2013-10-17 13:40:50 -0700 | [diff] [blame] | 224 | self._conn_lost += 1 |
| 225 | return |
Guido van Rossum | 27b7c7e | 2013-10-17 13:40:50 -0700 | [diff] [blame] | 226 | |
Guido van Rossum | ebb8e58 | 2013-12-04 12:12:07 -0800 | [diff] [blame] | 227 | # Observable states: |
| 228 | # 1. IDLE: _write_fut and _buffer both None |
| 229 | # 2. WRITING: _write_fut set; _buffer None |
| 230 | # 3. BACKED UP: _write_fut set; _buffer a bytearray |
| 231 | # We always copy the data, so the caller can't modify it |
| 232 | # while we're still waiting for the I/O to happen. |
| 233 | if self._write_fut is None: # IDLE -> WRITING |
| 234 | assert self._buffer is None |
| 235 | # Pass a copy, except if it's already immutable. |
| 236 | self._loop_writing(data=bytes(data)) |
| 237 | # XXX Should we pause the protocol at this point |
| 238 | # if len(data) > self._high_water? (That would |
| 239 | # require keeping track of the number of bytes passed |
| 240 | # to a send() that hasn't finished yet.) |
| 241 | elif not self._buffer: # WRITING -> BACKED UP |
| 242 | # Make a mutable copy which we can extend. |
| 243 | self._buffer = bytearray(data) |
| 244 | self._maybe_pause_protocol() |
| 245 | else: # BACKED UP |
| 246 | # Append to buffer (also copies). |
| 247 | self._buffer.extend(data) |
| 248 | self._maybe_pause_protocol() |
| 249 | |
| 250 | def _loop_writing(self, f=None, data=None): |
Guido van Rossum | 27b7c7e | 2013-10-17 13:40:50 -0700 | [diff] [blame] | 251 | try: |
| 252 | assert f is self._write_fut |
| 253 | self._write_fut = None |
| 254 | if f: |
| 255 | f.result() |
Guido van Rossum | ebb8e58 | 2013-12-04 12:12:07 -0800 | [diff] [blame] | 256 | if data is None: |
| 257 | data = self._buffer |
| 258 | self._buffer = None |
Guido van Rossum | 27b7c7e | 2013-10-17 13:40:50 -0700 | [diff] [blame] | 259 | if not data: |
| 260 | if self._closing: |
| 261 | self._loop.call_soon(self._call_connection_lost, None) |
| 262 | if self._eof_written: |
| 263 | self._sock.shutdown(socket.SHUT_WR) |
Guido van Rossum | ebb8e58 | 2013-12-04 12:12:07 -0800 | [diff] [blame] | 264 | else: |
| 265 | self._write_fut = self._loop._proactor.send(self._sock, data) |
| 266 | self._write_fut.add_done_callback(self._loop_writing) |
| 267 | # Now that we've reduced the buffer size, tell the |
| 268 | # protocol to resume writing if it was paused. Note that |
| 269 | # we do this last since the callback is called immediately |
| 270 | # and it may add more data to the buffer (even causing the |
| 271 | # protocol to be paused again). |
| 272 | self._maybe_resume_protocol() |
Guido van Rossum | 27b7c7e | 2013-10-17 13:40:50 -0700 | [diff] [blame] | 273 | except ConnectionResetError as exc: |
| 274 | self._force_close(exc) |
| 275 | except OSError as exc: |
| 276 | self._fatal_error(exc) |
| 277 | |
| 278 | def can_write_eof(self): |
| 279 | return True |
| 280 | |
| 281 | def write_eof(self): |
| 282 | self.close() |
| 283 | |
| 284 | def abort(self): |
| 285 | self._force_close(None) |
| 286 | |
| 287 | |
| 288 | class _ProactorDuplexPipeTransport(_ProactorReadPipeTransport, |
| 289 | _ProactorWritePipeTransport, |
| 290 | transports.Transport): |
| 291 | """Transport for duplex pipes.""" |
| 292 | |
| 293 | def can_write_eof(self): |
| 294 | return False |
| 295 | |
| 296 | def write_eof(self): |
| 297 | raise NotImplementedError |
| 298 | |
| 299 | |
| 300 | class _ProactorSocketTransport(_ProactorReadPipeTransport, |
| 301 | _ProactorWritePipeTransport, |
| 302 | transports.Transport): |
| 303 | """Transport for connected sockets.""" |
| 304 | |
| 305 | def _set_extra(self, sock): |
| 306 | self._extra['socket'] = sock |
| 307 | try: |
| 308 | self._extra['sockname'] = sock.getsockname() |
| 309 | except (socket.error, AttributeError): |
| 310 | pass |
| 311 | if 'peername' not in self._extra: |
| 312 | try: |
| 313 | self._extra['peername'] = sock.getpeername() |
| 314 | except (socket.error, AttributeError): |
| 315 | pass |
| 316 | |
| 317 | def can_write_eof(self): |
| 318 | return True |
| 319 | |
| 320 | def write_eof(self): |
| 321 | if self._closing or self._eof_written: |
| 322 | return |
| 323 | self._eof_written = True |
| 324 | if self._write_fut is None: |
| 325 | self._sock.shutdown(socket.SHUT_WR) |
| 326 | |
| 327 | |
| 328 | class BaseProactorEventLoop(base_events.BaseEventLoop): |
| 329 | |
| 330 | def __init__(self, proactor): |
| 331 | super().__init__() |
Guido van Rossum | fc29e0f | 2013-10-17 15:39:45 -0700 | [diff] [blame] | 332 | logger.debug('Using proactor: %s', proactor.__class__.__name__) |
Guido van Rossum | 27b7c7e | 2013-10-17 13:40:50 -0700 | [diff] [blame] | 333 | self._proactor = proactor |
| 334 | self._selector = proactor # convenient alias |
Victor Stinner | 7de2646 | 2014-01-11 00:03:21 +0100 | [diff] [blame] | 335 | self._self_reading_future = None |
| 336 | self._accept_futures = {} # socket file descriptor => Future |
Guido van Rossum | 27b7c7e | 2013-10-17 13:40:50 -0700 | [diff] [blame] | 337 | proactor.set_loop(self) |
| 338 | self._make_self_pipe() |
| 339 | |
| 340 | def _make_socket_transport(self, sock, protocol, waiter=None, |
| 341 | extra=None, server=None): |
| 342 | return _ProactorSocketTransport(self, sock, protocol, waiter, |
| 343 | extra, server) |
| 344 | |
| 345 | def _make_duplex_pipe_transport(self, sock, protocol, waiter=None, |
| 346 | extra=None): |
| 347 | return _ProactorDuplexPipeTransport(self, |
| 348 | sock, protocol, waiter, extra) |
| 349 | |
| 350 | def _make_read_pipe_transport(self, sock, protocol, waiter=None, |
| 351 | extra=None): |
| 352 | return _ProactorReadPipeTransport(self, sock, protocol, waiter, extra) |
| 353 | |
| 354 | def _make_write_pipe_transport(self, sock, protocol, waiter=None, |
Guido van Rossum | 5969128 | 2013-10-30 14:52:03 -0700 | [diff] [blame] | 355 | extra=None, check_for_hangup=True): |
| 356 | if check_for_hangup: |
| 357 | # We want connection_lost() to be called when other end closes |
| 358 | return _ProactorDuplexPipeTransport(self, |
| 359 | sock, protocol, waiter, extra) |
| 360 | else: |
| 361 | # If other end closes we may not notice for a long time |
| 362 | return _ProactorWritePipeTransport(self, sock, protocol, waiter, |
| 363 | extra) |
Guido van Rossum | 27b7c7e | 2013-10-17 13:40:50 -0700 | [diff] [blame] | 364 | |
| 365 | def close(self): |
| 366 | if self._proactor is not None: |
| 367 | self._close_self_pipe() |
| 368 | self._proactor.close() |
| 369 | self._proactor = None |
| 370 | self._selector = None |
Antoine Pitrou | 4ca7355 | 2013-10-20 00:54:10 +0200 | [diff] [blame] | 371 | super().close() |
Victor Stinner | 7de2646 | 2014-01-11 00:03:21 +0100 | [diff] [blame] | 372 | self._accept_futures.clear() |
Guido van Rossum | 27b7c7e | 2013-10-17 13:40:50 -0700 | [diff] [blame] | 373 | |
| 374 | def sock_recv(self, sock, n): |
| 375 | return self._proactor.recv(sock, n) |
| 376 | |
| 377 | def sock_sendall(self, sock, data): |
| 378 | return self._proactor.send(sock, data) |
| 379 | |
| 380 | def sock_connect(self, sock, address): |
| 381 | return self._proactor.connect(sock, address) |
| 382 | |
| 383 | def sock_accept(self, sock): |
| 384 | return self._proactor.accept(sock) |
| 385 | |
| 386 | def _socketpair(self): |
| 387 | raise NotImplementedError |
| 388 | |
| 389 | def _close_self_pipe(self): |
Victor Stinner | 7de2646 | 2014-01-11 00:03:21 +0100 | [diff] [blame] | 390 | if self._self_reading_future is not None: |
| 391 | self._self_reading_future.cancel() |
| 392 | self._self_reading_future = None |
Guido van Rossum | 27b7c7e | 2013-10-17 13:40:50 -0700 | [diff] [blame] | 393 | self._ssock.close() |
| 394 | self._ssock = None |
| 395 | self._csock.close() |
| 396 | self._csock = None |
| 397 | self._internal_fds -= 1 |
| 398 | |
| 399 | def _make_self_pipe(self): |
| 400 | # A self-socket, really. :-) |
| 401 | self._ssock, self._csock = self._socketpair() |
| 402 | self._ssock.setblocking(False) |
| 403 | self._csock.setblocking(False) |
| 404 | self._internal_fds += 1 |
| 405 | self.call_soon(self._loop_self_reading) |
| 406 | |
| 407 | def _loop_self_reading(self, f=None): |
| 408 | try: |
| 409 | if f is not None: |
| 410 | f.result() # may raise |
| 411 | f = self._proactor.recv(self._ssock, 4096) |
| 412 | except: |
| 413 | self.close() |
| 414 | raise |
| 415 | else: |
Victor Stinner | 7de2646 | 2014-01-11 00:03:21 +0100 | [diff] [blame] | 416 | self._self_reading_future = f |
Guido van Rossum | 27b7c7e | 2013-10-17 13:40:50 -0700 | [diff] [blame] | 417 | f.add_done_callback(self._loop_self_reading) |
| 418 | |
| 419 | def _write_to_self(self): |
| 420 | self._csock.send(b'x') |
| 421 | |
| 422 | def _start_serving(self, protocol_factory, sock, ssl=None, server=None): |
Guido van Rossum | ebb8e58 | 2013-12-04 12:12:07 -0800 | [diff] [blame] | 423 | if ssl: |
| 424 | raise ValueError('IocpEventLoop is incompatible with SSL.') |
Guido van Rossum | 27b7c7e | 2013-10-17 13:40:50 -0700 | [diff] [blame] | 425 | |
| 426 | def loop(f=None): |
| 427 | try: |
| 428 | if f is not None: |
| 429 | conn, addr = f.result() |
| 430 | protocol = protocol_factory() |
| 431 | self._make_socket_transport( |
| 432 | conn, protocol, |
| 433 | extra={'peername': addr}, server=server) |
| 434 | f = self._proactor.accept(sock) |
| 435 | except OSError: |
| 436 | if sock.fileno() != -1: |
Guido van Rossum | fc29e0f | 2013-10-17 15:39:45 -0700 | [diff] [blame] | 437 | logger.exception('Accept failed') |
Guido van Rossum | 27b7c7e | 2013-10-17 13:40:50 -0700 | [diff] [blame] | 438 | sock.close() |
| 439 | except futures.CancelledError: |
| 440 | sock.close() |
| 441 | else: |
Victor Stinner | 7de2646 | 2014-01-11 00:03:21 +0100 | [diff] [blame] | 442 | self._accept_futures[sock.fileno()] = f |
Guido van Rossum | 27b7c7e | 2013-10-17 13:40:50 -0700 | [diff] [blame] | 443 | f.add_done_callback(loop) |
| 444 | |
| 445 | self.call_soon(loop) |
| 446 | |
| 447 | def _process_events(self, event_list): |
| 448 | pass # XXX hard work currently done in poll |
| 449 | |
| 450 | def _stop_serving(self, sock): |
Victor Stinner | 7de2646 | 2014-01-11 00:03:21 +0100 | [diff] [blame] | 451 | for future in self._accept_futures.values(): |
| 452 | future.cancel() |
Guido van Rossum | 27b7c7e | 2013-10-17 13:40:50 -0700 | [diff] [blame] | 453 | self._proactor._stop_serving(sock) |
| 454 | sock.close() |