blob: eb4e6620fc65f58c94251b3096547c031981721d [file] [log] [blame]
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001"""
2Python implementation of the io module.
3"""
4
5import os
6import abc
7import codecs
Antoine Pitrou58fcf9f2011-11-21 20:16:44 +01008import errno
Serhiy Storchaka71fd2242015-04-10 16:16:16 +03009import stat
Serhiy Storchakaf0f55a02015-08-28 22:17:04 +030010import sys
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +000011# Import _thread instead of threading to reduce startup cost
Antoine Pitroua6a4dc82017-09-07 18:56:24 +020012from _thread import allocate_lock as Lock
Serhiy Storchakaf0f55a02015-08-28 22:17:04 +030013if sys.platform in {'win32', 'cygwin'}:
Serhiy Storchaka71fd2242015-04-10 16:16:16 +030014 from msvcrt import setmode as _setmode
15else:
16 _setmode = None
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +000017
18import io
Benjamin Petersonc3be11a2010-04-27 21:24:03 +000019from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +000020
Jesus Cea94363612012-06-22 18:32:07 +020021valid_seek_flags = {0, 1, 2} # Hardwired values
22if hasattr(os, 'SEEK_HOLE') :
23 valid_seek_flags.add(os.SEEK_HOLE)
24 valid_seek_flags.add(os.SEEK_DATA)
25
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +000026# open() uses st_blksize whenever we can
27DEFAULT_BUFFER_SIZE = 8 * 1024 # bytes
28
29# NOTE: Base classes defined here are registered with the "official" ABCs
Benjamin Peterson86fdbf32015-03-18 21:35:38 -050030# defined in io.py. We don't use real inheritance though, because we don't want
31# to inherit the C implementations.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +000032
Antoine Pitrou6b4883d2011-10-12 02:54:14 +020033# Rebind for compatibility
34BlockingIOError = BlockingIOError
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +000035
Victor Stinnerbc2aa812019-05-23 03:45:09 +020036# Does io.IOBase finalizer log the exception if the close() method fails?
37# The exception is ignored silently by default in release build.
38_IOBASE_EMITS_UNRAISABLE = (hasattr(sys, "gettotalrefcount") or sys.flags.dev_mode)
39
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +000040
Georg Brandl4d73b572011-01-13 07:13:06 +000041def open(file, mode="r", buffering=-1, encoding=None, errors=None,
Ross Lagerwall59142db2011-10-31 20:34:46 +020042 newline=None, closefd=True, opener=None):
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +000043
Andrew Svetlovf7a17b42012-12-25 16:47:37 +020044 r"""Open file and return a stream. Raise OSError upon failure.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +000045
46 file is either a text or byte string giving the name (and the path
47 if the file isn't in the current working directory) of the file to
48 be opened or an integer file descriptor of the file to be
49 wrapped. (If a file descriptor is given, it is closed when the
50 returned I/O object is closed, unless closefd is set to False.)
51
Charles-François Natalidc3044c2012-01-09 22:40:02 +010052 mode is an optional string that specifies the mode in which the file is
53 opened. It defaults to 'r' which means open for reading in text mode. Other
54 common values are 'w' for writing (truncating the file if it already
Charles-François Natalid612de12012-01-14 11:51:00 +010055 exists), 'x' for exclusive creation of a new file, and 'a' for appending
Charles-François Natalidc3044c2012-01-09 22:40:02 +010056 (which on some Unix systems, means that all writes append to the end of the
57 file regardless of the current seek position). In text mode, if encoding is
58 not specified the encoding used is platform dependent. (For reading and
59 writing raw bytes use binary mode and leave encoding unspecified.) The
60 available modes are:
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +000061
62 ========= ===============================================================
63 Character Meaning
64 --------- ---------------------------------------------------------------
65 'r' open for reading (default)
66 'w' open for writing, truncating the file first
Charles-François Natalidc3044c2012-01-09 22:40:02 +010067 'x' create a new file and open it for writing
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +000068 'a' open for writing, appending to the end of the file if it exists
69 'b' binary mode
70 't' text mode (default)
71 '+' open a disk file for updating (reading and writing)
Serhiy Storchaka6787a382013-11-23 22:12:06 +020072 'U' universal newline mode (deprecated)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +000073 ========= ===============================================================
74
75 The default mode is 'rt' (open for reading text). For binary random
76 access, the mode 'w+b' opens and truncates the file to 0 bytes, while
Charles-François Natalidc3044c2012-01-09 22:40:02 +010077 'r+b' opens the file without truncation. The 'x' mode implies 'w' and
78 raises an `FileExistsError` if the file already exists.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +000079
80 Python distinguishes between files opened in binary and text modes,
81 even when the underlying operating system doesn't. Files opened in
82 binary mode (appending 'b' to the mode argument) return contents as
83 bytes objects without any decoding. In text mode (the default, or when
84 't' is appended to the mode argument), the contents of the file are
85 returned as strings, the bytes having been first decoded using a
86 platform-dependent encoding or using the specified encoding if given.
87
Serhiy Storchaka6787a382013-11-23 22:12:06 +020088 'U' mode is deprecated and will raise an exception in future versions
89 of Python. It has no effect in Python 3. Use newline to control
90 universal newlines mode.
91
Antoine Pitroud5587bc2009-12-19 21:08:31 +000092 buffering is an optional integer used to set the buffering policy.
93 Pass 0 to switch buffering off (only allowed in binary mode), 1 to select
94 line buffering (only usable in text mode), and an integer > 1 to indicate
95 the size of a fixed-size chunk buffer. When no buffering argument is
96 given, the default buffering policy works as follows:
97
98 * Binary files are buffered in fixed-size chunks; the size of the buffer
99 is chosen using a heuristic trying to determine the underlying device's
100 "block size" and falling back on `io.DEFAULT_BUFFER_SIZE`.
101 On many systems, the buffer will typically be 4096 or 8192 bytes long.
102
103 * "Interactive" text files (files for which isatty() returns True)
104 use line buffering. Other text files use the policy described above
105 for binary files.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000106
Raymond Hettingercbb80892011-01-13 18:15:51 +0000107 encoding is the str name of the encoding used to decode or encode the
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000108 file. This should only be used in text mode. The default encoding is
109 platform dependent, but any encoding supported by Python can be
110 passed. See the codecs module for the list of supported encodings.
111
112 errors is an optional string that specifies how encoding errors are to
113 be handled---this argument should not be used in binary mode. Pass
114 'strict' to raise a ValueError exception if there is an encoding error
115 (the default of None has the same effect), or pass 'ignore' to ignore
116 errors. (Note that ignoring encoding errors can lead to data loss.)
117 See the documentation for codecs.register for a list of the permitted
118 encoding error strings.
119
Raymond Hettingercbb80892011-01-13 18:15:51 +0000120 newline is a string controlling how universal newlines works (it only
121 applies to text mode). It can be None, '', '\n', '\r', and '\r\n'. It works
122 as follows:
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000123
124 * On input, if newline is None, universal newlines mode is
125 enabled. Lines in the input can end in '\n', '\r', or '\r\n', and
126 these are translated into '\n' before being returned to the
127 caller. If it is '', universal newline mode is enabled, but line
128 endings are returned to the caller untranslated. If it has any of
129 the other legal values, input lines are only terminated by the given
130 string, and the line ending is returned to the caller untranslated.
131
132 * On output, if newline is None, any '\n' characters written are
133 translated to the system default line separator, os.linesep. If
134 newline is '', no translation takes place. If newline is any of the
135 other legal values, any '\n' characters written are translated to
136 the given string.
137
Raymond Hettingercbb80892011-01-13 18:15:51 +0000138 closedfd is a bool. If closefd is False, the underlying file descriptor will
139 be kept open when the file is closed. This does not work when a file name is
140 given and must be True in that case.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000141
Victor Stinnerdaf45552013-08-28 00:53:59 +0200142 The newly created file is non-inheritable.
143
Ross Lagerwall59142db2011-10-31 20:34:46 +0200144 A custom opener can be used by passing a callable as *opener*. The
145 underlying file descriptor for the file object is then obtained by calling
146 *opener* with (*file*, *flags*). *opener* must return an open file
147 descriptor (passing os.open as *opener* results in functionality similar to
148 passing None).
149
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000150 open() returns a file object whose type depends on the mode, and
151 through which the standard file operations such as reading and writing
152 are performed. When open() is used to open a file in a text mode ('w',
153 'r', 'wt', 'rt', etc.), it returns a TextIOWrapper. When used to open
154 a file in a binary mode, the returned class varies: in read binary
155 mode, it returns a BufferedReader; in write binary and append binary
156 modes, it returns a BufferedWriter, and in read/write mode, it returns
157 a BufferedRandom.
158
159 It is also possible to use a string or bytearray as a file for both
160 reading and writing. For strings StringIO can be used like a file
161 opened in a text mode, and for bytes a BytesIO can be used like a file
162 opened in a binary mode.
163 """
Ethan Furmand62548a2016-06-04 14:38:43 -0700164 if not isinstance(file, int):
165 file = os.fspath(file)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000166 if not isinstance(file, (str, bytes, int)):
167 raise TypeError("invalid file: %r" % file)
168 if not isinstance(mode, str):
169 raise TypeError("invalid mode: %r" % mode)
Benjamin Peterson95e392c2010-04-27 21:07:21 +0000170 if not isinstance(buffering, int):
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000171 raise TypeError("invalid buffering: %r" % buffering)
172 if encoding is not None and not isinstance(encoding, str):
173 raise TypeError("invalid encoding: %r" % encoding)
174 if errors is not None and not isinstance(errors, str):
175 raise TypeError("invalid errors: %r" % errors)
176 modes = set(mode)
Charles-François Natalidc3044c2012-01-09 22:40:02 +0100177 if modes - set("axrwb+tU") or len(mode) > len(modes):
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000178 raise ValueError("invalid mode: %r" % mode)
Charles-François Natalidc3044c2012-01-09 22:40:02 +0100179 creating = "x" in modes
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000180 reading = "r" in modes
181 writing = "w" in modes
182 appending = "a" in modes
183 updating = "+" in modes
184 text = "t" in modes
185 binary = "b" in modes
186 if "U" in modes:
Robert Collinsc94a1dc2015-07-26 06:43:13 +1200187 if creating or writing or appending or updating:
188 raise ValueError("mode U cannot be combined with 'x', 'w', 'a', or '+'")
Serhiy Storchaka6787a382013-11-23 22:12:06 +0200189 import warnings
190 warnings.warn("'U' mode is deprecated",
191 DeprecationWarning, 2)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000192 reading = True
193 if text and binary:
194 raise ValueError("can't have text and binary mode at once")
Charles-François Natalidc3044c2012-01-09 22:40:02 +0100195 if creating + reading + writing + appending > 1:
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000196 raise ValueError("can't have read/write/append mode at once")
Charles-François Natalidc3044c2012-01-09 22:40:02 +0100197 if not (creating or reading or writing or appending):
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000198 raise ValueError("must have exactly one of read/write/append mode")
199 if binary and encoding is not None:
200 raise ValueError("binary mode doesn't take an encoding argument")
201 if binary and errors is not None:
202 raise ValueError("binary mode doesn't take an errors argument")
203 if binary and newline is not None:
204 raise ValueError("binary mode doesn't take a newline argument")
Alexey Izbysheva2670562018-10-20 03:22:31 +0300205 if binary and buffering == 1:
206 import warnings
207 warnings.warn("line buffering (buffering=1) isn't supported in binary "
208 "mode, the default buffer size will be used",
209 RuntimeWarning, 2)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000210 raw = FileIO(file,
Charles-François Natalidc3044c2012-01-09 22:40:02 +0100211 (creating and "x" or "") +
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000212 (reading and "r" or "") +
213 (writing and "w" or "") +
214 (appending and "a" or "") +
215 (updating and "+" or ""),
Ross Lagerwall59142db2011-10-31 20:34:46 +0200216 closefd, opener=opener)
Serhiy Storchakaf10063e2014-06-09 13:32:34 +0300217 result = raw
218 try:
219 line_buffering = False
220 if buffering == 1 or buffering < 0 and raw.isatty():
221 buffering = -1
222 line_buffering = True
223 if buffering < 0:
224 buffering = DEFAULT_BUFFER_SIZE
225 try:
226 bs = os.fstat(raw.fileno()).st_blksize
227 except (OSError, AttributeError):
228 pass
229 else:
230 if bs > 1:
231 buffering = bs
232 if buffering < 0:
233 raise ValueError("invalid buffering size")
234 if buffering == 0:
235 if binary:
236 return result
237 raise ValueError("can't have unbuffered text I/O")
238 if updating:
239 buffer = BufferedRandom(raw, buffering)
240 elif creating or writing or appending:
241 buffer = BufferedWriter(raw, buffering)
242 elif reading:
243 buffer = BufferedReader(raw, buffering)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000244 else:
Serhiy Storchakaf10063e2014-06-09 13:32:34 +0300245 raise ValueError("unknown mode: %r" % mode)
246 result = buffer
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000247 if binary:
Serhiy Storchakaf10063e2014-06-09 13:32:34 +0300248 return result
249 text = TextIOWrapper(buffer, encoding, errors, newline, line_buffering)
250 result = text
251 text.mode = mode
252 return result
253 except:
254 result.close()
255 raise
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000256
Steve Dowerb82e17e2019-05-23 08:45:22 -0700257# Define a default pure-Python implementation for open_code()
258# that does not allow hooks. Warn on first use. Defined for tests.
259def _open_code_with_warning(path):
260 """Opens the provided file with mode ``'rb'``. This function
261 should be used when the intent is to treat the contents as
262 executable code.
263
264 ``path`` should be an absolute path.
265
266 When supported by the runtime, this function can be hooked
267 in order to allow embedders more control over code files.
268 This functionality is not supported on the current runtime.
269 """
270 import warnings
271 warnings.warn("_pyio.open_code() may not be using hooks",
272 RuntimeWarning, 2)
273 return open(path, "rb")
274
275try:
276 open_code = io.open_code
277except AttributeError:
278 open_code = _open_code_with_warning
279
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000280
281class DocDescriptor:
282 """Helper for builtins.open.__doc__
283 """
Miss Islington (bot)c71ae1a2019-08-29 02:02:51 -0700284 def __get__(self, obj, typ=None):
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000285 return (
Benjamin Petersonc3be11a2010-04-27 21:24:03 +0000286 "open(file, mode='r', buffering=-1, encoding=None, "
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000287 "errors=None, newline=None, closefd=True)\n\n" +
288 open.__doc__)
289
290class OpenWrapper:
291 """Wrapper for builtins.open
292
293 Trick so that open won't become a bound method when stored
294 as a class variable (as dbm.dumb does).
295
Nick Coghland6009512014-11-20 21:39:37 +1000296 See initstdio() in Python/pylifecycle.c.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000297 """
298 __doc__ = DocDescriptor()
299
300 def __new__(cls, *args, **kwargs):
301 return open(*args, **kwargs)
302
303
Antoine Pitrou0d739d72010-09-05 23:01:12 +0000304# In normal operation, both `UnsupportedOperation`s should be bound to the
305# same object.
306try:
307 UnsupportedOperation = io.UnsupportedOperation
308except AttributeError:
Serhiy Storchaka606ab862016-12-07 13:31:20 +0200309 class UnsupportedOperation(OSError, ValueError):
Antoine Pitrou0d739d72010-09-05 23:01:12 +0000310 pass
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000311
312
313class IOBase(metaclass=abc.ABCMeta):
314
315 """The abstract base class for all I/O classes, acting on streams of
316 bytes. There is no public constructor.
317
318 This class provides dummy implementations for many methods that
319 derived classes can override selectively; the default implementations
320 represent a file that cannot be read, written or seeked.
321
Steve Palmer7b97ab32019-04-09 05:35:27 +0100322 Even though IOBase does not declare read or write because
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000323 their signatures will vary, implementations and clients should
324 consider those methods part of the interface. Also, implementations
Amaury Forgeot d'Arc616453c2010-09-06 22:31:52 +0000325 may raise UnsupportedOperation when operations they do not support are
326 called.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000327
328 The basic type used for binary data read from or written to a file is
Steve Palmer7b97ab32019-04-09 05:35:27 +0100329 bytes. Other bytes-like objects are accepted as method arguments too.
330 Text I/O classes work with str data.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000331
332 Note that calling any method (even inquiries) on a closed stream is
Andrew Svetlovf7a17b42012-12-25 16:47:37 +0200333 undefined. Implementations may raise OSError in this case.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000334
335 IOBase (and its subclasses) support the iterator protocol, meaning
336 that an IOBase object can be iterated over yielding the lines in a
337 stream.
338
339 IOBase also supports the :keyword:`with` statement. In this example,
340 fp is closed after the suite of the with statement is complete:
341
342 with open('spam.txt', 'r') as fp:
343 fp.write('Spam and eggs!')
344 """
345
346 ### Internal ###
347
Raymond Hettinger3c940242011-01-12 23:39:31 +0000348 def _unsupported(self, name):
Andrew Svetlovf7a17b42012-12-25 16:47:37 +0200349 """Internal: raise an OSError exception for unsupported operations."""
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000350 raise UnsupportedOperation("%s.%s() not supported" %
351 (self.__class__.__name__, name))
352
353 ### Positioning ###
354
Georg Brandl4d73b572011-01-13 07:13:06 +0000355 def seek(self, pos, whence=0):
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000356 """Change stream position.
357
Terry Jan Reedyc30b7b12013-03-11 17:57:08 -0400358 Change the stream position to byte offset pos. Argument pos is
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000359 interpreted relative to the position indicated by whence. Values
Raymond Hettingercbb80892011-01-13 18:15:51 +0000360 for whence are ints:
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000361
362 * 0 -- start of stream (the default); offset should be zero or positive
363 * 1 -- current stream position; offset may be negative
364 * 2 -- end of stream; offset is usually negative
Jesus Cea94363612012-06-22 18:32:07 +0200365 Some operating systems / file systems could provide additional values.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000366
Raymond Hettingercbb80892011-01-13 18:15:51 +0000367 Return an int indicating the new absolute position.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000368 """
369 self._unsupported("seek")
370
Raymond Hettinger3c940242011-01-12 23:39:31 +0000371 def tell(self):
Raymond Hettingercbb80892011-01-13 18:15:51 +0000372 """Return an int indicating the current stream position."""
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000373 return self.seek(0, 1)
374
Georg Brandl4d73b572011-01-13 07:13:06 +0000375 def truncate(self, pos=None):
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000376 """Truncate file to size bytes.
377
378 Size defaults to the current IO position as reported by tell(). Return
379 the new size.
380 """
381 self._unsupported("truncate")
382
383 ### Flush and close ###
384
Raymond Hettinger3c940242011-01-12 23:39:31 +0000385 def flush(self):
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000386 """Flush write buffers, if applicable.
387
388 This is not implemented for read-only and non-blocking streams.
389 """
Antoine Pitrou6be88762010-05-03 16:48:20 +0000390 self._checkClosed()
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000391 # XXX Should this return the number of bytes written???
392
393 __closed = False
394
Raymond Hettinger3c940242011-01-12 23:39:31 +0000395 def close(self):
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000396 """Flush and close the IO object.
397
398 This method has no effect if the file is already closed.
399 """
400 if not self.__closed:
Benjamin Peterson68623612012-12-20 11:53:11 -0600401 try:
402 self.flush()
403 finally:
404 self.__closed = True
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000405
Raymond Hettinger3c940242011-01-12 23:39:31 +0000406 def __del__(self):
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000407 """Destructor. Calls close()."""
Victor Stinnerc15a6822019-06-13 00:23:49 +0200408 try:
409 closed = self.closed
Miss Islington (bot)102130a2019-08-29 01:13:29 -0700410 except AttributeError:
Victor Stinnerc15a6822019-06-13 00:23:49 +0200411 # If getting closed fails, then the object is probably
412 # in an unusable state, so ignore.
413 return
414
415 if closed:
416 return
417
Victor Stinnerbc2aa812019-05-23 03:45:09 +0200418 if _IOBASE_EMITS_UNRAISABLE:
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000419 self.close()
Victor Stinnerbc2aa812019-05-23 03:45:09 +0200420 else:
421 # The try/except block is in case this is called at program
422 # exit time, when it's possible that globals have already been
423 # deleted, and then the close() call might fail. Since
424 # there's nothing we can do about such failures and they annoy
425 # the end users, we suppress the traceback.
426 try:
427 self.close()
428 except:
429 pass
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000430
431 ### Inquiries ###
432
Raymond Hettinger3c940242011-01-12 23:39:31 +0000433 def seekable(self):
Raymond Hettingercbb80892011-01-13 18:15:51 +0000434 """Return a bool indicating whether object supports random access.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000435
Martin Panter754aab22016-03-31 07:21:56 +0000436 If False, seek(), tell() and truncate() will raise OSError.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000437 This method may need to do a test seek().
438 """
439 return False
440
441 def _checkSeekable(self, msg=None):
Amaury Forgeot d'Arcada99482010-09-06 22:23:13 +0000442 """Internal: raise UnsupportedOperation if file is not seekable
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000443 """
444 if not self.seekable():
Antoine Pitrou0d739d72010-09-05 23:01:12 +0000445 raise UnsupportedOperation("File or stream is not seekable."
446 if msg is None else msg)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000447
Raymond Hettinger3c940242011-01-12 23:39:31 +0000448 def readable(self):
Raymond Hettingercbb80892011-01-13 18:15:51 +0000449 """Return a bool indicating whether object was opened for reading.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000450
Martin Panter754aab22016-03-31 07:21:56 +0000451 If False, read() will raise OSError.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000452 """
453 return False
454
455 def _checkReadable(self, msg=None):
Amaury Forgeot d'Arcada99482010-09-06 22:23:13 +0000456 """Internal: raise UnsupportedOperation if file is not readable
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000457 """
458 if not self.readable():
Antoine Pitrou0d739d72010-09-05 23:01:12 +0000459 raise UnsupportedOperation("File or stream is not readable."
460 if msg is None else msg)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000461
Raymond Hettinger3c940242011-01-12 23:39:31 +0000462 def writable(self):
Raymond Hettingercbb80892011-01-13 18:15:51 +0000463 """Return a bool indicating whether object was opened for writing.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000464
Martin Panter754aab22016-03-31 07:21:56 +0000465 If False, write() and truncate() will raise OSError.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000466 """
467 return False
468
469 def _checkWritable(self, msg=None):
Amaury Forgeot d'Arcada99482010-09-06 22:23:13 +0000470 """Internal: raise UnsupportedOperation if file is not writable
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000471 """
472 if not self.writable():
Antoine Pitrou0d739d72010-09-05 23:01:12 +0000473 raise UnsupportedOperation("File or stream is not writable."
474 if msg is None else msg)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000475
476 @property
477 def closed(self):
478 """closed: bool. True iff the file has been closed.
479
480 For backwards compatibility, this is a property, not a predicate.
481 """
482 return self.__closed
483
484 def _checkClosed(self, msg=None):
Serhiy Storchaka6a7b3a72016-04-17 08:32:47 +0300485 """Internal: raise a ValueError if file is closed
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000486 """
487 if self.closed:
488 raise ValueError("I/O operation on closed file."
489 if msg is None else msg)
490
491 ### Context manager ###
492
Raymond Hettinger3c940242011-01-12 23:39:31 +0000493 def __enter__(self): # That's a forward reference
Raymond Hettingercbb80892011-01-13 18:15:51 +0000494 """Context management protocol. Returns self (an instance of IOBase)."""
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000495 self._checkClosed()
496 return self
497
Raymond Hettinger3c940242011-01-12 23:39:31 +0000498 def __exit__(self, *args):
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000499 """Context management protocol. Calls close()"""
500 self.close()
501
502 ### Lower-level APIs ###
503
504 # XXX Should these be present even if unimplemented?
505
Raymond Hettinger3c940242011-01-12 23:39:31 +0000506 def fileno(self):
Raymond Hettingercbb80892011-01-13 18:15:51 +0000507 """Returns underlying file descriptor (an int) if one exists.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000508
Andrew Svetlovf7a17b42012-12-25 16:47:37 +0200509 An OSError is raised if the IO object does not use a file descriptor.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000510 """
511 self._unsupported("fileno")
512
Raymond Hettinger3c940242011-01-12 23:39:31 +0000513 def isatty(self):
Raymond Hettingercbb80892011-01-13 18:15:51 +0000514 """Return a bool indicating whether this is an 'interactive' stream.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000515
516 Return False if it can't be determined.
517 """
518 self._checkClosed()
519 return False
520
521 ### Readline[s] and writelines ###
522
Serhiy Storchaka3c411542013-09-16 23:18:10 +0300523 def readline(self, size=-1):
Raymond Hettingercbb80892011-01-13 18:15:51 +0000524 r"""Read and return a line of bytes from the stream.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000525
Serhiy Storchaka3c411542013-09-16 23:18:10 +0300526 If size is specified, at most size bytes will be read.
527 Size should be an int.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000528
529 The line terminator is always b'\n' for binary files; for text
530 files, the newlines argument to open can be used to select the line
531 terminator(s) recognized.
532 """
533 # For backwards compatibility, a (slowish) readline().
534 if hasattr(self, "peek"):
535 def nreadahead():
536 readahead = self.peek(1)
537 if not readahead:
538 return 1
539 n = (readahead.find(b"\n") + 1) or len(readahead)
Serhiy Storchaka3c411542013-09-16 23:18:10 +0300540 if size >= 0:
541 n = min(n, size)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000542 return n
543 else:
544 def nreadahead():
545 return 1
Serhiy Storchaka3c411542013-09-16 23:18:10 +0300546 if size is None:
547 size = -1
Oren Milmande503602017-08-24 21:33:42 +0300548 else:
549 try:
550 size_index = size.__index__
551 except AttributeError:
552 raise TypeError(f"{size!r} is not an integer")
553 else:
554 size = size_index()
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000555 res = bytearray()
Serhiy Storchaka3c411542013-09-16 23:18:10 +0300556 while size < 0 or len(res) < size:
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000557 b = self.read(nreadahead())
558 if not b:
559 break
560 res += b
561 if res.endswith(b"\n"):
562 break
563 return bytes(res)
564
565 def __iter__(self):
566 self._checkClosed()
567 return self
568
569 def __next__(self):
570 line = self.readline()
571 if not line:
572 raise StopIteration
573 return line
574
575 def readlines(self, hint=None):
576 """Return a list of lines from the stream.
577
578 hint can be specified to control the number of lines read: no more
579 lines will be read if the total size (in bytes/characters) of all
580 lines so far exceeds hint.
581 """
582 if hint is None or hint <= 0:
583 return list(self)
584 n = 0
585 lines = []
586 for line in self:
587 lines.append(line)
588 n += len(line)
589 if n >= hint:
590 break
591 return lines
592
593 def writelines(self, lines):
Marcin Niemiraab865212019-04-22 21:13:51 +1000594 """Write a list of lines to the stream.
595
596 Line separators are not added, so it is usual for each of the lines
597 provided to have a line separator at the end.
598 """
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000599 self._checkClosed()
600 for line in lines:
601 self.write(line)
602
603io.IOBase.register(IOBase)
604
605
606class RawIOBase(IOBase):
607
608 """Base class for raw binary I/O."""
609
610 # The read() method is implemented by calling readinto(); derived
611 # classes that want to support read() only need to implement
612 # readinto() as a primitive operation. In general, readinto() can be
613 # more efficient than read().
614
615 # (It would be tempting to also provide an implementation of
616 # readinto() in terms of read(), in case the latter is a more suitable
617 # primitive operation, but that would lead to nasty recursion in case
618 # a subclass doesn't implement either.)
619
Serhiy Storchaka3c411542013-09-16 23:18:10 +0300620 def read(self, size=-1):
621 """Read and return up to size bytes, where size is an int.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000622
623 Returns an empty bytes object on EOF, or None if the object is
624 set not to block and has no data to read.
625 """
Serhiy Storchaka3c411542013-09-16 23:18:10 +0300626 if size is None:
627 size = -1
628 if size < 0:
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000629 return self.readall()
Serhiy Storchaka3c411542013-09-16 23:18:10 +0300630 b = bytearray(size.__index__())
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000631 n = self.readinto(b)
Antoine Pitrou328ec742010-09-14 18:37:24 +0000632 if n is None:
633 return None
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000634 del b[n:]
635 return bytes(b)
636
637 def readall(self):
638 """Read until EOF, using multiple read() call."""
639 res = bytearray()
640 while True:
641 data = self.read(DEFAULT_BUFFER_SIZE)
642 if not data:
643 break
644 res += data
Victor Stinnera80987f2011-05-25 22:47:16 +0200645 if res:
646 return bytes(res)
647 else:
648 # b'' or None
649 return data
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000650
Raymond Hettinger3c940242011-01-12 23:39:31 +0000651 def readinto(self, b):
Martin Panter6bb91f32016-05-28 00:41:57 +0000652 """Read bytes into a pre-allocated bytes-like object b.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000653
Raymond Hettingercbb80892011-01-13 18:15:51 +0000654 Returns an int representing the number of bytes read (0 for EOF), or
655 None if the object is set not to block and has no data to read.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000656 """
657 self._unsupported("readinto")
658
Raymond Hettinger3c940242011-01-12 23:39:31 +0000659 def write(self, b):
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000660 """Write the given buffer to the IO stream.
661
Martin Panter6bb91f32016-05-28 00:41:57 +0000662 Returns the number of bytes written, which may be less than the
663 length of b in bytes.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000664 """
665 self._unsupported("write")
666
667io.RawIOBase.register(RawIOBase)
668from _io import FileIO
669RawIOBase.register(FileIO)
670
671
672class BufferedIOBase(IOBase):
673
674 """Base class for buffered IO objects.
675
676 The main difference with RawIOBase is that the read() method
677 supports omitting the size argument, and does not have a default
678 implementation that defers to readinto().
679
680 In addition, read(), readinto() and write() may raise
681 BlockingIOError if the underlying raw stream is in non-blocking
682 mode and not ready; unlike their raw counterparts, they will never
683 return None.
684
685 A typical implementation should not inherit from a RawIOBase
686 implementation, but wrap one.
687 """
688
Martin Panterccb2c0e2016-10-20 23:48:14 +0000689 def read(self, size=-1):
Serhiy Storchaka3c411542013-09-16 23:18:10 +0300690 """Read and return up to size bytes, where size is an int.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000691
692 If the argument is omitted, None, or negative, reads and
693 returns all data until EOF.
694
695 If the argument is positive, and the underlying raw stream is
696 not 'interactive', multiple raw reads may be issued to satisfy
697 the byte count (unless EOF is reached first). But for
698 interactive raw streams (XXX and for pipes?), at most one raw
699 read will be issued, and a short result does not imply that
700 EOF is imminent.
701
702 Returns an empty bytes array on EOF.
703
704 Raises BlockingIOError if the underlying raw stream has no
705 data at the moment.
706 """
707 self._unsupported("read")
708
Martin Panterccb2c0e2016-10-20 23:48:14 +0000709 def read1(self, size=-1):
Serhiy Storchaka3c411542013-09-16 23:18:10 +0300710 """Read up to size bytes with at most one read() system call,
711 where size is an int.
Raymond Hettingercbb80892011-01-13 18:15:51 +0000712 """
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000713 self._unsupported("read1")
714
Raymond Hettinger3c940242011-01-12 23:39:31 +0000715 def readinto(self, b):
Martin Panter6bb91f32016-05-28 00:41:57 +0000716 """Read bytes into a pre-allocated bytes-like object b.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000717
718 Like read(), this may issue multiple reads to the underlying raw
719 stream, unless the latter is 'interactive'.
720
Raymond Hettingercbb80892011-01-13 18:15:51 +0000721 Returns an int representing the number of bytes read (0 for EOF).
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000722
723 Raises BlockingIOError if the underlying raw stream has no
724 data at the moment.
725 """
Benjamin Petersona96fea02014-06-22 14:17:44 -0700726
727 return self._readinto(b, read1=False)
728
729 def readinto1(self, b):
Martin Panter6bb91f32016-05-28 00:41:57 +0000730 """Read bytes into buffer *b*, using at most one system call
Benjamin Petersona96fea02014-06-22 14:17:44 -0700731
732 Returns an int representing the number of bytes read (0 for EOF).
733
734 Raises BlockingIOError if the underlying raw stream has no
735 data at the moment.
736 """
737
738 return self._readinto(b, read1=True)
739
740 def _readinto(self, b, read1):
741 if not isinstance(b, memoryview):
742 b = memoryview(b)
743 b = b.cast('B')
744
745 if read1:
746 data = self.read1(len(b))
747 else:
748 data = self.read(len(b))
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000749 n = len(data)
Benjamin Petersona96fea02014-06-22 14:17:44 -0700750
751 b[:n] = data
752
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000753 return n
754
Raymond Hettinger3c940242011-01-12 23:39:31 +0000755 def write(self, b):
Raymond Hettingercbb80892011-01-13 18:15:51 +0000756 """Write the given bytes buffer to the IO stream.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000757
Martin Panter6bb91f32016-05-28 00:41:57 +0000758 Return the number of bytes written, which is always the length of b
759 in bytes.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000760
761 Raises BlockingIOError if the buffer is full and the
762 underlying raw stream cannot accept more data at the moment.
763 """
764 self._unsupported("write")
765
Raymond Hettinger3c940242011-01-12 23:39:31 +0000766 def detach(self):
Benjamin Petersond2e0c792009-05-01 20:40:59 +0000767 """
768 Separate the underlying raw stream from the buffer and return it.
769
770 After the raw stream has been detached, the buffer is in an unusable
771 state.
772 """
773 self._unsupported("detach")
774
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000775io.BufferedIOBase.register(BufferedIOBase)
776
777
778class _BufferedIOMixin(BufferedIOBase):
779
780 """A mixin implementation of BufferedIOBase with an underlying raw stream.
781
782 This passes most requests on to the underlying raw stream. It
783 does *not* provide implementations of read(), readinto() or
784 write().
785 """
786
787 def __init__(self, raw):
Antoine Pitrou7f8f4182010-12-21 21:20:59 +0000788 self._raw = raw
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000789
790 ### Positioning ###
791
792 def seek(self, pos, whence=0):
793 new_position = self.raw.seek(pos, whence)
794 if new_position < 0:
Andrew Svetlovf7a17b42012-12-25 16:47:37 +0200795 raise OSError("seek() returned an invalid position")
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000796 return new_position
797
798 def tell(self):
799 pos = self.raw.tell()
800 if pos < 0:
Andrew Svetlovf7a17b42012-12-25 16:47:37 +0200801 raise OSError("tell() returned an invalid position")
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000802 return pos
803
804 def truncate(self, pos=None):
805 # Flush the stream. We're mixing buffered I/O with lower-level I/O,
806 # and a flush may be necessary to synch both views of the current
807 # file state.
808 self.flush()
809
810 if pos is None:
811 pos = self.tell()
812 # XXX: Should seek() be used, instead of passing the position
813 # XXX directly to truncate?
814 return self.raw.truncate(pos)
815
816 ### Flush and close ###
817
818 def flush(self):
Antoine Pitrou6be88762010-05-03 16:48:20 +0000819 if self.closed:
Jim Fasarakis-Hilliard1e73dbb2017-03-26 23:59:08 +0300820 raise ValueError("flush on closed file")
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000821 self.raw.flush()
822
823 def close(self):
Antoine Pitrou6be88762010-05-03 16:48:20 +0000824 if self.raw is not None and not self.closed:
Antoine Pitrou58fcf9f2011-11-21 20:16:44 +0100825 try:
826 # may raise BlockingIOError or BrokenPipeError etc
827 self.flush()
828 finally:
829 self.raw.close()
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000830
Benjamin Petersond2e0c792009-05-01 20:40:59 +0000831 def detach(self):
832 if self.raw is None:
833 raise ValueError("raw stream already detached")
834 self.flush()
Antoine Pitrou7f8f4182010-12-21 21:20:59 +0000835 raw = self._raw
836 self._raw = None
Benjamin Petersond2e0c792009-05-01 20:40:59 +0000837 return raw
838
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000839 ### Inquiries ###
840
841 def seekable(self):
842 return self.raw.seekable()
843
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000844 @property
Antoine Pitrou7f8f4182010-12-21 21:20:59 +0000845 def raw(self):
846 return self._raw
847
848 @property
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000849 def closed(self):
850 return self.raw.closed
851
852 @property
853 def name(self):
854 return self.raw.name
855
856 @property
857 def mode(self):
858 return self.raw.mode
859
Antoine Pitrou243757e2010-11-05 21:15:39 +0000860 def __getstate__(self):
Serhiy Storchaka0353b4e2018-10-31 02:28:07 +0200861 raise TypeError(f"cannot pickle {self.__class__.__name__!r} object")
Antoine Pitrou243757e2010-11-05 21:15:39 +0000862
Antoine Pitrou716c4442009-05-23 19:04:03 +0000863 def __repr__(self):
Serhiy Storchaka465e60e2014-07-25 23:36:00 +0300864 modname = self.__class__.__module__
865 clsname = self.__class__.__qualname__
Antoine Pitrou716c4442009-05-23 19:04:03 +0000866 try:
867 name = self.name
Miss Islington (bot)102130a2019-08-29 01:13:29 -0700868 except AttributeError:
Serhiy Storchaka465e60e2014-07-25 23:36:00 +0300869 return "<{}.{}>".format(modname, clsname)
Antoine Pitrou716c4442009-05-23 19:04:03 +0000870 else:
Serhiy Storchaka465e60e2014-07-25 23:36:00 +0300871 return "<{}.{} name={!r}>".format(modname, clsname, name)
Antoine Pitrou716c4442009-05-23 19:04:03 +0000872
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000873 ### Lower-level APIs ###
874
875 def fileno(self):
876 return self.raw.fileno()
877
878 def isatty(self):
879 return self.raw.isatty()
880
881
882class BytesIO(BufferedIOBase):
883
884 """Buffered I/O implementation using an in-memory bytes buffer."""
885
Victor Stinnera3568412019-05-28 01:44:21 +0200886 # Initialize _buffer as soon as possible since it's used by __del__()
887 # which calls close()
888 _buffer = None
889
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000890 def __init__(self, initial_bytes=None):
891 buf = bytearray()
892 if initial_bytes is not None:
893 buf += initial_bytes
894 self._buffer = buf
895 self._pos = 0
896
Alexandre Vassalotticf76e1a2009-07-22 03:24:36 +0000897 def __getstate__(self):
898 if self.closed:
899 raise ValueError("__getstate__ on closed file")
900 return self.__dict__.copy()
901
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000902 def getvalue(self):
903 """Return the bytes value (contents) of the buffer
904 """
905 if self.closed:
906 raise ValueError("getvalue on closed file")
907 return bytes(self._buffer)
908
Antoine Pitrou972ee132010-09-06 18:48:21 +0000909 def getbuffer(self):
910 """Return a readable and writable view of the buffer.
911 """
Serhiy Storchakac057c382015-02-03 02:00:18 +0200912 if self.closed:
913 raise ValueError("getbuffer on closed file")
Antoine Pitrou972ee132010-09-06 18:48:21 +0000914 return memoryview(self._buffer)
915
Serhiy Storchakac057c382015-02-03 02:00:18 +0200916 def close(self):
Victor Stinnera3568412019-05-28 01:44:21 +0200917 if self._buffer is not None:
918 self._buffer.clear()
Serhiy Storchakac057c382015-02-03 02:00:18 +0200919 super().close()
920
Martin Panterccb2c0e2016-10-20 23:48:14 +0000921 def read(self, size=-1):
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000922 if self.closed:
923 raise ValueError("read from closed file")
Serhiy Storchaka3c411542013-09-16 23:18:10 +0300924 if size is None:
925 size = -1
Oren Milmande503602017-08-24 21:33:42 +0300926 else:
927 try:
928 size_index = size.__index__
929 except AttributeError:
930 raise TypeError(f"{size!r} is not an integer")
931 else:
932 size = size_index()
Serhiy Storchaka3c411542013-09-16 23:18:10 +0300933 if size < 0:
934 size = len(self._buffer)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000935 if len(self._buffer) <= self._pos:
936 return b""
Serhiy Storchaka3c411542013-09-16 23:18:10 +0300937 newpos = min(len(self._buffer), self._pos + size)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000938 b = self._buffer[self._pos : newpos]
939 self._pos = newpos
940 return bytes(b)
941
Martin Panterccb2c0e2016-10-20 23:48:14 +0000942 def read1(self, size=-1):
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000943 """This is the same as read.
944 """
Serhiy Storchaka3c411542013-09-16 23:18:10 +0300945 return self.read(size)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000946
947 def write(self, b):
948 if self.closed:
949 raise ValueError("write to closed file")
950 if isinstance(b, str):
951 raise TypeError("can't write str to binary stream")
Martin Panter6bb91f32016-05-28 00:41:57 +0000952 with memoryview(b) as view:
953 n = view.nbytes # Size of any bytes-like object
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000954 if n == 0:
955 return 0
956 pos = self._pos
957 if pos > len(self._buffer):
958 # Inserts null bytes between the current end of the file
959 # and the new write position.
960 padding = b'\x00' * (pos - len(self._buffer))
961 self._buffer += padding
962 self._buffer[pos:pos + n] = b
963 self._pos += n
964 return n
965
966 def seek(self, pos, whence=0):
967 if self.closed:
968 raise ValueError("seek on closed file")
969 try:
Oren Milmande503602017-08-24 21:33:42 +0300970 pos_index = pos.__index__
971 except AttributeError:
972 raise TypeError(f"{pos!r} is not an integer")
973 else:
974 pos = pos_index()
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000975 if whence == 0:
976 if pos < 0:
977 raise ValueError("negative seek position %r" % (pos,))
978 self._pos = pos
979 elif whence == 1:
980 self._pos = max(0, self._pos + pos)
981 elif whence == 2:
982 self._pos = max(0, len(self._buffer) + pos)
983 else:
Jesus Cea94363612012-06-22 18:32:07 +0200984 raise ValueError("unsupported whence value")
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +0000985 return self._pos
986
987 def tell(self):
988 if self.closed:
989 raise ValueError("tell on closed file")
990 return self._pos
991
992 def truncate(self, pos=None):
993 if self.closed:
994 raise ValueError("truncate on closed file")
995 if pos is None:
996 pos = self._pos
Florent Xiclunab14930c2010-03-13 15:26:44 +0000997 else:
998 try:
Oren Milmande503602017-08-24 21:33:42 +0300999 pos_index = pos.__index__
1000 except AttributeError:
1001 raise TypeError(f"{pos!r} is not an integer")
1002 else:
1003 pos = pos_index()
Florent Xiclunab14930c2010-03-13 15:26:44 +00001004 if pos < 0:
1005 raise ValueError("negative truncate position %r" % (pos,))
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001006 del self._buffer[pos:]
Antoine Pitrou905a2ff2010-01-31 22:47:27 +00001007 return pos
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001008
1009 def readable(self):
Antoine Pitrou1d857452012-09-05 20:11:49 +02001010 if self.closed:
1011 raise ValueError("I/O operation on closed file.")
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001012 return True
1013
1014 def writable(self):
Antoine Pitrou1d857452012-09-05 20:11:49 +02001015 if self.closed:
1016 raise ValueError("I/O operation on closed file.")
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001017 return True
1018
1019 def seekable(self):
Antoine Pitrou1d857452012-09-05 20:11:49 +02001020 if self.closed:
1021 raise ValueError("I/O operation on closed file.")
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001022 return True
1023
1024
1025class BufferedReader(_BufferedIOMixin):
1026
1027 """BufferedReader(raw[, buffer_size])
1028
1029 A buffer for a readable, sequential BaseRawIO object.
1030
1031 The constructor creates a BufferedReader for the given readable raw
1032 stream and buffer_size. If buffer_size is omitted, DEFAULT_BUFFER_SIZE
1033 is used.
1034 """
1035
1036 def __init__(self, raw, buffer_size=DEFAULT_BUFFER_SIZE):
1037 """Create a new buffered reader using the given readable raw IO object.
1038 """
Antoine Pitroucf4c7492009-04-19 00:09:36 +00001039 if not raw.readable():
Andrew Svetlovf7a17b42012-12-25 16:47:37 +02001040 raise OSError('"raw" argument must be readable.')
Antoine Pitroucf4c7492009-04-19 00:09:36 +00001041
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001042 _BufferedIOMixin.__init__(self, raw)
1043 if buffer_size <= 0:
1044 raise ValueError("invalid buffer size")
1045 self.buffer_size = buffer_size
1046 self._reset_read_buf()
1047 self._read_lock = Lock()
1048
Martin Panter754aab22016-03-31 07:21:56 +00001049 def readable(self):
1050 return self.raw.readable()
1051
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001052 def _reset_read_buf(self):
1053 self._read_buf = b""
1054 self._read_pos = 0
1055
Serhiy Storchaka3c411542013-09-16 23:18:10 +03001056 def read(self, size=None):
1057 """Read size bytes.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001058
Serhiy Storchaka3c411542013-09-16 23:18:10 +03001059 Returns exactly size bytes of data unless the underlying raw IO
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001060 stream reaches EOF or if the call would block in non-blocking
Serhiy Storchaka3c411542013-09-16 23:18:10 +03001061 mode. If size is negative, read until EOF or until read() would
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001062 block.
1063 """
Serhiy Storchaka3c411542013-09-16 23:18:10 +03001064 if size is not None and size < -1:
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001065 raise ValueError("invalid number of bytes to read")
1066 with self._read_lock:
Serhiy Storchaka3c411542013-09-16 23:18:10 +03001067 return self._read_unlocked(size)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001068
1069 def _read_unlocked(self, n=None):
1070 nodata_val = b""
1071 empty_values = (b"", None)
1072 buf = self._read_buf
1073 pos = self._read_pos
1074
1075 # Special case for when the number of bytes to read is unspecified.
1076 if n is None or n == -1:
1077 self._reset_read_buf()
Victor Stinnerb57f1082011-05-26 00:19:38 +02001078 if hasattr(self.raw, 'readall'):
1079 chunk = self.raw.readall()
1080 if chunk is None:
1081 return buf[pos:] or None
1082 else:
1083 return buf[pos:] + chunk
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001084 chunks = [buf[pos:]] # Strip the consumed bytes.
1085 current_size = 0
1086 while True:
1087 # Read until EOF or until read() would block.
Charles-François Natali6e6c59b2015-02-07 13:27:50 +00001088 chunk = self.raw.read()
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001089 if chunk in empty_values:
1090 nodata_val = chunk
1091 break
1092 current_size += len(chunk)
1093 chunks.append(chunk)
1094 return b"".join(chunks) or nodata_val
1095
1096 # The number of bytes to read is specified, return at most n bytes.
1097 avail = len(buf) - pos # Length of the available buffered data.
1098 if n <= avail:
1099 # Fast path: the data to read is fully buffered.
1100 self._read_pos += n
1101 return buf[pos:pos+n]
1102 # Slow path: read from the stream until enough bytes are read,
1103 # or until an EOF occurs or until read() would block.
1104 chunks = [buf[pos:]]
1105 wanted = max(self.buffer_size, n)
1106 while avail < n:
Charles-François Natali6e6c59b2015-02-07 13:27:50 +00001107 chunk = self.raw.read(wanted)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001108 if chunk in empty_values:
1109 nodata_val = chunk
1110 break
1111 avail += len(chunk)
1112 chunks.append(chunk)
Martin Pantere26da7c2016-06-02 10:07:09 +00001113 # n is more than avail only when an EOF occurred or when
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001114 # read() would have blocked.
1115 n = min(n, avail)
1116 out = b"".join(chunks)
1117 self._read_buf = out[n:] # Save the extra data in the buffer.
1118 self._read_pos = 0
1119 return out[:n] if out else nodata_val
1120
Serhiy Storchaka3c411542013-09-16 23:18:10 +03001121 def peek(self, size=0):
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001122 """Returns buffered bytes without advancing the position.
1123
1124 The argument indicates a desired minimal number of bytes; we
1125 do at most one raw read to satisfy it. We never return more
1126 than self.buffer_size.
1127 """
1128 with self._read_lock:
Serhiy Storchaka3c411542013-09-16 23:18:10 +03001129 return self._peek_unlocked(size)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001130
1131 def _peek_unlocked(self, n=0):
1132 want = min(n, self.buffer_size)
1133 have = len(self._read_buf) - self._read_pos
1134 if have < want or have <= 0:
1135 to_read = self.buffer_size - have
Charles-François Natali6e6c59b2015-02-07 13:27:50 +00001136 current = self.raw.read(to_read)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001137 if current:
1138 self._read_buf = self._read_buf[self._read_pos:] + current
1139 self._read_pos = 0
1140 return self._read_buf[self._read_pos:]
1141
Martin Panterccb2c0e2016-10-20 23:48:14 +00001142 def read1(self, size=-1):
Serhiy Storchaka3c411542013-09-16 23:18:10 +03001143 """Reads up to size bytes, with at most one read() system call."""
1144 # Returns up to size bytes. If at least one byte is buffered, we
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001145 # only return buffered bytes. Otherwise, we do one raw read.
Serhiy Storchaka3c411542013-09-16 23:18:10 +03001146 if size < 0:
Martin Panterccb2c0e2016-10-20 23:48:14 +00001147 size = self.buffer_size
Serhiy Storchaka3c411542013-09-16 23:18:10 +03001148 if size == 0:
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001149 return b""
1150 with self._read_lock:
1151 self._peek_unlocked(1)
1152 return self._read_unlocked(
Serhiy Storchaka3c411542013-09-16 23:18:10 +03001153 min(size, len(self._read_buf) - self._read_pos))
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001154
Benjamin Petersona96fea02014-06-22 14:17:44 -07001155 # Implementing readinto() and readinto1() is not strictly necessary (we
1156 # could rely on the base class that provides an implementation in terms of
1157 # read() and read1()). We do it anyway to keep the _pyio implementation
1158 # similar to the io implementation (which implements the methods for
1159 # performance reasons).
1160 def _readinto(self, buf, read1):
1161 """Read data into *buf* with at most one system call."""
1162
Benjamin Petersona96fea02014-06-22 14:17:44 -07001163 # Need to create a memoryview object of type 'b', otherwise
1164 # we may not be able to assign bytes to it, and slicing it
1165 # would create a new object.
1166 if not isinstance(buf, memoryview):
1167 buf = memoryview(buf)
Martin Panter6bb91f32016-05-28 00:41:57 +00001168 if buf.nbytes == 0:
1169 return 0
Benjamin Petersona96fea02014-06-22 14:17:44 -07001170 buf = buf.cast('B')
1171
1172 written = 0
1173 with self._read_lock:
1174 while written < len(buf):
1175
1176 # First try to read from internal buffer
1177 avail = min(len(self._read_buf) - self._read_pos, len(buf))
1178 if avail:
1179 buf[written:written+avail] = \
1180 self._read_buf[self._read_pos:self._read_pos+avail]
1181 self._read_pos += avail
1182 written += avail
1183 if written == len(buf):
1184 break
1185
1186 # If remaining space in callers buffer is larger than
1187 # internal buffer, read directly into callers buffer
1188 if len(buf) - written > self.buffer_size:
1189 n = self.raw.readinto(buf[written:])
1190 if not n:
1191 break # eof
1192 written += n
1193
1194 # Otherwise refill internal buffer - unless we're
1195 # in read1 mode and already got some data
1196 elif not (read1 and written):
1197 if not self._peek_unlocked(1):
1198 break # eof
1199
1200 # In readinto1 mode, return as soon as we have some data
1201 if read1 and written:
1202 break
1203
1204 return written
1205
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001206 def tell(self):
1207 return _BufferedIOMixin.tell(self) - len(self._read_buf) + self._read_pos
1208
1209 def seek(self, pos, whence=0):
Jesus Cea94363612012-06-22 18:32:07 +02001210 if whence not in valid_seek_flags:
Jesus Cea990eff02012-04-26 17:05:31 +02001211 raise ValueError("invalid whence value")
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001212 with self._read_lock:
1213 if whence == 1:
1214 pos -= len(self._read_buf) - self._read_pos
1215 pos = _BufferedIOMixin.seek(self, pos, whence)
1216 self._reset_read_buf()
1217 return pos
1218
1219class BufferedWriter(_BufferedIOMixin):
1220
1221 """A buffer for a writeable sequential RawIO object.
1222
1223 The constructor creates a BufferedWriter for the given writeable raw
1224 stream. If the buffer_size is not given, it defaults to
Benjamin Peterson59406a92009-03-26 17:10:29 +00001225 DEFAULT_BUFFER_SIZE.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001226 """
1227
Florent Xicluna109d5732012-07-07 17:03:22 +02001228 def __init__(self, raw, buffer_size=DEFAULT_BUFFER_SIZE):
Antoine Pitroucf4c7492009-04-19 00:09:36 +00001229 if not raw.writable():
Andrew Svetlovf7a17b42012-12-25 16:47:37 +02001230 raise OSError('"raw" argument must be writable.')
Antoine Pitroucf4c7492009-04-19 00:09:36 +00001231
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001232 _BufferedIOMixin.__init__(self, raw)
1233 if buffer_size <= 0:
1234 raise ValueError("invalid buffer size")
1235 self.buffer_size = buffer_size
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001236 self._write_buf = bytearray()
1237 self._write_lock = Lock()
1238
Martin Panter754aab22016-03-31 07:21:56 +00001239 def writable(self):
1240 return self.raw.writable()
1241
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001242 def write(self, b):
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001243 if isinstance(b, str):
1244 raise TypeError("can't write str to binary stream")
1245 with self._write_lock:
benfogle9703f092017-11-10 16:03:40 -05001246 if self.closed:
1247 raise ValueError("write to closed file")
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001248 # XXX we can implement some more tricks to try and avoid
1249 # partial writes
1250 if len(self._write_buf) > self.buffer_size:
Antoine Pitrou58fcf9f2011-11-21 20:16:44 +01001251 # We're full, so let's pre-flush the buffer. (This may
1252 # raise BlockingIOError with characters_written == 0.)
1253 self._flush_unlocked()
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001254 before = len(self._write_buf)
1255 self._write_buf.extend(b)
1256 written = len(self._write_buf) - before
1257 if len(self._write_buf) > self.buffer_size:
1258 try:
1259 self._flush_unlocked()
1260 except BlockingIOError as e:
Benjamin Peterson394ee002009-03-05 22:33:59 +00001261 if len(self._write_buf) > self.buffer_size:
1262 # We've hit the buffer_size. We have to accept a partial
1263 # write and cut back our buffer.
1264 overage = len(self._write_buf) - self.buffer_size
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001265 written -= overage
Benjamin Peterson394ee002009-03-05 22:33:59 +00001266 self._write_buf = self._write_buf[:self.buffer_size]
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001267 raise BlockingIOError(e.errno, e.strerror, written)
1268 return written
1269
1270 def truncate(self, pos=None):
1271 with self._write_lock:
1272 self._flush_unlocked()
1273 if pos is None:
1274 pos = self.raw.tell()
1275 return self.raw.truncate(pos)
1276
1277 def flush(self):
1278 with self._write_lock:
1279 self._flush_unlocked()
1280
1281 def _flush_unlocked(self):
1282 if self.closed:
Jim Fasarakis-Hilliard1e73dbb2017-03-26 23:59:08 +03001283 raise ValueError("flush on closed file")
Antoine Pitrou58fcf9f2011-11-21 20:16:44 +01001284 while self._write_buf:
1285 try:
1286 n = self.raw.write(self._write_buf)
1287 except BlockingIOError:
1288 raise RuntimeError("self.raw should implement RawIOBase: it "
1289 "should not raise BlockingIOError")
Antoine Pitrou58fcf9f2011-11-21 20:16:44 +01001290 if n is None:
1291 raise BlockingIOError(
1292 errno.EAGAIN,
1293 "write could not complete without blocking", 0)
1294 if n > len(self._write_buf) or n < 0:
Andrew Svetlovf7a17b42012-12-25 16:47:37 +02001295 raise OSError("write() returned incorrect number of bytes")
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001296 del self._write_buf[:n]
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001297
1298 def tell(self):
1299 return _BufferedIOMixin.tell(self) + len(self._write_buf)
1300
1301 def seek(self, pos, whence=0):
Jesus Cea94363612012-06-22 18:32:07 +02001302 if whence not in valid_seek_flags:
1303 raise ValueError("invalid whence value")
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001304 with self._write_lock:
1305 self._flush_unlocked()
1306 return _BufferedIOMixin.seek(self, pos, whence)
1307
benfogle9703f092017-11-10 16:03:40 -05001308 def close(self):
1309 with self._write_lock:
1310 if self.raw is None or self.closed:
1311 return
1312 # We have to release the lock and call self.flush() (which will
1313 # probably just re-take the lock) in case flush has been overridden in
1314 # a subclass or the user set self.flush to something. This is the same
1315 # behavior as the C implementation.
1316 try:
1317 # may raise BlockingIOError or BrokenPipeError etc
1318 self.flush()
1319 finally:
1320 with self._write_lock:
1321 self.raw.close()
1322
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001323
1324class BufferedRWPair(BufferedIOBase):
1325
1326 """A buffered reader and writer object together.
1327
1328 A buffered reader object and buffered writer object put together to
1329 form a sequential IO object that can read and write. This is typically
1330 used with a socket or two-way pipe.
1331
1332 reader and writer are RawIOBase objects that are readable and
1333 writeable respectively. If the buffer_size is omitted it defaults to
Benjamin Peterson59406a92009-03-26 17:10:29 +00001334 DEFAULT_BUFFER_SIZE.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001335 """
1336
1337 # XXX The usefulness of this (compared to having two separate IO
1338 # objects) is questionable.
1339
Florent Xicluna109d5732012-07-07 17:03:22 +02001340 def __init__(self, reader, writer, buffer_size=DEFAULT_BUFFER_SIZE):
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001341 """Constructor.
1342
1343 The arguments are two RawIO instances.
1344 """
Antoine Pitroucf4c7492009-04-19 00:09:36 +00001345 if not reader.readable():
Andrew Svetlovf7a17b42012-12-25 16:47:37 +02001346 raise OSError('"reader" argument must be readable.')
Antoine Pitroucf4c7492009-04-19 00:09:36 +00001347
1348 if not writer.writable():
Andrew Svetlovf7a17b42012-12-25 16:47:37 +02001349 raise OSError('"writer" argument must be writable.')
Antoine Pitroucf4c7492009-04-19 00:09:36 +00001350
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001351 self.reader = BufferedReader(reader, buffer_size)
Benjamin Peterson59406a92009-03-26 17:10:29 +00001352 self.writer = BufferedWriter(writer, buffer_size)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001353
Martin Panterccb2c0e2016-10-20 23:48:14 +00001354 def read(self, size=-1):
Serhiy Storchaka3c411542013-09-16 23:18:10 +03001355 if size is None:
1356 size = -1
1357 return self.reader.read(size)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001358
1359 def readinto(self, b):
1360 return self.reader.readinto(b)
1361
1362 def write(self, b):
1363 return self.writer.write(b)
1364
Serhiy Storchaka3c411542013-09-16 23:18:10 +03001365 def peek(self, size=0):
1366 return self.reader.peek(size)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001367
Martin Panterccb2c0e2016-10-20 23:48:14 +00001368 def read1(self, size=-1):
Serhiy Storchaka3c411542013-09-16 23:18:10 +03001369 return self.reader.read1(size)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001370
Benjamin Petersona96fea02014-06-22 14:17:44 -07001371 def readinto1(self, b):
1372 return self.reader.readinto1(b)
1373
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001374 def readable(self):
1375 return self.reader.readable()
1376
1377 def writable(self):
1378 return self.writer.writable()
1379
1380 def flush(self):
1381 return self.writer.flush()
1382
1383 def close(self):
Serhiy Storchaka7665be62015-03-24 23:21:57 +02001384 try:
1385 self.writer.close()
1386 finally:
1387 self.reader.close()
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001388
1389 def isatty(self):
1390 return self.reader.isatty() or self.writer.isatty()
1391
1392 @property
1393 def closed(self):
1394 return self.writer.closed
1395
1396
1397class BufferedRandom(BufferedWriter, BufferedReader):
1398
1399 """A buffered interface to random access streams.
1400
1401 The constructor creates a reader and writer for a seekable stream,
1402 raw, given in the first argument. If the buffer_size is omitted it
Benjamin Peterson59406a92009-03-26 17:10:29 +00001403 defaults to DEFAULT_BUFFER_SIZE.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001404 """
1405
Florent Xicluna109d5732012-07-07 17:03:22 +02001406 def __init__(self, raw, buffer_size=DEFAULT_BUFFER_SIZE):
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001407 raw._checkSeekable()
1408 BufferedReader.__init__(self, raw, buffer_size)
Florent Xicluna109d5732012-07-07 17:03:22 +02001409 BufferedWriter.__init__(self, raw, buffer_size)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001410
1411 def seek(self, pos, whence=0):
Jesus Cea94363612012-06-22 18:32:07 +02001412 if whence not in valid_seek_flags:
1413 raise ValueError("invalid whence value")
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001414 self.flush()
1415 if self._read_buf:
1416 # Undo read ahead.
1417 with self._read_lock:
1418 self.raw.seek(self._read_pos - len(self._read_buf), 1)
1419 # First do the raw seek, then empty the read buffer, so that
1420 # if the raw seek fails, we don't lose buffered data forever.
1421 pos = self.raw.seek(pos, whence)
1422 with self._read_lock:
1423 self._reset_read_buf()
1424 if pos < 0:
Andrew Svetlovf7a17b42012-12-25 16:47:37 +02001425 raise OSError("seek() returned invalid position")
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001426 return pos
1427
1428 def tell(self):
1429 if self._write_buf:
1430 return BufferedWriter.tell(self)
1431 else:
1432 return BufferedReader.tell(self)
1433
1434 def truncate(self, pos=None):
1435 if pos is None:
1436 pos = self.tell()
1437 # Use seek to flush the read buffer.
Antoine Pitrou905a2ff2010-01-31 22:47:27 +00001438 return BufferedWriter.truncate(self, pos)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001439
Serhiy Storchaka3c411542013-09-16 23:18:10 +03001440 def read(self, size=None):
1441 if size is None:
1442 size = -1
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001443 self.flush()
Serhiy Storchaka3c411542013-09-16 23:18:10 +03001444 return BufferedReader.read(self, size)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001445
1446 def readinto(self, b):
1447 self.flush()
1448 return BufferedReader.readinto(self, b)
1449
Serhiy Storchaka3c411542013-09-16 23:18:10 +03001450 def peek(self, size=0):
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001451 self.flush()
Serhiy Storchaka3c411542013-09-16 23:18:10 +03001452 return BufferedReader.peek(self, size)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001453
Martin Panterccb2c0e2016-10-20 23:48:14 +00001454 def read1(self, size=-1):
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001455 self.flush()
Serhiy Storchaka3c411542013-09-16 23:18:10 +03001456 return BufferedReader.read1(self, size)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001457
Benjamin Petersona96fea02014-06-22 14:17:44 -07001458 def readinto1(self, b):
1459 self.flush()
1460 return BufferedReader.readinto1(self, b)
1461
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001462 def write(self, b):
1463 if self._read_buf:
1464 # Undo readahead
1465 with self._read_lock:
1466 self.raw.seek(self._read_pos - len(self._read_buf), 1)
1467 self._reset_read_buf()
1468 return BufferedWriter.write(self, b)
1469
1470
Serhiy Storchaka71fd2242015-04-10 16:16:16 +03001471class FileIO(RawIOBase):
1472 _fd = -1
1473 _created = False
1474 _readable = False
1475 _writable = False
1476 _appending = False
1477 _seekable = None
1478 _closefd = True
1479
1480 def __init__(self, file, mode='r', closefd=True, opener=None):
1481 """Open a file. The mode can be 'r' (default), 'w', 'x' or 'a' for reading,
1482 writing, exclusive creation or appending. The file will be created if it
1483 doesn't exist when opened for writing or appending; it will be truncated
1484 when opened for writing. A FileExistsError will be raised if it already
1485 exists when opened for creating. Opening a file for creating implies
1486 writing so this mode behaves in a similar way to 'w'. Add a '+' to the mode
1487 to allow simultaneous reading and writing. A custom opener can be used by
1488 passing a callable as *opener*. The underlying file descriptor for the file
1489 object is then obtained by calling opener with (*name*, *flags*).
1490 *opener* must return an open file descriptor (passing os.open as *opener*
1491 results in functionality similar to passing None).
1492 """
1493 if self._fd >= 0:
1494 # Have to close the existing file first.
1495 try:
1496 if self._closefd:
1497 os.close(self._fd)
1498 finally:
1499 self._fd = -1
1500
1501 if isinstance(file, float):
1502 raise TypeError('integer argument expected, got float')
1503 if isinstance(file, int):
1504 fd = file
1505 if fd < 0:
1506 raise ValueError('negative file descriptor')
1507 else:
1508 fd = -1
1509
1510 if not isinstance(mode, str):
1511 raise TypeError('invalid mode: %s' % (mode,))
1512 if not set(mode) <= set('xrwab+'):
1513 raise ValueError('invalid mode: %s' % (mode,))
1514 if sum(c in 'rwax' for c in mode) != 1 or mode.count('+') > 1:
1515 raise ValueError('Must have exactly one of create/read/write/append '
1516 'mode and at most one plus')
1517
1518 if 'x' in mode:
1519 self._created = True
1520 self._writable = True
1521 flags = os.O_EXCL | os.O_CREAT
1522 elif 'r' in mode:
1523 self._readable = True
1524 flags = 0
1525 elif 'w' in mode:
1526 self._writable = True
1527 flags = os.O_CREAT | os.O_TRUNC
1528 elif 'a' in mode:
1529 self._writable = True
1530 self._appending = True
1531 flags = os.O_APPEND | os.O_CREAT
1532
1533 if '+' in mode:
1534 self._readable = True
1535 self._writable = True
1536
1537 if self._readable and self._writable:
1538 flags |= os.O_RDWR
1539 elif self._readable:
1540 flags |= os.O_RDONLY
1541 else:
1542 flags |= os.O_WRONLY
1543
1544 flags |= getattr(os, 'O_BINARY', 0)
1545
1546 noinherit_flag = (getattr(os, 'O_NOINHERIT', 0) or
1547 getattr(os, 'O_CLOEXEC', 0))
1548 flags |= noinherit_flag
1549
1550 owned_fd = None
1551 try:
1552 if fd < 0:
1553 if not closefd:
1554 raise ValueError('Cannot use closefd=False with file name')
1555 if opener is None:
1556 fd = os.open(file, flags, 0o666)
1557 else:
1558 fd = opener(file, flags)
1559 if not isinstance(fd, int):
1560 raise TypeError('expected integer from opener')
1561 if fd < 0:
1562 raise OSError('Negative file descriptor')
1563 owned_fd = fd
1564 if not noinherit_flag:
1565 os.set_inheritable(fd, False)
1566
1567 self._closefd = closefd
1568 fdfstat = os.fstat(fd)
1569 try:
1570 if stat.S_ISDIR(fdfstat.st_mode):
1571 raise IsADirectoryError(errno.EISDIR,
1572 os.strerror(errno.EISDIR), file)
1573 except AttributeError:
1574 # Ignore the AttribueError if stat.S_ISDIR or errno.EISDIR
1575 # don't exist.
1576 pass
1577 self._blksize = getattr(fdfstat, 'st_blksize', 0)
1578 if self._blksize <= 1:
1579 self._blksize = DEFAULT_BUFFER_SIZE
1580
1581 if _setmode:
1582 # don't translate newlines (\r\n <=> \n)
1583 _setmode(fd, os.O_BINARY)
1584
1585 self.name = file
1586 if self._appending:
1587 # For consistent behaviour, we explicitly seek to the
1588 # end of file (otherwise, it might be done only on the
1589 # first write()).
1590 os.lseek(fd, 0, SEEK_END)
1591 except:
1592 if owned_fd is not None:
1593 os.close(owned_fd)
1594 raise
1595 self._fd = fd
1596
1597 def __del__(self):
1598 if self._fd >= 0 and self._closefd and not self.closed:
1599 import warnings
1600 warnings.warn('unclosed file %r' % (self,), ResourceWarning,
Victor Stinnere19558a2016-03-23 00:28:08 +01001601 stacklevel=2, source=self)
Serhiy Storchaka71fd2242015-04-10 16:16:16 +03001602 self.close()
1603
1604 def __getstate__(self):
Serhiy Storchaka0353b4e2018-10-31 02:28:07 +02001605 raise TypeError(f"cannot pickle {self.__class__.__name__!r} object")
Serhiy Storchaka71fd2242015-04-10 16:16:16 +03001606
1607 def __repr__(self):
1608 class_name = '%s.%s' % (self.__class__.__module__,
1609 self.__class__.__qualname__)
1610 if self.closed:
1611 return '<%s [closed]>' % class_name
1612 try:
1613 name = self.name
1614 except AttributeError:
1615 return ('<%s fd=%d mode=%r closefd=%r>' %
1616 (class_name, self._fd, self.mode, self._closefd))
1617 else:
1618 return ('<%s name=%r mode=%r closefd=%r>' %
1619 (class_name, name, self.mode, self._closefd))
1620
1621 def _checkReadable(self):
1622 if not self._readable:
1623 raise UnsupportedOperation('File not open for reading')
1624
1625 def _checkWritable(self, msg=None):
1626 if not self._writable:
1627 raise UnsupportedOperation('File not open for writing')
1628
1629 def read(self, size=None):
1630 """Read at most size bytes, returned as bytes.
1631
1632 Only makes one system call, so less data may be returned than requested
1633 In non-blocking mode, returns None if no data is available.
1634 Return an empty bytes object at EOF.
1635 """
1636 self._checkClosed()
1637 self._checkReadable()
1638 if size is None or size < 0:
1639 return self.readall()
1640 try:
1641 return os.read(self._fd, size)
1642 except BlockingIOError:
1643 return None
1644
1645 def readall(self):
1646 """Read all data from the file, returned as bytes.
1647
1648 In non-blocking mode, returns as much as is immediately available,
1649 or None if no data is available. Return an empty bytes object at EOF.
1650 """
1651 self._checkClosed()
1652 self._checkReadable()
1653 bufsize = DEFAULT_BUFFER_SIZE
1654 try:
1655 pos = os.lseek(self._fd, 0, SEEK_CUR)
1656 end = os.fstat(self._fd).st_size
1657 if end >= pos:
1658 bufsize = end - pos + 1
1659 except OSError:
1660 pass
1661
1662 result = bytearray()
1663 while True:
1664 if len(result) >= bufsize:
1665 bufsize = len(result)
1666 bufsize += max(bufsize, DEFAULT_BUFFER_SIZE)
1667 n = bufsize - len(result)
1668 try:
1669 chunk = os.read(self._fd, n)
1670 except BlockingIOError:
1671 if result:
1672 break
1673 return None
1674 if not chunk: # reached the end of the file
1675 break
1676 result += chunk
1677
1678 return bytes(result)
1679
1680 def readinto(self, b):
1681 """Same as RawIOBase.readinto()."""
1682 m = memoryview(b).cast('B')
1683 data = self.read(len(m))
1684 n = len(data)
1685 m[:n] = data
1686 return n
1687
1688 def write(self, b):
1689 """Write bytes b to file, return number written.
1690
1691 Only makes one system call, so not all of the data may be written.
1692 The number of bytes actually written is returned. In non-blocking mode,
1693 returns None if the write would block.
1694 """
1695 self._checkClosed()
1696 self._checkWritable()
1697 try:
1698 return os.write(self._fd, b)
1699 except BlockingIOError:
1700 return None
1701
1702 def seek(self, pos, whence=SEEK_SET):
1703 """Move to new file position.
1704
1705 Argument offset is a byte count. Optional argument whence defaults to
1706 SEEK_SET or 0 (offset from start of file, offset should be >= 0); other values
1707 are SEEK_CUR or 1 (move relative to current position, positive or negative),
1708 and SEEK_END or 2 (move relative to end of file, usually negative, although
1709 many platforms allow seeking beyond the end of a file).
1710
1711 Note that not all file objects are seekable.
1712 """
1713 if isinstance(pos, float):
1714 raise TypeError('an integer is required')
1715 self._checkClosed()
1716 return os.lseek(self._fd, pos, whence)
1717
1718 def tell(self):
1719 """tell() -> int. Current file position.
1720
1721 Can raise OSError for non seekable files."""
1722 self._checkClosed()
1723 return os.lseek(self._fd, 0, SEEK_CUR)
1724
1725 def truncate(self, size=None):
1726 """Truncate the file to at most size bytes.
1727
1728 Size defaults to the current file position, as returned by tell().
1729 The current file position is changed to the value of size.
1730 """
1731 self._checkClosed()
1732 self._checkWritable()
1733 if size is None:
1734 size = self.tell()
1735 os.ftruncate(self._fd, size)
1736 return size
1737
1738 def close(self):
1739 """Close the file.
1740
1741 A closed file cannot be used for further I/O operations. close() may be
1742 called more than once without error.
1743 """
1744 if not self.closed:
1745 try:
1746 if self._closefd:
1747 os.close(self._fd)
1748 finally:
1749 super().close()
1750
1751 def seekable(self):
1752 """True if file supports random-access."""
1753 self._checkClosed()
1754 if self._seekable is None:
1755 try:
1756 self.tell()
1757 except OSError:
1758 self._seekable = False
1759 else:
1760 self._seekable = True
1761 return self._seekable
1762
1763 def readable(self):
1764 """True if file was opened in a read mode."""
1765 self._checkClosed()
1766 return self._readable
1767
1768 def writable(self):
1769 """True if file was opened in a write mode."""
1770 self._checkClosed()
1771 return self._writable
1772
1773 def fileno(self):
1774 """Return the underlying file descriptor (an integer)."""
1775 self._checkClosed()
1776 return self._fd
1777
1778 def isatty(self):
1779 """True if the file is connected to a TTY device."""
1780 self._checkClosed()
1781 return os.isatty(self._fd)
1782
1783 @property
1784 def closefd(self):
1785 """True if the file descriptor will be closed by close()."""
1786 return self._closefd
1787
1788 @property
1789 def mode(self):
1790 """String giving the file mode"""
1791 if self._created:
1792 if self._readable:
1793 return 'xb+'
1794 else:
1795 return 'xb'
1796 elif self._appending:
1797 if self._readable:
1798 return 'ab+'
1799 else:
1800 return 'ab'
1801 elif self._readable:
1802 if self._writable:
1803 return 'rb+'
1804 else:
1805 return 'rb'
1806 else:
1807 return 'wb'
1808
1809
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001810class TextIOBase(IOBase):
1811
1812 """Base class for text I/O.
1813
1814 This class provides a character and line based interface to stream
Steve Palmer7b97ab32019-04-09 05:35:27 +01001815 I/O. There is no public constructor.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001816 """
1817
Serhiy Storchaka3c411542013-09-16 23:18:10 +03001818 def read(self, size=-1):
1819 """Read at most size characters from stream, where size is an int.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001820
Serhiy Storchaka3c411542013-09-16 23:18:10 +03001821 Read from underlying buffer until we have size characters or we hit EOF.
1822 If size is negative or omitted, read until EOF.
Raymond Hettingercbb80892011-01-13 18:15:51 +00001823
1824 Returns a string.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001825 """
1826 self._unsupported("read")
1827
Raymond Hettinger3c940242011-01-12 23:39:31 +00001828 def write(self, s):
Raymond Hettingercbb80892011-01-13 18:15:51 +00001829 """Write string s to stream and returning an int."""
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001830 self._unsupported("write")
1831
Georg Brandl4d73b572011-01-13 07:13:06 +00001832 def truncate(self, pos=None):
Raymond Hettingercbb80892011-01-13 18:15:51 +00001833 """Truncate size to pos, where pos is an int."""
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001834 self._unsupported("truncate")
1835
Raymond Hettinger3c940242011-01-12 23:39:31 +00001836 def readline(self):
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001837 """Read until newline or EOF.
1838
1839 Returns an empty string if EOF is hit immediately.
1840 """
1841 self._unsupported("readline")
1842
Raymond Hettinger3c940242011-01-12 23:39:31 +00001843 def detach(self):
Benjamin Petersond2e0c792009-05-01 20:40:59 +00001844 """
1845 Separate the underlying buffer from the TextIOBase and return it.
1846
1847 After the underlying buffer has been detached, the TextIO is in an
1848 unusable state.
1849 """
1850 self._unsupported("detach")
1851
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001852 @property
1853 def encoding(self):
1854 """Subclasses should override."""
1855 return None
1856
1857 @property
1858 def newlines(self):
1859 """Line endings translated so far.
1860
1861 Only line endings translated during reading are considered.
1862
1863 Subclasses should override.
1864 """
1865 return None
1866
Benjamin Peterson0926ad12009-06-06 18:02:12 +00001867 @property
1868 def errors(self):
1869 """Error setting of the decoder or encoder.
1870
1871 Subclasses should override."""
1872 return None
1873
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001874io.TextIOBase.register(TextIOBase)
1875
1876
1877class IncrementalNewlineDecoder(codecs.IncrementalDecoder):
1878 r"""Codec used when reading a file in universal newlines mode. It wraps
1879 another incremental decoder, translating \r\n and \r into \n. It also
1880 records the types of newlines encountered. When used with
1881 translate=False, it ensures that the newline sequence is returned in
1882 one piece.
1883 """
1884 def __init__(self, decoder, translate, errors='strict'):
1885 codecs.IncrementalDecoder.__init__(self, errors=errors)
1886 self.translate = translate
1887 self.decoder = decoder
1888 self.seennl = 0
1889 self.pendingcr = False
1890
1891 def decode(self, input, final=False):
1892 # decode input (with the eventual \r from a previous pass)
1893 if self.decoder is None:
1894 output = input
1895 else:
1896 output = self.decoder.decode(input, final=final)
1897 if self.pendingcr and (output or final):
1898 output = "\r" + output
1899 self.pendingcr = False
1900
1901 # retain last \r even when not translating data:
1902 # then readline() is sure to get \r\n in one pass
1903 if output.endswith("\r") and not final:
1904 output = output[:-1]
1905 self.pendingcr = True
1906
1907 # Record which newlines are read
1908 crlf = output.count('\r\n')
1909 cr = output.count('\r') - crlf
1910 lf = output.count('\n') - crlf
1911 self.seennl |= (lf and self._LF) | (cr and self._CR) \
1912 | (crlf and self._CRLF)
1913
1914 if self.translate:
1915 if crlf:
1916 output = output.replace("\r\n", "\n")
1917 if cr:
1918 output = output.replace("\r", "\n")
1919
1920 return output
1921
1922 def getstate(self):
1923 if self.decoder is None:
1924 buf = b""
1925 flag = 0
1926 else:
1927 buf, flag = self.decoder.getstate()
1928 flag <<= 1
1929 if self.pendingcr:
1930 flag |= 1
1931 return buf, flag
1932
1933 def setstate(self, state):
1934 buf, flag = state
1935 self.pendingcr = bool(flag & 1)
1936 if self.decoder is not None:
1937 self.decoder.setstate((buf, flag >> 1))
1938
1939 def reset(self):
1940 self.seennl = 0
1941 self.pendingcr = False
1942 if self.decoder is not None:
1943 self.decoder.reset()
1944
1945 _LF = 1
1946 _CR = 2
1947 _CRLF = 4
1948
1949 @property
1950 def newlines(self):
1951 return (None,
1952 "\n",
1953 "\r",
1954 ("\r", "\n"),
1955 "\r\n",
1956 ("\n", "\r\n"),
1957 ("\r", "\r\n"),
1958 ("\r", "\n", "\r\n")
1959 )[self.seennl]
1960
1961
1962class TextIOWrapper(TextIOBase):
1963
1964 r"""Character and line based layer over a BufferedIOBase object, buffer.
1965
1966 encoding gives the name of the encoding that the stream will be
Victor Stinnerf86a5e82012-06-05 13:43:22 +02001967 decoded or encoded with. It defaults to locale.getpreferredencoding(False).
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001968
1969 errors determines the strictness of encoding and decoding (see the
1970 codecs.register) and defaults to "strict".
1971
1972 newline can be None, '', '\n', '\r', or '\r\n'. It controls the
1973 handling of line endings. If it is None, universal newlines is
1974 enabled. With this enabled, on input, the lines endings '\n', '\r',
1975 or '\r\n' are translated to '\n' before being returned to the
1976 caller. Conversely, on output, '\n' is translated to the system
Éric Araujo39242302011-11-03 00:08:48 +01001977 default line separator, os.linesep. If newline is any other of its
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001978 legal values, that newline becomes the newline when the file is read
1979 and it is returned untranslated. On output, '\n' is converted to the
1980 newline.
1981
1982 If line_buffering is True, a call to flush is implied when a call to
1983 write contains a newline character.
1984 """
1985
1986 _CHUNK_SIZE = 2048
1987
Victor Stinnera3568412019-05-28 01:44:21 +02001988 # Initialize _buffer as soon as possible since it's used by __del__()
1989 # which calls close()
1990 _buffer = None
1991
Andrew Svetlov4e9e9c12012-08-13 16:09:54 +03001992 # The write_through argument has no effect here since this
1993 # implementation always writes through. The argument is present only
1994 # so that the signature can match the signature of the C version.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001995 def __init__(self, buffer, encoding=None, errors=None, newline=None,
Antoine Pitroue96ec682011-07-23 21:46:35 +02001996 line_buffering=False, write_through=False):
INADA Naoki507434f2017-12-21 09:59:53 +09001997 self._check_newline(newline)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00001998 if encoding is None:
1999 try:
2000 encoding = os.device_encoding(buffer.fileno())
2001 except (AttributeError, UnsupportedOperation):
2002 pass
2003 if encoding is None:
2004 try:
2005 import locale
Brett Cannoncd171c82013-07-04 17:43:24 -04002006 except ImportError:
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002007 # Importing locale may fail if Python is being built
2008 encoding = "ascii"
2009 else:
Victor Stinnerf86a5e82012-06-05 13:43:22 +02002010 encoding = locale.getpreferredencoding(False)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002011
2012 if not isinstance(encoding, str):
2013 raise ValueError("invalid encoding: %r" % encoding)
2014
Nick Coghlana9b15242014-02-04 22:11:18 +10002015 if not codecs.lookup(encoding)._is_text_encoding:
2016 msg = ("%r is not a text encoding; "
2017 "use codecs.open() to handle arbitrary codecs")
2018 raise LookupError(msg % encoding)
2019
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002020 if errors is None:
2021 errors = "strict"
2022 else:
2023 if not isinstance(errors, str):
2024 raise ValueError("invalid errors: %r" % errors)
2025
Antoine Pitrou7f8f4182010-12-21 21:20:59 +00002026 self._buffer = buffer
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002027 self._decoded_chars = '' # buffer for text returned from decoder
2028 self._decoded_chars_used = 0 # offset into _decoded_chars for read()
2029 self._snapshot = None # info for reconstructing decoder state
2030 self._seekable = self._telling = self.buffer.seekable()
Antoine Pitroue96ec682011-07-23 21:46:35 +02002031 self._has_read1 = hasattr(self.buffer, 'read1')
INADA Naoki507434f2017-12-21 09:59:53 +09002032 self._configure(encoding, errors, newline,
2033 line_buffering, write_through)
2034
2035 def _check_newline(self, newline):
2036 if newline is not None and not isinstance(newline, str):
2037 raise TypeError("illegal newline type: %r" % (type(newline),))
2038 if newline not in (None, "", "\n", "\r", "\r\n"):
2039 raise ValueError("illegal newline value: %r" % (newline,))
2040
2041 def _configure(self, encoding=None, errors=None, newline=None,
2042 line_buffering=False, write_through=False):
2043 self._encoding = encoding
2044 self._errors = errors
2045 self._encoder = None
2046 self._decoder = None
Antoine Pitrou211b81d2011-02-25 20:27:33 +00002047 self._b2cratio = 0.0
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002048
INADA Naoki507434f2017-12-21 09:59:53 +09002049 self._readuniversal = not newline
2050 self._readtranslate = newline is None
2051 self._readnl = newline
2052 self._writetranslate = newline != ''
2053 self._writenl = newline or os.linesep
2054
2055 self._line_buffering = line_buffering
2056 self._write_through = write_through
2057
2058 # don't write a BOM in the middle of a file
Antoine Pitroue4501852009-05-14 18:55:55 +00002059 if self._seekable and self.writable():
2060 position = self.buffer.tell()
2061 if position != 0:
2062 try:
2063 self._get_encoder().setstate(0)
2064 except LookupError:
2065 # Sometimes the encoder doesn't exist
2066 pass
2067
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002068 # self._snapshot is either None, or a tuple (dec_flags, next_input)
2069 # where dec_flags is the second (integer) item of the decoder state
2070 # and next_input is the chunk of input bytes that comes next after the
2071 # snapshot point. We use this to reconstruct decoder states in tell().
2072
2073 # Naming convention:
2074 # - "bytes_..." for integer variables that count input bytes
2075 # - "chars_..." for integer variables that count decoded characters
2076
Benjamin Petersonc4c0eae2009-03-09 00:07:03 +00002077 def __repr__(self):
Serhiy Storchaka465e60e2014-07-25 23:36:00 +03002078 result = "<{}.{}".format(self.__class__.__module__,
2079 self.__class__.__qualname__)
Antoine Pitrou716c4442009-05-23 19:04:03 +00002080 try:
2081 name = self.name
Miss Islington (bot)102130a2019-08-29 01:13:29 -07002082 except AttributeError:
Antoine Pitroua4815ca2011-01-09 20:38:15 +00002083 pass
Antoine Pitrou716c4442009-05-23 19:04:03 +00002084 else:
Antoine Pitroua4815ca2011-01-09 20:38:15 +00002085 result += " name={0!r}".format(name)
2086 try:
2087 mode = self.mode
Miss Islington (bot)102130a2019-08-29 01:13:29 -07002088 except AttributeError:
Antoine Pitroua4815ca2011-01-09 20:38:15 +00002089 pass
2090 else:
2091 result += " mode={0!r}".format(mode)
2092 return result + " encoding={0!r}>".format(self.encoding)
Benjamin Petersonc4c0eae2009-03-09 00:07:03 +00002093
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002094 @property
2095 def encoding(self):
2096 return self._encoding
2097
2098 @property
2099 def errors(self):
2100 return self._errors
2101
2102 @property
2103 def line_buffering(self):
2104 return self._line_buffering
2105
Antoine Pitrou7f8f4182010-12-21 21:20:59 +00002106 @property
Antoine Pitrou3c2817b2017-06-03 12:32:28 +02002107 def write_through(self):
2108 return self._write_through
2109
2110 @property
Antoine Pitrou7f8f4182010-12-21 21:20:59 +00002111 def buffer(self):
2112 return self._buffer
2113
INADA Naoki507434f2017-12-21 09:59:53 +09002114 def reconfigure(self, *,
2115 encoding=None, errors=None, newline=Ellipsis,
2116 line_buffering=None, write_through=None):
Antoine Pitrou3c2817b2017-06-03 12:32:28 +02002117 """Reconfigure the text stream with new parameters.
2118
2119 This also flushes the stream.
2120 """
INADA Naoki507434f2017-12-21 09:59:53 +09002121 if (self._decoder is not None
2122 and (encoding is not None or errors is not None
2123 or newline is not Ellipsis)):
2124 raise UnsupportedOperation(
2125 "It is not possible to set the encoding or newline of stream "
2126 "after the first read")
2127
2128 if errors is None:
2129 if encoding is None:
2130 errors = self._errors
2131 else:
2132 errors = 'strict'
2133 elif not isinstance(errors, str):
2134 raise TypeError("invalid errors: %r" % errors)
2135
2136 if encoding is None:
2137 encoding = self._encoding
2138 else:
2139 if not isinstance(encoding, str):
2140 raise TypeError("invalid encoding: %r" % encoding)
2141
2142 if newline is Ellipsis:
2143 newline = self._readnl
2144 self._check_newline(newline)
2145
Antoine Pitrou3c2817b2017-06-03 12:32:28 +02002146 if line_buffering is None:
2147 line_buffering = self.line_buffering
2148 if write_through is None:
2149 write_through = self.write_through
INADA Naoki507434f2017-12-21 09:59:53 +09002150
Antoine Pitrou3c2817b2017-06-03 12:32:28 +02002151 self.flush()
INADA Naoki507434f2017-12-21 09:59:53 +09002152 self._configure(encoding, errors, newline,
2153 line_buffering, write_through)
Antoine Pitrou3c2817b2017-06-03 12:32:28 +02002154
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002155 def seekable(self):
Antoine Pitrou1d857452012-09-05 20:11:49 +02002156 if self.closed:
2157 raise ValueError("I/O operation on closed file.")
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002158 return self._seekable
2159
2160 def readable(self):
2161 return self.buffer.readable()
2162
2163 def writable(self):
2164 return self.buffer.writable()
2165
2166 def flush(self):
2167 self.buffer.flush()
2168 self._telling = self._seekable
2169
2170 def close(self):
Antoine Pitrou6be88762010-05-03 16:48:20 +00002171 if self.buffer is not None and not self.closed:
Benjamin Peterson68623612012-12-20 11:53:11 -06002172 try:
2173 self.flush()
2174 finally:
2175 self.buffer.close()
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002176
2177 @property
2178 def closed(self):
2179 return self.buffer.closed
2180
2181 @property
2182 def name(self):
2183 return self.buffer.name
2184
2185 def fileno(self):
2186 return self.buffer.fileno()
2187
2188 def isatty(self):
2189 return self.buffer.isatty()
2190
Raymond Hettinger00fa0392011-01-13 02:52:26 +00002191 def write(self, s):
Raymond Hettingercbb80892011-01-13 18:15:51 +00002192 'Write data, where s is a str'
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002193 if self.closed:
2194 raise ValueError("write to closed file")
2195 if not isinstance(s, str):
2196 raise TypeError("can't write %s to text stream" %
2197 s.__class__.__name__)
2198 length = len(s)
2199 haslf = (self._writetranslate or self._line_buffering) and "\n" in s
2200 if haslf and self._writetranslate and self._writenl != "\n":
2201 s = s.replace("\n", self._writenl)
2202 encoder = self._encoder or self._get_encoder()
2203 # XXX What if we were just reading?
2204 b = encoder.encode(s)
2205 self.buffer.write(b)
2206 if self._line_buffering and (haslf or "\r" in s):
2207 self.flush()
Zackery Spytz23db9352018-06-29 04:14:58 -06002208 self._set_decoded_chars('')
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002209 self._snapshot = None
2210 if self._decoder:
2211 self._decoder.reset()
2212 return length
2213
2214 def _get_encoder(self):
2215 make_encoder = codecs.getincrementalencoder(self._encoding)
2216 self._encoder = make_encoder(self._errors)
2217 return self._encoder
2218
2219 def _get_decoder(self):
2220 make_decoder = codecs.getincrementaldecoder(self._encoding)
2221 decoder = make_decoder(self._errors)
2222 if self._readuniversal:
2223 decoder = IncrementalNewlineDecoder(decoder, self._readtranslate)
2224 self._decoder = decoder
2225 return decoder
2226
2227 # The following three methods implement an ADT for _decoded_chars.
2228 # Text returned from the decoder is buffered here until the client
2229 # requests it by calling our read() or readline() method.
2230 def _set_decoded_chars(self, chars):
2231 """Set the _decoded_chars buffer."""
2232 self._decoded_chars = chars
2233 self._decoded_chars_used = 0
2234
2235 def _get_decoded_chars(self, n=None):
2236 """Advance into the _decoded_chars buffer."""
2237 offset = self._decoded_chars_used
2238 if n is None:
2239 chars = self._decoded_chars[offset:]
2240 else:
2241 chars = self._decoded_chars[offset:offset + n]
2242 self._decoded_chars_used += len(chars)
2243 return chars
2244
2245 def _rewind_decoded_chars(self, n):
2246 """Rewind the _decoded_chars buffer."""
2247 if self._decoded_chars_used < n:
2248 raise AssertionError("rewind decoded_chars out of bounds")
2249 self._decoded_chars_used -= n
2250
2251 def _read_chunk(self):
2252 """
2253 Read and decode the next chunk of data from the BufferedReader.
2254 """
2255
2256 # The return value is True unless EOF was reached. The decoded
2257 # string is placed in self._decoded_chars (replacing its previous
2258 # value). The entire input chunk is sent to the decoder, though
2259 # some of it may remain buffered in the decoder, yet to be
2260 # converted.
2261
2262 if self._decoder is None:
2263 raise ValueError("no decoder")
2264
2265 if self._telling:
2266 # To prepare for tell(), we need to snapshot a point in the
2267 # file where the decoder's input buffer is empty.
2268
2269 dec_buffer, dec_flags = self._decoder.getstate()
2270 # Given this, we know there was a valid snapshot point
2271 # len(dec_buffer) bytes ago with decoder state (b'', dec_flags).
2272
2273 # Read a chunk, decode it, and put the result in self._decoded_chars.
Antoine Pitroue96ec682011-07-23 21:46:35 +02002274 if self._has_read1:
2275 input_chunk = self.buffer.read1(self._CHUNK_SIZE)
2276 else:
2277 input_chunk = self.buffer.read(self._CHUNK_SIZE)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002278 eof = not input_chunk
Antoine Pitrou211b81d2011-02-25 20:27:33 +00002279 decoded_chars = self._decoder.decode(input_chunk, eof)
2280 self._set_decoded_chars(decoded_chars)
2281 if decoded_chars:
2282 self._b2cratio = len(input_chunk) / len(self._decoded_chars)
2283 else:
2284 self._b2cratio = 0.0
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002285
2286 if self._telling:
2287 # At the snapshot point, len(dec_buffer) bytes before the read,
2288 # the next input to be decoded is dec_buffer + input_chunk.
2289 self._snapshot = (dec_flags, dec_buffer + input_chunk)
2290
2291 return not eof
2292
2293 def _pack_cookie(self, position, dec_flags=0,
2294 bytes_to_feed=0, need_eof=0, chars_to_skip=0):
2295 # The meaning of a tell() cookie is: seek to position, set the
2296 # decoder flags to dec_flags, read bytes_to_feed bytes, feed them
2297 # into the decoder with need_eof as the EOF flag, then skip
2298 # chars_to_skip characters of the decoded result. For most simple
2299 # decoders, tell() will often just give a byte offset in the file.
2300 return (position | (dec_flags<<64) | (bytes_to_feed<<128) |
2301 (chars_to_skip<<192) | bool(need_eof)<<256)
2302
2303 def _unpack_cookie(self, bigint):
2304 rest, position = divmod(bigint, 1<<64)
2305 rest, dec_flags = divmod(rest, 1<<64)
2306 rest, bytes_to_feed = divmod(rest, 1<<64)
2307 need_eof, chars_to_skip = divmod(rest, 1<<64)
2308 return position, dec_flags, bytes_to_feed, need_eof, chars_to_skip
2309
2310 def tell(self):
2311 if not self._seekable:
Antoine Pitrou0d739d72010-09-05 23:01:12 +00002312 raise UnsupportedOperation("underlying stream is not seekable")
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002313 if not self._telling:
Andrew Svetlovf7a17b42012-12-25 16:47:37 +02002314 raise OSError("telling position disabled by next() call")
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002315 self.flush()
2316 position = self.buffer.tell()
2317 decoder = self._decoder
2318 if decoder is None or self._snapshot is None:
2319 if self._decoded_chars:
2320 # This should never happen.
2321 raise AssertionError("pending decoded text")
2322 return position
2323
2324 # Skip backward to the snapshot point (see _read_chunk).
2325 dec_flags, next_input = self._snapshot
2326 position -= len(next_input)
2327
2328 # How many decoded characters have been used up since the snapshot?
2329 chars_to_skip = self._decoded_chars_used
2330 if chars_to_skip == 0:
2331 # We haven't moved from the snapshot point.
2332 return self._pack_cookie(position, dec_flags)
2333
2334 # Starting from the snapshot position, we will walk the decoder
2335 # forward until it gives us enough decoded characters.
2336 saved_state = decoder.getstate()
2337 try:
Antoine Pitrou211b81d2011-02-25 20:27:33 +00002338 # Fast search for an acceptable start point, close to our
2339 # current pos.
2340 # Rationale: calling decoder.decode() has a large overhead
2341 # regardless of chunk size; we want the number of such calls to
Raymond Hettinger14010182018-09-13 21:17:40 -07002342 # be O(1) in most situations (common decoders, sensible input).
Antoine Pitrou211b81d2011-02-25 20:27:33 +00002343 # Actually, it will be exactly 1 for fixed-size codecs (all
2344 # 8-bit codecs, also UTF-16 and UTF-32).
2345 skip_bytes = int(self._b2cratio * chars_to_skip)
2346 skip_back = 1
2347 assert skip_bytes <= len(next_input)
2348 while skip_bytes > 0:
2349 decoder.setstate((b'', dec_flags))
2350 # Decode up to temptative start point
2351 n = len(decoder.decode(next_input[:skip_bytes]))
2352 if n <= chars_to_skip:
2353 b, d = decoder.getstate()
2354 if not b:
2355 # Before pos and no bytes buffered in decoder => OK
2356 dec_flags = d
2357 chars_to_skip -= n
2358 break
2359 # Skip back by buffered amount and reset heuristic
2360 skip_bytes -= len(b)
2361 skip_back = 1
2362 else:
2363 # We're too far ahead, skip back a bit
2364 skip_bytes -= skip_back
2365 skip_back = skip_back * 2
2366 else:
2367 skip_bytes = 0
2368 decoder.setstate((b'', dec_flags))
2369
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002370 # Note our initial start point.
Antoine Pitrou211b81d2011-02-25 20:27:33 +00002371 start_pos = position + skip_bytes
2372 start_flags = dec_flags
2373 if chars_to_skip == 0:
2374 # We haven't moved from the start point.
2375 return self._pack_cookie(start_pos, start_flags)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002376
2377 # Feed the decoder one byte at a time. As we go, note the
2378 # nearest "safe start point" before the current location
2379 # (a point where the decoder has nothing buffered, so seek()
2380 # can safely start from there and advance to this location).
Antoine Pitrou211b81d2011-02-25 20:27:33 +00002381 bytes_fed = 0
2382 need_eof = 0
2383 # Chars decoded since `start_pos`
2384 chars_decoded = 0
2385 for i in range(skip_bytes, len(next_input)):
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002386 bytes_fed += 1
Antoine Pitrou211b81d2011-02-25 20:27:33 +00002387 chars_decoded += len(decoder.decode(next_input[i:i+1]))
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002388 dec_buffer, dec_flags = decoder.getstate()
2389 if not dec_buffer and chars_decoded <= chars_to_skip:
2390 # Decoder buffer is empty, so this is a safe start point.
2391 start_pos += bytes_fed
2392 chars_to_skip -= chars_decoded
2393 start_flags, bytes_fed, chars_decoded = dec_flags, 0, 0
2394 if chars_decoded >= chars_to_skip:
2395 break
2396 else:
2397 # We didn't get enough decoded data; signal EOF to get more.
2398 chars_decoded += len(decoder.decode(b'', final=True))
2399 need_eof = 1
2400 if chars_decoded < chars_to_skip:
Andrew Svetlovf7a17b42012-12-25 16:47:37 +02002401 raise OSError("can't reconstruct logical file position")
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002402
2403 # The returned cookie corresponds to the last safe start point.
2404 return self._pack_cookie(
2405 start_pos, start_flags, bytes_fed, need_eof, chars_to_skip)
2406 finally:
2407 decoder.setstate(saved_state)
2408
2409 def truncate(self, pos=None):
2410 self.flush()
2411 if pos is None:
2412 pos = self.tell()
Antoine Pitrou905a2ff2010-01-31 22:47:27 +00002413 return self.buffer.truncate(pos)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002414
Benjamin Petersond2e0c792009-05-01 20:40:59 +00002415 def detach(self):
2416 if self.buffer is None:
2417 raise ValueError("buffer is already detached")
2418 self.flush()
Antoine Pitrou7f8f4182010-12-21 21:20:59 +00002419 buffer = self._buffer
2420 self._buffer = None
Benjamin Petersond2e0c792009-05-01 20:40:59 +00002421 return buffer
2422
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002423 def seek(self, cookie, whence=0):
Antoine Pitrou85e3ee72015-04-13 20:01:21 +02002424 def _reset_encoder(position):
2425 """Reset the encoder (merely useful for proper BOM handling)"""
2426 try:
2427 encoder = self._encoder or self._get_encoder()
2428 except LookupError:
2429 # Sometimes the encoder doesn't exist
2430 pass
2431 else:
2432 if position != 0:
2433 encoder.setstate(0)
2434 else:
2435 encoder.reset()
2436
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002437 if self.closed:
2438 raise ValueError("tell on closed file")
2439 if not self._seekable:
Antoine Pitrou0d739d72010-09-05 23:01:12 +00002440 raise UnsupportedOperation("underlying stream is not seekable")
ngie-eign848037c2019-03-02 23:28:26 -08002441 if whence == SEEK_CUR:
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002442 if cookie != 0:
Antoine Pitrou0d739d72010-09-05 23:01:12 +00002443 raise UnsupportedOperation("can't do nonzero cur-relative seeks")
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002444 # Seeking to the current position should attempt to
2445 # sync the underlying buffer with the current position.
2446 whence = 0
2447 cookie = self.tell()
ngie-eign848037c2019-03-02 23:28:26 -08002448 elif whence == SEEK_END:
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002449 if cookie != 0:
Antoine Pitrou0d739d72010-09-05 23:01:12 +00002450 raise UnsupportedOperation("can't do nonzero end-relative seeks")
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002451 self.flush()
ngie-eign848037c2019-03-02 23:28:26 -08002452 position = self.buffer.seek(0, whence)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002453 self._set_decoded_chars('')
2454 self._snapshot = None
2455 if self._decoder:
2456 self._decoder.reset()
Antoine Pitrou85e3ee72015-04-13 20:01:21 +02002457 _reset_encoder(position)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002458 return position
2459 if whence != 0:
Jesus Cea94363612012-06-22 18:32:07 +02002460 raise ValueError("unsupported whence (%r)" % (whence,))
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002461 if cookie < 0:
2462 raise ValueError("negative seek position %r" % (cookie,))
2463 self.flush()
2464
2465 # The strategy of seek() is to go back to the safe start point
2466 # and replay the effect of read(chars_to_skip) from there.
2467 start_pos, dec_flags, bytes_to_feed, need_eof, chars_to_skip = \
2468 self._unpack_cookie(cookie)
2469
2470 # Seek back to the safe start point.
2471 self.buffer.seek(start_pos)
2472 self._set_decoded_chars('')
2473 self._snapshot = None
2474
2475 # Restore the decoder to its state from the safe start point.
Benjamin Peterson9363a652009-03-05 00:42:09 +00002476 if cookie == 0 and self._decoder:
2477 self._decoder.reset()
2478 elif self._decoder or dec_flags or chars_to_skip:
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002479 self._decoder = self._decoder or self._get_decoder()
2480 self._decoder.setstate((b'', dec_flags))
2481 self._snapshot = (dec_flags, b'')
2482
2483 if chars_to_skip:
2484 # Just like _read_chunk, feed the decoder and save a snapshot.
2485 input_chunk = self.buffer.read(bytes_to_feed)
2486 self._set_decoded_chars(
2487 self._decoder.decode(input_chunk, need_eof))
2488 self._snapshot = (dec_flags, input_chunk)
2489
2490 # Skip chars_to_skip of the decoded characters.
2491 if len(self._decoded_chars) < chars_to_skip:
Andrew Svetlovf7a17b42012-12-25 16:47:37 +02002492 raise OSError("can't restore logical file position")
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002493 self._decoded_chars_used = chars_to_skip
2494
Antoine Pitrou85e3ee72015-04-13 20:01:21 +02002495 _reset_encoder(cookie)
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002496 return cookie
2497
Serhiy Storchaka3c411542013-09-16 23:18:10 +03002498 def read(self, size=None):
Benjamin Petersona1b49012009-03-31 23:11:32 +00002499 self._checkReadable()
Serhiy Storchaka3c411542013-09-16 23:18:10 +03002500 if size is None:
2501 size = -1
Oren Milmande503602017-08-24 21:33:42 +03002502 else:
2503 try:
2504 size_index = size.__index__
2505 except AttributeError:
2506 raise TypeError(f"{size!r} is not an integer")
2507 else:
2508 size = size_index()
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002509 decoder = self._decoder or self._get_decoder()
Serhiy Storchaka3c411542013-09-16 23:18:10 +03002510 if size < 0:
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002511 # Read everything.
2512 result = (self._get_decoded_chars() +
2513 decoder.decode(self.buffer.read(), final=True))
2514 self._set_decoded_chars('')
2515 self._snapshot = None
2516 return result
2517 else:
Serhiy Storchaka3c411542013-09-16 23:18:10 +03002518 # Keep reading chunks until we have size characters to return.
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002519 eof = False
Serhiy Storchaka3c411542013-09-16 23:18:10 +03002520 result = self._get_decoded_chars(size)
2521 while len(result) < size and not eof:
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002522 eof = not self._read_chunk()
Serhiy Storchaka3c411542013-09-16 23:18:10 +03002523 result += self._get_decoded_chars(size - len(result))
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002524 return result
2525
2526 def __next__(self):
2527 self._telling = False
2528 line = self.readline()
2529 if not line:
2530 self._snapshot = None
2531 self._telling = self._seekable
2532 raise StopIteration
2533 return line
2534
Serhiy Storchaka3c411542013-09-16 23:18:10 +03002535 def readline(self, size=None):
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002536 if self.closed:
2537 raise ValueError("read from closed file")
Serhiy Storchaka3c411542013-09-16 23:18:10 +03002538 if size is None:
2539 size = -1
Oren Milmande503602017-08-24 21:33:42 +03002540 else:
2541 try:
2542 size_index = size.__index__
2543 except AttributeError:
2544 raise TypeError(f"{size!r} is not an integer")
2545 else:
2546 size = size_index()
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002547
2548 # Grab all the decoded text (we will rewind any extra bits later).
2549 line = self._get_decoded_chars()
2550
2551 start = 0
2552 # Make the decoder if it doesn't already exist.
2553 if not self._decoder:
2554 self._get_decoder()
2555
2556 pos = endpos = None
2557 while True:
2558 if self._readtranslate:
2559 # Newlines are already translated, only search for \n
2560 pos = line.find('\n', start)
2561 if pos >= 0:
2562 endpos = pos + 1
2563 break
2564 else:
2565 start = len(line)
2566
2567 elif self._readuniversal:
2568 # Universal newline search. Find any of \r, \r\n, \n
2569 # The decoder ensures that \r\n are not split in two pieces
2570
2571 # In C we'd look for these in parallel of course.
2572 nlpos = line.find("\n", start)
2573 crpos = line.find("\r", start)
2574 if crpos == -1:
2575 if nlpos == -1:
2576 # Nothing found
2577 start = len(line)
2578 else:
2579 # Found \n
2580 endpos = nlpos + 1
2581 break
2582 elif nlpos == -1:
2583 # Found lone \r
2584 endpos = crpos + 1
2585 break
2586 elif nlpos < crpos:
2587 # Found \n
2588 endpos = nlpos + 1
2589 break
2590 elif nlpos == crpos + 1:
2591 # Found \r\n
2592 endpos = crpos + 2
2593 break
2594 else:
2595 # Found \r
2596 endpos = crpos + 1
2597 break
2598 else:
2599 # non-universal
2600 pos = line.find(self._readnl)
2601 if pos >= 0:
2602 endpos = pos + len(self._readnl)
2603 break
2604
Serhiy Storchaka3c411542013-09-16 23:18:10 +03002605 if size >= 0 and len(line) >= size:
2606 endpos = size # reached length size
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002607 break
2608
2609 # No line ending seen yet - get more data'
2610 while self._read_chunk():
2611 if self._decoded_chars:
2612 break
2613 if self._decoded_chars:
2614 line += self._get_decoded_chars()
2615 else:
2616 # end of file
2617 self._set_decoded_chars('')
2618 self._snapshot = None
2619 return line
2620
Serhiy Storchaka3c411542013-09-16 23:18:10 +03002621 if size >= 0 and endpos > size:
2622 endpos = size # don't exceed size
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002623
2624 # Rewind _decoded_chars to just after the line ending we found.
2625 self._rewind_decoded_chars(len(line) - endpos)
2626 return line[:endpos]
2627
2628 @property
2629 def newlines(self):
2630 return self._decoder.newlines if self._decoder else None
2631
2632
2633class StringIO(TextIOWrapper):
2634 """Text I/O implementation using an in-memory buffer.
2635
2636 The initial_value argument sets the value of object. The newline
2637 argument is like the one of TextIOWrapper's constructor.
2638 """
2639
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002640 def __init__(self, initial_value="", newline="\n"):
2641 super(StringIO, self).__init__(BytesIO(),
2642 encoding="utf-8",
Serhiy Storchakac92ea762014-01-29 11:33:26 +02002643 errors="surrogatepass",
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002644 newline=newline)
Antoine Pitrou11446482009-04-04 14:09:30 +00002645 # Issue #5645: make universal newlines semantics the same as in the
2646 # C version, even under Windows.
2647 if newline is None:
2648 self._writetranslate = False
Alexandre Vassalottid2bb18b2009-07-22 03:07:33 +00002649 if initial_value is not None:
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002650 if not isinstance(initial_value, str):
Alexandre Vassalottid2bb18b2009-07-22 03:07:33 +00002651 raise TypeError("initial_value must be str or None, not {0}"
2652 .format(type(initial_value).__name__))
Benjamin Peterson4fa88fa2009-03-04 00:14:51 +00002653 self.write(initial_value)
2654 self.seek(0)
2655
2656 def getvalue(self):
2657 self.flush()
Antoine Pitrou57839a62014-02-02 23:37:29 +01002658 decoder = self._decoder or self._get_decoder()
2659 old_state = decoder.getstate()
2660 decoder.reset()
2661 try:
2662 return decoder.decode(self.buffer.getvalue(), final=True)
2663 finally:
2664 decoder.setstate(old_state)
Benjamin Peterson9fd459a2009-03-09 00:09:44 +00002665
2666 def __repr__(self):
2667 # TextIOWrapper tells the encoding in its repr. In StringIO,
Martin Panter7462b6492015-11-02 03:37:02 +00002668 # that's an implementation detail.
Benjamin Peterson9fd459a2009-03-09 00:09:44 +00002669 return object.__repr__(self)
Benjamin Petersonb487e632009-03-21 03:08:31 +00002670
2671 @property
Benjamin Peterson0926ad12009-06-06 18:02:12 +00002672 def errors(self):
2673 return None
2674
2675 @property
Benjamin Petersonb487e632009-03-21 03:08:31 +00002676 def encoding(self):
2677 return None
Benjamin Petersond2e0c792009-05-01 20:40:59 +00002678
2679 def detach(self):
2680 # This doesn't make sense on StringIO.
2681 self._unsupported("detach")