Guido van Rossum | 4b8c6ea | 2000-02-04 15:39:30 +0000 | [diff] [blame] | 1 | """Functions that read and write gzipped files. |
| 2 | |
Guido van Rossum | 54f22ed | 2000-02-04 15:10:34 +0000 | [diff] [blame] | 3 | The user of the file doesn't have to worry about the compression, |
| 4 | but random access is not allowed.""" |
| 5 | |
| 6 | # based on Andrew Kuchling's minigzip.py distributed with the zlib module |
| 7 | |
Lars Gustäbel | 1440df2 | 2009-10-29 09:39:47 +0000 | [diff] [blame] | 8 | import struct, sys, time, os |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 9 | import zlib |
Georg Brandl | 1a3284e | 2007-12-02 09:40:06 +0000 | [diff] [blame] | 10 | import builtins |
Antoine Pitrou | b1f8835 | 2010-01-03 22:37:40 +0000 | [diff] [blame] | 11 | import io |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 12 | |
Antoine Pitrou | 79c5ef1 | 2010-08-17 21:10:05 +0000 | [diff] [blame] | 13 | __all__ = ["GzipFile", "open", "compress", "decompress"] |
Skip Montanaro | 2dd4276 | 2001-01-23 15:35:05 +0000 | [diff] [blame] | 14 | |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 15 | FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16 |
| 16 | |
| 17 | READ, WRITE = 1, 2 |
| 18 | |
Nadeem Vawda | 7e12620 | 2012-05-06 15:04:01 +0200 | [diff] [blame] | 19 | def open(filename, mode="rb", compresslevel=9, |
| 20 | encoding=None, errors=None, newline=None): |
| 21 | """Open a gzip-compressed file in binary or text mode. |
| 22 | |
Nadeem Vawda | 6872101 | 2012-06-04 23:21:38 +0200 | [diff] [blame] | 23 | The filename argument can be an actual filename (a str or bytes object), or |
| 24 | an existing file object to read from or write to. |
| 25 | |
Nadeem Vawda | 7e12620 | 2012-05-06 15:04:01 +0200 | [diff] [blame] | 26 | The mode argument can be "r", "rb", "w", "wb", "a" or "ab" for binary mode, |
| 27 | or "rt", "wt" or "at" for text mode. The default mode is "rb", and the |
| 28 | default compresslevel is 9. |
| 29 | |
| 30 | For binary mode, this function is equivalent to the GzipFile constructor: |
| 31 | GzipFile(filename, mode, compresslevel). In this case, the encoding, errors |
| 32 | and newline arguments must not be provided. |
| 33 | |
| 34 | For text mode, a GzipFile object is created, and wrapped in an |
| 35 | io.TextIOWrapper instance with the specified encoding, error handling |
| 36 | behavior, and line ending(s). |
| 37 | |
| 38 | """ |
| 39 | if "t" in mode: |
| 40 | if "b" in mode: |
| 41 | raise ValueError("Invalid mode: %r" % (mode,)) |
| 42 | else: |
| 43 | if encoding is not None: |
| 44 | raise ValueError("Argument 'encoding' not supported in binary mode") |
| 45 | if errors is not None: |
| 46 | raise ValueError("Argument 'errors' not supported in binary mode") |
| 47 | if newline is not None: |
| 48 | raise ValueError("Argument 'newline' not supported in binary mode") |
Nadeem Vawda | 6872101 | 2012-06-04 23:21:38 +0200 | [diff] [blame] | 49 | |
| 50 | gz_mode = mode.replace("t", "") |
| 51 | if isinstance(filename, (str, bytes)): |
| 52 | binary_file = GzipFile(filename, gz_mode, compresslevel) |
| 53 | elif hasattr(filename, "read") or hasattr(filename, "write"): |
| 54 | binary_file = GzipFile(None, gz_mode, compresslevel, filename) |
| 55 | else: |
| 56 | raise TypeError("filename must be a str or bytes object, or a file") |
| 57 | |
Nadeem Vawda | 7e12620 | 2012-05-06 15:04:01 +0200 | [diff] [blame] | 58 | if "t" in mode: |
| 59 | return io.TextIOWrapper(binary_file, encoding, errors, newline) |
| 60 | else: |
| 61 | return binary_file |
| 62 | |
Guido van Rossum | 95bdd0b | 1999-04-12 14:34:16 +0000 | [diff] [blame] | 63 | def write32u(output, value): |
Tim Peters | fb0ea52 | 2002-11-04 19:50:11 +0000 | [diff] [blame] | 64 | # The L format writes the bit pattern correctly whether signed |
| 65 | # or unsigned. |
Guido van Rossum | 95bdd0b | 1999-04-12 14:34:16 +0000 | [diff] [blame] | 66 | output.write(struct.pack("<L", value)) |
| 67 | |
Antoine Pitrou | 7b96984 | 2010-09-23 16:22:51 +0000 | [diff] [blame] | 68 | class _PaddedFile: |
| 69 | """Minimal read-only file object that prepends a string to the contents |
| 70 | of an actual file. Shouldn't be used outside of gzip.py, as it lacks |
| 71 | essential functionality.""" |
| 72 | |
| 73 | def __init__(self, f, prepend=b''): |
| 74 | self._buffer = prepend |
| 75 | self._length = len(prepend) |
| 76 | self.file = f |
| 77 | self._read = 0 |
| 78 | |
| 79 | def read(self, size): |
| 80 | if self._read is None: |
| 81 | return self.file.read(size) |
| 82 | if self._read + size <= self._length: |
| 83 | read = self._read |
| 84 | self._read += size |
| 85 | return self._buffer[read:self._read] |
| 86 | else: |
| 87 | read = self._read |
| 88 | self._read = None |
| 89 | return self._buffer[read:] + \ |
| 90 | self.file.read(size-self._length+read) |
| 91 | |
| 92 | def prepend(self, prepend=b'', readprevious=False): |
| 93 | if self._read is None: |
| 94 | self._buffer = prepend |
| 95 | elif readprevious and len(prepend) <= self._read: |
| 96 | self._read -= len(prepend) |
| 97 | return |
| 98 | else: |
| 99 | self._buffer = self._buffer[read:] + prepend |
| 100 | self._length = len(self._buffer) |
| 101 | self._read = 0 |
| 102 | |
| 103 | def unused(self): |
| 104 | if self._read is None: |
| 105 | return b'' |
| 106 | return self._buffer[self._read:] |
| 107 | |
| 108 | def seek(self, offset, whence=0): |
| 109 | # This is only ever called with offset=whence=0 |
| 110 | if whence == 1 and self._read is not None: |
| 111 | if 0 <= offset + self._read <= self._length: |
| 112 | self._read += offset |
| 113 | return |
| 114 | else: |
| 115 | offset += self._length - self._read |
| 116 | self._read = None |
| 117 | self._buffer = None |
| 118 | return self.file.seek(offset, whence) |
| 119 | |
| 120 | def __getattr__(self, name): |
Georg Brandl | 9f1c1dc | 2010-11-20 11:25:01 +0000 | [diff] [blame] | 121 | return getattr(self.file, name) |
Antoine Pitrou | 7b96984 | 2010-09-23 16:22:51 +0000 | [diff] [blame] | 122 | |
| 123 | |
Antoine Pitrou | b1f8835 | 2010-01-03 22:37:40 +0000 | [diff] [blame] | 124 | class GzipFile(io.BufferedIOBase): |
Raymond Hettinger | aef22fb | 2002-05-29 16:18:42 +0000 | [diff] [blame] | 125 | """The GzipFile class simulates most of the methods of a file object with |
Guido van Rossum | 97c5fcc | 2002-08-06 17:03:25 +0000 | [diff] [blame] | 126 | the exception of the readinto() and truncate() methods. |
Raymond Hettinger | aef22fb | 2002-05-29 16:18:42 +0000 | [diff] [blame] | 127 | |
Nadeem Vawda | 30d94b7 | 2012-02-11 23:45:10 +0200 | [diff] [blame] | 128 | This class only supports opening files in binary mode. If you need to open a |
Nadeem Vawda | 83a4dd3 | 2012-06-30 13:34:28 +0200 | [diff] [blame] | 129 | compressed file in text mode, use the gzip.open() function. |
Nadeem Vawda | 30d94b7 | 2012-02-11 23:45:10 +0200 | [diff] [blame] | 130 | |
Raymond Hettinger | aef22fb | 2002-05-29 16:18:42 +0000 | [diff] [blame] | 131 | """ |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 132 | |
Guido van Rossum | 68de379 | 1997-07-19 20:22:23 +0000 | [diff] [blame] | 133 | myfileobj = None |
Andrew M. Kuchling | 01cb47b | 2005-06-09 14:19:32 +0000 | [diff] [blame] | 134 | max_read_chunk = 10 * 1024 * 1024 # 10Mb |
Guido van Rossum | 68de379 | 1997-07-19 20:22:23 +0000 | [diff] [blame] | 135 | |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 136 | def __init__(self, filename=None, mode=None, |
Antoine Pitrou | 42db3ef | 2009-01-04 21:37:59 +0000 | [diff] [blame] | 137 | compresslevel=9, fileobj=None, mtime=None): |
Raymond Hettinger | aef22fb | 2002-05-29 16:18:42 +0000 | [diff] [blame] | 138 | """Constructor for the GzipFile class. |
| 139 | |
| 140 | At least one of fileobj and filename must be given a |
| 141 | non-trivial value. |
| 142 | |
| 143 | The new class instance is based on fileobj, which can be a regular |
Serhiy Storchaka | 50254c5 | 2013-08-29 11:35:43 +0300 | [diff] [blame] | 144 | file, an io.BytesIO object, or any other object which simulates a file. |
Raymond Hettinger | aef22fb | 2002-05-29 16:18:42 +0000 | [diff] [blame] | 145 | It defaults to None, in which case filename is opened to provide |
| 146 | a file object. |
| 147 | |
| 148 | When fileobj is not None, the filename argument is only used to be |
| 149 | included in the gzip file header, which may includes the original |
| 150 | filename of the uncompressed file. It defaults to the filename of |
| 151 | fileobj, if discernible; otherwise, it defaults to the empty string, |
| 152 | and in this case the original filename is not included in the header. |
| 153 | |
| 154 | The mode argument can be any of 'r', 'rb', 'a', 'ab', 'w', or 'wb', |
| 155 | depending on whether the file will be read or written. The default |
| 156 | is the mode of fileobj if discernible; otherwise, the default is 'rb'. |
Nadeem Vawda | 30d94b7 | 2012-02-11 23:45:10 +0200 | [diff] [blame] | 157 | A mode of 'r' is equivalent to one of 'rb', and similarly for 'w' and |
| 158 | 'wb', and 'a' and 'ab'. |
Raymond Hettinger | aef22fb | 2002-05-29 16:18:42 +0000 | [diff] [blame] | 159 | |
Nadeem Vawda | 19e568d | 2012-11-11 14:04:14 +0100 | [diff] [blame] | 160 | The compresslevel argument is an integer from 0 to 9 controlling the |
Raymond Hettinger | aef22fb | 2002-05-29 16:18:42 +0000 | [diff] [blame] | 161 | level of compression; 1 is fastest and produces the least compression, |
Nadeem Vawda | 19e568d | 2012-11-11 14:04:14 +0100 | [diff] [blame] | 162 | and 9 is slowest and produces the most compression. 0 is no compression |
| 163 | at all. The default is 9. |
Raymond Hettinger | aef22fb | 2002-05-29 16:18:42 +0000 | [diff] [blame] | 164 | |
Antoine Pitrou | 42db3ef | 2009-01-04 21:37:59 +0000 | [diff] [blame] | 165 | The mtime argument is an optional numeric timestamp to be written |
| 166 | to the stream when compressing. All gzip compressed streams |
| 167 | are required to contain a timestamp. If omitted or None, the |
| 168 | current time is used. This module ignores the timestamp when |
| 169 | decompressing; however, some programs, such as gunzip, make use |
| 170 | of it. The format of the timestamp is the same as that of the |
| 171 | return value of time.time() and of the st_mtime member of the |
| 172 | object returned by os.stat(). |
| 173 | |
Raymond Hettinger | aef22fb | 2002-05-29 16:18:42 +0000 | [diff] [blame] | 174 | """ |
| 175 | |
Nadeem Vawda | 30d94b7 | 2012-02-11 23:45:10 +0200 | [diff] [blame] | 176 | if mode and ('t' in mode or 'U' in mode): |
Nadeem Vawda | be66af4 | 2012-02-12 00:06:02 +0200 | [diff] [blame] | 177 | raise ValueError("Invalid mode: {!r}".format(mode)) |
Skip Montanaro | 12424bc | 2002-05-23 01:43:05 +0000 | [diff] [blame] | 178 | if mode and 'b' not in mode: |
| 179 | mode += 'b' |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 180 | if fileobj is None: |
Georg Brandl | 1a3284e | 2007-12-02 09:40:06 +0000 | [diff] [blame] | 181 | fileobj = self.myfileobj = builtins.open(filename, mode or 'rb') |
Guido van Rossum | 68de379 | 1997-07-19 20:22:23 +0000 | [diff] [blame] | 182 | if filename is None: |
Nadeem Vawda | 103e811 | 2012-06-20 01:35:22 +0200 | [diff] [blame] | 183 | filename = getattr(fileobj, 'name', '') |
| 184 | if not isinstance(filename, (str, bytes)): |
Nadeem Vawda | 892b0b9 | 2012-01-18 09:25:58 +0200 | [diff] [blame] | 185 | filename = '' |
Guido van Rossum | 68de379 | 1997-07-19 20:22:23 +0000 | [diff] [blame] | 186 | if mode is None: |
Nadeem Vawda | be66af4 | 2012-02-12 00:06:02 +0200 | [diff] [blame] | 187 | mode = getattr(fileobj, 'mode', 'rb') |
Guido van Rossum | 68de379 | 1997-07-19 20:22:23 +0000 | [diff] [blame] | 188 | |
Nadeem Vawda | be66af4 | 2012-02-12 00:06:02 +0200 | [diff] [blame] | 189 | if mode.startswith('r'): |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 190 | self.mode = READ |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 191 | # Set flag indicating start of a new member |
Guido van Rossum | 8ca162f | 2002-04-07 06:36:23 +0000 | [diff] [blame] | 192 | self._new_member = True |
Antoine Pitrou | b1f8835 | 2010-01-03 22:37:40 +0000 | [diff] [blame] | 193 | # Buffer data read from gzip file. extrastart is offset in |
| 194 | # stream where buffer starts. extrasize is number of |
| 195 | # bytes remaining in buffer from current stream position. |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 196 | self.extrabuf = b"" |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 197 | self.extrasize = 0 |
Antoine Pitrou | b1f8835 | 2010-01-03 22:37:40 +0000 | [diff] [blame] | 198 | self.extrastart = 0 |
Thomas Wouters | cf297e4 | 2007-02-23 15:07:44 +0000 | [diff] [blame] | 199 | self.name = filename |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 200 | # Starts small, scales exponentially |
| 201 | self.min_readsize = 100 |
Antoine Pitrou | 7b96984 | 2010-09-23 16:22:51 +0000 | [diff] [blame] | 202 | fileobj = _PaddedFile(fileobj) |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 203 | |
Nadeem Vawda | be66af4 | 2012-02-12 00:06:02 +0200 | [diff] [blame] | 204 | elif mode.startswith(('w', 'a')): |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 205 | self.mode = WRITE |
| 206 | self._init_write(filename) |
| 207 | self.compress = zlib.compressobj(compresslevel, |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 208 | zlib.DEFLATED, |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 209 | -zlib.MAX_WBITS, |
| 210 | zlib.DEF_MEM_LEVEL, |
| 211 | 0) |
| 212 | else: |
Nadeem Vawda | be66af4 | 2012-02-12 00:06:02 +0200 | [diff] [blame] | 213 | raise ValueError("Invalid mode: {!r}".format(mode)) |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 214 | |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 215 | self.fileobj = fileobj |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 216 | self.offset = 0 |
Antoine Pitrou | 42db3ef | 2009-01-04 21:37:59 +0000 | [diff] [blame] | 217 | self.mtime = mtime |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 218 | |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 219 | if self.mode == WRITE: |
| 220 | self._write_gzip_header() |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 221 | |
Thomas Wouters | cf297e4 | 2007-02-23 15:07:44 +0000 | [diff] [blame] | 222 | @property |
| 223 | def filename(self): |
| 224 | import warnings |
Philip Jenvey | a394f2d | 2009-05-08 03:57:12 +0000 | [diff] [blame] | 225 | warnings.warn("use the name attribute", DeprecationWarning, 2) |
Thomas Wouters | cf297e4 | 2007-02-23 15:07:44 +0000 | [diff] [blame] | 226 | if self.mode == WRITE and self.name[-3:] != ".gz": |
| 227 | return self.name + ".gz" |
| 228 | return self.name |
| 229 | |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 230 | def __repr__(self): |
Antoine Pitrou | c3ed2e7 | 2010-09-29 10:49:46 +0000 | [diff] [blame] | 231 | fileobj = self.fileobj |
| 232 | if isinstance(fileobj, _PaddedFile): |
| 233 | fileobj = fileobj.file |
| 234 | s = repr(fileobj) |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 235 | return '<gzip ' + s[1:-1] + ' ' + hex(id(self)) + '>' |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 236 | |
Antoine Pitrou | 7980eaa | 2010-10-06 21:21:18 +0000 | [diff] [blame] | 237 | def _check_closed(self): |
| 238 | """Raises a ValueError if the underlying file object has been closed. |
| 239 | |
| 240 | """ |
| 241 | if self.closed: |
| 242 | raise ValueError('I/O operation on closed file.') |
| 243 | |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 244 | def _init_write(self, filename): |
Thomas Wouters | cf297e4 | 2007-02-23 15:07:44 +0000 | [diff] [blame] | 245 | self.name = filename |
Antoine Pitrou | 77b338b | 2009-12-14 18:00:06 +0000 | [diff] [blame] | 246 | self.crc = zlib.crc32(b"") & 0xffffffff |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 247 | self.size = 0 |
| 248 | self.writebuf = [] |
| 249 | self.bufsize = 0 |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 250 | |
| 251 | def _write_gzip_header(self): |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 252 | self.fileobj.write(b'\037\213') # magic header |
| 253 | self.fileobj.write(b'\010') # compression method |
Lars Gustäbel | 5590d8c | 2007-08-10 12:02:32 +0000 | [diff] [blame] | 254 | try: |
Lars Gustäbel | ead7056 | 2007-08-13 09:05:16 +0000 | [diff] [blame] | 255 | # RFC 1952 requires the FNAME field to be Latin-1. Do not |
| 256 | # include filenames that cannot be represented that way. |
Lars Gustäbel | 1440df2 | 2009-10-29 09:39:47 +0000 | [diff] [blame] | 257 | fname = os.path.basename(self.name) |
Nadeem Vawda | 103e811 | 2012-06-20 01:35:22 +0200 | [diff] [blame] | 258 | if not isinstance(fname, bytes): |
| 259 | fname = fname.encode('latin-1') |
Lars Gustäbel | ead7056 | 2007-08-13 09:05:16 +0000 | [diff] [blame] | 260 | if fname.endswith(b'.gz'): |
| 261 | fname = fname[:-3] |
Lars Gustäbel | 5590d8c | 2007-08-10 12:02:32 +0000 | [diff] [blame] | 262 | except UnicodeEncodeError: |
Lars Gustäbel | ead7056 | 2007-08-13 09:05:16 +0000 | [diff] [blame] | 263 | fname = b'' |
| 264 | flags = 0 |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 265 | if fname: |
| 266 | flags = FNAME |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 267 | self.fileobj.write(chr(flags).encode('latin-1')) |
Antoine Pitrou | 42db3ef | 2009-01-04 21:37:59 +0000 | [diff] [blame] | 268 | mtime = self.mtime |
| 269 | if mtime is None: |
| 270 | mtime = time.time() |
| 271 | write32u(self.fileobj, int(mtime)) |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 272 | self.fileobj.write(b'\002') |
| 273 | self.fileobj.write(b'\377') |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 274 | if fname: |
Lars Gustäbel | 5590d8c | 2007-08-10 12:02:32 +0000 | [diff] [blame] | 275 | self.fileobj.write(fname + b'\000') |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 276 | |
| 277 | def _init_read(self): |
Antoine Pitrou | 77b338b | 2009-12-14 18:00:06 +0000 | [diff] [blame] | 278 | self.crc = zlib.crc32(b"") & 0xffffffff |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 279 | self.size = 0 |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 280 | |
Serhiy Storchaka | 7c3922f | 2013-01-22 17:01:59 +0200 | [diff] [blame] | 281 | def _read_exact(self, n): |
| 282 | data = self.fileobj.read(n) |
| 283 | while len(data) < n: |
| 284 | b = self.fileobj.read(n - len(data)) |
| 285 | if not b: |
| 286 | raise EOFError("Compressed file ended before the " |
| 287 | "end-of-stream marker was reached") |
| 288 | data += b |
| 289 | return data |
| 290 | |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 291 | def _read_gzip_header(self): |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 292 | magic = self.fileobj.read(2) |
Antoine Pitrou | 7b96984 | 2010-09-23 16:22:51 +0000 | [diff] [blame] | 293 | if magic == b'': |
Serhiy Storchaka | 7c3922f | 2013-01-22 17:01:59 +0200 | [diff] [blame] | 294 | return False |
Antoine Pitrou | 7b96984 | 2010-09-23 16:22:51 +0000 | [diff] [blame] | 295 | |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 296 | if magic != b'\037\213': |
Andrew Svetlov | f7a17b4 | 2012-12-25 16:47:37 +0200 | [diff] [blame] | 297 | raise OSError('Not a gzipped file') |
Serhiy Storchaka | 7c3922f | 2013-01-22 17:01:59 +0200 | [diff] [blame] | 298 | |
| 299 | method, flag, self.mtime = struct.unpack("<BBIxx", self._read_exact(8)) |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 300 | if method != 8: |
Andrew Svetlov | f7a17b4 | 2012-12-25 16:47:37 +0200 | [diff] [blame] | 301 | raise OSError('Unknown compression method') |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 302 | |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 303 | if flag & FEXTRA: |
| 304 | # Read & discard the extra field, if present |
Serhiy Storchaka | 7e69f00 | 2013-04-08 22:35:02 +0300 | [diff] [blame] | 305 | extra_len, = struct.unpack("<H", self._read_exact(2)) |
| 306 | self._read_exact(extra_len) |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 307 | if flag & FNAME: |
| 308 | # Read and discard a null-terminated string containing the filename |
Guido van Rossum | 8ca162f | 2002-04-07 06:36:23 +0000 | [diff] [blame] | 309 | while True: |
Tim Peters | fb0ea52 | 2002-11-04 19:50:11 +0000 | [diff] [blame] | 310 | s = self.fileobj.read(1) |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 311 | if not s or s==b'\000': |
Tim Peters | fb0ea52 | 2002-11-04 19:50:11 +0000 | [diff] [blame] | 312 | break |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 313 | if flag & FCOMMENT: |
| 314 | # Read and discard a null-terminated string containing a comment |
Guido van Rossum | 8ca162f | 2002-04-07 06:36:23 +0000 | [diff] [blame] | 315 | while True: |
Tim Peters | fb0ea52 | 2002-11-04 19:50:11 +0000 | [diff] [blame] | 316 | s = self.fileobj.read(1) |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 317 | if not s or s==b'\000': |
Tim Peters | fb0ea52 | 2002-11-04 19:50:11 +0000 | [diff] [blame] | 318 | break |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 319 | if flag & FHCRC: |
Serhiy Storchaka | 7c3922f | 2013-01-22 17:01:59 +0200 | [diff] [blame] | 320 | self._read_exact(2) # Read & discard the 16-bit header CRC |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 321 | |
Antoine Pitrou | 7b96984 | 2010-09-23 16:22:51 +0000 | [diff] [blame] | 322 | unused = self.fileobj.unused() |
| 323 | if unused: |
| 324 | uncompress = self.decompress.decompress(unused) |
| 325 | self._add_read_data(uncompress) |
Serhiy Storchaka | 7c3922f | 2013-01-22 17:01:59 +0200 | [diff] [blame] | 326 | return True |
Antoine Pitrou | 7b96984 | 2010-09-23 16:22:51 +0000 | [diff] [blame] | 327 | |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 328 | def write(self,data): |
Antoine Pitrou | 7980eaa | 2010-10-06 21:21:18 +0000 | [diff] [blame] | 329 | self._check_closed() |
Martin v. Löwis | db04489 | 2002-03-11 06:46:52 +0000 | [diff] [blame] | 330 | if self.mode != WRITE: |
| 331 | import errno |
Andrew Svetlov | f7a17b4 | 2012-12-25 16:47:37 +0200 | [diff] [blame] | 332 | raise OSError(errno.EBADF, "write() on read-only GzipFile object") |
Tim Peters | 863ac44 | 2002-04-16 01:38:40 +0000 | [diff] [blame] | 333 | |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 334 | if self.fileobj is None: |
Collin Winter | ce36ad8 | 2007-08-30 01:19:48 +0000 | [diff] [blame] | 335 | raise ValueError("write() on closed GzipFile object") |
Antoine Pitrou | b1f8835 | 2010-01-03 22:37:40 +0000 | [diff] [blame] | 336 | |
| 337 | # Convert data type if called by io.BufferedWriter. |
| 338 | if isinstance(data, memoryview): |
| 339 | data = data.tobytes() |
| 340 | |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 341 | if len(data) > 0: |
| 342 | self.size = self.size + len(data) |
Christian Heimes | fe337bf | 2008-03-23 21:54:12 +0000 | [diff] [blame] | 343 | self.crc = zlib.crc32(data, self.crc) & 0xffffffff |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 344 | self.fileobj.write( self.compress.compress(data) ) |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 345 | self.offset += len(data) |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 346 | |
Antoine Pitrou | b1f8835 | 2010-01-03 22:37:40 +0000 | [diff] [blame] | 347 | return len(data) |
| 348 | |
Guido van Rossum | 5606801 | 2000-02-02 16:51:06 +0000 | [diff] [blame] | 349 | def read(self, size=-1): |
Antoine Pitrou | 7980eaa | 2010-10-06 21:21:18 +0000 | [diff] [blame] | 350 | self._check_closed() |
Martin v. Löwis | db04489 | 2002-03-11 06:46:52 +0000 | [diff] [blame] | 351 | if self.mode != READ: |
| 352 | import errno |
Andrew Svetlov | f7a17b4 | 2012-12-25 16:47:37 +0200 | [diff] [blame] | 353 | raise OSError(errno.EBADF, "read() on write-only GzipFile object") |
Tim Peters | 863ac44 | 2002-04-16 01:38:40 +0000 | [diff] [blame] | 354 | |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 355 | if self.extrasize <= 0 and self.fileobj is None: |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 356 | return b'' |
Guido van Rossum | b16a3b8 | 1998-01-27 19:29:45 +0000 | [diff] [blame] | 357 | |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 358 | readsize = 1024 |
Guido van Rossum | 5606801 | 2000-02-02 16:51:06 +0000 | [diff] [blame] | 359 | if size < 0: # get the whole thing |
Serhiy Storchaka | 7c3922f | 2013-01-22 17:01:59 +0200 | [diff] [blame] | 360 | while self._read(readsize): |
| 361 | readsize = min(self.max_read_chunk, readsize * 2) |
| 362 | size = self.extrasize |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 363 | else: # just get some more of it |
Serhiy Storchaka | 7c3922f | 2013-01-22 17:01:59 +0200 | [diff] [blame] | 364 | while size > self.extrasize: |
| 365 | if not self._read(readsize): |
| 366 | if size > self.extrasize: |
| 367 | size = self.extrasize |
| 368 | break |
| 369 | readsize = min(self.max_read_chunk, readsize * 2) |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 370 | |
Antoine Pitrou | b1f8835 | 2010-01-03 22:37:40 +0000 | [diff] [blame] | 371 | offset = self.offset - self.extrastart |
| 372 | chunk = self.extrabuf[offset: offset + size] |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 373 | self.extrasize = self.extrasize - size |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 374 | |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 375 | self.offset += size |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 376 | return chunk |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 377 | |
Antoine Pitrou | 4ec4b0c | 2011-04-04 21:00:37 +0200 | [diff] [blame] | 378 | def read1(self, size=-1): |
| 379 | self._check_closed() |
| 380 | if self.mode != READ: |
| 381 | import errno |
Andrew Svetlov | f7a17b4 | 2012-12-25 16:47:37 +0200 | [diff] [blame] | 382 | raise OSError(errno.EBADF, "read1() on write-only GzipFile object") |
Antoine Pitrou | 4ec4b0c | 2011-04-04 21:00:37 +0200 | [diff] [blame] | 383 | |
| 384 | if self.extrasize <= 0 and self.fileobj is None: |
| 385 | return b'' |
| 386 | |
Serhiy Storchaka | 57f9b7a | 2013-01-22 17:07:49 +0200 | [diff] [blame] | 387 | # For certain input data, a single call to _read() may not return |
| 388 | # any data. In this case, retry until we get some data or reach EOF. |
| 389 | while self.extrasize <= 0 and self._read(): |
Antoine Pitrou | 4ec4b0c | 2011-04-04 21:00:37 +0200 | [diff] [blame] | 390 | pass |
| 391 | if size < 0 or size > self.extrasize: |
| 392 | size = self.extrasize |
| 393 | |
| 394 | offset = self.offset - self.extrastart |
| 395 | chunk = self.extrabuf[offset: offset + size] |
| 396 | self.extrasize -= size |
| 397 | self.offset += size |
| 398 | return chunk |
| 399 | |
Antoine Pitrou | c3ed2e7 | 2010-09-29 10:49:46 +0000 | [diff] [blame] | 400 | def peek(self, n): |
| 401 | if self.mode != READ: |
| 402 | import errno |
Andrew Svetlov | f7a17b4 | 2012-12-25 16:47:37 +0200 | [diff] [blame] | 403 | raise OSError(errno.EBADF, "peek() on write-only GzipFile object") |
Antoine Pitrou | c3ed2e7 | 2010-09-29 10:49:46 +0000 | [diff] [blame] | 404 | |
Antoine Pitrou | 7b998e9 | 2010-10-04 21:55:14 +0000 | [diff] [blame] | 405 | # Do not return ridiculously small buffers, for one common idiom |
| 406 | # is to call peek(1) and expect more bytes in return. |
Antoine Pitrou | c3ed2e7 | 2010-09-29 10:49:46 +0000 | [diff] [blame] | 407 | if n < 100: |
| 408 | n = 100 |
| 409 | if self.extrasize == 0: |
| 410 | if self.fileobj is None: |
| 411 | return b'' |
Serhiy Storchaka | 7c3922f | 2013-01-22 17:01:59 +0200 | [diff] [blame] | 412 | # Ensure that we don't return b"" if we haven't reached EOF. |
| 413 | # 1024 is the same buffering heuristic used in read() |
| 414 | while self.extrasize == 0 and self._read(max(n, 1024)): |
Antoine Pitrou | c3ed2e7 | 2010-09-29 10:49:46 +0000 | [diff] [blame] | 415 | pass |
| 416 | offset = self.offset - self.extrastart |
| 417 | remaining = self.extrasize |
| 418 | assert remaining == len(self.extrabuf) - offset |
| 419 | return self.extrabuf[offset:offset + n] |
| 420 | |
Guido van Rossum | b16a3b8 | 1998-01-27 19:29:45 +0000 | [diff] [blame] | 421 | def _unread(self, buf): |
Guido van Rossum | 84c6fc9 | 1998-08-03 15:41:39 +0000 | [diff] [blame] | 422 | self.extrasize = len(buf) + self.extrasize |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 423 | self.offset -= len(buf) |
Guido van Rossum | b16a3b8 | 1998-01-27 19:29:45 +0000 | [diff] [blame] | 424 | |
| 425 | def _read(self, size=1024): |
Tim Peters | fb0ea52 | 2002-11-04 19:50:11 +0000 | [diff] [blame] | 426 | if self.fileobj is None: |
Serhiy Storchaka | 7c3922f | 2013-01-22 17:01:59 +0200 | [diff] [blame] | 427 | return False |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 428 | |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 429 | if self._new_member: |
Andrew M. Kuchling | 41616ee | 2000-07-29 20:15:26 +0000 | [diff] [blame] | 430 | # If the _new_member flag is set, we have to |
| 431 | # jump to the next member, if there is one. |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 432 | self._init_read() |
Serhiy Storchaka | 7c3922f | 2013-01-22 17:01:59 +0200 | [diff] [blame] | 433 | if not self._read_gzip_header(): |
| 434 | return False |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 435 | self.decompress = zlib.decompressobj(-zlib.MAX_WBITS) |
Guido van Rossum | 8ca162f | 2002-04-07 06:36:23 +0000 | [diff] [blame] | 436 | self._new_member = False |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 437 | |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 438 | # Read a chunk of data from the file |
| 439 | buf = self.fileobj.read(size) |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 440 | |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 441 | # If the EOF has been reached, flush the decompression object |
| 442 | # and mark this object as finished. |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 443 | |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 444 | if buf == b"": |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 445 | uncompress = self.decompress.flush() |
Antoine Pitrou | 7b96984 | 2010-09-23 16:22:51 +0000 | [diff] [blame] | 446 | # Prepend the already read bytes to the fileobj to they can be |
| 447 | # seen by _read_eof() |
| 448 | self.fileobj.prepend(self.decompress.unused_data, True) |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 449 | self._read_eof() |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 450 | self._add_read_data( uncompress ) |
Serhiy Storchaka | 7c3922f | 2013-01-22 17:01:59 +0200 | [diff] [blame] | 451 | return False |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 452 | |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 453 | uncompress = self.decompress.decompress(buf) |
| 454 | self._add_read_data( uncompress ) |
| 455 | |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 456 | if self.decompress.unused_data != b"": |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 457 | # Ending case: we've come to the end of a member in the file, |
| 458 | # so seek back to the start of the unused data, finish up |
| 459 | # this member, and read a new gzip header. |
Antoine Pitrou | 7b96984 | 2010-09-23 16:22:51 +0000 | [diff] [blame] | 460 | # Prepend the already read bytes to the fileobj to they can be |
| 461 | # seen by _read_eof() and _read_gzip_header() |
| 462 | self.fileobj.prepend(self.decompress.unused_data, True) |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 463 | # Check the CRC and file size, and set the flag so we read |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 464 | # a new member on the next call |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 465 | self._read_eof() |
Guido van Rossum | 8ca162f | 2002-04-07 06:36:23 +0000 | [diff] [blame] | 466 | self._new_member = True |
Serhiy Storchaka | 7c3922f | 2013-01-22 17:01:59 +0200 | [diff] [blame] | 467 | return True |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 468 | |
| 469 | def _add_read_data(self, data): |
Christian Heimes | fe337bf | 2008-03-23 21:54:12 +0000 | [diff] [blame] | 470 | self.crc = zlib.crc32(data, self.crc) & 0xffffffff |
Antoine Pitrou | b1f8835 | 2010-01-03 22:37:40 +0000 | [diff] [blame] | 471 | offset = self.offset - self.extrastart |
| 472 | self.extrabuf = self.extrabuf[offset:] + data |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 473 | self.extrasize = self.extrasize + len(data) |
Antoine Pitrou | b1f8835 | 2010-01-03 22:37:40 +0000 | [diff] [blame] | 474 | self.extrastart = self.offset |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 475 | self.size = self.size + len(data) |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 476 | |
| 477 | def _read_eof(self): |
Antoine Pitrou | 7b96984 | 2010-09-23 16:22:51 +0000 | [diff] [blame] | 478 | # We've read to the end of the file |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 479 | # We check the that the computed CRC and size of the |
Tim Peters | 9288f95 | 2002-11-05 20:38:55 +0000 | [diff] [blame] | 480 | # uncompressed data matches the stored values. Note that the size |
| 481 | # stored is the true file size mod 2**32. |
Serhiy Storchaka | 7c3922f | 2013-01-22 17:01:59 +0200 | [diff] [blame] | 482 | crc32, isize = struct.unpack("<II", self._read_exact(8)) |
Christian Heimes | fe337bf | 2008-03-23 21:54:12 +0000 | [diff] [blame] | 483 | if crc32 != self.crc: |
Andrew Svetlov | f7a17b4 | 2012-12-25 16:47:37 +0200 | [diff] [blame] | 484 | raise OSError("CRC check failed %s != %s" % (hex(crc32), |
Christian Heimes | fe337bf | 2008-03-23 21:54:12 +0000 | [diff] [blame] | 485 | hex(self.crc))) |
Christian Heimes | 1dc5400 | 2008-03-24 02:19:29 +0000 | [diff] [blame] | 486 | elif isize != (self.size & 0xffffffff): |
Andrew Svetlov | f7a17b4 | 2012-12-25 16:47:37 +0200 | [diff] [blame] | 487 | raise OSError("Incorrect length of data produced") |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 488 | |
Antoine Pitrou | 8e33fd7 | 2010-01-13 14:37:26 +0000 | [diff] [blame] | 489 | # Gzip files can be padded with zeroes and still have archives. |
| 490 | # Consume all zero bytes and set the file position to the first |
| 491 | # non-zero byte. See http://www.gzip.org/#faq8 |
| 492 | c = b"\x00" |
| 493 | while c == b"\x00": |
| 494 | c = self.fileobj.read(1) |
| 495 | if c: |
Antoine Pitrou | 7b96984 | 2010-09-23 16:22:51 +0000 | [diff] [blame] | 496 | self.fileobj.prepend(c, True) |
Antoine Pitrou | 8e33fd7 | 2010-01-13 14:37:26 +0000 | [diff] [blame] | 497 | |
Antoine Pitrou | b1f8835 | 2010-01-03 22:37:40 +0000 | [diff] [blame] | 498 | @property |
| 499 | def closed(self): |
| 500 | return self.fileobj is None |
| 501 | |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 502 | def close(self): |
Georg Brandl | b533e26 | 2008-05-25 18:19:30 +0000 | [diff] [blame] | 503 | if self.fileobj is None: |
| 504 | return |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 505 | if self.mode == WRITE: |
| 506 | self.fileobj.write(self.compress.flush()) |
Christian Heimes | fe337bf | 2008-03-23 21:54:12 +0000 | [diff] [blame] | 507 | write32u(self.fileobj, self.crc) |
Tim Peters | 9288f95 | 2002-11-05 20:38:55 +0000 | [diff] [blame] | 508 | # self.size may exceed 2GB, or even 4GB |
Christian Heimes | 1dc5400 | 2008-03-24 02:19:29 +0000 | [diff] [blame] | 509 | write32u(self.fileobj, self.size & 0xffffffff) |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 510 | self.fileobj = None |
| 511 | elif self.mode == READ: |
| 512 | self.fileobj = None |
| 513 | if self.myfileobj: |
| 514 | self.myfileobj.close() |
| 515 | self.myfileobj = None |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 516 | |
Martin v. Löwis | f2a8d63 | 2005-03-03 08:35:22 +0000 | [diff] [blame] | 517 | def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH): |
Antoine Pitrou | 7980eaa | 2010-10-06 21:21:18 +0000 | [diff] [blame] | 518 | self._check_closed() |
Martin v. Löwis | f2a8d63 | 2005-03-03 08:35:22 +0000 | [diff] [blame] | 519 | if self.mode == WRITE: |
Tim Peters | eba28be | 2005-03-28 01:08:02 +0000 | [diff] [blame] | 520 | # Ensure the compressor's buffer is flushed |
| 521 | self.fileobj.write(self.compress.flush(zlib_mode)) |
Mark Dickinson | a9eb87a | 2010-05-04 18:47:04 +0000 | [diff] [blame] | 522 | self.fileobj.flush() |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 523 | |
Tim Peters | 5cfb05e | 2004-07-27 21:02:02 +0000 | [diff] [blame] | 524 | def fileno(self): |
| 525 | """Invoke the underlying file object's fileno() method. |
| 526 | |
| 527 | This will raise AttributeError if the underlying file object |
| 528 | doesn't support fileno(). |
| 529 | """ |
| 530 | return self.fileobj.fileno() |
| 531 | |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 532 | def rewind(self): |
| 533 | '''Return the uncompressed stream file position indicator to the |
Tim Peters | ab9ba27 | 2001-08-09 21:40:30 +0000 | [diff] [blame] | 534 | beginning of the file''' |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 535 | if self.mode != READ: |
Andrew Svetlov | f7a17b4 | 2012-12-25 16:47:37 +0200 | [diff] [blame] | 536 | raise OSError("Can't rewind in write mode") |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 537 | self.fileobj.seek(0) |
Guido van Rossum | 8ca162f | 2002-04-07 06:36:23 +0000 | [diff] [blame] | 538 | self._new_member = True |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 539 | self.extrabuf = b"" |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 540 | self.extrasize = 0 |
Antoine Pitrou | b1f8835 | 2010-01-03 22:37:40 +0000 | [diff] [blame] | 541 | self.extrastart = 0 |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 542 | self.offset = 0 |
| 543 | |
Antoine Pitrou | b1f8835 | 2010-01-03 22:37:40 +0000 | [diff] [blame] | 544 | def readable(self): |
| 545 | return self.mode == READ |
| 546 | |
| 547 | def writable(self): |
| 548 | return self.mode == WRITE |
| 549 | |
| 550 | def seekable(self): |
| 551 | return True |
| 552 | |
Thomas Wouters | 89f507f | 2006-12-13 04:49:30 +0000 | [diff] [blame] | 553 | def seek(self, offset, whence=0): |
| 554 | if whence: |
| 555 | if whence == 1: |
| 556 | offset = self.offset + offset |
| 557 | else: |
| 558 | raise ValueError('Seek from end not supported') |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 559 | if self.mode == WRITE: |
| 560 | if offset < self.offset: |
Andrew Svetlov | f7a17b4 | 2012-12-25 16:47:37 +0200 | [diff] [blame] | 561 | raise OSError('Negative seek in write mode') |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 562 | count = offset - self.offset |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 563 | chunk = bytes(1024) |
Tim Peters | fb0ea52 | 2002-11-04 19:50:11 +0000 | [diff] [blame] | 564 | for i in range(count // 1024): |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 565 | self.write(chunk) |
| 566 | self.write(bytes(count % 1024)) |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 567 | elif self.mode == READ: |
| 568 | if offset < self.offset: |
| 569 | # for negative seek, rewind and do positive seek |
| 570 | self.rewind() |
| 571 | count = offset - self.offset |
Tim Peters | fb0ea52 | 2002-11-04 19:50:11 +0000 | [diff] [blame] | 572 | for i in range(count // 1024): |
| 573 | self.read(1024) |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 574 | self.read(count % 1024) |
| 575 | |
Antoine Pitrou | b1f8835 | 2010-01-03 22:37:40 +0000 | [diff] [blame] | 576 | return self.offset |
| 577 | |
Andrew M. Kuchling | 41616ee | 2000-07-29 20:15:26 +0000 | [diff] [blame] | 578 | def readline(self, size=-1): |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 579 | if size < 0: |
Antoine Pitrou | b1f8835 | 2010-01-03 22:37:40 +0000 | [diff] [blame] | 580 | # Shortcut common case - newline found in buffer. |
| 581 | offset = self.offset - self.extrastart |
| 582 | i = self.extrabuf.find(b'\n', offset) + 1 |
| 583 | if i > 0: |
| 584 | self.extrasize -= i - offset |
| 585 | self.offset += i - offset |
| 586 | return self.extrabuf[offset: i] |
| 587 | |
Christian Heimes | a37d4c6 | 2007-12-04 23:02:19 +0000 | [diff] [blame] | 588 | size = sys.maxsize |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 589 | readsize = self.min_readsize |
| 590 | else: |
| 591 | readsize = size |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 592 | bufs = [] |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 593 | while size != 0: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 594 | c = self.read(readsize) |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 595 | i = c.find(b'\n') |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 596 | |
| 597 | # We set i=size to break out of the loop under two |
| 598 | # conditions: 1) there's no newline, and the chunk is |
| 599 | # larger than size, or 2) there is a newline, but the |
| 600 | # resulting line would be longer than 'size'. |
| 601 | if (size <= i) or (i == -1 and len(c) > size): |
| 602 | i = size - 1 |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 603 | |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 604 | if i >= 0 or c == b'': |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 605 | bufs.append(c[:i + 1]) # Add portion of last chunk |
| 606 | self._unread(c[i + 1:]) # Push back rest of chunk |
| 607 | break |
Andrew M. Kuchling | 41616ee | 2000-07-29 20:15:26 +0000 | [diff] [blame] | 608 | |
| 609 | # Append chunk to list, decrease 'size', |
| 610 | bufs.append(c) |
| 611 | size = size - len(c) |
| 612 | readsize = min(size, readsize * 2) |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 613 | if readsize > self.min_readsize: |
| 614 | self.min_readsize = min(readsize, self.min_readsize * 2, 512) |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 615 | return b''.join(bufs) # Return resulting line |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 616 | |
Guido van Rossum | 51ca6e3 | 1997-12-30 20:09:08 +0000 | [diff] [blame] | 617 | |
Antoine Pitrou | 79c5ef1 | 2010-08-17 21:10:05 +0000 | [diff] [blame] | 618 | def compress(data, compresslevel=9): |
| 619 | """Compress data in one shot and return the compressed string. |
Nadeem Vawda | 19e568d | 2012-11-11 14:04:14 +0100 | [diff] [blame] | 620 | Optional argument is the compression level, in range of 0-9. |
Antoine Pitrou | 79c5ef1 | 2010-08-17 21:10:05 +0000 | [diff] [blame] | 621 | """ |
| 622 | buf = io.BytesIO() |
| 623 | with GzipFile(fileobj=buf, mode='wb', compresslevel=compresslevel) as f: |
| 624 | f.write(data) |
| 625 | return buf.getvalue() |
| 626 | |
| 627 | def decompress(data): |
| 628 | """Decompress a gzip compressed string in one shot. |
| 629 | Return the decompressed string. |
| 630 | """ |
| 631 | with GzipFile(fileobj=io.BytesIO(data)) as f: |
| 632 | return f.read() |
| 633 | |
| 634 | |
Guido van Rossum | 51ca6e3 | 1997-12-30 20:09:08 +0000 | [diff] [blame] | 635 | def _test(): |
| 636 | # Act like gzip; with -d, act like gunzip. |
| 637 | # The input file is not deleted, however, nor are any other gzip |
| 638 | # options or features supported. |
Guido van Rossum | 51ca6e3 | 1997-12-30 20:09:08 +0000 | [diff] [blame] | 639 | args = sys.argv[1:] |
| 640 | decompress = args and args[0] == "-d" |
| 641 | if decompress: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 642 | args = args[1:] |
Guido van Rossum | 51ca6e3 | 1997-12-30 20:09:08 +0000 | [diff] [blame] | 643 | if not args: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 644 | args = ["-"] |
Guido van Rossum | 51ca6e3 | 1997-12-30 20:09:08 +0000 | [diff] [blame] | 645 | for arg in args: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 646 | if decompress: |
| 647 | if arg == "-": |
Antoine Pitrou | 9d625c2 | 2009-01-04 21:11:10 +0000 | [diff] [blame] | 648 | f = GzipFile(filename="", mode="rb", fileobj=sys.stdin.buffer) |
| 649 | g = sys.stdout.buffer |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 650 | else: |
| 651 | if arg[-3:] != ".gz": |
Guido van Rossum | be19ed7 | 2007-02-09 05:37:30 +0000 | [diff] [blame] | 652 | print("filename doesn't end in .gz:", repr(arg)) |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 653 | continue |
| 654 | f = open(arg, "rb") |
Georg Brandl | 1a3284e | 2007-12-02 09:40:06 +0000 | [diff] [blame] | 655 | g = builtins.open(arg[:-3], "wb") |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 656 | else: |
| 657 | if arg == "-": |
Antoine Pitrou | 9d625c2 | 2009-01-04 21:11:10 +0000 | [diff] [blame] | 658 | f = sys.stdin.buffer |
| 659 | g = GzipFile(filename="", mode="wb", fileobj=sys.stdout.buffer) |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 660 | else: |
Georg Brandl | 1a3284e | 2007-12-02 09:40:06 +0000 | [diff] [blame] | 661 | f = builtins.open(arg, "rb") |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 662 | g = open(arg + ".gz", "wb") |
Guido van Rossum | 8ca162f | 2002-04-07 06:36:23 +0000 | [diff] [blame] | 663 | while True: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 664 | chunk = f.read(1024) |
| 665 | if not chunk: |
| 666 | break |
| 667 | g.write(chunk) |
Antoine Pitrou | ecc4757 | 2012-08-30 00:29:24 +0200 | [diff] [blame] | 668 | if g is not sys.stdout.buffer: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 669 | g.close() |
Antoine Pitrou | ecc4757 | 2012-08-30 00:29:24 +0200 | [diff] [blame] | 670 | if f is not sys.stdin.buffer: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 671 | f.close() |
Guido van Rossum | 51ca6e3 | 1997-12-30 20:09:08 +0000 | [diff] [blame] | 672 | |
| 673 | if __name__ == '__main__': |
| 674 | _test() |