Guido van Rossum | 4b8c6ea | 2000-02-04 15:39:30 +0000 | [diff] [blame] | 1 | """Functions that read and write gzipped files. |
| 2 | |
Guido van Rossum | 54f22ed | 2000-02-04 15:10:34 +0000 | [diff] [blame] | 3 | The user of the file doesn't have to worry about the compression, |
| 4 | but random access is not allowed.""" |
| 5 | |
| 6 | # based on Andrew Kuchling's minigzip.py distributed with the zlib module |
| 7 | |
Lars Gustäbel | 8c06ccc | 2009-10-29 09:15:00 +0000 | [diff] [blame] | 8 | import struct, sys, time, os |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 9 | import zlib |
Antoine Pitrou | 673ddf9 | 2010-01-03 22:29:56 +0000 | [diff] [blame] | 10 | import io |
Guido van Rossum | 68de379 | 1997-07-19 20:22:23 +0000 | [diff] [blame] | 11 | import __builtin__ |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 12 | |
Skip Montanaro | 2dd4276 | 2001-01-23 15:35:05 +0000 | [diff] [blame] | 13 | __all__ = ["GzipFile","open"] |
| 14 | |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 15 | FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16 |
| 16 | |
| 17 | READ, WRITE = 1, 2 |
| 18 | |
Guido van Rossum | 95bdd0b | 1999-04-12 14:34:16 +0000 | [diff] [blame] | 19 | def write32u(output, value): |
Tim Peters | fb0ea52 | 2002-11-04 19:50:11 +0000 | [diff] [blame] | 20 | # The L format writes the bit pattern correctly whether signed |
| 21 | # or unsigned. |
Guido van Rossum | 95bdd0b | 1999-04-12 14:34:16 +0000 | [diff] [blame] | 22 | output.write(struct.pack("<L", value)) |
| 23 | |
Benjamin Peterson | 6e165b4 | 2013-05-11 13:17:13 -0500 | [diff] [blame] | 24 | def read32(input): |
| 25 | return struct.unpack("<I", input.read(4))[0] |
| 26 | |
Fred Drake | fa1591c | 1999-04-05 18:37:59 +0000 | [diff] [blame] | 27 | def open(filename, mode="rb", compresslevel=9): |
Raymond Hettinger | aef22fb | 2002-05-29 16:18:42 +0000 | [diff] [blame] | 28 | """Shorthand for GzipFile(filename, mode, compresslevel). |
| 29 | |
| 30 | The filename argument is required; mode defaults to 'rb' |
| 31 | and compresslevel defaults to 9. |
| 32 | |
| 33 | """ |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 34 | return GzipFile(filename, mode, compresslevel) |
| 35 | |
Antoine Pitrou | 673ddf9 | 2010-01-03 22:29:56 +0000 | [diff] [blame] | 36 | class GzipFile(io.BufferedIOBase): |
Raymond Hettinger | aef22fb | 2002-05-29 16:18:42 +0000 | [diff] [blame] | 37 | """The GzipFile class simulates most of the methods of a file object with |
Guido van Rossum | 97c5fcc | 2002-08-06 17:03:25 +0000 | [diff] [blame] | 38 | the exception of the readinto() and truncate() methods. |
Raymond Hettinger | aef22fb | 2002-05-29 16:18:42 +0000 | [diff] [blame] | 39 | |
| 40 | """ |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 41 | |
Guido van Rossum | 68de379 | 1997-07-19 20:22:23 +0000 | [diff] [blame] | 42 | myfileobj = None |
Andrew M. Kuchling | 01cb47b | 2005-06-09 14:19:32 +0000 | [diff] [blame] | 43 | max_read_chunk = 10 * 1024 * 1024 # 10Mb |
Guido van Rossum | 68de379 | 1997-07-19 20:22:23 +0000 | [diff] [blame] | 44 | |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 45 | def __init__(self, filename=None, mode=None, |
Antoine Pitrou | f0d2c3f | 2009-01-04 21:29:23 +0000 | [diff] [blame] | 46 | compresslevel=9, fileobj=None, mtime=None): |
Raymond Hettinger | aef22fb | 2002-05-29 16:18:42 +0000 | [diff] [blame] | 47 | """Constructor for the GzipFile class. |
| 48 | |
| 49 | At least one of fileobj and filename must be given a |
| 50 | non-trivial value. |
| 51 | |
| 52 | The new class instance is based on fileobj, which can be a regular |
| 53 | file, a StringIO object, or any other object which simulates a file. |
| 54 | It defaults to None, in which case filename is opened to provide |
| 55 | a file object. |
| 56 | |
| 57 | When fileobj is not None, the filename argument is only used to be |
Martin Panter | 5b48fa9 | 2016-04-19 04:03:41 +0000 | [diff] [blame] | 58 | included in the gzip file header, which may include the original |
Raymond Hettinger | aef22fb | 2002-05-29 16:18:42 +0000 | [diff] [blame] | 59 | filename of the uncompressed file. It defaults to the filename of |
| 60 | fileobj, if discernible; otherwise, it defaults to the empty string, |
| 61 | and in this case the original filename is not included in the header. |
| 62 | |
| 63 | The mode argument can be any of 'r', 'rb', 'a', 'ab', 'w', or 'wb', |
| 64 | depending on whether the file will be read or written. The default |
| 65 | is the mode of fileobj if discernible; otherwise, the default is 'rb'. |
| 66 | Be aware that only the 'rb', 'ab', and 'wb' values should be used |
| 67 | for cross-platform portability. |
| 68 | |
Nadeem Vawda | 04050b8 | 2012-11-11 13:52:10 +0100 | [diff] [blame] | 69 | The compresslevel argument is an integer from 0 to 9 controlling the |
Raymond Hettinger | aef22fb | 2002-05-29 16:18:42 +0000 | [diff] [blame] | 70 | level of compression; 1 is fastest and produces the least compression, |
Nadeem Vawda | 04050b8 | 2012-11-11 13:52:10 +0100 | [diff] [blame] | 71 | and 9 is slowest and produces the most compression. 0 is no compression |
| 72 | at all. The default is 9. |
Raymond Hettinger | aef22fb | 2002-05-29 16:18:42 +0000 | [diff] [blame] | 73 | |
Antoine Pitrou | f0d2c3f | 2009-01-04 21:29:23 +0000 | [diff] [blame] | 74 | The mtime argument is an optional numeric timestamp to be written |
| 75 | to the stream when compressing. All gzip compressed streams |
| 76 | are required to contain a timestamp. If omitted or None, the |
| 77 | current time is used. This module ignores the timestamp when |
| 78 | decompressing; however, some programs, such as gunzip, make use |
| 79 | of it. The format of the timestamp is the same as that of the |
| 80 | return value of time.time() and of the st_mtime member of the |
| 81 | object returned by os.stat(). |
| 82 | |
Raymond Hettinger | aef22fb | 2002-05-29 16:18:42 +0000 | [diff] [blame] | 83 | """ |
| 84 | |
Nadeem Vawda | dd72b3f | 2012-10-21 18:15:05 +0200 | [diff] [blame] | 85 | # Make sure we don't inadvertently enable universal newlines on the |
| 86 | # underlying file object - in read mode, this causes data corruption. |
| 87 | if mode: |
| 88 | mode = mode.replace('U', '') |
Skip Montanaro | 12424bc | 2002-05-23 01:43:05 +0000 | [diff] [blame] | 89 | # guarantee the file is opened in binary mode on platforms |
| 90 | # that care about that sort of thing |
| 91 | if mode and 'b' not in mode: |
| 92 | mode += 'b' |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 93 | if fileobj is None: |
Fred Drake | 9bb76d1 | 1999-04-05 18:33:40 +0000 | [diff] [blame] | 94 | fileobj = self.myfileobj = __builtin__.open(filename, mode or 'rb') |
Guido van Rossum | 68de379 | 1997-07-19 20:22:23 +0000 | [diff] [blame] | 95 | if filename is None: |
Nadeem Vawda | d7664de | 2012-01-19 00:40:46 +0200 | [diff] [blame] | 96 | # Issue #13781: os.fdopen() creates a fileobj with a bogus name |
| 97 | # attribute. Avoid saving this in the gzip header's filename field. |
| 98 | if hasattr(fileobj, 'name') and fileobj.name != '<fdopen>': |
| 99 | filename = fileobj.name |
| 100 | else: |
| 101 | filename = '' |
Guido van Rossum | 68de379 | 1997-07-19 20:22:23 +0000 | [diff] [blame] | 102 | if mode is None: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 103 | if hasattr(fileobj, 'mode'): mode = fileobj.mode |
Fred Drake | 9bb76d1 | 1999-04-05 18:33:40 +0000 | [diff] [blame] | 104 | else: mode = 'rb' |
Guido van Rossum | 68de379 | 1997-07-19 20:22:23 +0000 | [diff] [blame] | 105 | |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 106 | if mode[0:1] == 'r': |
| 107 | self.mode = READ |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 108 | # Set flag indicating start of a new member |
Guido van Rossum | 8ca162f | 2002-04-07 06:36:23 +0000 | [diff] [blame] | 109 | self._new_member = True |
Antoine Pitrou | 673ddf9 | 2010-01-03 22:29:56 +0000 | [diff] [blame] | 110 | # Buffer data read from gzip file. extrastart is offset in |
| 111 | # stream where buffer starts. extrasize is number of |
| 112 | # bytes remaining in buffer from current stream position. |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 113 | self.extrabuf = "" |
| 114 | self.extrasize = 0 |
Antoine Pitrou | 673ddf9 | 2010-01-03 22:29:56 +0000 | [diff] [blame] | 115 | self.extrastart = 0 |
Lars Gustäbel | 5b1a785 | 2007-02-13 16:09:24 +0000 | [diff] [blame] | 116 | self.name = filename |
Bob Ippolito | d82c310 | 2006-05-22 15:59:12 +0000 | [diff] [blame] | 117 | # Starts small, scales exponentially |
| 118 | self.min_readsize = 100 |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 119 | |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 120 | elif mode[0:1] == 'w' or mode[0:1] == 'a': |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 121 | self.mode = WRITE |
| 122 | self._init_write(filename) |
| 123 | self.compress = zlib.compressobj(compresslevel, |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 124 | zlib.DEFLATED, |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 125 | -zlib.MAX_WBITS, |
| 126 | zlib.DEF_MEM_LEVEL, |
| 127 | 0) |
| 128 | else: |
Martin v. Löwis | db04489 | 2002-03-11 06:46:52 +0000 | [diff] [blame] | 129 | raise IOError, "Mode " + mode + " not supported" |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 130 | |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 131 | self.fileobj = fileobj |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 132 | self.offset = 0 |
Antoine Pitrou | f0d2c3f | 2009-01-04 21:29:23 +0000 | [diff] [blame] | 133 | self.mtime = mtime |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 134 | |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 135 | if self.mode == WRITE: |
| 136 | self._write_gzip_header() |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 137 | |
Lars Gustäbel | 5b1a785 | 2007-02-13 16:09:24 +0000 | [diff] [blame] | 138 | @property |
| 139 | def filename(self): |
| 140 | import warnings |
Philip Jenvey | d846f1d | 2009-05-08 02:28:39 +0000 | [diff] [blame] | 141 | warnings.warn("use the name attribute", DeprecationWarning, 2) |
Lars Gustäbel | 5b1a785 | 2007-02-13 16:09:24 +0000 | [diff] [blame] | 142 | if self.mode == WRITE and self.name[-3:] != ".gz": |
| 143 | return self.name + ".gz" |
| 144 | return self.name |
| 145 | |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 146 | def __repr__(self): |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 147 | s = repr(self.fileobj) |
| 148 | return '<gzip ' + s[1:-1] + ' ' + hex(id(self)) + '>' |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 149 | |
Antoine Pitrou | 76a66aa | 2010-10-06 21:26:52 +0000 | [diff] [blame] | 150 | def _check_closed(self): |
| 151 | """Raises a ValueError if the underlying file object has been closed. |
| 152 | |
| 153 | """ |
| 154 | if self.closed: |
| 155 | raise ValueError('I/O operation on closed file.') |
| 156 | |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 157 | def _init_write(self, filename): |
Lars Gustäbel | 5b1a785 | 2007-02-13 16:09:24 +0000 | [diff] [blame] | 158 | self.name = filename |
Gregory P. Smith | 79b4ba8 | 2008-03-23 21:04:43 +0000 | [diff] [blame] | 159 | self.crc = zlib.crc32("") & 0xffffffffL |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 160 | self.size = 0 |
| 161 | self.writebuf = [] |
| 162 | self.bufsize = 0 |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 163 | |
| 164 | def _write_gzip_header(self): |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 165 | self.fileobj.write('\037\213') # magic header |
| 166 | self.fileobj.write('\010') # compression method |
Serhiy Storchaka | 54edfb3 | 2014-10-12 22:23:28 +0300 | [diff] [blame] | 167 | try: |
| 168 | # RFC 1952 requires the FNAME field to be Latin-1. Do not |
| 169 | # include filenames that cannot be represented that way. |
| 170 | fname = os.path.basename(self.name) |
| 171 | if not isinstance(fname, str): |
| 172 | fname = fname.encode('latin-1') |
| 173 | if fname.endswith('.gz'): |
| 174 | fname = fname[:-3] |
| 175 | except UnicodeEncodeError: |
| 176 | fname = '' |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 177 | flags = 0 |
Lars Gustäbel | f19c1b5 | 2007-02-13 16:24:00 +0000 | [diff] [blame] | 178 | if fname: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 179 | flags = FNAME |
| 180 | self.fileobj.write(chr(flags)) |
Antoine Pitrou | f0d2c3f | 2009-01-04 21:29:23 +0000 | [diff] [blame] | 181 | mtime = self.mtime |
| 182 | if mtime is None: |
| 183 | mtime = time.time() |
| 184 | write32u(self.fileobj, long(mtime)) |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 185 | self.fileobj.write('\002') |
| 186 | self.fileobj.write('\377') |
Lars Gustäbel | f19c1b5 | 2007-02-13 16:24:00 +0000 | [diff] [blame] | 187 | if fname: |
| 188 | self.fileobj.write(fname + '\000') |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 189 | |
| 190 | def _init_read(self): |
Gregory P. Smith | 79b4ba8 | 2008-03-23 21:04:43 +0000 | [diff] [blame] | 191 | self.crc = zlib.crc32("") & 0xffffffffL |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 192 | self.size = 0 |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 193 | |
| 194 | def _read_gzip_header(self): |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 195 | magic = self.fileobj.read(2) |
| 196 | if magic != '\037\213': |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 197 | raise IOError, 'Not a gzipped file' |
Benjamin Peterson | 6e165b4 | 2013-05-11 13:17:13 -0500 | [diff] [blame] | 198 | method = ord( self.fileobj.read(1) ) |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 199 | if method != 8: |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 200 | raise IOError, 'Unknown compression method' |
Benjamin Peterson | 6e165b4 | 2013-05-11 13:17:13 -0500 | [diff] [blame] | 201 | flag = ord( self.fileobj.read(1) ) |
| 202 | self.mtime = read32(self.fileobj) |
| 203 | # extraflag = self.fileobj.read(1) |
| 204 | # os = self.fileobj.read(1) |
| 205 | self.fileobj.read(2) |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 206 | |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 207 | if flag & FEXTRA: |
| 208 | # Read & discard the extra field, if present |
Benjamin Peterson | 6e165b4 | 2013-05-11 13:17:13 -0500 | [diff] [blame] | 209 | xlen = ord(self.fileobj.read(1)) |
| 210 | xlen = xlen + 256*ord(self.fileobj.read(1)) |
| 211 | self.fileobj.read(xlen) |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 212 | if flag & FNAME: |
| 213 | # Read and discard a null-terminated string containing the filename |
Guido van Rossum | 8ca162f | 2002-04-07 06:36:23 +0000 | [diff] [blame] | 214 | while True: |
Tim Peters | fb0ea52 | 2002-11-04 19:50:11 +0000 | [diff] [blame] | 215 | s = self.fileobj.read(1) |
| 216 | if not s or s=='\000': |
| 217 | break |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 218 | if flag & FCOMMENT: |
| 219 | # Read and discard a null-terminated string containing a comment |
Guido van Rossum | 8ca162f | 2002-04-07 06:36:23 +0000 | [diff] [blame] | 220 | while True: |
Tim Peters | fb0ea52 | 2002-11-04 19:50:11 +0000 | [diff] [blame] | 221 | s = self.fileobj.read(1) |
| 222 | if not s or s=='\000': |
| 223 | break |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 224 | if flag & FHCRC: |
Benjamin Peterson | 6e165b4 | 2013-05-11 13:17:13 -0500 | [diff] [blame] | 225 | self.fileobj.read(2) # Read & discard the 16-bit header CRC |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 226 | |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 227 | def write(self,data): |
Antoine Pitrou | 76a66aa | 2010-10-06 21:26:52 +0000 | [diff] [blame] | 228 | self._check_closed() |
Martin v. Löwis | db04489 | 2002-03-11 06:46:52 +0000 | [diff] [blame] | 229 | if self.mode != WRITE: |
| 230 | import errno |
| 231 | raise IOError(errno.EBADF, "write() on read-only GzipFile object") |
Tim Peters | 863ac44 | 2002-04-16 01:38:40 +0000 | [diff] [blame] | 232 | |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 233 | if self.fileobj is None: |
| 234 | raise ValueError, "write() on closed GzipFile object" |
Antoine Pitrou | 673ddf9 | 2010-01-03 22:29:56 +0000 | [diff] [blame] | 235 | |
| 236 | # Convert data type if called by io.BufferedWriter. |
| 237 | if isinstance(data, memoryview): |
| 238 | data = data.tobytes() |
| 239 | |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 240 | if len(data) > 0: |
Serhiy Storchaka | f689f10 | 2015-03-23 15:25:18 +0200 | [diff] [blame] | 241 | self.fileobj.write(self.compress.compress(data)) |
| 242 | self.size += len(data) |
Gregory P. Smith | 79b4ba8 | 2008-03-23 21:04:43 +0000 | [diff] [blame] | 243 | self.crc = zlib.crc32(data, self.crc) & 0xffffffffL |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 244 | self.offset += len(data) |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 245 | |
Antoine Pitrou | 673ddf9 | 2010-01-03 22:29:56 +0000 | [diff] [blame] | 246 | return len(data) |
| 247 | |
Guido van Rossum | 5606801 | 2000-02-02 16:51:06 +0000 | [diff] [blame] | 248 | def read(self, size=-1): |
Antoine Pitrou | 76a66aa | 2010-10-06 21:26:52 +0000 | [diff] [blame] | 249 | self._check_closed() |
Martin v. Löwis | db04489 | 2002-03-11 06:46:52 +0000 | [diff] [blame] | 250 | if self.mode != READ: |
| 251 | import errno |
Brett Cannon | edfb302 | 2003-12-04 19:28:06 +0000 | [diff] [blame] | 252 | raise IOError(errno.EBADF, "read() on write-only GzipFile object") |
Tim Peters | 863ac44 | 2002-04-16 01:38:40 +0000 | [diff] [blame] | 253 | |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 254 | if self.extrasize <= 0 and self.fileobj is None: |
| 255 | return '' |
Guido van Rossum | b16a3b8 | 1998-01-27 19:29:45 +0000 | [diff] [blame] | 256 | |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 257 | readsize = 1024 |
Guido van Rossum | 5606801 | 2000-02-02 16:51:06 +0000 | [diff] [blame] | 258 | if size < 0: # get the whole thing |
Benjamin Peterson | 6e165b4 | 2013-05-11 13:17:13 -0500 | [diff] [blame] | 259 | try: |
| 260 | while True: |
| 261 | self._read(readsize) |
| 262 | readsize = min(self.max_read_chunk, readsize * 2) |
| 263 | except EOFError: |
| 264 | size = self.extrasize |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 265 | else: # just get some more of it |
Benjamin Peterson | 6e165b4 | 2013-05-11 13:17:13 -0500 | [diff] [blame] | 266 | try: |
| 267 | while size > self.extrasize: |
| 268 | self._read(readsize) |
| 269 | readsize = min(self.max_read_chunk, readsize * 2) |
| 270 | except EOFError: |
| 271 | if size > self.extrasize: |
| 272 | size = self.extrasize |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 273 | |
Antoine Pitrou | 673ddf9 | 2010-01-03 22:29:56 +0000 | [diff] [blame] | 274 | offset = self.offset - self.extrastart |
| 275 | chunk = self.extrabuf[offset: offset + size] |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 276 | self.extrasize = self.extrasize - size |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 277 | |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 278 | self.offset += size |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 279 | return chunk |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 280 | |
Guido van Rossum | b16a3b8 | 1998-01-27 19:29:45 +0000 | [diff] [blame] | 281 | def _unread(self, buf): |
Guido van Rossum | 84c6fc9 | 1998-08-03 15:41:39 +0000 | [diff] [blame] | 282 | self.extrasize = len(buf) + self.extrasize |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 283 | self.offset -= len(buf) |
Guido van Rossum | b16a3b8 | 1998-01-27 19:29:45 +0000 | [diff] [blame] | 284 | |
| 285 | def _read(self, size=1024): |
Tim Peters | fb0ea52 | 2002-11-04 19:50:11 +0000 | [diff] [blame] | 286 | if self.fileobj is None: |
Benjamin Peterson | 6e165b4 | 2013-05-11 13:17:13 -0500 | [diff] [blame] | 287 | raise EOFError, "Reached EOF" |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 288 | |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 289 | if self._new_member: |
Andrew M. Kuchling | 41616ee | 2000-07-29 20:15:26 +0000 | [diff] [blame] | 290 | # If the _new_member flag is set, we have to |
| 291 | # jump to the next member, if there is one. |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 292 | # |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 293 | # First, check if we're at the end of the file; |
| 294 | # if so, it's time to stop; no more members to read. |
| 295 | pos = self.fileobj.tell() # Save current position |
| 296 | self.fileobj.seek(0, 2) # Seek to end of file |
| 297 | if pos == self.fileobj.tell(): |
Benjamin Peterson | 6e165b4 | 2013-05-11 13:17:13 -0500 | [diff] [blame] | 298 | raise EOFError, "Reached EOF" |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 299 | else: |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 300 | self.fileobj.seek( pos ) # Return to original position |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 301 | |
| 302 | self._init_read() |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 303 | self._read_gzip_header() |
| 304 | self.decompress = zlib.decompressobj(-zlib.MAX_WBITS) |
Guido van Rossum | 8ca162f | 2002-04-07 06:36:23 +0000 | [diff] [blame] | 305 | self._new_member = False |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 306 | |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 307 | # Read a chunk of data from the file |
| 308 | buf = self.fileobj.read(size) |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 309 | |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 310 | # If the EOF has been reached, flush the decompression object |
| 311 | # and mark this object as finished. |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 312 | |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 313 | if buf == "": |
| 314 | uncompress = self.decompress.flush() |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 315 | self._read_eof() |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 316 | self._add_read_data( uncompress ) |
Benjamin Peterson | 6e165b4 | 2013-05-11 13:17:13 -0500 | [diff] [blame] | 317 | raise EOFError, 'Reached EOF' |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 318 | |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 319 | uncompress = self.decompress.decompress(buf) |
| 320 | self._add_read_data( uncompress ) |
| 321 | |
| 322 | if self.decompress.unused_data != "": |
| 323 | # Ending case: we've come to the end of a member in the file, |
| 324 | # so seek back to the start of the unused data, finish up |
| 325 | # this member, and read a new gzip header. |
| 326 | # (The number of bytes to seek back is the length of the unused |
Benjamin Peterson | 6e165b4 | 2013-05-11 13:17:13 -0500 | [diff] [blame] | 327 | # data, minus 8 because _read_eof() will rewind a further 8 bytes) |
| 328 | self.fileobj.seek( -len(self.decompress.unused_data)+8, 1) |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 329 | |
| 330 | # Check the CRC and file size, and set the flag so we read |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 331 | # a new member on the next call |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 332 | self._read_eof() |
Guido van Rossum | 8ca162f | 2002-04-07 06:36:23 +0000 | [diff] [blame] | 333 | self._new_member = True |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 334 | |
| 335 | def _add_read_data(self, data): |
Gregory P. Smith | 79b4ba8 | 2008-03-23 21:04:43 +0000 | [diff] [blame] | 336 | self.crc = zlib.crc32(data, self.crc) & 0xffffffffL |
Antoine Pitrou | 673ddf9 | 2010-01-03 22:29:56 +0000 | [diff] [blame] | 337 | offset = self.offset - self.extrastart |
| 338 | self.extrabuf = self.extrabuf[offset:] + data |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 339 | self.extrasize = self.extrasize + len(data) |
Antoine Pitrou | 673ddf9 | 2010-01-03 22:29:56 +0000 | [diff] [blame] | 340 | self.extrastart = self.offset |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 341 | self.size = self.size + len(data) |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 342 | |
| 343 | def _read_eof(self): |
Benjamin Peterson | 6e165b4 | 2013-05-11 13:17:13 -0500 | [diff] [blame] | 344 | # We've read to the end of the file, so we have to rewind in order |
| 345 | # to reread the 8 bytes containing the CRC and the file size. |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 346 | # We check the that the computed CRC and size of the |
Tim Peters | 9288f95 | 2002-11-05 20:38:55 +0000 | [diff] [blame] | 347 | # uncompressed data matches the stored values. Note that the size |
| 348 | # stored is the true file size mod 2**32. |
Benjamin Peterson | 6e165b4 | 2013-05-11 13:17:13 -0500 | [diff] [blame] | 349 | self.fileobj.seek(-8, 1) |
| 350 | crc32 = read32(self.fileobj) |
| 351 | isize = read32(self.fileobj) # may exceed 2GB |
Gregory P. Smith | 79b4ba8 | 2008-03-23 21:04:43 +0000 | [diff] [blame] | 352 | if crc32 != self.crc: |
| 353 | raise IOError("CRC check failed %s != %s" % (hex(crc32), |
| 354 | hex(self.crc))) |
Gregory P. Smith | ac830e9 | 2008-03-23 23:43:02 +0000 | [diff] [blame] | 355 | elif isize != (self.size & 0xffffffffL): |
Andrew M. Kuchling | 64edd6a | 2003-02-05 21:35:07 +0000 | [diff] [blame] | 356 | raise IOError, "Incorrect length of data produced" |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 357 | |
Antoine Pitrou | 5a9112c | 2010-01-13 14:32:10 +0000 | [diff] [blame] | 358 | # Gzip files can be padded with zeroes and still have archives. |
| 359 | # Consume all zero bytes and set the file position to the first |
| 360 | # non-zero byte. See http://www.gzip.org/#faq8 |
| 361 | c = "\x00" |
| 362 | while c == "\x00": |
| 363 | c = self.fileobj.read(1) |
| 364 | if c: |
| 365 | self.fileobj.seek(-1, 1) |
| 366 | |
Antoine Pitrou | 673ddf9 | 2010-01-03 22:29:56 +0000 | [diff] [blame] | 367 | @property |
| 368 | def closed(self): |
| 369 | return self.fileobj is None |
| 370 | |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 371 | def close(self): |
Serhiy Storchaka | 1aa2c0f | 2015-04-10 13:24:10 +0300 | [diff] [blame] | 372 | fileobj = self.fileobj |
| 373 | if fileobj is None: |
Georg Brandl | e08e3d0 | 2008-05-25 08:07:37 +0000 | [diff] [blame] | 374 | return |
Serhiy Storchaka | 1aa2c0f | 2015-04-10 13:24:10 +0300 | [diff] [blame] | 375 | self.fileobj = None |
| 376 | try: |
| 377 | if self.mode == WRITE: |
| 378 | fileobj.write(self.compress.flush()) |
| 379 | write32u(fileobj, self.crc) |
| 380 | # self.size may exceed 2GB, or even 4GB |
| 381 | write32u(fileobj, self.size & 0xffffffffL) |
| 382 | finally: |
| 383 | myfileobj = self.myfileobj |
| 384 | if myfileobj: |
| 385 | self.myfileobj = None |
| 386 | myfileobj.close() |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 387 | |
Martin v. Löwis | f2a8d63 | 2005-03-03 08:35:22 +0000 | [diff] [blame] | 388 | def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH): |
Antoine Pitrou | 76a66aa | 2010-10-06 21:26:52 +0000 | [diff] [blame] | 389 | self._check_closed() |
Martin v. Löwis | f2a8d63 | 2005-03-03 08:35:22 +0000 | [diff] [blame] | 390 | if self.mode == WRITE: |
Tim Peters | eba28be | 2005-03-28 01:08:02 +0000 | [diff] [blame] | 391 | # Ensure the compressor's buffer is flushed |
| 392 | self.fileobj.write(self.compress.flush(zlib_mode)) |
Mark Dickinson | 3b0b4ff | 2010-05-04 18:45:27 +0000 | [diff] [blame] | 393 | self.fileobj.flush() |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 394 | |
Tim Peters | 5cfb05e | 2004-07-27 21:02:02 +0000 | [diff] [blame] | 395 | def fileno(self): |
| 396 | """Invoke the underlying file object's fileno() method. |
| 397 | |
| 398 | This will raise AttributeError if the underlying file object |
| 399 | doesn't support fileno(). |
| 400 | """ |
| 401 | return self.fileobj.fileno() |
| 402 | |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 403 | def rewind(self): |
| 404 | '''Return the uncompressed stream file position indicator to the |
Tim Peters | ab9ba27 | 2001-08-09 21:40:30 +0000 | [diff] [blame] | 405 | beginning of the file''' |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 406 | if self.mode != READ: |
| 407 | raise IOError("Can't rewind in write mode") |
| 408 | self.fileobj.seek(0) |
Guido van Rossum | 8ca162f | 2002-04-07 06:36:23 +0000 | [diff] [blame] | 409 | self._new_member = True |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 410 | self.extrabuf = "" |
| 411 | self.extrasize = 0 |
Antoine Pitrou | 673ddf9 | 2010-01-03 22:29:56 +0000 | [diff] [blame] | 412 | self.extrastart = 0 |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 413 | self.offset = 0 |
| 414 | |
Antoine Pitrou | 673ddf9 | 2010-01-03 22:29:56 +0000 | [diff] [blame] | 415 | def readable(self): |
| 416 | return self.mode == READ |
| 417 | |
| 418 | def writable(self): |
| 419 | return self.mode == WRITE |
| 420 | |
| 421 | def seekable(self): |
| 422 | return True |
| 423 | |
Martin v. Löwis | 065f0c8 | 2006-11-12 10:41:39 +0000 | [diff] [blame] | 424 | def seek(self, offset, whence=0): |
| 425 | if whence: |
| 426 | if whence == 1: |
| 427 | offset = self.offset + offset |
| 428 | else: |
| 429 | raise ValueError('Seek from end not supported') |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 430 | if self.mode == WRITE: |
| 431 | if offset < self.offset: |
| 432 | raise IOError('Negative seek in write mode') |
| 433 | count = offset - self.offset |
Chris Withers | 2cc0b07 | 2012-11-09 15:48:17 +0000 | [diff] [blame] | 434 | for i in xrange(count // 1024): |
Tim Peters | fb0ea52 | 2002-11-04 19:50:11 +0000 | [diff] [blame] | 435 | self.write(1024 * '\0') |
| 436 | self.write((count % 1024) * '\0') |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 437 | elif self.mode == READ: |
| 438 | if offset < self.offset: |
| 439 | # for negative seek, rewind and do positive seek |
| 440 | self.rewind() |
| 441 | count = offset - self.offset |
Chris Withers | 2cc0b07 | 2012-11-09 15:48:17 +0000 | [diff] [blame] | 442 | for i in xrange(count // 1024): |
Tim Peters | fb0ea52 | 2002-11-04 19:50:11 +0000 | [diff] [blame] | 443 | self.read(1024) |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 444 | self.read(count % 1024) |
| 445 | |
Antoine Pitrou | 673ddf9 | 2010-01-03 22:29:56 +0000 | [diff] [blame] | 446 | return self.offset |
| 447 | |
Andrew M. Kuchling | 41616ee | 2000-07-29 20:15:26 +0000 | [diff] [blame] | 448 | def readline(self, size=-1): |
Bob Ippolito | d82c310 | 2006-05-22 15:59:12 +0000 | [diff] [blame] | 449 | if size < 0: |
Antoine Pitrou | 673ddf9 | 2010-01-03 22:29:56 +0000 | [diff] [blame] | 450 | # Shortcut common case - newline found in buffer. |
| 451 | offset = self.offset - self.extrastart |
| 452 | i = self.extrabuf.find('\n', offset) + 1 |
| 453 | if i > 0: |
| 454 | self.extrasize -= i - offset |
| 455 | self.offset += i - offset |
| 456 | return self.extrabuf[offset: i] |
| 457 | |
Bob Ippolito | d82c310 | 2006-05-22 15:59:12 +0000 | [diff] [blame] | 458 | size = sys.maxint |
| 459 | readsize = self.min_readsize |
| 460 | else: |
| 461 | readsize = size |
Bob Ippolito | b975973 | 2006-05-22 15:22:46 +0000 | [diff] [blame] | 462 | bufs = [] |
Bob Ippolito | d82c310 | 2006-05-22 15:59:12 +0000 | [diff] [blame] | 463 | while size != 0: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 464 | c = self.read(readsize) |
Eric S. Raymond | ee5e61d | 2001-02-09 09:10:35 +0000 | [diff] [blame] | 465 | i = c.find('\n') |
Bob Ippolito | d82c310 | 2006-05-22 15:59:12 +0000 | [diff] [blame] | 466 | |
| 467 | # We set i=size to break out of the loop under two |
| 468 | # conditions: 1) there's no newline, and the chunk is |
| 469 | # larger than size, or 2) there is a newline, but the |
| 470 | # resulting line would be longer than 'size'. |
| 471 | if (size <= i) or (i == -1 and len(c) > size): |
| 472 | i = size - 1 |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 473 | |
Andrew M. Kuchling | 41616ee | 2000-07-29 20:15:26 +0000 | [diff] [blame] | 474 | if i >= 0 or c == '': |
Bob Ippolito | d82c310 | 2006-05-22 15:59:12 +0000 | [diff] [blame] | 475 | bufs.append(c[:i + 1]) # Add portion of last chunk |
| 476 | self._unread(c[i + 1:]) # Push back rest of chunk |
| 477 | break |
Andrew M. Kuchling | 41616ee | 2000-07-29 20:15:26 +0000 | [diff] [blame] | 478 | |
Bob Ippolito | b975973 | 2006-05-22 15:22:46 +0000 | [diff] [blame] | 479 | # Append chunk to list, decrease 'size', |
| 480 | bufs.append(c) |
| 481 | size = size - len(c) |
| 482 | readsize = min(size, readsize * 2) |
Bob Ippolito | d82c310 | 2006-05-22 15:59:12 +0000 | [diff] [blame] | 483 | if readsize > self.min_readsize: |
| 484 | self.min_readsize = min(readsize, self.min_readsize * 2, 512) |
| 485 | return ''.join(bufs) # Return resulting line |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 486 | |
Guido van Rossum | 51ca6e3 | 1997-12-30 20:09:08 +0000 | [diff] [blame] | 487 | |
| 488 | def _test(): |
| 489 | # Act like gzip; with -d, act like gunzip. |
| 490 | # The input file is not deleted, however, nor are any other gzip |
| 491 | # options or features supported. |
Guido van Rossum | 51ca6e3 | 1997-12-30 20:09:08 +0000 | [diff] [blame] | 492 | args = sys.argv[1:] |
| 493 | decompress = args and args[0] == "-d" |
| 494 | if decompress: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 495 | args = args[1:] |
Guido van Rossum | 51ca6e3 | 1997-12-30 20:09:08 +0000 | [diff] [blame] | 496 | if not args: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 497 | args = ["-"] |
Guido van Rossum | 51ca6e3 | 1997-12-30 20:09:08 +0000 | [diff] [blame] | 498 | for arg in args: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 499 | if decompress: |
| 500 | if arg == "-": |
| 501 | f = GzipFile(filename="", mode="rb", fileobj=sys.stdin) |
| 502 | g = sys.stdout |
| 503 | else: |
| 504 | if arg[-3:] != ".gz": |
Walter Dörwald | 70a6b49 | 2004-02-12 17:35:32 +0000 | [diff] [blame] | 505 | print "filename doesn't end in .gz:", repr(arg) |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 506 | continue |
| 507 | f = open(arg, "rb") |
| 508 | g = __builtin__.open(arg[:-3], "wb") |
| 509 | else: |
| 510 | if arg == "-": |
| 511 | f = sys.stdin |
| 512 | g = GzipFile(filename="", mode="wb", fileobj=sys.stdout) |
| 513 | else: |
| 514 | f = __builtin__.open(arg, "rb") |
| 515 | g = open(arg + ".gz", "wb") |
Guido van Rossum | 8ca162f | 2002-04-07 06:36:23 +0000 | [diff] [blame] | 516 | while True: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 517 | chunk = f.read(1024) |
| 518 | if not chunk: |
| 519 | break |
| 520 | g.write(chunk) |
| 521 | if g is not sys.stdout: |
| 522 | g.close() |
| 523 | if f is not sys.stdin: |
| 524 | f.close() |
Guido van Rossum | 51ca6e3 | 1997-12-30 20:09:08 +0000 | [diff] [blame] | 525 | |
| 526 | if __name__ == '__main__': |
| 527 | _test() |