Guido van Rossum | 4b8c6ea | 2000-02-04 15:39:30 +0000 | [diff] [blame] | 1 | """Functions that read and write gzipped files. |
| 2 | |
Guido van Rossum | 54f22ed | 2000-02-04 15:10:34 +0000 | [diff] [blame] | 3 | The user of the file doesn't have to worry about the compression, |
| 4 | but random access is not allowed.""" |
| 5 | |
| 6 | # based on Andrew Kuchling's minigzip.py distributed with the zlib module |
| 7 | |
Tim Peters | 49667c2 | 2004-07-27 21:05:21 +0000 | [diff] [blame] | 8 | import struct, sys, time |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 9 | import zlib |
Guido van Rossum | 68de379 | 1997-07-19 20:22:23 +0000 | [diff] [blame] | 10 | import __builtin__ |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 11 | |
Skip Montanaro | 2dd4276 | 2001-01-23 15:35:05 +0000 | [diff] [blame] | 12 | __all__ = ["GzipFile","open"] |
| 13 | |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 14 | FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16 |
| 15 | |
| 16 | READ, WRITE = 1, 2 |
| 17 | |
Tim Peters | fb0ea52 | 2002-11-04 19:50:11 +0000 | [diff] [blame] | 18 | def U32(i): |
| 19 | """Return i as an unsigned integer, assuming it fits in 32 bits. |
| 20 | |
| 21 | If it's >= 2GB when viewed as a 32-bit unsigned int, return a long. |
| 22 | """ |
| 23 | if i < 0: |
Guido van Rossum | e2a383d | 2007-01-15 16:59:06 +0000 | [diff] [blame] | 24 | i += 1 << 32 |
Tim Peters | fb0ea52 | 2002-11-04 19:50:11 +0000 | [diff] [blame] | 25 | return i |
| 26 | |
Tim Peters | 9288f95 | 2002-11-05 20:38:55 +0000 | [diff] [blame] | 27 | def LOWU32(i): |
| 28 | """Return the low-order 32 bits of an int, as a non-negative int.""" |
Guido van Rossum | e2a383d | 2007-01-15 16:59:06 +0000 | [diff] [blame] | 29 | return i & 0xFFFFFFFF |
Tim Peters | 9288f95 | 2002-11-05 20:38:55 +0000 | [diff] [blame] | 30 | |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 31 | def write32(output, value): |
Jeremy Hylton | c19f997 | 1999-03-23 23:05:34 +0000 | [diff] [blame] | 32 | output.write(struct.pack("<l", value)) |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 33 | |
Guido van Rossum | 95bdd0b | 1999-04-12 14:34:16 +0000 | [diff] [blame] | 34 | def write32u(output, value): |
Tim Peters | fb0ea52 | 2002-11-04 19:50:11 +0000 | [diff] [blame] | 35 | # The L format writes the bit pattern correctly whether signed |
| 36 | # or unsigned. |
Guido van Rossum | 95bdd0b | 1999-04-12 14:34:16 +0000 | [diff] [blame] | 37 | output.write(struct.pack("<L", value)) |
| 38 | |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 39 | def read32(input): |
Jeremy Hylton | c19f997 | 1999-03-23 23:05:34 +0000 | [diff] [blame] | 40 | return struct.unpack("<l", input.read(4))[0] |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 41 | |
Fred Drake | fa1591c | 1999-04-05 18:37:59 +0000 | [diff] [blame] | 42 | def open(filename, mode="rb", compresslevel=9): |
Raymond Hettinger | aef22fb | 2002-05-29 16:18:42 +0000 | [diff] [blame] | 43 | """Shorthand for GzipFile(filename, mode, compresslevel). |
| 44 | |
| 45 | The filename argument is required; mode defaults to 'rb' |
| 46 | and compresslevel defaults to 9. |
| 47 | |
| 48 | """ |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 49 | return GzipFile(filename, mode, compresslevel) |
| 50 | |
| 51 | class GzipFile: |
Raymond Hettinger | aef22fb | 2002-05-29 16:18:42 +0000 | [diff] [blame] | 52 | """The GzipFile class simulates most of the methods of a file object with |
Guido van Rossum | 97c5fcc | 2002-08-06 17:03:25 +0000 | [diff] [blame] | 53 | the exception of the readinto() and truncate() methods. |
Raymond Hettinger | aef22fb | 2002-05-29 16:18:42 +0000 | [diff] [blame] | 54 | |
| 55 | """ |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 56 | |
Guido van Rossum | 68de379 | 1997-07-19 20:22:23 +0000 | [diff] [blame] | 57 | myfileobj = None |
Andrew M. Kuchling | 01cb47b | 2005-06-09 14:19:32 +0000 | [diff] [blame] | 58 | max_read_chunk = 10 * 1024 * 1024 # 10Mb |
Guido van Rossum | 68de379 | 1997-07-19 20:22:23 +0000 | [diff] [blame] | 59 | |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 60 | def __init__(self, filename=None, mode=None, |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 61 | compresslevel=9, fileobj=None): |
Raymond Hettinger | aef22fb | 2002-05-29 16:18:42 +0000 | [diff] [blame] | 62 | """Constructor for the GzipFile class. |
| 63 | |
| 64 | At least one of fileobj and filename must be given a |
| 65 | non-trivial value. |
| 66 | |
| 67 | The new class instance is based on fileobj, which can be a regular |
| 68 | file, a StringIO object, or any other object which simulates a file. |
| 69 | It defaults to None, in which case filename is opened to provide |
| 70 | a file object. |
| 71 | |
| 72 | When fileobj is not None, the filename argument is only used to be |
| 73 | included in the gzip file header, which may includes the original |
| 74 | filename of the uncompressed file. It defaults to the filename of |
| 75 | fileobj, if discernible; otherwise, it defaults to the empty string, |
| 76 | and in this case the original filename is not included in the header. |
| 77 | |
| 78 | The mode argument can be any of 'r', 'rb', 'a', 'ab', 'w', or 'wb', |
| 79 | depending on whether the file will be read or written. The default |
| 80 | is the mode of fileobj if discernible; otherwise, the default is 'rb'. |
| 81 | Be aware that only the 'rb', 'ab', and 'wb' values should be used |
| 82 | for cross-platform portability. |
| 83 | |
| 84 | The compresslevel argument is an integer from 1 to 9 controlling the |
| 85 | level of compression; 1 is fastest and produces the least compression, |
| 86 | and 9 is slowest and produces the most compression. The default is 9. |
| 87 | |
| 88 | """ |
| 89 | |
Skip Montanaro | 12424bc | 2002-05-23 01:43:05 +0000 | [diff] [blame] | 90 | # guarantee the file is opened in binary mode on platforms |
| 91 | # that care about that sort of thing |
| 92 | if mode and 'b' not in mode: |
| 93 | mode += 'b' |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 94 | if fileobj is None: |
Fred Drake | 9bb76d1 | 1999-04-05 18:33:40 +0000 | [diff] [blame] | 95 | fileobj = self.myfileobj = __builtin__.open(filename, mode or 'rb') |
Guido van Rossum | 68de379 | 1997-07-19 20:22:23 +0000 | [diff] [blame] | 96 | if filename is None: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 97 | if hasattr(fileobj, 'name'): filename = fileobj.name |
| 98 | else: filename = '' |
Guido van Rossum | 68de379 | 1997-07-19 20:22:23 +0000 | [diff] [blame] | 99 | if mode is None: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 100 | if hasattr(fileobj, 'mode'): mode = fileobj.mode |
Fred Drake | 9bb76d1 | 1999-04-05 18:33:40 +0000 | [diff] [blame] | 101 | else: mode = 'rb' |
Guido van Rossum | 68de379 | 1997-07-19 20:22:23 +0000 | [diff] [blame] | 102 | |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 103 | if mode[0:1] == 'r': |
| 104 | self.mode = READ |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 105 | # Set flag indicating start of a new member |
Guido van Rossum | 8ca162f | 2002-04-07 06:36:23 +0000 | [diff] [blame] | 106 | self._new_member = True |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 107 | self.extrabuf = b"" |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 108 | self.extrasize = 0 |
Thomas Wouters | cf297e4 | 2007-02-23 15:07:44 +0000 | [diff] [blame] | 109 | self.name = filename |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 110 | # Starts small, scales exponentially |
| 111 | self.min_readsize = 100 |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 112 | |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 113 | elif mode[0:1] == 'w' or mode[0:1] == 'a': |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 114 | self.mode = WRITE |
| 115 | self._init_write(filename) |
| 116 | self.compress = zlib.compressobj(compresslevel, |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 117 | zlib.DEFLATED, |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 118 | -zlib.MAX_WBITS, |
| 119 | zlib.DEF_MEM_LEVEL, |
| 120 | 0) |
| 121 | else: |
Martin v. Löwis | db04489 | 2002-03-11 06:46:52 +0000 | [diff] [blame] | 122 | raise IOError, "Mode " + mode + " not supported" |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 123 | |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 124 | self.fileobj = fileobj |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 125 | self.offset = 0 |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 126 | |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 127 | if self.mode == WRITE: |
| 128 | self._write_gzip_header() |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 129 | |
Thomas Wouters | cf297e4 | 2007-02-23 15:07:44 +0000 | [diff] [blame] | 130 | @property |
| 131 | def filename(self): |
| 132 | import warnings |
| 133 | warnings.warn("use the name attribute", DeprecationWarning) |
| 134 | if self.mode == WRITE and self.name[-3:] != ".gz": |
| 135 | return self.name + ".gz" |
| 136 | return self.name |
| 137 | |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 138 | def __repr__(self): |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 139 | s = repr(self.fileobj) |
| 140 | return '<gzip ' + s[1:-1] + ' ' + hex(id(self)) + '>' |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 141 | |
| 142 | def _init_write(self, filename): |
Thomas Wouters | cf297e4 | 2007-02-23 15:07:44 +0000 | [diff] [blame] | 143 | self.name = filename |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 144 | self.crc = zlib.crc32("") |
| 145 | self.size = 0 |
| 146 | self.writebuf = [] |
| 147 | self.bufsize = 0 |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 148 | |
| 149 | def _write_gzip_header(self): |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 150 | self.fileobj.write(b'\037\213') # magic header |
| 151 | self.fileobj.write(b'\010') # compression method |
Lars Gustäbel | 5590d8c | 2007-08-10 12:02:32 +0000 | [diff] [blame] | 152 | try: |
Lars Gustäbel | ead7056 | 2007-08-13 09:05:16 +0000 | [diff] [blame] | 153 | # RFC 1952 requires the FNAME field to be Latin-1. Do not |
| 154 | # include filenames that cannot be represented that way. |
| 155 | fname = self.name.encode('latin-1') |
| 156 | if fname.endswith(b'.gz'): |
| 157 | fname = fname[:-3] |
Lars Gustäbel | 5590d8c | 2007-08-10 12:02:32 +0000 | [diff] [blame] | 158 | except UnicodeEncodeError: |
Lars Gustäbel | ead7056 | 2007-08-13 09:05:16 +0000 | [diff] [blame] | 159 | fname = b'' |
| 160 | flags = 0 |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 161 | if fname: |
| 162 | flags = FNAME |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 163 | self.fileobj.write(chr(flags).encode('latin-1')) |
Guido van Rossum | e2a383d | 2007-01-15 16:59:06 +0000 | [diff] [blame] | 164 | write32u(self.fileobj, int(time.time())) |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 165 | self.fileobj.write(b'\002') |
| 166 | self.fileobj.write(b'\377') |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 167 | if fname: |
Lars Gustäbel | 5590d8c | 2007-08-10 12:02:32 +0000 | [diff] [blame] | 168 | self.fileobj.write(fname + b'\000') |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 169 | |
| 170 | def _init_read(self): |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 171 | self.crc = zlib.crc32("") |
| 172 | self.size = 0 |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 173 | |
| 174 | def _read_gzip_header(self): |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 175 | magic = self.fileobj.read(2) |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 176 | if magic != b'\037\213': |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 177 | raise IOError, 'Not a gzipped file' |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 178 | method = ord( self.fileobj.read(1) ) |
| 179 | if method != 8: |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 180 | raise IOError, 'Unknown compression method' |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 181 | flag = ord( self.fileobj.read(1) ) |
| 182 | # modtime = self.fileobj.read(4) |
| 183 | # extraflag = self.fileobj.read(1) |
| 184 | # os = self.fileobj.read(1) |
| 185 | self.fileobj.read(6) |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 186 | |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 187 | if flag & FEXTRA: |
| 188 | # Read & discard the extra field, if present |
Tim Peters | fb0ea52 | 2002-11-04 19:50:11 +0000 | [diff] [blame] | 189 | xlen = ord(self.fileobj.read(1)) |
| 190 | xlen = xlen + 256*ord(self.fileobj.read(1)) |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 191 | self.fileobj.read(xlen) |
| 192 | if flag & FNAME: |
| 193 | # Read and discard a null-terminated string containing the filename |
Guido van Rossum | 8ca162f | 2002-04-07 06:36:23 +0000 | [diff] [blame] | 194 | while True: |
Tim Peters | fb0ea52 | 2002-11-04 19:50:11 +0000 | [diff] [blame] | 195 | s = self.fileobj.read(1) |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 196 | if not s or s==b'\000': |
Tim Peters | fb0ea52 | 2002-11-04 19:50:11 +0000 | [diff] [blame] | 197 | break |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 198 | if flag & FCOMMENT: |
| 199 | # Read and discard a null-terminated string containing a comment |
Guido van Rossum | 8ca162f | 2002-04-07 06:36:23 +0000 | [diff] [blame] | 200 | while True: |
Tim Peters | fb0ea52 | 2002-11-04 19:50:11 +0000 | [diff] [blame] | 201 | s = self.fileobj.read(1) |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 202 | if not s or s==b'\000': |
Tim Peters | fb0ea52 | 2002-11-04 19:50:11 +0000 | [diff] [blame] | 203 | break |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 204 | if flag & FHCRC: |
| 205 | self.fileobj.read(2) # Read & discard the 16-bit header CRC |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 206 | |
| 207 | |
| 208 | def write(self,data): |
Martin v. Löwis | db04489 | 2002-03-11 06:46:52 +0000 | [diff] [blame] | 209 | if self.mode != WRITE: |
| 210 | import errno |
| 211 | raise IOError(errno.EBADF, "write() on read-only GzipFile object") |
Tim Peters | 863ac44 | 2002-04-16 01:38:40 +0000 | [diff] [blame] | 212 | |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 213 | if self.fileobj is None: |
| 214 | raise ValueError, "write() on closed GzipFile object" |
| 215 | if len(data) > 0: |
| 216 | self.size = self.size + len(data) |
| 217 | self.crc = zlib.crc32(data, self.crc) |
| 218 | self.fileobj.write( self.compress.compress(data) ) |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 219 | self.offset += len(data) |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 220 | |
Guido van Rossum | 5606801 | 2000-02-02 16:51:06 +0000 | [diff] [blame] | 221 | def read(self, size=-1): |
Martin v. Löwis | db04489 | 2002-03-11 06:46:52 +0000 | [diff] [blame] | 222 | if self.mode != READ: |
| 223 | import errno |
Brett Cannon | edfb302 | 2003-12-04 19:28:06 +0000 | [diff] [blame] | 224 | raise IOError(errno.EBADF, "read() on write-only GzipFile object") |
Tim Peters | 863ac44 | 2002-04-16 01:38:40 +0000 | [diff] [blame] | 225 | |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 226 | if self.extrasize <= 0 and self.fileobj is None: |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 227 | return b'' |
Guido van Rossum | b16a3b8 | 1998-01-27 19:29:45 +0000 | [diff] [blame] | 228 | |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 229 | readsize = 1024 |
Guido van Rossum | 5606801 | 2000-02-02 16:51:06 +0000 | [diff] [blame] | 230 | if size < 0: # get the whole thing |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 231 | try: |
Guido van Rossum | 8ca162f | 2002-04-07 06:36:23 +0000 | [diff] [blame] | 232 | while True: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 233 | self._read(readsize) |
Andrew M. Kuchling | 01cb47b | 2005-06-09 14:19:32 +0000 | [diff] [blame] | 234 | readsize = min(self.max_read_chunk, readsize * 2) |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 235 | except EOFError: |
| 236 | size = self.extrasize |
| 237 | else: # just get some more of it |
| 238 | try: |
| 239 | while size > self.extrasize: |
| 240 | self._read(readsize) |
Andrew M. Kuchling | 01cb47b | 2005-06-09 14:19:32 +0000 | [diff] [blame] | 241 | readsize = min(self.max_read_chunk, readsize * 2) |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 242 | except EOFError: |
Guido van Rossum | 84c6fc9 | 1998-08-03 15:41:39 +0000 | [diff] [blame] | 243 | if size > self.extrasize: |
| 244 | size = self.extrasize |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 245 | |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 246 | chunk = self.extrabuf[:size] |
| 247 | self.extrabuf = self.extrabuf[size:] |
| 248 | self.extrasize = self.extrasize - size |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 249 | |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 250 | self.offset += size |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 251 | return chunk |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 252 | |
Guido van Rossum | b16a3b8 | 1998-01-27 19:29:45 +0000 | [diff] [blame] | 253 | def _unread(self, buf): |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 254 | self.extrabuf = buf + self.extrabuf |
Guido van Rossum | 84c6fc9 | 1998-08-03 15:41:39 +0000 | [diff] [blame] | 255 | self.extrasize = len(buf) + self.extrasize |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 256 | self.offset -= len(buf) |
Guido van Rossum | b16a3b8 | 1998-01-27 19:29:45 +0000 | [diff] [blame] | 257 | |
| 258 | def _read(self, size=1024): |
Tim Peters | fb0ea52 | 2002-11-04 19:50:11 +0000 | [diff] [blame] | 259 | if self.fileobj is None: |
| 260 | raise EOFError, "Reached EOF" |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 261 | |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 262 | if self._new_member: |
Andrew M. Kuchling | 41616ee | 2000-07-29 20:15:26 +0000 | [diff] [blame] | 263 | # If the _new_member flag is set, we have to |
| 264 | # jump to the next member, if there is one. |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 265 | # |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 266 | # First, check if we're at the end of the file; |
| 267 | # if so, it's time to stop; no more members to read. |
| 268 | pos = self.fileobj.tell() # Save current position |
| 269 | self.fileobj.seek(0, 2) # Seek to end of file |
| 270 | if pos == self.fileobj.tell(): |
Andrew M. Kuchling | 2d813e5 | 1999-09-06 16:34:51 +0000 | [diff] [blame] | 271 | raise EOFError, "Reached EOF" |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 272 | else: |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 273 | self.fileobj.seek( pos ) # Return to original position |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 274 | |
| 275 | self._init_read() |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 276 | self._read_gzip_header() |
| 277 | self.decompress = zlib.decompressobj(-zlib.MAX_WBITS) |
Guido van Rossum | 8ca162f | 2002-04-07 06:36:23 +0000 | [diff] [blame] | 278 | self._new_member = False |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 279 | |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 280 | # Read a chunk of data from the file |
| 281 | buf = self.fileobj.read(size) |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 282 | |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 283 | # If the EOF has been reached, flush the decompression object |
| 284 | # and mark this object as finished. |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 285 | |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 286 | if buf == b"": |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 287 | uncompress = self.decompress.flush() |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 288 | self._read_eof() |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 289 | self._add_read_data( uncompress ) |
| 290 | raise EOFError, 'Reached EOF' |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 291 | |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 292 | uncompress = self.decompress.decompress(buf) |
| 293 | self._add_read_data( uncompress ) |
| 294 | |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 295 | if self.decompress.unused_data != b"": |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 296 | # Ending case: we've come to the end of a member in the file, |
| 297 | # so seek back to the start of the unused data, finish up |
| 298 | # this member, and read a new gzip header. |
| 299 | # (The number of bytes to seek back is the length of the unused |
| 300 | # data, minus 8 because _read_eof() will rewind a further 8 bytes) |
| 301 | self.fileobj.seek( -len(self.decompress.unused_data)+8, 1) |
| 302 | |
| 303 | # Check the CRC and file size, and set the flag so we read |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 304 | # a new member on the next call |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 305 | self._read_eof() |
Guido van Rossum | 8ca162f | 2002-04-07 06:36:23 +0000 | [diff] [blame] | 306 | self._new_member = True |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 307 | |
| 308 | def _add_read_data(self, data): |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 309 | self.crc = zlib.crc32(data, self.crc) |
| 310 | self.extrabuf = self.extrabuf + data |
| 311 | self.extrasize = self.extrasize + len(data) |
| 312 | self.size = self.size + len(data) |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 313 | |
| 314 | def _read_eof(self): |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 315 | # We've read to the end of the file, so we have to rewind in order |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 316 | # to reread the 8 bytes containing the CRC and the file size. |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 317 | # We check the that the computed CRC and size of the |
Tim Peters | 9288f95 | 2002-11-05 20:38:55 +0000 | [diff] [blame] | 318 | # uncompressed data matches the stored values. Note that the size |
| 319 | # stored is the true file size mod 2**32. |
Andrew M. Kuchling | f4f119c | 1999-03-25 21:49:14 +0000 | [diff] [blame] | 320 | self.fileobj.seek(-8, 1) |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 321 | crc32 = read32(self.fileobj) |
Tim Peters | fb0ea52 | 2002-11-04 19:50:11 +0000 | [diff] [blame] | 322 | isize = U32(read32(self.fileobj)) # may exceed 2GB |
| 323 | if U32(crc32) != U32(self.crc): |
Andrew M. Kuchling | 64edd6a | 2003-02-05 21:35:07 +0000 | [diff] [blame] | 324 | raise IOError, "CRC check failed" |
Tim Peters | 9288f95 | 2002-11-05 20:38:55 +0000 | [diff] [blame] | 325 | elif isize != LOWU32(self.size): |
Andrew M. Kuchling | 64edd6a | 2003-02-05 21:35:07 +0000 | [diff] [blame] | 326 | raise IOError, "Incorrect length of data produced" |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 327 | |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 328 | def close(self): |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 329 | if self.mode == WRITE: |
| 330 | self.fileobj.write(self.compress.flush()) |
Thomas Wouters | 0e3f591 | 2006-08-11 14:57:12 +0000 | [diff] [blame] | 331 | # The native zlib crc is an unsigned 32-bit integer, but |
| 332 | # the Python wrapper implicitly casts that to a signed C |
| 333 | # long. So, on a 32-bit box self.crc may "look negative", |
| 334 | # while the same crc on a 64-bit box may "look positive". |
| 335 | # To avoid irksome warnings from the `struct` module, force |
| 336 | # it to look positive on all boxes. |
| 337 | write32u(self.fileobj, LOWU32(self.crc)) |
Tim Peters | 9288f95 | 2002-11-05 20:38:55 +0000 | [diff] [blame] | 338 | # self.size may exceed 2GB, or even 4GB |
| 339 | write32u(self.fileobj, LOWU32(self.size)) |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 340 | self.fileobj = None |
| 341 | elif self.mode == READ: |
| 342 | self.fileobj = None |
| 343 | if self.myfileobj: |
| 344 | self.myfileobj.close() |
| 345 | self.myfileobj = None |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 346 | |
Andrew M. Kuchling | 916fcc3 | 1999-08-10 13:19:30 +0000 | [diff] [blame] | 347 | def __del__(self): |
Jeremy Hylton | e298c30 | 2000-05-08 16:59:59 +0000 | [diff] [blame] | 348 | try: |
| 349 | if (self.myfileobj is None and |
| 350 | self.fileobj is None): |
| 351 | return |
| 352 | except AttributeError: |
| 353 | return |
| 354 | self.close() |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 355 | |
Martin v. Löwis | f2a8d63 | 2005-03-03 08:35:22 +0000 | [diff] [blame] | 356 | def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH): |
| 357 | if self.mode == WRITE: |
Tim Peters | eba28be | 2005-03-28 01:08:02 +0000 | [diff] [blame] | 358 | # Ensure the compressor's buffer is flushed |
| 359 | self.fileobj.write(self.compress.flush(zlib_mode)) |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 360 | self.fileobj.flush() |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 361 | |
Tim Peters | 5cfb05e | 2004-07-27 21:02:02 +0000 | [diff] [blame] | 362 | def fileno(self): |
| 363 | """Invoke the underlying file object's fileno() method. |
| 364 | |
| 365 | This will raise AttributeError if the underlying file object |
| 366 | doesn't support fileno(). |
| 367 | """ |
| 368 | return self.fileobj.fileno() |
| 369 | |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 370 | def isatty(self): |
Guido van Rossum | 8ca162f | 2002-04-07 06:36:23 +0000 | [diff] [blame] | 371 | return False |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 372 | |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 373 | def tell(self): |
| 374 | return self.offset |
| 375 | |
| 376 | def rewind(self): |
| 377 | '''Return the uncompressed stream file position indicator to the |
Tim Peters | ab9ba27 | 2001-08-09 21:40:30 +0000 | [diff] [blame] | 378 | beginning of the file''' |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 379 | if self.mode != READ: |
| 380 | raise IOError("Can't rewind in write mode") |
| 381 | self.fileobj.seek(0) |
Guido van Rossum | 8ca162f | 2002-04-07 06:36:23 +0000 | [diff] [blame] | 382 | self._new_member = True |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 383 | self.extrabuf = b"" |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 384 | self.extrasize = 0 |
| 385 | self.offset = 0 |
| 386 | |
Thomas Wouters | 89f507f | 2006-12-13 04:49:30 +0000 | [diff] [blame] | 387 | def seek(self, offset, whence=0): |
| 388 | if whence: |
| 389 | if whence == 1: |
| 390 | offset = self.offset + offset |
| 391 | else: |
| 392 | raise ValueError('Seek from end not supported') |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 393 | if self.mode == WRITE: |
| 394 | if offset < self.offset: |
| 395 | raise IOError('Negative seek in write mode') |
| 396 | count = offset - self.offset |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 397 | chunk = bytes(1024) |
Tim Peters | fb0ea52 | 2002-11-04 19:50:11 +0000 | [diff] [blame] | 398 | for i in range(count // 1024): |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 399 | self.write(chunk) |
| 400 | self.write(bytes(count % 1024)) |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 401 | elif self.mode == READ: |
| 402 | if offset < self.offset: |
| 403 | # for negative seek, rewind and do positive seek |
| 404 | self.rewind() |
| 405 | count = offset - self.offset |
Tim Peters | fb0ea52 | 2002-11-04 19:50:11 +0000 | [diff] [blame] | 406 | for i in range(count // 1024): |
| 407 | self.read(1024) |
Martin v. Löwis | 8cc965c | 2001-08-09 07:21:56 +0000 | [diff] [blame] | 408 | self.read(count % 1024) |
| 409 | |
Andrew M. Kuchling | 41616ee | 2000-07-29 20:15:26 +0000 | [diff] [blame] | 410 | def readline(self, size=-1): |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 411 | if size < 0: |
| 412 | size = sys.maxint |
| 413 | readsize = self.min_readsize |
| 414 | else: |
| 415 | readsize = size |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 416 | bufs = [] |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 417 | while size != 0: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 418 | c = self.read(readsize) |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 419 | i = c.find(b'\n') |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 420 | |
| 421 | # We set i=size to break out of the loop under two |
| 422 | # conditions: 1) there's no newline, and the chunk is |
| 423 | # larger than size, or 2) there is a newline, but the |
| 424 | # resulting line would be longer than 'size'. |
| 425 | if (size <= i) or (i == -1 and len(c) > size): |
| 426 | i = size - 1 |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 427 | |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 428 | if i >= 0 or c == b'': |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 429 | bufs.append(c[:i + 1]) # Add portion of last chunk |
| 430 | self._unread(c[i + 1:]) # Push back rest of chunk |
| 431 | break |
Andrew M. Kuchling | 41616ee | 2000-07-29 20:15:26 +0000 | [diff] [blame] | 432 | |
| 433 | # Append chunk to list, decrease 'size', |
| 434 | bufs.append(c) |
| 435 | size = size - len(c) |
| 436 | readsize = min(size, readsize * 2) |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 437 | if readsize > self.min_readsize: |
| 438 | self.min_readsize = min(readsize, self.min_readsize * 2, 512) |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 439 | return b''.join(bufs) # Return resulting line |
Tim Peters | 07e99cb | 2001-01-14 23:47:14 +0000 | [diff] [blame] | 440 | |
Andrew M. Kuchling | 41616ee | 2000-07-29 20:15:26 +0000 | [diff] [blame] | 441 | def readlines(self, sizehint=0): |
| 442 | # Negative numbers result in reading all the lines |
Tim Peters | fb0ea52 | 2002-11-04 19:50:11 +0000 | [diff] [blame] | 443 | if sizehint <= 0: |
| 444 | sizehint = sys.maxint |
Andrew M. Kuchling | 41616ee | 2000-07-29 20:15:26 +0000 | [diff] [blame] | 445 | L = [] |
| 446 | while sizehint > 0: |
| 447 | line = self.readline() |
Walter Dörwald | 5b1284d | 2007-06-06 16:43:59 +0000 | [diff] [blame] | 448 | if line == b"": |
Tim Peters | fb0ea52 | 2002-11-04 19:50:11 +0000 | [diff] [blame] | 449 | break |
Guido van Rossum | 8ca162f | 2002-04-07 06:36:23 +0000 | [diff] [blame] | 450 | L.append(line) |
Andrew M. Kuchling | 41616ee | 2000-07-29 20:15:26 +0000 | [diff] [blame] | 451 | sizehint = sizehint - len(line) |
| 452 | |
| 453 | return L |
Guido van Rossum | 1526219 | 1997-04-30 16:04:57 +0000 | [diff] [blame] | 454 | |
Guido van Rossum | 68de379 | 1997-07-19 20:22:23 +0000 | [diff] [blame] | 455 | def writelines(self, L): |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 456 | for line in L: |
| 457 | self.write(line) |
Guido van Rossum | 51ca6e3 | 1997-12-30 20:09:08 +0000 | [diff] [blame] | 458 | |
Neil Schemenauer | cacbdf6 | 2002-03-20 18:36:00 +0000 | [diff] [blame] | 459 | def __iter__(self): |
| 460 | return self |
| 461 | |
Georg Brandl | a18af4e | 2007-04-21 15:47:16 +0000 | [diff] [blame] | 462 | def __next__(self): |
Neil Schemenauer | cacbdf6 | 2002-03-20 18:36:00 +0000 | [diff] [blame] | 463 | line = self.readline() |
| 464 | if line: |
| 465 | return line |
| 466 | else: |
| 467 | raise StopIteration |
| 468 | |
Guido van Rossum | 51ca6e3 | 1997-12-30 20:09:08 +0000 | [diff] [blame] | 469 | |
| 470 | def _test(): |
| 471 | # Act like gzip; with -d, act like gunzip. |
| 472 | # The input file is not deleted, however, nor are any other gzip |
| 473 | # options or features supported. |
Guido van Rossum | 51ca6e3 | 1997-12-30 20:09:08 +0000 | [diff] [blame] | 474 | args = sys.argv[1:] |
| 475 | decompress = args and args[0] == "-d" |
| 476 | if decompress: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 477 | args = args[1:] |
Guido van Rossum | 51ca6e3 | 1997-12-30 20:09:08 +0000 | [diff] [blame] | 478 | if not args: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 479 | args = ["-"] |
Guido van Rossum | 51ca6e3 | 1997-12-30 20:09:08 +0000 | [diff] [blame] | 480 | for arg in args: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 481 | if decompress: |
| 482 | if arg == "-": |
| 483 | f = GzipFile(filename="", mode="rb", fileobj=sys.stdin) |
| 484 | g = sys.stdout |
| 485 | else: |
| 486 | if arg[-3:] != ".gz": |
Guido van Rossum | be19ed7 | 2007-02-09 05:37:30 +0000 | [diff] [blame] | 487 | print("filename doesn't end in .gz:", repr(arg)) |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 488 | continue |
| 489 | f = open(arg, "rb") |
| 490 | g = __builtin__.open(arg[:-3], "wb") |
| 491 | else: |
| 492 | if arg == "-": |
| 493 | f = sys.stdin |
| 494 | g = GzipFile(filename="", mode="wb", fileobj=sys.stdout) |
| 495 | else: |
| 496 | f = __builtin__.open(arg, "rb") |
| 497 | g = open(arg + ".gz", "wb") |
Guido van Rossum | 8ca162f | 2002-04-07 06:36:23 +0000 | [diff] [blame] | 498 | while True: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 499 | chunk = f.read(1024) |
| 500 | if not chunk: |
| 501 | break |
| 502 | g.write(chunk) |
| 503 | if g is not sys.stdout: |
| 504 | g.close() |
| 505 | if f is not sys.stdin: |
| 506 | f.close() |
Guido van Rossum | 51ca6e3 | 1997-12-30 20:09:08 +0000 | [diff] [blame] | 507 | |
| 508 | if __name__ == '__main__': |
| 509 | _test() |