| """Functions that read and write gzipped files. |
| |
| The user of the file doesn't have to worry about the compression, |
| but random access is not allowed.""" |
| |
| # based on Andrew Kuchling's minigzip.py distributed with the zlib module |
| |
| import struct, sys, time, os |
| import zlib |
| import builtins |
| import io |
| import _compression |
| |
| __all__ = ["GzipFile", "open", "compress", "decompress"] |
| |
| FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16 |
| |
| READ, WRITE = 1, 2 |
| |
| def open(filename, mode="rb", compresslevel=9, |
| encoding=None, errors=None, newline=None): |
| """Open a gzip-compressed file in binary or text mode. |
| |
| The filename argument can be an actual filename (a str or bytes object), or |
| an existing file object to read from or write to. |
| |
| The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or "ab" for |
| binary mode, or "rt", "wt", "xt" or "at" for text mode. The default mode is |
| "rb", and the default compresslevel is 9. |
| |
| For binary mode, this function is equivalent to the GzipFile constructor: |
| GzipFile(filename, mode, compresslevel). In this case, the encoding, errors |
| and newline arguments must not be provided. |
| |
| For text mode, a GzipFile object is created, and wrapped in an |
| io.TextIOWrapper instance with the specified encoding, error handling |
| behavior, and line ending(s). |
| |
| """ |
| if "t" in mode: |
| if "b" in mode: |
| raise ValueError("Invalid mode: %r" % (mode,)) |
| else: |
| if encoding is not None: |
| raise ValueError("Argument 'encoding' not supported in binary mode") |
| if errors is not None: |
| raise ValueError("Argument 'errors' not supported in binary mode") |
| if newline is not None: |
| raise ValueError("Argument 'newline' not supported in binary mode") |
| |
| gz_mode = mode.replace("t", "") |
| if isinstance(filename, (str, bytes, os.PathLike)): |
| binary_file = GzipFile(filename, gz_mode, compresslevel) |
| elif hasattr(filename, "read") or hasattr(filename, "write"): |
| binary_file = GzipFile(None, gz_mode, compresslevel, filename) |
| else: |
| raise TypeError("filename must be a str or bytes object, or a file") |
| |
| if "t" in mode: |
| return io.TextIOWrapper(binary_file, encoding, errors, newline) |
| else: |
| return binary_file |
| |
| def write32u(output, value): |
| # The L format writes the bit pattern correctly whether signed |
| # or unsigned. |
| output.write(struct.pack("<L", value)) |
| |
| class _PaddedFile: |
| """Minimal read-only file object that prepends a string to the contents |
| of an actual file. Shouldn't be used outside of gzip.py, as it lacks |
| essential functionality.""" |
| |
| def __init__(self, f, prepend=b''): |
| self._buffer = prepend |
| self._length = len(prepend) |
| self.file = f |
| self._read = 0 |
| |
| def read(self, size): |
| if self._read is None: |
| return self.file.read(size) |
| if self._read + size <= self._length: |
| read = self._read |
| self._read += size |
| return self._buffer[read:self._read] |
| else: |
| read = self._read |
| self._read = None |
| return self._buffer[read:] + \ |
| self.file.read(size-self._length+read) |
| |
| def prepend(self, prepend=b''): |
| if self._read is None: |
| self._buffer = prepend |
| else: # Assume data was read since the last prepend() call |
| self._read -= len(prepend) |
| return |
| self._length = len(self._buffer) |
| self._read = 0 |
| |
| def seek(self, off): |
| self._read = None |
| self._buffer = None |
| return self.file.seek(off) |
| |
| def seekable(self): |
| return True # Allows fast-forwarding even in unseekable streams |
| |
| class GzipFile(_compression.BaseStream): |
| """The GzipFile class simulates most of the methods of a file object with |
| the exception of the truncate() method. |
| |
| This class only supports opening files in binary mode. If you need to open a |
| compressed file in text mode, use the gzip.open() function. |
| |
| """ |
| |
| # Overridden with internal file object to be closed, if only a filename |
| # is passed in |
| myfileobj = None |
| |
| def __init__(self, filename=None, mode=None, |
| compresslevel=9, fileobj=None, mtime=None): |
| """Constructor for the GzipFile class. |
| |
| At least one of fileobj and filename must be given a |
| non-trivial value. |
| |
| The new class instance is based on fileobj, which can be a regular |
| file, an io.BytesIO object, or any other object which simulates a file. |
| It defaults to None, in which case filename is opened to provide |
| a file object. |
| |
| When fileobj is not None, the filename argument is only used to be |
| included in the gzip file header, which may include the original |
| filename of the uncompressed file. It defaults to the filename of |
| fileobj, if discernible; otherwise, it defaults to the empty string, |
| and in this case the original filename is not included in the header. |
| |
| The mode argument can be any of 'r', 'rb', 'a', 'ab', 'w', 'wb', 'x', or |
| 'xb' depending on whether the file will be read or written. The default |
| is the mode of fileobj if discernible; otherwise, the default is 'rb'. |
| A mode of 'r' is equivalent to one of 'rb', and similarly for 'w' and |
| 'wb', 'a' and 'ab', and 'x' and 'xb'. |
| |
| The compresslevel argument is an integer from 0 to 9 controlling the |
| level of compression; 1 is fastest and produces the least compression, |
| and 9 is slowest and produces the most compression. 0 is no compression |
| at all. The default is 9. |
| |
| The mtime argument is an optional numeric timestamp to be written |
| to the last modification time field in the stream when compressing. |
| If omitted or None, the current time is used. |
| |
| """ |
| |
| if mode and ('t' in mode or 'U' in mode): |
| raise ValueError("Invalid mode: {!r}".format(mode)) |
| if mode and 'b' not in mode: |
| mode += 'b' |
| if fileobj is None: |
| fileobj = self.myfileobj = builtins.open(filename, mode or 'rb') |
| if filename is None: |
| filename = getattr(fileobj, 'name', '') |
| if not isinstance(filename, (str, bytes)): |
| filename = '' |
| else: |
| filename = os.fspath(filename) |
| if mode is None: |
| mode = getattr(fileobj, 'mode', 'rb') |
| |
| if mode.startswith('r'): |
| self.mode = READ |
| raw = _GzipReader(fileobj) |
| self._buffer = io.BufferedReader(raw) |
| self.name = filename |
| |
| elif mode.startswith(('w', 'a', 'x')): |
| self.mode = WRITE |
| self._init_write(filename) |
| self.compress = zlib.compressobj(compresslevel, |
| zlib.DEFLATED, |
| -zlib.MAX_WBITS, |
| zlib.DEF_MEM_LEVEL, |
| 0) |
| self._write_mtime = mtime |
| else: |
| raise ValueError("Invalid mode: {!r}".format(mode)) |
| |
| self.fileobj = fileobj |
| |
| if self.mode == WRITE: |
| self._write_gzip_header() |
| |
| @property |
| def filename(self): |
| import warnings |
| warnings.warn("use the name attribute", DeprecationWarning, 2) |
| if self.mode == WRITE and self.name[-3:] != ".gz": |
| return self.name + ".gz" |
| return self.name |
| |
| @property |
| def mtime(self): |
| """Last modification time read from stream, or None""" |
| return self._buffer.raw._last_mtime |
| |
| def __repr__(self): |
| s = repr(self.fileobj) |
| return '<gzip ' + s[1:-1] + ' ' + hex(id(self)) + '>' |
| |
| def _init_write(self, filename): |
| self.name = filename |
| self.crc = zlib.crc32(b"") |
| self.size = 0 |
| self.writebuf = [] |
| self.bufsize = 0 |
| self.offset = 0 # Current file offset for seek(), tell(), etc |
| |
| def _write_gzip_header(self): |
| self.fileobj.write(b'\037\213') # magic header |
| self.fileobj.write(b'\010') # compression method |
| try: |
| # RFC 1952 requires the FNAME field to be Latin-1. Do not |
| # include filenames that cannot be represented that way. |
| fname = os.path.basename(self.name) |
| if not isinstance(fname, bytes): |
| fname = fname.encode('latin-1') |
| if fname.endswith(b'.gz'): |
| fname = fname[:-3] |
| except UnicodeEncodeError: |
| fname = b'' |
| flags = 0 |
| if fname: |
| flags = FNAME |
| self.fileobj.write(chr(flags).encode('latin-1')) |
| mtime = self._write_mtime |
| if mtime is None: |
| mtime = time.time() |
| write32u(self.fileobj, int(mtime)) |
| self.fileobj.write(b'\002') |
| self.fileobj.write(b'\377') |
| if fname: |
| self.fileobj.write(fname + b'\000') |
| |
| def write(self,data): |
| self._check_not_closed() |
| if self.mode != WRITE: |
| import errno |
| raise OSError(errno.EBADF, "write() on read-only GzipFile object") |
| |
| if self.fileobj is None: |
| raise ValueError("write() on closed GzipFile object") |
| |
| if isinstance(data, bytes): |
| length = len(data) |
| else: |
| # accept any data that supports the buffer protocol |
| data = memoryview(data) |
| length = data.nbytes |
| |
| if length > 0: |
| self.fileobj.write(self.compress.compress(data)) |
| self.size += length |
| self.crc = zlib.crc32(data, self.crc) |
| self.offset += length |
| |
| return length |
| |
| def read(self, size=-1): |
| self._check_not_closed() |
| if self.mode != READ: |
| import errno |
| raise OSError(errno.EBADF, "read() on write-only GzipFile object") |
| return self._buffer.read(size) |
| |
| def read1(self, size=-1): |
| """Implements BufferedIOBase.read1() |
| |
| Reads up to a buffer's worth of data is size is negative.""" |
| self._check_not_closed() |
| if self.mode != READ: |
| import errno |
| raise OSError(errno.EBADF, "read1() on write-only GzipFile object") |
| |
| if size < 0: |
| size = io.DEFAULT_BUFFER_SIZE |
| return self._buffer.read1(size) |
| |
| def peek(self, n): |
| self._check_not_closed() |
| if self.mode != READ: |
| import errno |
| raise OSError(errno.EBADF, "peek() on write-only GzipFile object") |
| return self._buffer.peek(n) |
| |
| @property |
| def closed(self): |
| return self.fileobj is None |
| |
| def close(self): |
| fileobj = self.fileobj |
| if fileobj is None: |
| return |
| self.fileobj = None |
| try: |
| if self.mode == WRITE: |
| fileobj.write(self.compress.flush()) |
| write32u(fileobj, self.crc) |
| # self.size may exceed 2GB, or even 4GB |
| write32u(fileobj, self.size & 0xffffffff) |
| elif self.mode == READ: |
| self._buffer.close() |
| finally: |
| myfileobj = self.myfileobj |
| if myfileobj: |
| self.myfileobj = None |
| myfileobj.close() |
| |
| def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH): |
| self._check_not_closed() |
| if self.mode == WRITE: |
| # Ensure the compressor's buffer is flushed |
| self.fileobj.write(self.compress.flush(zlib_mode)) |
| self.fileobj.flush() |
| |
| def fileno(self): |
| """Invoke the underlying file object's fileno() method. |
| |
| This will raise AttributeError if the underlying file object |
| doesn't support fileno(). |
| """ |
| return self.fileobj.fileno() |
| |
| def rewind(self): |
| '''Return the uncompressed stream file position indicator to the |
| beginning of the file''' |
| if self.mode != READ: |
| raise OSError("Can't rewind in write mode") |
| self._buffer.seek(0) |
| |
| def readable(self): |
| return self.mode == READ |
| |
| def writable(self): |
| return self.mode == WRITE |
| |
| def seekable(self): |
| return True |
| |
| def seek(self, offset, whence=io.SEEK_SET): |
| if self.mode == WRITE: |
| if whence != io.SEEK_SET: |
| if whence == io.SEEK_CUR: |
| offset = self.offset + offset |
| else: |
| raise ValueError('Seek from end not supported') |
| if offset < self.offset: |
| raise OSError('Negative seek in write mode') |
| count = offset - self.offset |
| chunk = b'\0' * 1024 |
| for i in range(count // 1024): |
| self.write(chunk) |
| self.write(b'\0' * (count % 1024)) |
| elif self.mode == READ: |
| self._check_not_closed() |
| return self._buffer.seek(offset, whence) |
| |
| return self.offset |
| |
| def readline(self, size=-1): |
| self._check_not_closed() |
| return self._buffer.readline(size) |
| |
| |
| class _GzipReader(_compression.DecompressReader): |
| def __init__(self, fp): |
| super().__init__(_PaddedFile(fp), zlib.decompressobj, |
| wbits=-zlib.MAX_WBITS) |
| # Set flag indicating start of a new member |
| self._new_member = True |
| self._last_mtime = None |
| |
| def _init_read(self): |
| self._crc = zlib.crc32(b"") |
| self._stream_size = 0 # Decompressed size of unconcatenated stream |
| |
| def _read_exact(self, n): |
| '''Read exactly *n* bytes from `self._fp` |
| |
| This method is required because self._fp may be unbuffered, |
| i.e. return short reads. |
| ''' |
| |
| data = self._fp.read(n) |
| while len(data) < n: |
| b = self._fp.read(n - len(data)) |
| if not b: |
| raise EOFError("Compressed file ended before the " |
| "end-of-stream marker was reached") |
| data += b |
| return data |
| |
| def _read_gzip_header(self): |
| magic = self._fp.read(2) |
| if magic == b'': |
| return False |
| |
| if magic != b'\037\213': |
| raise OSError('Not a gzipped file (%r)' % magic) |
| |
| (method, flag, |
| self._last_mtime) = struct.unpack("<BBIxx", self._read_exact(8)) |
| if method != 8: |
| raise OSError('Unknown compression method') |
| |
| if flag & FEXTRA: |
| # Read & discard the extra field, if present |
| extra_len, = struct.unpack("<H", self._read_exact(2)) |
| self._read_exact(extra_len) |
| if flag & FNAME: |
| # Read and discard a null-terminated string containing the filename |
| while True: |
| s = self._fp.read(1) |
| if not s or s==b'\000': |
| break |
| if flag & FCOMMENT: |
| # Read and discard a null-terminated string containing a comment |
| while True: |
| s = self._fp.read(1) |
| if not s or s==b'\000': |
| break |
| if flag & FHCRC: |
| self._read_exact(2) # Read & discard the 16-bit header CRC |
| return True |
| |
| def read(self, size=-1): |
| if size < 0: |
| return self.readall() |
| # size=0 is special because decompress(max_length=0) is not supported |
| if not size: |
| return b"" |
| |
| # For certain input data, a single |
| # call to decompress() may not return |
| # any data. In this case, retry until we get some data or reach EOF. |
| while True: |
| if self._decompressor.eof: |
| # Ending case: we've come to the end of a member in the file, |
| # so finish up this member, and read a new gzip header. |
| # Check the CRC and file size, and set the flag so we read |
| # a new member |
| self._read_eof() |
| self._new_member = True |
| self._decompressor = self._decomp_factory( |
| **self._decomp_args) |
| |
| if self._new_member: |
| # If the _new_member flag is set, we have to |
| # jump to the next member, if there is one. |
| self._init_read() |
| if not self._read_gzip_header(): |
| self._size = self._pos |
| return b"" |
| self._new_member = False |
| |
| # Read a chunk of data from the file |
| buf = self._fp.read(io.DEFAULT_BUFFER_SIZE) |
| |
| uncompress = self._decompressor.decompress(buf, size) |
| if self._decompressor.unconsumed_tail != b"": |
| self._fp.prepend(self._decompressor.unconsumed_tail) |
| elif self._decompressor.unused_data != b"": |
| # Prepend the already read bytes to the fileobj so they can |
| # be seen by _read_eof() and _read_gzip_header() |
| self._fp.prepend(self._decompressor.unused_data) |
| |
| if uncompress != b"": |
| break |
| if buf == b"": |
| raise EOFError("Compressed file ended before the " |
| "end-of-stream marker was reached") |
| |
| self._add_read_data( uncompress ) |
| self._pos += len(uncompress) |
| return uncompress |
| |
| def _add_read_data(self, data): |
| self._crc = zlib.crc32(data, self._crc) |
| self._stream_size = self._stream_size + len(data) |
| |
| def _read_eof(self): |
| # We've read to the end of the file |
| # We check the that the computed CRC and size of the |
| # uncompressed data matches the stored values. Note that the size |
| # stored is the true file size mod 2**32. |
| crc32, isize = struct.unpack("<II", self._read_exact(8)) |
| if crc32 != self._crc: |
| raise OSError("CRC check failed %s != %s" % (hex(crc32), |
| hex(self._crc))) |
| elif isize != (self._stream_size & 0xffffffff): |
| raise OSError("Incorrect length of data produced") |
| |
| # Gzip files can be padded with zeroes and still have archives. |
| # Consume all zero bytes and set the file position to the first |
| # non-zero byte. See http://www.gzip.org/#faq8 |
| c = b"\x00" |
| while c == b"\x00": |
| c = self._fp.read(1) |
| if c: |
| self._fp.prepend(c) |
| |
| def _rewind(self): |
| super()._rewind() |
| self._new_member = True |
| |
| def compress(data, compresslevel=9): |
| """Compress data in one shot and return the compressed string. |
| Optional argument is the compression level, in range of 0-9. |
| """ |
| buf = io.BytesIO() |
| with GzipFile(fileobj=buf, mode='wb', compresslevel=compresslevel) as f: |
| f.write(data) |
| return buf.getvalue() |
| |
| def decompress(data): |
| """Decompress a gzip compressed string in one shot. |
| Return the decompressed string. |
| """ |
| with GzipFile(fileobj=io.BytesIO(data)) as f: |
| return f.read() |
| |
| |
| def _test(): |
| # Act like gzip; with -d, act like gunzip. |
| # The input file is not deleted, however, nor are any other gzip |
| # options or features supported. |
| args = sys.argv[1:] |
| decompress = args and args[0] == "-d" |
| if decompress: |
| args = args[1:] |
| if not args: |
| args = ["-"] |
| for arg in args: |
| if decompress: |
| if arg == "-": |
| f = GzipFile(filename="", mode="rb", fileobj=sys.stdin.buffer) |
| g = sys.stdout.buffer |
| else: |
| if arg[-3:] != ".gz": |
| print("filename doesn't end in .gz:", repr(arg)) |
| continue |
| f = open(arg, "rb") |
| g = builtins.open(arg[:-3], "wb") |
| else: |
| if arg == "-": |
| f = sys.stdin.buffer |
| g = GzipFile(filename="", mode="wb", fileobj=sys.stdout.buffer) |
| else: |
| f = builtins.open(arg, "rb") |
| g = open(arg + ".gz", "wb") |
| while True: |
| chunk = f.read(1024) |
| if not chunk: |
| break |
| g.write(chunk) |
| if g is not sys.stdout.buffer: |
| g.close() |
| if f is not sys.stdin.buffer: |
| f.close() |
| |
| if __name__ == '__main__': |
| _test() |