blob: 7143f0b5e2ca2661d144c24a2f1cdcbd9e4fa0cf [file] [log] [blame]
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001#!/usr/bin/env python
2# -*- coding: iso-8859-1 -*-
3#-------------------------------------------------------------------
4# tarfile.py
5#-------------------------------------------------------------------
6# Copyright (C) 2002 Lars Gustäbel <lars@gustaebel.de>
7# All rights reserved.
8#
9# Permission is hereby granted, free of charge, to any person
10# obtaining a copy of this software and associated documentation
11# files (the "Software"), to deal in the Software without
12# restriction, including without limitation the rights to use,
13# copy, modify, merge, publish, distribute, sublicense, and/or sell
14# copies of the Software, and to permit persons to whom the
15# Software is furnished to do so, subject to the following
16# conditions:
17#
18# The above copyright notice and this permission notice shall be
19# included in all copies or substantial portions of the Software.
20#
21# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
22# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
23# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
24# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
25# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
26# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
28# OTHER DEALINGS IN THE SOFTWARE.
29#
30"""Read from and write to tar format archives.
31"""
32
33__version__ = "$Revision$"
34# $Source$
35
Lars Gustäbelc64e4022007-03-13 10:47:19 +000036version = "0.9.0"
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +000037__author__ = "Lars Gustäbel (lars@gustaebel.de)"
38__date__ = "$Date$"
39__cvsid__ = "$Id$"
40__credits__ = "Gustavo Niemeyer, Niels Gustäbel, Richard Townsend."
41
42#---------
43# Imports
44#---------
45import sys
46import os
47import shutil
48import stat
49import errno
50import time
51import struct
Georg Brandl3354f282006-10-29 09:16:12 +000052import copy
Lars Gustäbelc64e4022007-03-13 10:47:19 +000053import re
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +000054
Jack Jansencfc49022003-03-07 13:37:32 +000055if sys.platform == 'mac':
56 # This module needs work for MacOS9, especially in the area of pathname
57 # handling. In many places it is assumed a simple substitution of / by the
58 # local os.path.sep is good enough to convert pathnames, but this does not
59 # work with the mac rooted:path:name versus :nonrooted:path:name syntax
60 raise ImportError, "tarfile does not work for platform==mac"
61
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +000062try:
63 import grp, pwd
64except ImportError:
65 grp = pwd = None
66
67# from tarfile import *
68__all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError"]
69
70#---------------------------------------------------------
71# tar constants
72#---------------------------------------------------------
Lars Gustäbelc64e4022007-03-13 10:47:19 +000073NUL = "\0" # the null character
74BLOCKSIZE = 512 # length of processing blocks
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +000075RECORDSIZE = BLOCKSIZE * 20 # length of records
Lars Gustäbelc64e4022007-03-13 10:47:19 +000076GNU_MAGIC = "ustar \0" # magic gnu tar string
77POSIX_MAGIC = "ustar\x0000" # magic posix tar string
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +000078
Lars Gustäbelc64e4022007-03-13 10:47:19 +000079LENGTH_NAME = 100 # maximum length of a filename
80LENGTH_LINK = 100 # maximum length of a linkname
81LENGTH_PREFIX = 155 # maximum length of the prefix field
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +000082
Lars Gustäbelc64e4022007-03-13 10:47:19 +000083REGTYPE = "0" # regular file
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +000084AREGTYPE = "\0" # regular file
Lars Gustäbelc64e4022007-03-13 10:47:19 +000085LNKTYPE = "1" # link (inside tarfile)
86SYMTYPE = "2" # symbolic link
87CHRTYPE = "3" # character special device
88BLKTYPE = "4" # block special device
89DIRTYPE = "5" # directory
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +000090FIFOTYPE = "6" # fifo special device
91CONTTYPE = "7" # contiguous file
92
Lars Gustäbelc64e4022007-03-13 10:47:19 +000093GNUTYPE_LONGNAME = "L" # GNU tar longname
94GNUTYPE_LONGLINK = "K" # GNU tar longlink
95GNUTYPE_SPARSE = "S" # GNU tar sparse file
96
97XHDTYPE = "x" # POSIX.1-2001 extended header
98XGLTYPE = "g" # POSIX.1-2001 global header
99SOLARIS_XHDTYPE = "X" # Solaris extended header
100
101USTAR_FORMAT = 0 # POSIX.1-1988 (ustar) format
102GNU_FORMAT = 1 # GNU tar format
103PAX_FORMAT = 2 # POSIX.1-2001 (pax) format
104DEFAULT_FORMAT = GNU_FORMAT
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000105
106#---------------------------------------------------------
107# tarfile constants
108#---------------------------------------------------------
Lars Gustäbelc64e4022007-03-13 10:47:19 +0000109# File types that tarfile supports:
110SUPPORTED_TYPES = (REGTYPE, AREGTYPE, LNKTYPE,
111 SYMTYPE, DIRTYPE, FIFOTYPE,
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000112 CONTTYPE, CHRTYPE, BLKTYPE,
113 GNUTYPE_LONGNAME, GNUTYPE_LONGLINK,
114 GNUTYPE_SPARSE)
115
Lars Gustäbelc64e4022007-03-13 10:47:19 +0000116# File types that will be treated as a regular file.
117REGULAR_TYPES = (REGTYPE, AREGTYPE,
118 CONTTYPE, GNUTYPE_SPARSE)
119
120# File types that are part of the GNU tar format.
121GNU_TYPES = (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK,
122 GNUTYPE_SPARSE)
123
124# Fields from a pax header that override a TarInfo attribute.
125PAX_FIELDS = ("path", "linkpath", "size", "mtime",
126 "uid", "gid", "uname", "gname")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000127
Lars Gustäbela0fcb932007-05-27 19:49:30 +0000128# Fields in a pax header that are numbers, all other fields
129# are treated as strings.
130PAX_NUMBER_FIELDS = {
131 "atime": float,
132 "ctime": float,
133 "mtime": float,
134 "uid": int,
135 "gid": int,
136 "size": int
137}
138
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000139#---------------------------------------------------------
140# Bits used in the mode field, values in octal.
141#---------------------------------------------------------
142S_IFLNK = 0120000 # symbolic link
143S_IFREG = 0100000 # regular file
144S_IFBLK = 0060000 # block device
145S_IFDIR = 0040000 # directory
146S_IFCHR = 0020000 # character device
147S_IFIFO = 0010000 # fifo
148
149TSUID = 04000 # set UID on execution
150TSGID = 02000 # set GID on execution
151TSVTX = 01000 # reserved
152
153TUREAD = 0400 # read by owner
154TUWRITE = 0200 # write by owner
155TUEXEC = 0100 # execute/search by owner
156TGREAD = 0040 # read by group
157TGWRITE = 0020 # write by group
158TGEXEC = 0010 # execute/search by group
159TOREAD = 0004 # read by other
160TOWRITE = 0002 # write by other
161TOEXEC = 0001 # execute/search by other
162
163#---------------------------------------------------------
Lars Gustäbelc64e4022007-03-13 10:47:19 +0000164# initialization
165#---------------------------------------------------------
166ENCODING = sys.getfilesystemencoding()
167if ENCODING is None:
Lars Gustäbela0fcb932007-05-27 19:49:30 +0000168 ENCODING = sys.getdefaultencoding()
Lars Gustäbelc64e4022007-03-13 10:47:19 +0000169
170#---------------------------------------------------------
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000171# Some useful functions
172#---------------------------------------------------------
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000173
Georg Brandl38c6a222006-05-10 16:26:03 +0000174def stn(s, length):
175 """Convert a python string to a null-terminated string buffer.
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000176 """
Georg Brandla32e0a02006-10-24 16:54:16 +0000177 return s[:length] + (length - len(s)) * NUL
Georg Brandl38c6a222006-05-10 16:26:03 +0000178
Lars Gustäbelc64e4022007-03-13 10:47:19 +0000179def nts(s):
180 """Convert a null-terminated string field to a python string.
181 """
182 # Use the string up to the first null char.
183 p = s.find("\0")
184 if p == -1:
185 return s
186 return s[:p]
187
Georg Brandl38c6a222006-05-10 16:26:03 +0000188def nti(s):
189 """Convert a number field to a python number.
190 """
191 # There are two possible encodings for a number field, see
192 # itn() below.
193 if s[0] != chr(0200):
Georg Brandlded1c4d2006-12-20 11:55:16 +0000194 try:
Lars Gustäbelc64e4022007-03-13 10:47:19 +0000195 n = int(nts(s) or "0", 8)
Georg Brandlded1c4d2006-12-20 11:55:16 +0000196 except ValueError:
197 raise HeaderError("invalid header")
Georg Brandl38c6a222006-05-10 16:26:03 +0000198 else:
199 n = 0L
200 for i in xrange(len(s) - 1):
201 n <<= 8
202 n += ord(s[i + 1])
203 return n
204
Lars Gustäbelc64e4022007-03-13 10:47:19 +0000205def itn(n, digits=8, format=DEFAULT_FORMAT):
Georg Brandl38c6a222006-05-10 16:26:03 +0000206 """Convert a python number to a number field.
207 """
208 # POSIX 1003.1-1988 requires numbers to be encoded as a string of
209 # octal digits followed by a null-byte, this allows values up to
210 # (8**(digits-1))-1. GNU tar allows storing numbers greater than
211 # that if necessary. A leading 0200 byte indicates this particular
212 # encoding, the following digits-1 bytes are a big-endian
213 # representation. This allows values up to (256**(digits-1))-1.
214 if 0 <= n < 8 ** (digits - 1):
215 s = "%0*o" % (digits - 1, n) + NUL
216 else:
Lars Gustäbelc64e4022007-03-13 10:47:19 +0000217 if format != GNU_FORMAT or n >= 256 ** (digits - 1):
Georg Brandle4751e32006-05-18 06:11:19 +0000218 raise ValueError("overflow in number field")
Georg Brandl38c6a222006-05-10 16:26:03 +0000219
220 if n < 0:
221 # XXX We mimic GNU tar's behaviour with negative numbers,
222 # this could raise OverflowError.
223 n = struct.unpack("L", struct.pack("l", n))[0]
224
225 s = ""
226 for i in xrange(digits - 1):
227 s = chr(n & 0377) + s
228 n >>= 8
229 s = chr(0200) + s
230 return s
231
Lars Gustäbela0fcb932007-05-27 19:49:30 +0000232def uts(s, encoding, errors):
233 """Convert a unicode object to a string.
234 """
235 if errors == "utf-8":
236 # An extra error handler similar to the -o invalid=UTF-8 option
237 # in POSIX.1-2001. Replace untranslatable characters with their
238 # UTF-8 representation.
239 try:
240 return s.encode(encoding, "strict")
241 except UnicodeEncodeError:
242 x = []
243 for c in s:
244 try:
245 x.append(c.encode(encoding, "strict"))
246 except UnicodeEncodeError:
247 x.append(c.encode("utf8"))
248 return "".join(x)
249 else:
250 return s.encode(encoding, errors)
251
Georg Brandl38c6a222006-05-10 16:26:03 +0000252def calc_chksums(buf):
253 """Calculate the checksum for a member's header by summing up all
254 characters except for the chksum field which is treated as if
255 it was filled with spaces. According to the GNU tar sources,
256 some tars (Sun and NeXT) calculate chksum with signed char,
257 which will be different if there are chars in the buffer with
258 the high bit set. So we calculate two checksums, unsigned and
259 signed.
260 """
261 unsigned_chksum = 256 + sum(struct.unpack("148B", buf[:148]) + struct.unpack("356B", buf[156:512]))
262 signed_chksum = 256 + sum(struct.unpack("148b", buf[:148]) + struct.unpack("356b", buf[156:512]))
263 return unsigned_chksum, signed_chksum
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000264
265def copyfileobj(src, dst, length=None):
266 """Copy length bytes from fileobj src to fileobj dst.
267 If length is None, copy the entire content.
268 """
269 if length == 0:
270 return
271 if length is None:
272 shutil.copyfileobj(src, dst)
273 return
274
275 BUFSIZE = 16 * 1024
276 blocks, remainder = divmod(length, BUFSIZE)
277 for b in xrange(blocks):
278 buf = src.read(BUFSIZE)
279 if len(buf) < BUFSIZE:
Georg Brandle4751e32006-05-18 06:11:19 +0000280 raise IOError("end of file reached")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000281 dst.write(buf)
282
283 if remainder != 0:
284 buf = src.read(remainder)
285 if len(buf) < remainder:
Georg Brandle4751e32006-05-18 06:11:19 +0000286 raise IOError("end of file reached")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000287 dst.write(buf)
288 return
289
290filemode_table = (
Andrew M. Kuchling8bc462f2004-10-20 11:48:42 +0000291 ((S_IFLNK, "l"),
292 (S_IFREG, "-"),
293 (S_IFBLK, "b"),
294 (S_IFDIR, "d"),
295 (S_IFCHR, "c"),
296 (S_IFIFO, "p")),
297
298 ((TUREAD, "r"),),
299 ((TUWRITE, "w"),),
300 ((TUEXEC|TSUID, "s"),
301 (TSUID, "S"),
302 (TUEXEC, "x")),
303
304 ((TGREAD, "r"),),
305 ((TGWRITE, "w"),),
306 ((TGEXEC|TSGID, "s"),
307 (TSGID, "S"),
308 (TGEXEC, "x")),
309
310 ((TOREAD, "r"),),
311 ((TOWRITE, "w"),),
312 ((TOEXEC|TSVTX, "t"),
313 (TSVTX, "T"),
314 (TOEXEC, "x"))
315)
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000316
317def filemode(mode):
318 """Convert a file's mode to a string of the form
319 -rwxrwxrwx.
320 Used by TarFile.list()
321 """
Andrew M. Kuchling8bc462f2004-10-20 11:48:42 +0000322 perm = []
323 for table in filemode_table:
324 for bit, char in table:
325 if mode & bit == bit:
326 perm.append(char)
327 break
328 else:
329 perm.append("-")
330 return "".join(perm)
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000331
332if os.sep != "/":
333 normpath = lambda path: os.path.normpath(path).replace(os.sep, "/")
334else:
335 normpath = os.path.normpath
336
337class TarError(Exception):
338 """Base exception."""
339 pass
340class ExtractError(TarError):
341 """General exception for extract errors."""
342 pass
343class ReadError(TarError):
344 """Exception for unreadble tar archives."""
345 pass
346class CompressionError(TarError):
347 """Exception for unavailable compression methods."""
348 pass
349class StreamError(TarError):
350 """Exception for unsupported operations on stream-like TarFiles."""
351 pass
Georg Brandlebbeed72006-12-19 22:06:46 +0000352class HeaderError(TarError):
353 """Exception for invalid headers."""
354 pass
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000355
356#---------------------------
357# internal stream interface
358#---------------------------
359class _LowLevelFile:
360 """Low-level file object. Supports reading and writing.
361 It is used instead of a regular file object for streaming
362 access.
363 """
364
365 def __init__(self, name, mode):
366 mode = {
367 "r": os.O_RDONLY,
368 "w": os.O_WRONLY | os.O_CREAT | os.O_TRUNC,
369 }[mode]
370 if hasattr(os, "O_BINARY"):
371 mode |= os.O_BINARY
372 self.fd = os.open(name, mode)
373
374 def close(self):
375 os.close(self.fd)
376
377 def read(self, size):
378 return os.read(self.fd, size)
379
380 def write(self, s):
381 os.write(self.fd, s)
382
383class _Stream:
384 """Class that serves as an adapter between TarFile and
385 a stream-like object. The stream-like object only
386 needs to have a read() or write() method and is accessed
387 blockwise. Use of gzip or bzip2 compression is possible.
388 A stream-like object could be for example: sys.stdin,
389 sys.stdout, a socket, a tape device etc.
390
391 _Stream is intended to be used only internally.
392 """
393
Martin v. Löwis78be7df2005-03-05 12:47:42 +0000394 def __init__(self, name, mode, comptype, fileobj, bufsize):
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000395 """Construct a _Stream object.
396 """
397 self._extfileobj = True
398 if fileobj is None:
399 fileobj = _LowLevelFile(name, mode)
400 self._extfileobj = False
401
Martin v. Löwis78be7df2005-03-05 12:47:42 +0000402 if comptype == '*':
403 # Enable transparent compression detection for the
404 # stream interface
405 fileobj = _StreamProxy(fileobj)
406 comptype = fileobj.getcomptype()
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000407
Martin v. Löwis78be7df2005-03-05 12:47:42 +0000408 self.name = name or ""
409 self.mode = mode
410 self.comptype = comptype
411 self.fileobj = fileobj
412 self.bufsize = bufsize
413 self.buf = ""
414 self.pos = 0L
415 self.closed = False
416
417 if comptype == "gz":
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000418 try:
419 import zlib
420 except ImportError:
Georg Brandle4751e32006-05-18 06:11:19 +0000421 raise CompressionError("zlib module is not available")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000422 self.zlib = zlib
423 self.crc = zlib.crc32("")
424 if mode == "r":
425 self._init_read_gz()
426 else:
427 self._init_write_gz()
428
Martin v. Löwis78be7df2005-03-05 12:47:42 +0000429 if comptype == "bz2":
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000430 try:
431 import bz2
432 except ImportError:
Georg Brandle4751e32006-05-18 06:11:19 +0000433 raise CompressionError("bz2 module is not available")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000434 if mode == "r":
435 self.dbuf = ""
436 self.cmp = bz2.BZ2Decompressor()
437 else:
438 self.cmp = bz2.BZ2Compressor()
439
440 def __del__(self):
Martin v. Löwis78be7df2005-03-05 12:47:42 +0000441 if hasattr(self, "closed") and not self.closed:
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000442 self.close()
443
444 def _init_write_gz(self):
445 """Initialize for writing with gzip compression.
446 """
447 self.cmp = self.zlib.compressobj(9, self.zlib.DEFLATED,
448 -self.zlib.MAX_WBITS,
449 self.zlib.DEF_MEM_LEVEL,
450 0)
451 timestamp = struct.pack("<L", long(time.time()))
452 self.__write("\037\213\010\010%s\002\377" % timestamp)
453 if self.name.endswith(".gz"):
454 self.name = self.name[:-3]
455 self.__write(self.name + NUL)
456
457 def write(self, s):
458 """Write string s to the stream.
459 """
Martin v. Löwis78be7df2005-03-05 12:47:42 +0000460 if self.comptype == "gz":
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000461 self.crc = self.zlib.crc32(s, self.crc)
462 self.pos += len(s)
Martin v. Löwis78be7df2005-03-05 12:47:42 +0000463 if self.comptype != "tar":
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000464 s = self.cmp.compress(s)
465 self.__write(s)
466
467 def __write(self, s):
468 """Write string s to the stream if a whole new block
469 is ready to be written.
470 """
471 self.buf += s
472 while len(self.buf) > self.bufsize:
473 self.fileobj.write(self.buf[:self.bufsize])
474 self.buf = self.buf[self.bufsize:]
475
476 def close(self):
477 """Close the _Stream object. No operation should be
478 done on it afterwards.
479 """
480 if self.closed:
481 return
482
Martin v. Löwis78be7df2005-03-05 12:47:42 +0000483 if self.mode == "w" and self.comptype != "tar":
Martin v. Löwisc234a522004-08-22 21:28:33 +0000484 self.buf += self.cmp.flush()
Martin v. Löwis78be7df2005-03-05 12:47:42 +0000485
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000486 if self.mode == "w" and self.buf:
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000487 self.fileobj.write(self.buf)
488 self.buf = ""
Martin v. Löwis78be7df2005-03-05 12:47:42 +0000489 if self.comptype == "gz":
Tim Petersa05f6e22006-08-02 05:20:08 +0000490 # The native zlib crc is an unsigned 32-bit integer, but
491 # the Python wrapper implicitly casts that to a signed C
492 # long. So, on a 32-bit box self.crc may "look negative",
493 # while the same crc on a 64-bit box may "look positive".
494 # To avoid irksome warnings from the `struct` module, force
495 # it to look positive on all boxes.
496 self.fileobj.write(struct.pack("<L", self.crc & 0xffffffffL))
Andrew M. Kuchling10a44492003-10-24 17:38:34 +0000497 self.fileobj.write(struct.pack("<L", self.pos & 0xffffFFFFL))
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000498
499 if not self._extfileobj:
500 self.fileobj.close()
501
502 self.closed = True
503
504 def _init_read_gz(self):
505 """Initialize for reading a gzip compressed fileobj.
506 """
507 self.cmp = self.zlib.decompressobj(-self.zlib.MAX_WBITS)
508 self.dbuf = ""
509
510 # taken from gzip.GzipFile with some alterations
511 if self.__read(2) != "\037\213":
Georg Brandle4751e32006-05-18 06:11:19 +0000512 raise ReadError("not a gzip file")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000513 if self.__read(1) != "\010":
Georg Brandle4751e32006-05-18 06:11:19 +0000514 raise CompressionError("unsupported compression method")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000515
516 flag = ord(self.__read(1))
517 self.__read(6)
518
519 if flag & 4:
520 xlen = ord(self.__read(1)) + 256 * ord(self.__read(1))
521 self.read(xlen)
522 if flag & 8:
523 while True:
524 s = self.__read(1)
525 if not s or s == NUL:
526 break
527 if flag & 16:
528 while True:
529 s = self.__read(1)
530 if not s or s == NUL:
531 break
532 if flag & 2:
533 self.__read(2)
534
535 def tell(self):
536 """Return the stream's file pointer position.
537 """
538 return self.pos
539
540 def seek(self, pos=0):
541 """Set the stream's file pointer to pos. Negative seeking
542 is forbidden.
543 """
544 if pos - self.pos >= 0:
545 blocks, remainder = divmod(pos - self.pos, self.bufsize)
546 for i in xrange(blocks):
547 self.read(self.bufsize)
548 self.read(remainder)
549 else:
Georg Brandle4751e32006-05-18 06:11:19 +0000550 raise StreamError("seeking backwards is not allowed")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000551 return self.pos
552
553 def read(self, size=None):
554 """Return the next size number of bytes from the stream.
555 If size is not defined, return all bytes of the stream
556 up to EOF.
557 """
558 if size is None:
559 t = []
560 while True:
561 buf = self._read(self.bufsize)
562 if not buf:
563 break
564 t.append(buf)
565 buf = "".join(t)
566 else:
567 buf = self._read(size)
568 self.pos += len(buf)
569 return buf
570
571 def _read(self, size):
572 """Return size bytes from the stream.
573 """
Martin v. Löwis78be7df2005-03-05 12:47:42 +0000574 if self.comptype == "tar":
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000575 return self.__read(size)
576
577 c = len(self.dbuf)
578 t = [self.dbuf]
579 while c < size:
580 buf = self.__read(self.bufsize)
581 if not buf:
582 break
Lars Gustäbelc64e4022007-03-13 10:47:19 +0000583 try:
584 buf = self.cmp.decompress(buf)
585 except IOError:
586 raise ReadError("invalid compressed data")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000587 t.append(buf)
588 c += len(buf)
589 t = "".join(t)
590 self.dbuf = t[size:]
591 return t[:size]
592
593 def __read(self, size):
594 """Return size bytes from stream. If internal buffer is empty,
595 read another block from the stream.
596 """
597 c = len(self.buf)
598 t = [self.buf]
599 while c < size:
600 buf = self.fileobj.read(self.bufsize)
601 if not buf:
602 break
603 t.append(buf)
604 c += len(buf)
605 t = "".join(t)
606 self.buf = t[size:]
607 return t[:size]
608# class _Stream
609
Martin v. Löwis78be7df2005-03-05 12:47:42 +0000610class _StreamProxy(object):
611 """Small proxy class that enables transparent compression
612 detection for the Stream interface (mode 'r|*').
613 """
614
615 def __init__(self, fileobj):
616 self.fileobj = fileobj
617 self.buf = self.fileobj.read(BLOCKSIZE)
618
619 def read(self, size):
620 self.read = self.fileobj.read
621 return self.buf
622
623 def getcomptype(self):
624 if self.buf.startswith("\037\213\010"):
625 return "gz"
626 if self.buf.startswith("BZh91"):
627 return "bz2"
628 return "tar"
629
630 def close(self):
631 self.fileobj.close()
632# class StreamProxy
633
Georg Brandl49c8f4c2006-05-15 19:30:35 +0000634class _BZ2Proxy(object):
635 """Small proxy class that enables external file object
636 support for "r:bz2" and "w:bz2" modes. This is actually
637 a workaround for a limitation in bz2 module's BZ2File
638 class which (unlike gzip.GzipFile) has no support for
639 a file object argument.
640 """
641
642 blocksize = 16 * 1024
643
644 def __init__(self, fileobj, mode):
645 self.fileobj = fileobj
646 self.mode = mode
Lars Gustäbelc64e4022007-03-13 10:47:19 +0000647 self.name = getattr(self.fileobj, "name", None)
Georg Brandl49c8f4c2006-05-15 19:30:35 +0000648 self.init()
649
650 def init(self):
651 import bz2
652 self.pos = 0
653 if self.mode == "r":
654 self.bz2obj = bz2.BZ2Decompressor()
655 self.fileobj.seek(0)
656 self.buf = ""
657 else:
658 self.bz2obj = bz2.BZ2Compressor()
659
660 def read(self, size):
661 b = [self.buf]
662 x = len(self.buf)
663 while x < size:
664 try:
665 raw = self.fileobj.read(self.blocksize)
666 data = self.bz2obj.decompress(raw)
667 b.append(data)
668 except EOFError:
669 break
670 x += len(data)
671 self.buf = "".join(b)
672
673 buf = self.buf[:size]
674 self.buf = self.buf[size:]
675 self.pos += len(buf)
676 return buf
677
678 def seek(self, pos):
679 if pos < self.pos:
680 self.init()
681 self.read(pos - self.pos)
682
683 def tell(self):
684 return self.pos
685
686 def write(self, data):
687 self.pos += len(data)
688 raw = self.bz2obj.compress(data)
689 self.fileobj.write(raw)
690
691 def close(self):
692 if self.mode == "w":
693 raw = self.bz2obj.flush()
694 self.fileobj.write(raw)
Georg Brandle8953182006-05-27 14:02:03 +0000695 self.fileobj.close()
Georg Brandl49c8f4c2006-05-15 19:30:35 +0000696# class _BZ2Proxy
697
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000698#------------------------
699# Extraction file object
700#------------------------
Lars Gustäbel6baa5022006-12-23 16:40:13 +0000701class _FileInFile(object):
702 """A thin wrapper around an existing file object that
703 provides a part of its data as an individual file
704 object.
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000705 """
706
Lars Gustäbel6baa5022006-12-23 16:40:13 +0000707 def __init__(self, fileobj, offset, size, sparse=None):
708 self.fileobj = fileobj
709 self.offset = offset
710 self.size = size
711 self.sparse = sparse
712 self.position = 0
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000713
Lars Gustäbel6baa5022006-12-23 16:40:13 +0000714 def tell(self):
715 """Return the current file position.
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000716 """
Lars Gustäbel6baa5022006-12-23 16:40:13 +0000717 return self.position
718
719 def seek(self, position):
720 """Seek to a position in the file.
721 """
722 self.position = position
723
724 def read(self, size=None):
725 """Read data from the file.
726 """
727 if size is None:
728 size = self.size - self.position
729 else:
730 size = min(size, self.size - self.position)
731
732 if self.sparse is None:
733 return self.readnormal(size)
734 else:
735 return self.readsparse(size)
736
737 def readnormal(self, size):
738 """Read operation for regular files.
739 """
740 self.fileobj.seek(self.offset + self.position)
741 self.position += size
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000742 return self.fileobj.read(size)
743
Lars Gustäbel6baa5022006-12-23 16:40:13 +0000744 def readsparse(self, size):
745 """Read operation for sparse files.
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000746 """
Lars Gustäbel6baa5022006-12-23 16:40:13 +0000747 data = []
748 while size > 0:
749 buf = self.readsparsesection(size)
750 if not buf:
751 break
752 size -= len(buf)
753 data.append(buf)
754 return "".join(data)
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000755
Lars Gustäbel6baa5022006-12-23 16:40:13 +0000756 def readsparsesection(self, size):
757 """Read a single section of a sparse file.
758 """
759 section = self.sparse.find(self.position)
760
761 if section is None:
762 return ""
763
764 size = min(size, section.offset + section.size - self.position)
765
766 if isinstance(section, _data):
767 realpos = section.realpos + self.position - section.offset
768 self.fileobj.seek(self.offset + realpos)
769 self.position += size
770 return self.fileobj.read(size)
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000771 else:
Lars Gustäbel6baa5022006-12-23 16:40:13 +0000772 self.position += size
773 return NUL * size
774#class _FileInFile
775
776
777class ExFileObject(object):
778 """File-like object for reading an archive member.
779 Is returned by TarFile.extractfile().
780 """
781 blocksize = 1024
782
783 def __init__(self, tarfile, tarinfo):
784 self.fileobj = _FileInFile(tarfile.fileobj,
785 tarinfo.offset_data,
786 tarinfo.size,
787 getattr(tarinfo, "sparse", None))
788 self.name = tarinfo.name
789 self.mode = "r"
790 self.closed = False
791 self.size = tarinfo.size
792
793 self.position = 0
794 self.buffer = ""
795
796 def read(self, size=None):
797 """Read at most size bytes from the file. If size is not
798 present or None, read all data until EOF is reached.
799 """
800 if self.closed:
801 raise ValueError("I/O operation on closed file")
802
803 buf = ""
804 if self.buffer:
805 if size is None:
806 buf = self.buffer
807 self.buffer = ""
808 else:
809 buf = self.buffer[:size]
810 self.buffer = self.buffer[size:]
811
812 if size is None:
813 buf += self.fileobj.read()
814 else:
815 buf += self.fileobj.read(size - len(buf))
816
817 self.position += len(buf)
818 return buf
819
820 def readline(self, size=-1):
821 """Read one entire line from the file. If size is present
822 and non-negative, return a string with at most that
823 size, which may be an incomplete line.
824 """
825 if self.closed:
826 raise ValueError("I/O operation on closed file")
827
828 if "\n" in self.buffer:
829 pos = self.buffer.find("\n") + 1
830 else:
831 buffers = [self.buffer]
832 while True:
833 buf = self.fileobj.read(self.blocksize)
834 buffers.append(buf)
835 if not buf or "\n" in buf:
836 self.buffer = "".join(buffers)
837 pos = self.buffer.find("\n") + 1
838 if pos == 0:
839 # no newline found.
840 pos = len(self.buffer)
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000841 break
Lars Gustäbel6baa5022006-12-23 16:40:13 +0000842
843 if size != -1:
844 pos = min(size, pos)
845
846 buf = self.buffer[:pos]
847 self.buffer = self.buffer[pos:]
848 self.position += len(buf)
849 return buf
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000850
851 def readlines(self):
Lars Gustäbel6baa5022006-12-23 16:40:13 +0000852 """Return a list with all remaining lines.
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000853 """
854 result = []
855 while True:
856 line = self.readline()
857 if not line: break
858 result.append(line)
859 return result
860
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000861 def tell(self):
862 """Return the current file position.
863 """
Lars Gustäbel6baa5022006-12-23 16:40:13 +0000864 if self.closed:
865 raise ValueError("I/O operation on closed file")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000866
Lars Gustäbel6baa5022006-12-23 16:40:13 +0000867 return self.position
868
869 def seek(self, pos, whence=os.SEEK_SET):
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000870 """Seek to a position in the file.
871 """
Lars Gustäbel6baa5022006-12-23 16:40:13 +0000872 if self.closed:
873 raise ValueError("I/O operation on closed file")
874
875 if whence == os.SEEK_SET:
876 self.position = min(max(pos, 0), self.size)
877 elif whence == os.SEEK_CUR:
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000878 if pos < 0:
Lars Gustäbel6baa5022006-12-23 16:40:13 +0000879 self.position = max(self.position + pos, 0)
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000880 else:
Lars Gustäbel6baa5022006-12-23 16:40:13 +0000881 self.position = min(self.position + pos, self.size)
882 elif whence == os.SEEK_END:
883 self.position = max(min(self.size + pos, self.size), 0)
884 else:
885 raise ValueError("Invalid argument")
886
887 self.buffer = ""
888 self.fileobj.seek(self.position)
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000889
890 def close(self):
891 """Close the file object.
892 """
893 self.closed = True
Martin v. Löwisdf241532005-03-03 08:17:42 +0000894
895 def __iter__(self):
Lars Gustäbel6baa5022006-12-23 16:40:13 +0000896 """Get an iterator over the file's lines.
Martin v. Löwisdf241532005-03-03 08:17:42 +0000897 """
Lars Gustäbel6baa5022006-12-23 16:40:13 +0000898 while True:
899 line = self.readline()
900 if not line:
901 break
902 yield line
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000903#class ExFileObject
904
905#------------------
906# Exported Classes
907#------------------
908class TarInfo(object):
909 """Informational class which holds the details about an
910 archive member given by a tar header block.
911 TarInfo objects are returned by TarFile.getmember(),
912 TarFile.getmembers() and TarFile.gettarinfo() and are
913 usually created internally.
914 """
915
916 def __init__(self, name=""):
917 """Construct a TarInfo object. name is the optional name
918 of the member.
919 """
Lars Gustäbelc64e4022007-03-13 10:47:19 +0000920 self.name = name # member name
921 self.mode = 0644 # file permissions
Georg Brandl38c6a222006-05-10 16:26:03 +0000922 self.uid = 0 # user id
923 self.gid = 0 # group id
924 self.size = 0 # file size
925 self.mtime = 0 # modification time
926 self.chksum = 0 # header checksum
927 self.type = REGTYPE # member type
928 self.linkname = "" # link name
Lars Gustäbelc64e4022007-03-13 10:47:19 +0000929 self.uname = "root" # user name
930 self.gname = "root" # group name
Georg Brandl38c6a222006-05-10 16:26:03 +0000931 self.devmajor = 0 # device major number
932 self.devminor = 0 # device minor number
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000933
Georg Brandl38c6a222006-05-10 16:26:03 +0000934 self.offset = 0 # the tar header starts here
935 self.offset_data = 0 # the file's data starts here
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000936
Lars Gustäbelc64e4022007-03-13 10:47:19 +0000937 self.pax_headers = {} # pax header information
938
939 # In pax headers the "name" and "linkname" field are called
940 # "path" and "linkpath".
941 def _getpath(self):
942 return self.name
943 def _setpath(self, name):
944 self.name = name
945 path = property(_getpath, _setpath)
946
947 def _getlinkpath(self):
948 return self.linkname
949 def _setlinkpath(self, linkname):
950 self.linkname = linkname
951 linkpath = property(_getlinkpath, _setlinkpath)
952
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +0000953 def __repr__(self):
954 return "<%s %r at %#x>" % (self.__class__.__name__,self.name,id(self))
955
Lars Gustäbela0fcb932007-05-27 19:49:30 +0000956 def get_info(self, encoding, errors):
Lars Gustäbelc64e4022007-03-13 10:47:19 +0000957 """Return the TarInfo's attributes as a dictionary.
958 """
959 info = {
960 "name": normpath(self.name),
961 "mode": self.mode & 07777,
962 "uid": self.uid,
963 "gid": self.gid,
964 "size": self.size,
965 "mtime": self.mtime,
966 "chksum": self.chksum,
967 "type": self.type,
968 "linkname": normpath(self.linkname) if self.linkname else "",
969 "uname": self.uname,
970 "gname": self.gname,
971 "devmajor": self.devmajor,
972 "devminor": self.devminor
973 }
974
975 if info["type"] == DIRTYPE and not info["name"].endswith("/"):
976 info["name"] += "/"
977
Lars Gustäbela0fcb932007-05-27 19:49:30 +0000978 for key in ("name", "linkname", "uname", "gname"):
979 if type(info[key]) is unicode:
980 info[key] = info[key].encode(encoding, errors)
981
Lars Gustäbelc64e4022007-03-13 10:47:19 +0000982 return info
983
Lars Gustäbela0fcb932007-05-27 19:49:30 +0000984 def tobuf(self, format=DEFAULT_FORMAT, encoding=ENCODING, errors="strict"):
Lars Gustäbelc64e4022007-03-13 10:47:19 +0000985 """Return a tar header as a string of 512 byte blocks.
986 """
Lars Gustäbela0fcb932007-05-27 19:49:30 +0000987 info = self.get_info(encoding, errors)
988
Lars Gustäbelc64e4022007-03-13 10:47:19 +0000989 if format == USTAR_FORMAT:
Lars Gustäbela0fcb932007-05-27 19:49:30 +0000990 return self.create_ustar_header(info)
Lars Gustäbelc64e4022007-03-13 10:47:19 +0000991 elif format == GNU_FORMAT:
Lars Gustäbela0fcb932007-05-27 19:49:30 +0000992 return self.create_gnu_header(info)
Lars Gustäbelc64e4022007-03-13 10:47:19 +0000993 elif format == PAX_FORMAT:
Lars Gustäbela0fcb932007-05-27 19:49:30 +0000994 return self.create_pax_header(info, encoding, errors)
Lars Gustäbelc64e4022007-03-13 10:47:19 +0000995 else:
996 raise ValueError("invalid format")
997
Lars Gustäbela0fcb932007-05-27 19:49:30 +0000998 def create_ustar_header(self, info):
Lars Gustäbelc64e4022007-03-13 10:47:19 +0000999 """Return the object as a ustar header block.
1000 """
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001001 info["magic"] = POSIX_MAGIC
1002
1003 if len(info["linkname"]) > LENGTH_LINK:
1004 raise ValueError("linkname is too long")
1005
1006 if len(info["name"]) > LENGTH_NAME:
1007 info["prefix"], info["name"] = self._posix_split_name(info["name"])
1008
1009 return self._create_header(info, USTAR_FORMAT)
1010
Lars Gustäbela0fcb932007-05-27 19:49:30 +00001011 def create_gnu_header(self, info):
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001012 """Return the object as a GNU header block sequence.
1013 """
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001014 info["magic"] = GNU_MAGIC
1015
1016 buf = ""
1017 if len(info["linkname"]) > LENGTH_LINK:
1018 buf += self._create_gnu_long_header(info["linkname"], GNUTYPE_LONGLINK)
1019
1020 if len(info["name"]) > LENGTH_NAME:
1021 buf += self._create_gnu_long_header(info["name"], GNUTYPE_LONGNAME)
1022
1023 return buf + self._create_header(info, GNU_FORMAT)
1024
Lars Gustäbela0fcb932007-05-27 19:49:30 +00001025 def create_pax_header(self, info, encoding, errors):
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001026 """Return the object as a ustar header block. If it cannot be
1027 represented this way, prepend a pax extended header sequence
1028 with supplement information.
1029 """
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001030 info["magic"] = POSIX_MAGIC
1031 pax_headers = self.pax_headers.copy()
1032
1033 # Test string fields for values that exceed the field length or cannot
1034 # be represented in ASCII encoding.
1035 for name, hname, length in (
1036 ("name", "path", LENGTH_NAME), ("linkname", "linkpath", LENGTH_LINK),
1037 ("uname", "uname", 32), ("gname", "gname", 32)):
1038
Lars Gustäbela0fcb932007-05-27 19:49:30 +00001039 if hname in pax_headers:
1040 # The pax header has priority.
1041 continue
1042
1043 val = info[name].decode(encoding, errors)
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001044
1045 # Try to encode the string as ASCII.
1046 try:
1047 val.encode("ascii")
1048 except UnicodeEncodeError:
1049 pax_headers[hname] = val
1050 continue
1051
Lars Gustäbela0fcb932007-05-27 19:49:30 +00001052 if len(info[name]) > length:
1053 pax_headers[hname] = val
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001054
1055 # Test number fields for values that exceed the field limit or values
1056 # that like to be stored as float.
1057 for name, digits in (("uid", 8), ("gid", 8), ("size", 12), ("mtime", 12)):
Lars Gustäbela0fcb932007-05-27 19:49:30 +00001058 if name in pax_headers:
1059 # The pax header has priority. Avoid overflow.
1060 info[name] = 0
1061 continue
1062
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001063 val = info[name]
1064 if not 0 <= val < 8 ** (digits - 1) or isinstance(val, float):
1065 pax_headers[name] = unicode(val)
1066 info[name] = 0
1067
Lars Gustäbela0fcb932007-05-27 19:49:30 +00001068 # Create a pax extended header if necessary.
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001069 if pax_headers:
1070 buf = self._create_pax_generic_header(pax_headers)
1071 else:
1072 buf = ""
1073
1074 return buf + self._create_header(info, USTAR_FORMAT)
1075
1076 @classmethod
Lars Gustäbela0fcb932007-05-27 19:49:30 +00001077 def create_pax_global_header(cls, pax_headers):
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001078 """Return the object as a pax global header block sequence.
1079 """
Lars Gustäbela0fcb932007-05-27 19:49:30 +00001080 return cls._create_pax_generic_header(pax_headers, type=XGLTYPE)
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001081
1082 def _posix_split_name(self, name):
1083 """Split a name longer than 100 chars into a prefix
1084 and a name part.
1085 """
1086 prefix = name[:LENGTH_PREFIX + 1]
1087 while prefix and prefix[-1] != "/":
1088 prefix = prefix[:-1]
1089
1090 name = name[len(prefix):]
1091 prefix = prefix[:-1]
1092
1093 if not prefix or len(name) > LENGTH_NAME:
1094 raise ValueError("name is too long")
1095 return prefix, name
1096
1097 @staticmethod
1098 def _create_header(info, format):
1099 """Return a header block. info is a dictionary with file
1100 information, format must be one of the *_FORMAT constants.
1101 """
1102 parts = [
1103 stn(info.get("name", ""), 100),
1104 itn(info.get("mode", 0) & 07777, 8, format),
1105 itn(info.get("uid", 0), 8, format),
1106 itn(info.get("gid", 0), 8, format),
1107 itn(info.get("size", 0), 12, format),
1108 itn(info.get("mtime", 0), 12, format),
1109 " ", # checksum field
1110 info.get("type", REGTYPE),
1111 stn(info.get("linkname", ""), 100),
Lars Gustäbela0fcb932007-05-27 19:49:30 +00001112 stn(info.get("magic", POSIX_MAGIC), 8),
1113 stn(info.get("uname", "root"), 32),
1114 stn(info.get("gname", "root"), 32),
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001115 itn(info.get("devmajor", 0), 8, format),
1116 itn(info.get("devminor", 0), 8, format),
1117 stn(info.get("prefix", ""), 155)
1118 ]
1119
1120 buf = struct.pack("%ds" % BLOCKSIZE, "".join(parts))
1121 chksum = calc_chksums(buf[-BLOCKSIZE:])[0]
1122 buf = buf[:-364] + "%06o\0" % chksum + buf[-357:]
1123 return buf
1124
1125 @staticmethod
1126 def _create_payload(payload):
1127 """Return the string payload filled with zero bytes
1128 up to the next 512 byte border.
1129 """
1130 blocks, remainder = divmod(len(payload), BLOCKSIZE)
1131 if remainder > 0:
1132 payload += (BLOCKSIZE - remainder) * NUL
1133 return payload
1134
1135 @classmethod
1136 def _create_gnu_long_header(cls, name, type):
1137 """Return a GNUTYPE_LONGNAME or GNUTYPE_LONGLINK sequence
1138 for name.
1139 """
1140 name += NUL
1141
1142 info = {}
1143 info["name"] = "././@LongLink"
1144 info["type"] = type
1145 info["size"] = len(name)
1146 info["magic"] = GNU_MAGIC
1147
1148 # create extended header + name blocks.
1149 return cls._create_header(info, USTAR_FORMAT) + \
1150 cls._create_payload(name)
1151
1152 @classmethod
1153 def _create_pax_generic_header(cls, pax_headers, type=XHDTYPE):
1154 """Return a POSIX.1-2001 extended or global header sequence
1155 that contains a list of keyword, value pairs. The values
1156 must be unicode objects.
1157 """
1158 records = []
1159 for keyword, value in pax_headers.iteritems():
1160 keyword = keyword.encode("utf8")
1161 value = value.encode("utf8")
1162 l = len(keyword) + len(value) + 3 # ' ' + '=' + '\n'
1163 n = p = 0
1164 while True:
1165 n = l + len(str(p))
1166 if n == p:
1167 break
1168 p = n
1169 records.append("%d %s=%s\n" % (p, keyword, value))
1170 records = "".join(records)
1171
1172 # We use a hardcoded "././@PaxHeader" name like star does
1173 # instead of the one that POSIX recommends.
1174 info = {}
1175 info["name"] = "././@PaxHeader"
1176 info["type"] = type
1177 info["size"] = len(records)
1178 info["magic"] = POSIX_MAGIC
1179
1180 # Create pax header + record blocks.
1181 return cls._create_header(info, USTAR_FORMAT) + \
1182 cls._create_payload(records)
1183
Guido van Rossum75b64e62005-01-16 00:16:11 +00001184 @classmethod
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001185 def frombuf(cls, buf):
1186 """Construct a TarInfo object from a 512 byte string buffer.
1187 """
Georg Brandl38c6a222006-05-10 16:26:03 +00001188 if len(buf) != BLOCKSIZE:
Georg Brandlebbeed72006-12-19 22:06:46 +00001189 raise HeaderError("truncated header")
Georg Brandl38c6a222006-05-10 16:26:03 +00001190 if buf.count(NUL) == BLOCKSIZE:
Georg Brandlebbeed72006-12-19 22:06:46 +00001191 raise HeaderError("empty header")
1192
Georg Brandlded1c4d2006-12-20 11:55:16 +00001193 chksum = nti(buf[148:156])
Georg Brandlebbeed72006-12-19 22:06:46 +00001194 if chksum not in calc_chksums(buf):
1195 raise HeaderError("bad checksum")
Georg Brandl38c6a222006-05-10 16:26:03 +00001196
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001197 obj = cls()
1198 obj.buf = buf
1199 obj.name = nts(buf[0:100])
1200 obj.mode = nti(buf[100:108])
1201 obj.uid = nti(buf[108:116])
1202 obj.gid = nti(buf[116:124])
1203 obj.size = nti(buf[124:136])
1204 obj.mtime = nti(buf[136:148])
1205 obj.chksum = chksum
1206 obj.type = buf[156:157]
1207 obj.linkname = nts(buf[157:257])
1208 obj.uname = nts(buf[265:297])
1209 obj.gname = nts(buf[297:329])
1210 obj.devmajor = nti(buf[329:337])
1211 obj.devminor = nti(buf[337:345])
1212 prefix = nts(buf[345:500])
Georg Brandl3354f282006-10-29 09:16:12 +00001213
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001214 # Old V7 tar format represents a directory as a regular
1215 # file with a trailing slash.
1216 if obj.type == AREGTYPE and obj.name.endswith("/"):
1217 obj.type = DIRTYPE
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001218
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001219 # Remove redundant slashes from directories.
1220 if obj.isdir():
1221 obj.name = obj.name.rstrip("/")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001222
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001223 # Reconstruct a ustar longname.
1224 if prefix and obj.type not in GNU_TYPES:
1225 obj.name = prefix + "/" + obj.name
1226 return obj
1227
1228 @classmethod
1229 def fromtarfile(cls, tarfile):
1230 """Return the next TarInfo object from TarFile object
1231 tarfile.
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001232 """
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001233 buf = tarfile.fileobj.read(BLOCKSIZE)
1234 if not buf:
1235 return
1236 obj = cls.frombuf(buf)
1237 obj.offset = tarfile.fileobj.tell() - BLOCKSIZE
1238 return obj._proc_member(tarfile)
Georg Brandl3354f282006-10-29 09:16:12 +00001239
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001240 #--------------------------------------------------------------------------
1241 # The following are methods that are called depending on the type of a
1242 # member. The entry point is _proc_member() which can be overridden in a
1243 # subclass to add custom _proc_*() methods. A _proc_*() method MUST
1244 # implement the following
1245 # operations:
1246 # 1. Set self.offset_data to the position where the data blocks begin,
1247 # if there is data that follows.
1248 # 2. Set tarfile.offset to the position where the next member's header will
1249 # begin.
1250 # 3. Return self or another valid TarInfo object.
1251 def _proc_member(self, tarfile):
1252 """Choose the right processing method depending on
1253 the type and call it.
Georg Brandl3354f282006-10-29 09:16:12 +00001254 """
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001255 if self.type in (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK):
1256 return self._proc_gnulong(tarfile)
1257 elif self.type == GNUTYPE_SPARSE:
1258 return self._proc_sparse(tarfile)
1259 elif self.type in (XHDTYPE, XGLTYPE, SOLARIS_XHDTYPE):
1260 return self._proc_pax(tarfile)
1261 else:
1262 return self._proc_builtin(tarfile)
Georg Brandl3354f282006-10-29 09:16:12 +00001263
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001264 def _proc_builtin(self, tarfile):
1265 """Process a builtin type or an unknown type which
1266 will be treated as a regular file.
1267 """
1268 self.offset_data = tarfile.fileobj.tell()
1269 offset = self.offset_data
1270 if self.isreg() or self.type not in SUPPORTED_TYPES:
1271 # Skip the following data blocks.
1272 offset += self._block(self.size)
1273 tarfile.offset = offset
Georg Brandl3354f282006-10-29 09:16:12 +00001274
Lars Gustäbela0fcb932007-05-27 19:49:30 +00001275 # Patch the TarInfo object with saved global
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001276 # header information.
Lars Gustäbela0fcb932007-05-27 19:49:30 +00001277 self._apply_pax_info(tarfile.pax_headers, tarfile.encoding, tarfile.errors)
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001278
1279 return self
1280
1281 def _proc_gnulong(self, tarfile):
1282 """Process the blocks that hold a GNU longname
1283 or longlink member.
1284 """
1285 buf = tarfile.fileobj.read(self._block(self.size))
1286
1287 # Fetch the next header and process it.
Lars Gustäbela0fcb932007-05-27 19:49:30 +00001288 next = self.fromtarfile(tarfile)
1289 if next is None:
1290 raise HeaderError("missing subsequent header")
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001291
1292 # Patch the TarInfo object from the next header with
1293 # the longname information.
1294 next.offset = self.offset
1295 if self.type == GNUTYPE_LONGNAME:
Lars Gustäbela0fcb932007-05-27 19:49:30 +00001296 next.name = nts(buf)
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001297 elif self.type == GNUTYPE_LONGLINK:
Lars Gustäbela0fcb932007-05-27 19:49:30 +00001298 next.linkname = nts(buf)
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001299
1300 return next
1301
1302 def _proc_sparse(self, tarfile):
1303 """Process a GNU sparse header plus extra headers.
1304 """
1305 buf = self.buf
1306 sp = _ringbuffer()
1307 pos = 386
1308 lastpos = 0L
1309 realpos = 0L
1310 # There are 4 possible sparse structs in the
1311 # first header.
1312 for i in xrange(4):
1313 try:
1314 offset = nti(buf[pos:pos + 12])
1315 numbytes = nti(buf[pos + 12:pos + 24])
1316 except ValueError:
1317 break
1318 if offset > lastpos:
1319 sp.append(_hole(lastpos, offset - lastpos))
1320 sp.append(_data(offset, numbytes, realpos))
1321 realpos += numbytes
1322 lastpos = offset + numbytes
1323 pos += 24
1324
1325 isextended = ord(buf[482])
1326 origsize = nti(buf[483:495])
1327
1328 # If the isextended flag is given,
1329 # there are extra headers to process.
1330 while isextended == 1:
1331 buf = tarfile.fileobj.read(BLOCKSIZE)
1332 pos = 0
1333 for i in xrange(21):
1334 try:
1335 offset = nti(buf[pos:pos + 12])
1336 numbytes = nti(buf[pos + 12:pos + 24])
1337 except ValueError:
1338 break
1339 if offset > lastpos:
1340 sp.append(_hole(lastpos, offset - lastpos))
1341 sp.append(_data(offset, numbytes, realpos))
1342 realpos += numbytes
1343 lastpos = offset + numbytes
1344 pos += 24
1345 isextended = ord(buf[504])
1346
1347 if lastpos < origsize:
1348 sp.append(_hole(lastpos, origsize - lastpos))
1349
1350 self.sparse = sp
1351
1352 self.offset_data = tarfile.fileobj.tell()
1353 tarfile.offset = self.offset_data + self._block(self.size)
1354 self.size = origsize
1355
1356 return self
1357
1358 def _proc_pax(self, tarfile):
1359 """Process an extended or global header as described in
1360 POSIX.1-2001.
1361 """
1362 # Read the header information.
1363 buf = tarfile.fileobj.read(self._block(self.size))
1364
1365 # A pax header stores supplemental information for either
1366 # the following file (extended) or all following files
1367 # (global).
1368 if self.type == XGLTYPE:
1369 pax_headers = tarfile.pax_headers
1370 else:
1371 pax_headers = tarfile.pax_headers.copy()
1372
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001373 # Parse pax header information. A record looks like that:
1374 # "%d %s=%s\n" % (length, keyword, value). length is the size
1375 # of the complete record including the length field itself and
Lars Gustäbela0fcb932007-05-27 19:49:30 +00001376 # the newline. keyword and value are both UTF-8 encoded strings.
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001377 regex = re.compile(r"(\d+) ([^=]+)=", re.U)
1378 pos = 0
1379 while True:
1380 match = regex.match(buf, pos)
1381 if not match:
1382 break
1383
1384 length, keyword = match.groups()
1385 length = int(length)
1386 value = buf[match.end(2) + 1:match.start(1) + length - 1]
1387
1388 keyword = keyword.decode("utf8")
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001389 value = value.decode("utf8")
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001390
1391 pax_headers[keyword] = value
1392 pos += length
1393
Lars Gustäbela0fcb932007-05-27 19:49:30 +00001394 # Fetch the next header.
1395 next = self.fromtarfile(tarfile)
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001396
Lars Gustäbela0fcb932007-05-27 19:49:30 +00001397 if self.type in (XHDTYPE, SOLARIS_XHDTYPE):
1398 if next is None:
1399 raise HeaderError("missing subsequent header")
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001400
Lars Gustäbela0fcb932007-05-27 19:49:30 +00001401 # Patch the TarInfo object with the extended header info.
1402 next._apply_pax_info(pax_headers, tarfile.encoding, tarfile.errors)
1403 next.offset = self.offset
1404
1405 if pax_headers.has_key("size"):
1406 # If the extended header replaces the size field,
1407 # we need to recalculate the offset where the next
1408 # header starts.
1409 offset = next.offset_data
1410 if next.isreg() or next.type not in SUPPORTED_TYPES:
1411 offset += next._block(next.size)
1412 tarfile.offset = offset
1413
1414 return next
1415
1416 def _apply_pax_info(self, pax_headers, encoding, errors):
1417 """Replace fields with supplemental information from a previous
1418 pax extended or global header.
1419 """
1420 for keyword, value in pax_headers.iteritems():
1421 if keyword not in PAX_FIELDS:
1422 continue
1423
1424 if keyword == "path":
1425 value = value.rstrip("/")
1426
1427 if keyword in PAX_NUMBER_FIELDS:
1428 try:
1429 value = PAX_NUMBER_FIELDS[keyword](value)
1430 except ValueError:
1431 value = 0
1432 else:
1433 value = uts(value, encoding, errors)
1434
1435 setattr(self, keyword, value)
1436
1437 self.pax_headers = pax_headers.copy()
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001438
1439 def _block(self, count):
1440 """Round up a byte count by BLOCKSIZE and return it,
1441 e.g. _block(834) => 1024.
1442 """
1443 blocks, remainder = divmod(count, BLOCKSIZE)
1444 if remainder:
1445 blocks += 1
1446 return blocks * BLOCKSIZE
Georg Brandl3354f282006-10-29 09:16:12 +00001447
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001448 def isreg(self):
1449 return self.type in REGULAR_TYPES
1450 def isfile(self):
1451 return self.isreg()
1452 def isdir(self):
1453 return self.type == DIRTYPE
1454 def issym(self):
1455 return self.type == SYMTYPE
1456 def islnk(self):
1457 return self.type == LNKTYPE
1458 def ischr(self):
1459 return self.type == CHRTYPE
1460 def isblk(self):
1461 return self.type == BLKTYPE
1462 def isfifo(self):
1463 return self.type == FIFOTYPE
1464 def issparse(self):
1465 return self.type == GNUTYPE_SPARSE
1466 def isdev(self):
1467 return self.type in (CHRTYPE, BLKTYPE, FIFOTYPE)
1468# class TarInfo
1469
1470class TarFile(object):
1471 """The TarFile Class provides an interface to tar archives.
1472 """
1473
1474 debug = 0 # May be set from 0 (no msgs) to 3 (all msgs)
1475
1476 dereference = False # If true, add content of linked file to the
1477 # tar file, else the link.
1478
1479 ignore_zeros = False # If true, skips empty or invalid blocks and
1480 # continues processing.
1481
1482 errorlevel = 0 # If 0, fatal errors only appear in debug
1483 # messages (if debug >= 0). If > 0, errors
1484 # are passed to the caller as exceptions.
1485
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001486 format = DEFAULT_FORMAT # The format to use when creating an archive.
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001487
Lars Gustäbela0fcb932007-05-27 19:49:30 +00001488 encoding = ENCODING # Encoding for 8-bit character strings.
1489
1490 errors = None # Error handler for unicode conversion.
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001491
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001492 tarinfo = TarInfo # The default TarInfo class to use.
1493
1494 fileobject = ExFileObject # The default ExFileObject class to use.
1495
1496 def __init__(self, name=None, mode="r", fileobj=None, format=None,
1497 tarinfo=None, dereference=None, ignore_zeros=None, encoding=None,
Lars Gustäbela0fcb932007-05-27 19:49:30 +00001498 errors=None, pax_headers=None, debug=None, errorlevel=None):
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001499 """Open an (uncompressed) tar archive `name'. `mode' is either 'r' to
1500 read from an existing archive, 'a' to append data to an existing
1501 file or 'w' to create a new file overwriting an existing one. `mode'
1502 defaults to 'r'.
1503 If `fileobj' is given, it is used for reading or writing data. If it
1504 can be determined, `mode' is overridden by `fileobj's mode.
1505 `fileobj' is not closed, when TarFile is closed.
1506 """
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001507 if len(mode) > 1 or mode not in "raw":
Georg Brandle4751e32006-05-18 06:11:19 +00001508 raise ValueError("mode must be 'r', 'a' or 'w'")
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001509 self.mode = mode
1510 self._mode = {"r": "rb", "a": "r+b", "w": "wb"}[mode]
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001511
1512 if not fileobj:
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001513 if self.mode == "a" and not os.path.exists(name):
Lars Gustäbel3f8aca12007-02-06 18:38:13 +00001514 # Create nonexistent files in append mode.
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001515 self.mode = "w"
1516 self._mode = "wb"
Brett Cannon6cef0762007-05-25 20:17:15 +00001517 fileobj = bltn_open(name, self._mode)
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001518 self._extfileobj = False
1519 else:
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001520 if name is None and hasattr(fileobj, "name"):
1521 name = fileobj.name
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001522 if hasattr(fileobj, "mode"):
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001523 self._mode = fileobj.mode
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001524 self._extfileobj = True
Lars Gustäbel0f4a14b2007-08-28 12:31:09 +00001525 self.name = os.path.abspath(name) if name else None
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001526 self.fileobj = fileobj
1527
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001528 # Init attributes.
1529 if format is not None:
1530 self.format = format
1531 if tarinfo is not None:
1532 self.tarinfo = tarinfo
1533 if dereference is not None:
1534 self.dereference = dereference
1535 if ignore_zeros is not None:
1536 self.ignore_zeros = ignore_zeros
1537 if encoding is not None:
1538 self.encoding = encoding
Lars Gustäbela0fcb932007-05-27 19:49:30 +00001539
1540 if errors is not None:
1541 self.errors = errors
1542 elif mode == "r":
1543 self.errors = "utf-8"
1544 else:
1545 self.errors = "strict"
1546
1547 if pax_headers is not None and self.format == PAX_FORMAT:
1548 self.pax_headers = pax_headers
1549 else:
1550 self.pax_headers = {}
1551
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001552 if debug is not None:
1553 self.debug = debug
1554 if errorlevel is not None:
1555 self.errorlevel = errorlevel
1556
1557 # Init datastructures.
Georg Brandl38c6a222006-05-10 16:26:03 +00001558 self.closed = False
1559 self.members = [] # list of members as TarInfo objects
1560 self._loaded = False # flag if all members have been read
Lars Gustäbel77b2d632007-12-01 21:02:12 +00001561 self.offset = self.fileobj.tell()
1562 # current position in the archive file
Georg Brandl38c6a222006-05-10 16:26:03 +00001563 self.inodes = {} # dictionary caching the inodes of
1564 # archive members already added
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001565
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001566 if self.mode == "r":
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001567 self.firstmember = None
1568 self.firstmember = self.next()
1569
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001570 if self.mode == "a":
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001571 # Move to the end of the archive,
1572 # before the first empty block.
1573 self.firstmember = None
1574 while True:
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001575 if self.next() is None:
Lars Gustäbel3f8aca12007-02-06 18:38:13 +00001576 if self.offset > 0:
1577 self.fileobj.seek(- BLOCKSIZE, 1)
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001578 break
1579
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001580 if self.mode in "aw":
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001581 self._loaded = True
1582
Lars Gustäbela0fcb932007-05-27 19:49:30 +00001583 if self.pax_headers:
1584 buf = self.tarinfo.create_pax_global_header(self.pax_headers.copy())
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001585 self.fileobj.write(buf)
1586 self.offset += len(buf)
1587
1588 def _getposix(self):
1589 return self.format == USTAR_FORMAT
1590 def _setposix(self, value):
1591 import warnings
1592 warnings.warn("use the format attribute instead", DeprecationWarning)
1593 if value:
1594 self.format = USTAR_FORMAT
1595 else:
1596 self.format = GNU_FORMAT
1597 posix = property(_getposix, _setposix)
1598
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001599 #--------------------------------------------------------------------------
1600 # Below are the classmethods which act as alternate constructors to the
1601 # TarFile class. The open() method is the only one that is needed for
1602 # public use; it is the "super"-constructor and is able to select an
1603 # adequate "sub"-constructor for a particular compression using the mapping
1604 # from OPEN_METH.
1605 #
1606 # This concept allows one to subclass TarFile without losing the comfort of
1607 # the super-constructor. A sub-constructor is registered and made available
1608 # by adding it to the mapping in OPEN_METH.
1609
Guido van Rossum75b64e62005-01-16 00:16:11 +00001610 @classmethod
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001611 def open(cls, name=None, mode="r", fileobj=None, bufsize=RECORDSIZE, **kwargs):
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001612 """Open a tar archive for reading, writing or appending. Return
1613 an appropriate TarFile class.
1614
1615 mode:
Martin v. Löwis78be7df2005-03-05 12:47:42 +00001616 'r' or 'r:*' open for reading with transparent compression
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001617 'r:' open for reading exclusively uncompressed
1618 'r:gz' open for reading with gzip compression
1619 'r:bz2' open for reading with bzip2 compression
Lars Gustäbel3f8aca12007-02-06 18:38:13 +00001620 'a' or 'a:' open for appending, creating the file if necessary
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001621 'w' or 'w:' open for writing without compression
1622 'w:gz' open for writing with gzip compression
1623 'w:bz2' open for writing with bzip2 compression
Martin v. Löwis78be7df2005-03-05 12:47:42 +00001624
1625 'r|*' open a stream of tar blocks with transparent compression
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001626 'r|' open an uncompressed stream of tar blocks for reading
1627 'r|gz' open a gzip compressed stream of tar blocks
1628 'r|bz2' open a bzip2 compressed stream of tar blocks
1629 'w|' open an uncompressed stream for writing
1630 'w|gz' open a gzip compressed stream for writing
1631 'w|bz2' open a bzip2 compressed stream for writing
1632 """
1633
1634 if not name and not fileobj:
Georg Brandle4751e32006-05-18 06:11:19 +00001635 raise ValueError("nothing to open")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001636
Martin v. Löwis78be7df2005-03-05 12:47:42 +00001637 if mode in ("r", "r:*"):
1638 # Find out which *open() is appropriate for opening the file.
1639 for comptype in cls.OPEN_METH:
1640 func = getattr(cls, cls.OPEN_METH[comptype])
Lars Gustäbela7ba6fc2006-12-27 10:30:46 +00001641 if fileobj is not None:
1642 saved_pos = fileobj.tell()
Martin v. Löwis78be7df2005-03-05 12:47:42 +00001643 try:
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001644 return func(name, "r", fileobj, **kwargs)
1645 except (ReadError, CompressionError), e:
Lars Gustäbela7ba6fc2006-12-27 10:30:46 +00001646 if fileobj is not None:
1647 fileobj.seek(saved_pos)
Martin v. Löwis78be7df2005-03-05 12:47:42 +00001648 continue
Georg Brandle4751e32006-05-18 06:11:19 +00001649 raise ReadError("file could not be opened successfully")
Martin v. Löwis78be7df2005-03-05 12:47:42 +00001650
1651 elif ":" in mode:
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001652 filemode, comptype = mode.split(":", 1)
1653 filemode = filemode or "r"
1654 comptype = comptype or "tar"
1655
1656 # Select the *open() function according to
1657 # given compression.
1658 if comptype in cls.OPEN_METH:
1659 func = getattr(cls, cls.OPEN_METH[comptype])
1660 else:
Georg Brandle4751e32006-05-18 06:11:19 +00001661 raise CompressionError("unknown compression type %r" % comptype)
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001662 return func(name, filemode, fileobj, **kwargs)
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001663
1664 elif "|" in mode:
1665 filemode, comptype = mode.split("|", 1)
1666 filemode = filemode or "r"
1667 comptype = comptype or "tar"
1668
1669 if filemode not in "rw":
Georg Brandle4751e32006-05-18 06:11:19 +00001670 raise ValueError("mode must be 'r' or 'w'")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001671
1672 t = cls(name, filemode,
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001673 _Stream(name, filemode, comptype, fileobj, bufsize),
1674 **kwargs)
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001675 t._extfileobj = False
1676 return t
1677
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001678 elif mode in "aw":
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001679 return cls.taropen(name, mode, fileobj, **kwargs)
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001680
Georg Brandle4751e32006-05-18 06:11:19 +00001681 raise ValueError("undiscernible mode")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001682
Guido van Rossum75b64e62005-01-16 00:16:11 +00001683 @classmethod
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001684 def taropen(cls, name, mode="r", fileobj=None, **kwargs):
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001685 """Open uncompressed tar archive name for reading or writing.
1686 """
1687 if len(mode) > 1 or mode not in "raw":
Georg Brandle4751e32006-05-18 06:11:19 +00001688 raise ValueError("mode must be 'r', 'a' or 'w'")
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001689 return cls(name, mode, fileobj, **kwargs)
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001690
Guido van Rossum75b64e62005-01-16 00:16:11 +00001691 @classmethod
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001692 def gzopen(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs):
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001693 """Open gzip compressed tar archive name for reading or writing.
1694 Appending is not allowed.
1695 """
1696 if len(mode) > 1 or mode not in "rw":
Georg Brandle4751e32006-05-18 06:11:19 +00001697 raise ValueError("mode must be 'r' or 'w'")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001698
1699 try:
1700 import gzip
Neal Norwitz4ec68242003-04-11 03:05:56 +00001701 gzip.GzipFile
1702 except (ImportError, AttributeError):
Georg Brandle4751e32006-05-18 06:11:19 +00001703 raise CompressionError("gzip module is not available")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001704
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001705 if fileobj is None:
Brett Cannon6cef0762007-05-25 20:17:15 +00001706 fileobj = bltn_open(name, mode + "b")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001707
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001708 try:
Lars Gustäbela4b23812006-12-23 17:57:23 +00001709 t = cls.taropen(name, mode,
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001710 gzip.GzipFile(name, mode, compresslevel, fileobj),
1711 **kwargs)
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001712 except IOError:
Georg Brandle4751e32006-05-18 06:11:19 +00001713 raise ReadError("not a gzip file")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001714 t._extfileobj = False
1715 return t
1716
Guido van Rossum75b64e62005-01-16 00:16:11 +00001717 @classmethod
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001718 def bz2open(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs):
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001719 """Open bzip2 compressed tar archive name for reading or writing.
1720 Appending is not allowed.
1721 """
1722 if len(mode) > 1 or mode not in "rw":
Georg Brandle4751e32006-05-18 06:11:19 +00001723 raise ValueError("mode must be 'r' or 'w'.")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001724
1725 try:
1726 import bz2
1727 except ImportError:
Georg Brandle4751e32006-05-18 06:11:19 +00001728 raise CompressionError("bz2 module is not available")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001729
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001730 if fileobj is not None:
Georg Brandl49c8f4c2006-05-15 19:30:35 +00001731 fileobj = _BZ2Proxy(fileobj, mode)
1732 else:
1733 fileobj = bz2.BZ2File(name, mode, compresslevel=compresslevel)
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001734
1735 try:
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001736 t = cls.taropen(name, mode, fileobj, **kwargs)
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001737 except IOError:
Georg Brandle4751e32006-05-18 06:11:19 +00001738 raise ReadError("not a bzip2 file")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001739 t._extfileobj = False
1740 return t
1741
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001742 # All *open() methods are registered here.
1743 OPEN_METH = {
1744 "tar": "taropen", # uncompressed tar
1745 "gz": "gzopen", # gzip compressed tar
1746 "bz2": "bz2open" # bzip2 compressed tar
1747 }
1748
1749 #--------------------------------------------------------------------------
1750 # The public methods which TarFile provides:
1751
1752 def close(self):
1753 """Close the TarFile. In write-mode, two finishing zero blocks are
1754 appended to the archive.
1755 """
1756 if self.closed:
1757 return
1758
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001759 if self.mode in "aw":
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001760 self.fileobj.write(NUL * (BLOCKSIZE * 2))
1761 self.offset += (BLOCKSIZE * 2)
1762 # fill up the end with zero-blocks
1763 # (like option -b20 for tar does)
1764 blocks, remainder = divmod(self.offset, RECORDSIZE)
1765 if remainder > 0:
1766 self.fileobj.write(NUL * (RECORDSIZE - remainder))
1767
1768 if not self._extfileobj:
1769 self.fileobj.close()
1770 self.closed = True
1771
1772 def getmember(self, name):
1773 """Return a TarInfo object for member `name'. If `name' can not be
1774 found in the archive, KeyError is raised. If a member occurs more
1775 than once in the archive, its last occurence is assumed to be the
1776 most up-to-date version.
1777 """
Martin v. Löwisf3c56112004-09-18 09:08:52 +00001778 tarinfo = self._getmember(name)
1779 if tarinfo is None:
Georg Brandle4751e32006-05-18 06:11:19 +00001780 raise KeyError("filename %r not found" % name)
Martin v. Löwisf3c56112004-09-18 09:08:52 +00001781 return tarinfo
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001782
1783 def getmembers(self):
1784 """Return the members of the archive as a list of TarInfo objects. The
1785 list has the same order as the members in the archive.
1786 """
1787 self._check()
1788 if not self._loaded: # if we want to obtain a list of
1789 self._load() # all members, we first have to
1790 # scan the whole archive.
1791 return self.members
1792
1793 def getnames(self):
1794 """Return the members of the archive as a list of their names. It has
1795 the same order as the list returned by getmembers().
1796 """
Martin v. Löwisf3c56112004-09-18 09:08:52 +00001797 return [tarinfo.name for tarinfo in self.getmembers()]
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001798
1799 def gettarinfo(self, name=None, arcname=None, fileobj=None):
1800 """Create a TarInfo object for either the file `name' or the file
1801 object `fileobj' (using os.fstat on its file descriptor). You can
1802 modify some of the TarInfo's attributes before you add it using
1803 addfile(). If given, `arcname' specifies an alternative name for the
1804 file in the archive.
1805 """
1806 self._check("aw")
1807
1808 # When fileobj is given, replace name by
1809 # fileobj's real name.
1810 if fileobj is not None:
1811 name = fileobj.name
1812
1813 # Building the name of the member in the archive.
1814 # Backward slashes are converted to forward slashes,
1815 # Absolute paths are turned to relative paths.
1816 if arcname is None:
1817 arcname = name
1818 arcname = normpath(arcname)
1819 drv, arcname = os.path.splitdrive(arcname)
1820 while arcname[0:1] == "/":
1821 arcname = arcname[1:]
1822
1823 # Now, fill the TarInfo object with
1824 # information specific for the file.
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001825 tarinfo = self.tarinfo()
1826 tarinfo.tarfile = self
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001827
1828 # Use os.stat or os.lstat, depending on platform
1829 # and if symlinks shall be resolved.
1830 if fileobj is None:
1831 if hasattr(os, "lstat") and not self.dereference:
1832 statres = os.lstat(name)
1833 else:
1834 statres = os.stat(name)
1835 else:
1836 statres = os.fstat(fileobj.fileno())
1837 linkname = ""
1838
1839 stmd = statres.st_mode
1840 if stat.S_ISREG(stmd):
1841 inode = (statres.st_ino, statres.st_dev)
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001842 if not self.dereference and statres.st_nlink > 1 and \
1843 inode in self.inodes and arcname != self.inodes[inode]:
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001844 # Is it a hardlink to an already
1845 # archived file?
1846 type = LNKTYPE
1847 linkname = self.inodes[inode]
1848 else:
1849 # The inode is added only if its valid.
1850 # For win32 it is always 0.
1851 type = REGTYPE
1852 if inode[0]:
1853 self.inodes[inode] = arcname
1854 elif stat.S_ISDIR(stmd):
1855 type = DIRTYPE
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001856 elif stat.S_ISFIFO(stmd):
1857 type = FIFOTYPE
1858 elif stat.S_ISLNK(stmd):
1859 type = SYMTYPE
1860 linkname = os.readlink(name)
1861 elif stat.S_ISCHR(stmd):
1862 type = CHRTYPE
1863 elif stat.S_ISBLK(stmd):
1864 type = BLKTYPE
1865 else:
1866 return None
1867
1868 # Fill the TarInfo object with all
1869 # information we can get.
Martin v. Löwis5dbdc592005-08-27 10:07:56 +00001870 tarinfo.name = arcname
1871 tarinfo.mode = stmd
1872 tarinfo.uid = statres.st_uid
1873 tarinfo.gid = statres.st_gid
1874 if stat.S_ISREG(stmd):
Martin v. Löwis61d77e02004-08-20 06:35:46 +00001875 tarinfo.size = statres.st_size
Martin v. Löwis5dbdc592005-08-27 10:07:56 +00001876 else:
1877 tarinfo.size = 0L
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001878 tarinfo.mtime = statres.st_mtime
Martin v. Löwis5dbdc592005-08-27 10:07:56 +00001879 tarinfo.type = type
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001880 tarinfo.linkname = linkname
1881 if pwd:
1882 try:
1883 tarinfo.uname = pwd.getpwuid(tarinfo.uid)[0]
1884 except KeyError:
1885 pass
1886 if grp:
1887 try:
1888 tarinfo.gname = grp.getgrgid(tarinfo.gid)[0]
1889 except KeyError:
1890 pass
1891
1892 if type in (CHRTYPE, BLKTYPE):
1893 if hasattr(os, "major") and hasattr(os, "minor"):
1894 tarinfo.devmajor = os.major(statres.st_rdev)
1895 tarinfo.devminor = os.minor(statres.st_rdev)
1896 return tarinfo
1897
1898 def list(self, verbose=True):
1899 """Print a table of contents to sys.stdout. If `verbose' is False, only
1900 the names of the members are printed. If it is True, an `ls -l'-like
1901 output is produced.
1902 """
1903 self._check()
1904
1905 for tarinfo in self:
1906 if verbose:
1907 print filemode(tarinfo.mode),
1908 print "%s/%s" % (tarinfo.uname or tarinfo.uid,
1909 tarinfo.gname or tarinfo.gid),
1910 if tarinfo.ischr() or tarinfo.isblk():
1911 print "%10s" % ("%d,%d" \
1912 % (tarinfo.devmajor, tarinfo.devminor)),
1913 else:
1914 print "%10d" % tarinfo.size,
1915 print "%d-%02d-%02d %02d:%02d:%02d" \
1916 % time.localtime(tarinfo.mtime)[:6],
1917
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001918 print tarinfo.name + ("/" if tarinfo.isdir() else ""),
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001919
1920 if verbose:
1921 if tarinfo.issym():
1922 print "->", tarinfo.linkname,
1923 if tarinfo.islnk():
1924 print "link to", tarinfo.linkname,
1925 print
1926
Lars Gustäbel104490e2007-06-18 11:42:11 +00001927 def add(self, name, arcname=None, recursive=True, exclude=None):
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001928 """Add the file `name' to the archive. `name' may be any type of file
1929 (directory, fifo, symbolic link, etc.). If given, `arcname'
1930 specifies an alternative name for the file in the archive.
1931 Directories are added recursively by default. This can be avoided by
Lars Gustäbel104490e2007-06-18 11:42:11 +00001932 setting `recursive' to False. `exclude' is a function that should
1933 return True for each filename to be excluded.
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001934 """
1935 self._check("aw")
1936
1937 if arcname is None:
1938 arcname = name
1939
Lars Gustäbel104490e2007-06-18 11:42:11 +00001940 # Exclude pathnames.
1941 if exclude is not None and exclude(name):
1942 self._dbg(2, "tarfile: Excluded %r" % name)
1943 return
1944
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001945 # Skip if somebody tries to archive the archive...
Lars Gustäbela4b23812006-12-23 17:57:23 +00001946 if self.name is not None and os.path.abspath(name) == self.name:
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001947 self._dbg(2, "tarfile: Skipped %r" % name)
1948 return
1949
1950 # Special case: The user wants to add the current
1951 # working directory.
1952 if name == ".":
1953 if recursive:
1954 if arcname == ".":
1955 arcname = ""
Lars Gustäbelc64e4022007-03-13 10:47:19 +00001956 for f in os.listdir(name):
Lars Gustäbel104490e2007-06-18 11:42:11 +00001957 self.add(f, os.path.join(arcname, f), recursive, exclude)
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001958 return
1959
1960 self._dbg(1, name)
1961
1962 # Create a TarInfo object from the file.
1963 tarinfo = self.gettarinfo(name, arcname)
1964
1965 if tarinfo is None:
1966 self._dbg(1, "tarfile: Unsupported type %r" % name)
1967 return
1968
1969 # Append the tar header and data to the archive.
1970 if tarinfo.isreg():
Brett Cannon6cef0762007-05-25 20:17:15 +00001971 f = bltn_open(name, "rb")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001972 self.addfile(tarinfo, f)
1973 f.close()
1974
Martin v. Löwis5dbdc592005-08-27 10:07:56 +00001975 elif tarinfo.isdir():
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001976 self.addfile(tarinfo)
1977 if recursive:
1978 for f in os.listdir(name):
Lars Gustäbel104490e2007-06-18 11:42:11 +00001979 self.add(os.path.join(name, f), os.path.join(arcname, f), recursive, exclude)
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001980
Martin v. Löwis5dbdc592005-08-27 10:07:56 +00001981 else:
1982 self.addfile(tarinfo)
1983
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001984 def addfile(self, tarinfo, fileobj=None):
1985 """Add the TarInfo object `tarinfo' to the archive. If `fileobj' is
1986 given, tarinfo.size bytes are read from it and added to the archive.
1987 You can create TarInfo objects using gettarinfo().
1988 On Windows platforms, `fileobj' should always be opened with mode
1989 'rb' to avoid irritation about the file size.
1990 """
1991 self._check("aw")
1992
Georg Brandl3354f282006-10-29 09:16:12 +00001993 tarinfo = copy.copy(tarinfo)
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001994
Lars Gustäbela0fcb932007-05-27 19:49:30 +00001995 buf = tarinfo.tobuf(self.format, self.encoding, self.errors)
Georg Brandl3354f282006-10-29 09:16:12 +00001996 self.fileobj.write(buf)
1997 self.offset += len(buf)
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00001998
1999 # If there's data to follow, append it.
2000 if fileobj is not None:
2001 copyfileobj(fileobj, self.fileobj, tarinfo.size)
2002 blocks, remainder = divmod(tarinfo.size, BLOCKSIZE)
2003 if remainder > 0:
2004 self.fileobj.write(NUL * (BLOCKSIZE - remainder))
2005 blocks += 1
2006 self.offset += blocks * BLOCKSIZE
2007
Martin v. Löwisf3c56112004-09-18 09:08:52 +00002008 self.members.append(tarinfo)
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002009
Martin v. Löwis00a73e72005-03-04 19:40:34 +00002010 def extractall(self, path=".", members=None):
2011 """Extract all members from the archive to the current working
2012 directory and set owner, modification time and permissions on
2013 directories afterwards. `path' specifies a different directory
2014 to extract to. `members' is optional and must be a subset of the
2015 list returned by getmembers().
2016 """
2017 directories = []
2018
2019 if members is None:
2020 members = self
2021
2022 for tarinfo in members:
2023 if tarinfo.isdir():
2024 # Extract directory with a safe mode, so that
2025 # all files below can be extracted as well.
2026 try:
Lars Gustäbelc64e4022007-03-13 10:47:19 +00002027 os.makedirs(os.path.join(path, tarinfo.name), 0700)
Martin v. Löwis00a73e72005-03-04 19:40:34 +00002028 except EnvironmentError:
2029 pass
2030 directories.append(tarinfo)
2031 else:
2032 self.extract(tarinfo, path)
2033
2034 # Reverse sort directories.
2035 directories.sort(lambda a, b: cmp(a.name, b.name))
2036 directories.reverse()
2037
2038 # Set correct owner, mtime and filemode on directories.
2039 for tarinfo in directories:
Lars Gustäbel2ee1c762008-01-04 14:00:33 +00002040 dirpath = os.path.join(path, tarinfo.name)
Martin v. Löwis00a73e72005-03-04 19:40:34 +00002041 try:
Lars Gustäbel2ee1c762008-01-04 14:00:33 +00002042 self.chown(tarinfo, dirpath)
2043 self.utime(tarinfo, dirpath)
2044 self.chmod(tarinfo, dirpath)
Martin v. Löwis00a73e72005-03-04 19:40:34 +00002045 except ExtractError, e:
2046 if self.errorlevel > 1:
2047 raise
2048 else:
2049 self._dbg(1, "tarfile: %s" % e)
2050
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002051 def extract(self, member, path=""):
2052 """Extract a member from the archive to the current working directory,
2053 using its full name. Its file information is extracted as accurately
2054 as possible. `member' may be a filename or a TarInfo object. You can
2055 specify a different directory using `path'.
2056 """
2057 self._check("r")
2058
Lars Gustäbelc64e4022007-03-13 10:47:19 +00002059 if isinstance(member, basestring):
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002060 tarinfo = self.getmember(member)
Lars Gustäbelc64e4022007-03-13 10:47:19 +00002061 else:
2062 tarinfo = member
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002063
Neal Norwitza4f651a2004-07-20 22:07:44 +00002064 # Prepare the link target for makelink().
2065 if tarinfo.islnk():
2066 tarinfo._link_target = os.path.join(path, tarinfo.linkname)
2067
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002068 try:
2069 self._extract_member(tarinfo, os.path.join(path, tarinfo.name))
2070 except EnvironmentError, e:
2071 if self.errorlevel > 0:
2072 raise
2073 else:
2074 if e.filename is None:
2075 self._dbg(1, "tarfile: %s" % e.strerror)
2076 else:
2077 self._dbg(1, "tarfile: %s %r" % (e.strerror, e.filename))
2078 except ExtractError, e:
2079 if self.errorlevel > 1:
2080 raise
2081 else:
2082 self._dbg(1, "tarfile: %s" % e)
2083
2084 def extractfile(self, member):
2085 """Extract a member from the archive as a file object. `member' may be
2086 a filename or a TarInfo object. If `member' is a regular file, a
2087 file-like object is returned. If `member' is a link, a file-like
2088 object is constructed from the link's target. If `member' is none of
2089 the above, None is returned.
2090 The file-like object is read-only and provides the following
2091 methods: read(), readline(), readlines(), seek() and tell()
2092 """
2093 self._check("r")
2094
Lars Gustäbelc64e4022007-03-13 10:47:19 +00002095 if isinstance(member, basestring):
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002096 tarinfo = self.getmember(member)
Lars Gustäbelc64e4022007-03-13 10:47:19 +00002097 else:
2098 tarinfo = member
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002099
2100 if tarinfo.isreg():
2101 return self.fileobject(self, tarinfo)
2102
2103 elif tarinfo.type not in SUPPORTED_TYPES:
2104 # If a member's type is unknown, it is treated as a
2105 # regular file.
2106 return self.fileobject(self, tarinfo)
2107
2108 elif tarinfo.islnk() or tarinfo.issym():
2109 if isinstance(self.fileobj, _Stream):
2110 # A small but ugly workaround for the case that someone tries
2111 # to extract a (sym)link as a file-object from a non-seekable
2112 # stream of tar blocks.
Georg Brandle4751e32006-05-18 06:11:19 +00002113 raise StreamError("cannot extract (sym)link as file object")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002114 else:
Georg Brandl7eb4b7d2005-07-22 21:49:32 +00002115 # A (sym)link's file object is its target's file object.
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002116 return self.extractfile(self._getmember(tarinfo.linkname,
2117 tarinfo))
2118 else:
2119 # If there's no data associated with the member (directory, chrdev,
2120 # blkdev, etc.), return None instead of a file object.
2121 return None
2122
2123 def _extract_member(self, tarinfo, targetpath):
2124 """Extract the TarInfo object tarinfo to a physical
2125 file called targetpath.
2126 """
2127 # Fetch the TarInfo object for the given name
2128 # and build the destination pathname, replacing
2129 # forward slashes to platform specific separators.
2130 if targetpath[-1:] == "/":
2131 targetpath = targetpath[:-1]
2132 targetpath = os.path.normpath(targetpath)
2133
2134 # Create all upper directories.
2135 upperdirs = os.path.dirname(targetpath)
2136 if upperdirs and not os.path.exists(upperdirs):
Lars Gustäbeld2e22902007-01-23 11:17:33 +00002137 os.makedirs(upperdirs)
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002138
2139 if tarinfo.islnk() or tarinfo.issym():
2140 self._dbg(1, "%s -> %s" % (tarinfo.name, tarinfo.linkname))
2141 else:
2142 self._dbg(1, tarinfo.name)
2143
2144 if tarinfo.isreg():
2145 self.makefile(tarinfo, targetpath)
2146 elif tarinfo.isdir():
2147 self.makedir(tarinfo, targetpath)
2148 elif tarinfo.isfifo():
2149 self.makefifo(tarinfo, targetpath)
2150 elif tarinfo.ischr() or tarinfo.isblk():
2151 self.makedev(tarinfo, targetpath)
2152 elif tarinfo.islnk() or tarinfo.issym():
2153 self.makelink(tarinfo, targetpath)
2154 elif tarinfo.type not in SUPPORTED_TYPES:
2155 self.makeunknown(tarinfo, targetpath)
2156 else:
2157 self.makefile(tarinfo, targetpath)
2158
2159 self.chown(tarinfo, targetpath)
2160 if not tarinfo.issym():
2161 self.chmod(tarinfo, targetpath)
2162 self.utime(tarinfo, targetpath)
2163
2164 #--------------------------------------------------------------------------
2165 # Below are the different file methods. They are called via
2166 # _extract_member() when extract() is called. They can be replaced in a
2167 # subclass to implement other functionality.
2168
2169 def makedir(self, tarinfo, targetpath):
2170 """Make a directory called targetpath.
2171 """
2172 try:
2173 os.mkdir(targetpath)
2174 except EnvironmentError, e:
2175 if e.errno != errno.EEXIST:
2176 raise
2177
2178 def makefile(self, tarinfo, targetpath):
2179 """Make a file called targetpath.
2180 """
2181 source = self.extractfile(tarinfo)
Brett Cannon6cef0762007-05-25 20:17:15 +00002182 target = bltn_open(targetpath, "wb")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002183 copyfileobj(source, target)
2184 source.close()
2185 target.close()
2186
2187 def makeunknown(self, tarinfo, targetpath):
2188 """Make a file from a TarInfo object with an unknown type
2189 at targetpath.
2190 """
2191 self.makefile(tarinfo, targetpath)
2192 self._dbg(1, "tarfile: Unknown file type %r, " \
2193 "extracted as regular file." % tarinfo.type)
2194
2195 def makefifo(self, tarinfo, targetpath):
2196 """Make a fifo called targetpath.
2197 """
2198 if hasattr(os, "mkfifo"):
2199 os.mkfifo(targetpath)
2200 else:
Georg Brandle4751e32006-05-18 06:11:19 +00002201 raise ExtractError("fifo not supported by system")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002202
2203 def makedev(self, tarinfo, targetpath):
2204 """Make a character or block device called targetpath.
2205 """
2206 if not hasattr(os, "mknod") or not hasattr(os, "makedev"):
Georg Brandle4751e32006-05-18 06:11:19 +00002207 raise ExtractError("special devices not supported by system")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002208
2209 mode = tarinfo.mode
2210 if tarinfo.isblk():
2211 mode |= stat.S_IFBLK
2212 else:
2213 mode |= stat.S_IFCHR
2214
2215 os.mknod(targetpath, mode,
2216 os.makedev(tarinfo.devmajor, tarinfo.devminor))
2217
2218 def makelink(self, tarinfo, targetpath):
2219 """Make a (symbolic) link called targetpath. If it cannot be created
2220 (platform limitation), we try to make a copy of the referenced file
2221 instead of a link.
2222 """
2223 linkpath = tarinfo.linkname
2224 try:
2225 if tarinfo.issym():
2226 os.symlink(linkpath, targetpath)
2227 else:
Neal Norwitza4f651a2004-07-20 22:07:44 +00002228 # See extract().
2229 os.link(tarinfo._link_target, targetpath)
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002230 except AttributeError:
2231 if tarinfo.issym():
2232 linkpath = os.path.join(os.path.dirname(tarinfo.name),
2233 linkpath)
2234 linkpath = normpath(linkpath)
2235
2236 try:
2237 self._extract_member(self.getmember(linkpath), targetpath)
2238 except (EnvironmentError, KeyError), e:
2239 linkpath = os.path.normpath(linkpath)
2240 try:
2241 shutil.copy2(linkpath, targetpath)
2242 except EnvironmentError, e:
Georg Brandle4751e32006-05-18 06:11:19 +00002243 raise IOError("link could not be created")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002244
2245 def chown(self, tarinfo, targetpath):
2246 """Set owner of targetpath according to tarinfo.
2247 """
2248 if pwd and hasattr(os, "geteuid") and os.geteuid() == 0:
2249 # We have to be root to do so.
2250 try:
2251 g = grp.getgrnam(tarinfo.gname)[2]
2252 except KeyError:
2253 try:
2254 g = grp.getgrgid(tarinfo.gid)[2]
2255 except KeyError:
2256 g = os.getgid()
2257 try:
2258 u = pwd.getpwnam(tarinfo.uname)[2]
2259 except KeyError:
2260 try:
2261 u = pwd.getpwuid(tarinfo.uid)[2]
2262 except KeyError:
2263 u = os.getuid()
2264 try:
2265 if tarinfo.issym() and hasattr(os, "lchown"):
2266 os.lchown(targetpath, u, g)
2267 else:
Andrew MacIntyre7970d202003-02-19 12:51:34 +00002268 if sys.platform != "os2emx":
2269 os.chown(targetpath, u, g)
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002270 except EnvironmentError, e:
Georg Brandle4751e32006-05-18 06:11:19 +00002271 raise ExtractError("could not change owner")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002272
2273 def chmod(self, tarinfo, targetpath):
2274 """Set file permissions of targetpath according to tarinfo.
2275 """
Jack Jansen834eff62003-03-07 12:47:06 +00002276 if hasattr(os, 'chmod'):
2277 try:
2278 os.chmod(targetpath, tarinfo.mode)
2279 except EnvironmentError, e:
Georg Brandle4751e32006-05-18 06:11:19 +00002280 raise ExtractError("could not change mode")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002281
2282 def utime(self, tarinfo, targetpath):
2283 """Set modification time of targetpath according to tarinfo.
2284 """
Jack Jansen834eff62003-03-07 12:47:06 +00002285 if not hasattr(os, 'utime'):
Tim Petersf9347782003-03-07 15:36:41 +00002286 return
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002287 if sys.platform == "win32" and tarinfo.isdir():
2288 # According to msdn.microsoft.com, it is an error (EACCES)
2289 # to use utime() on directories.
2290 return
2291 try:
2292 os.utime(targetpath, (tarinfo.mtime, tarinfo.mtime))
2293 except EnvironmentError, e:
Georg Brandle4751e32006-05-18 06:11:19 +00002294 raise ExtractError("could not change modification time")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002295
2296 #--------------------------------------------------------------------------
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002297 def next(self):
2298 """Return the next member of the archive as a TarInfo object, when
2299 TarFile is opened for reading. Return None if there is no more
2300 available.
2301 """
2302 self._check("ra")
2303 if self.firstmember is not None:
2304 m = self.firstmember
2305 self.firstmember = None
2306 return m
2307
2308 # Read the next block.
Andrew M. Kuchling864bba12004-07-10 22:02:11 +00002309 self.fileobj.seek(self.offset)
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002310 while True:
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002311 try:
Lars Gustäbelc64e4022007-03-13 10:47:19 +00002312 tarinfo = self.tarinfo.fromtarfile(self)
2313 if tarinfo is None:
2314 return
2315 self.members.append(tarinfo)
Georg Brandl38c6a222006-05-10 16:26:03 +00002316
Georg Brandlebbeed72006-12-19 22:06:46 +00002317 except HeaderError, e:
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002318 if self.ignore_zeros:
Georg Brandlebbeed72006-12-19 22:06:46 +00002319 self._dbg(2, "0x%X: %s" % (self.offset, e))
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002320 self.offset += BLOCKSIZE
2321 continue
2322 else:
Andrew M. Kuchling864bba12004-07-10 22:02:11 +00002323 if self.offset == 0:
Georg Brandlebbeed72006-12-19 22:06:46 +00002324 raise ReadError(str(e))
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002325 return None
2326 break
2327
Georg Brandl38c6a222006-05-10 16:26:03 +00002328 return tarinfo
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002329
2330 #--------------------------------------------------------------------------
2331 # Little helper methods:
2332
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002333 def _getmember(self, name, tarinfo=None):
2334 """Find an archive member by name from bottom to top.
2335 If tarinfo is given, it is used as the starting point.
2336 """
Martin v. Löwisf3c56112004-09-18 09:08:52 +00002337 # Ensure that all members have been loaded.
2338 members = self.getmembers()
2339
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002340 if tarinfo is None:
Martin v. Löwisf3c56112004-09-18 09:08:52 +00002341 end = len(members)
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002342 else:
Martin v. Löwisf3c56112004-09-18 09:08:52 +00002343 end = members.index(tarinfo)
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002344
2345 for i in xrange(end - 1, -1, -1):
Martin v. Löwisf3c56112004-09-18 09:08:52 +00002346 if name == members[i].name:
2347 return members[i]
Andrew M. Kuchling864bba12004-07-10 22:02:11 +00002348
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002349 def _load(self):
2350 """Read through the entire archive file and look for readable
2351 members.
2352 """
2353 while True:
2354 tarinfo = self.next()
2355 if tarinfo is None:
2356 break
2357 self._loaded = True
2358
2359 def _check(self, mode=None):
2360 """Check if TarFile is still open, and if the operation's mode
2361 corresponds to TarFile's mode.
2362 """
2363 if self.closed:
Georg Brandle4751e32006-05-18 06:11:19 +00002364 raise IOError("%s is closed" % self.__class__.__name__)
Lars Gustäbelc64e4022007-03-13 10:47:19 +00002365 if mode is not None and self.mode not in mode:
2366 raise IOError("bad operation for mode %r" % self.mode)
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002367
2368 def __iter__(self):
2369 """Provide an iterator object.
2370 """
2371 if self._loaded:
2372 return iter(self.members)
2373 else:
2374 return TarIter(self)
2375
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002376 def _dbg(self, level, msg):
2377 """Write debugging output to sys.stderr.
2378 """
2379 if level <= self.debug:
2380 print >> sys.stderr, msg
2381# class TarFile
2382
2383class TarIter:
2384 """Iterator Class.
2385
2386 for tarinfo in TarFile(...):
2387 suite...
2388 """
2389
2390 def __init__(self, tarfile):
2391 """Construct a TarIter object.
2392 """
2393 self.tarfile = tarfile
Martin v. Löwis637431b2005-03-03 23:12:42 +00002394 self.index = 0
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002395 def __iter__(self):
2396 """Return iterator object.
2397 """
2398 return self
2399 def next(self):
2400 """Return the next item using TarFile's next() method.
2401 When all members have been read, set TarFile as _loaded.
2402 """
Martin v. Löwis637431b2005-03-03 23:12:42 +00002403 # Fix for SF #1100429: Under rare circumstances it can
2404 # happen that getmembers() is called during iteration,
2405 # which will cause TarIter to stop prematurely.
2406 if not self.tarfile._loaded:
2407 tarinfo = self.tarfile.next()
2408 if not tarinfo:
2409 self.tarfile._loaded = True
2410 raise StopIteration
2411 else:
2412 try:
2413 tarinfo = self.tarfile.members[self.index]
2414 except IndexError:
2415 raise StopIteration
2416 self.index += 1
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002417 return tarinfo
2418
2419# Helper classes for sparse file support
2420class _section:
2421 """Base class for _data and _hole.
2422 """
2423 def __init__(self, offset, size):
2424 self.offset = offset
2425 self.size = size
2426 def __contains__(self, offset):
2427 return self.offset <= offset < self.offset + self.size
2428
2429class _data(_section):
2430 """Represent a data section in a sparse file.
2431 """
2432 def __init__(self, offset, size, realpos):
2433 _section.__init__(self, offset, size)
2434 self.realpos = realpos
2435
2436class _hole(_section):
2437 """Represent a hole section in a sparse file.
2438 """
2439 pass
2440
2441class _ringbuffer(list):
2442 """Ringbuffer class which increases performance
2443 over a regular list.
2444 """
2445 def __init__(self):
2446 self.idx = 0
2447 def find(self, offset):
2448 idx = self.idx
2449 while True:
2450 item = self[idx]
2451 if offset in item:
2452 break
2453 idx += 1
2454 if idx == len(self):
2455 idx = 0
2456 if idx == self.idx:
2457 # End of File
2458 return None
2459 self.idx = idx
2460 return item
2461
2462#---------------------------------------------
2463# zipfile compatible TarFile class
2464#---------------------------------------------
2465TAR_PLAIN = 0 # zipfile.ZIP_STORED
2466TAR_GZIPPED = 8 # zipfile.ZIP_DEFLATED
2467class TarFileCompat:
2468 """TarFile class compatible with standard module zipfile's
2469 ZipFile class.
2470 """
2471 def __init__(self, file, mode="r", compression=TAR_PLAIN):
2472 if compression == TAR_PLAIN:
2473 self.tarfile = TarFile.taropen(file, mode)
2474 elif compression == TAR_GZIPPED:
2475 self.tarfile = TarFile.gzopen(file, mode)
2476 else:
Georg Brandle4751e32006-05-18 06:11:19 +00002477 raise ValueError("unknown compression constant")
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002478 if mode[0:1] == "r":
2479 members = self.tarfile.getmembers()
Raymond Hettingera1d09e22005-09-11 16:34:05 +00002480 for m in members:
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002481 m.filename = m.name
2482 m.file_size = m.size
2483 m.date_time = time.gmtime(m.mtime)[:6]
2484 def namelist(self):
2485 return map(lambda m: m.name, self.infolist())
2486 def infolist(self):
2487 return filter(lambda m: m.type in REGULAR_TYPES,
2488 self.tarfile.getmembers())
2489 def printdir(self):
2490 self.tarfile.list()
2491 def testzip(self):
2492 return
2493 def getinfo(self, name):
2494 return self.tarfile.getmember(name)
2495 def read(self, name):
2496 return self.tarfile.extractfile(self.tarfile.getmember(name)).read()
2497 def write(self, filename, arcname=None, compress_type=None):
2498 self.tarfile.add(filename, arcname)
2499 def writestr(self, zinfo, bytes):
Raymond Hettingera6172712004-12-31 19:15:26 +00002500 try:
2501 from cStringIO import StringIO
2502 except ImportError:
2503 from StringIO import StringIO
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002504 import calendar
2505 zinfo.name = zinfo.filename
2506 zinfo.size = zinfo.file_size
2507 zinfo.mtime = calendar.timegm(zinfo.date_time)
Raymond Hettingera6172712004-12-31 19:15:26 +00002508 self.tarfile.addfile(zinfo, StringIO(bytes))
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002509 def close(self):
2510 self.tarfile.close()
2511#class TarFileCompat
2512
2513#--------------------
2514# exported functions
2515#--------------------
2516def is_tarfile(name):
2517 """Return True if name points to a tar archive that we
2518 are able to handle, else return False.
2519 """
2520 try:
2521 t = open(name)
2522 t.close()
2523 return True
2524 except TarError:
2525 return False
2526
Brett Cannon6cef0762007-05-25 20:17:15 +00002527bltn_open = open
Neal Norwitzb9ef4ae2003-01-05 23:19:43 +00002528open = TarFile.open