| #! /usr/bin/env python |
| |
| """Mirror a remote ftp subtree into a local directory tree. |
| |
| usage: ftpmirror [-v] [-q] [-i] [-m] [-n] [-r] [-s pat] |
| [-l username [-p passwd [-a account]]] |
| hostname[:port] [remotedir [localdir]] |
| -v: verbose |
| -q: quiet |
| -i: interactive mode |
| -m: macintosh server (NCSA telnet 2.4) (implies -n -s '*.o') |
| -n: don't log in |
| -r: remove local files/directories no longer pertinent |
| -l username [-p passwd [-a account]]: login info (default .netrc or anonymous) |
| -s pat: skip files matching pattern |
| hostname: remote host w/ optional port separated by ':' |
| remotedir: remote directory (default initial) |
| localdir: local directory (default current) |
| """ |
| |
| import os |
| import sys |
| import time |
| import getopt |
| import ftplib |
| import netrc |
| from fnmatch import fnmatch |
| |
| # Print usage message and exit |
| def usage(*args): |
| sys.stdout = sys.stderr |
| for msg in args: print msg |
| print __doc__ |
| sys.exit(2) |
| |
| verbose = 1 # 0 for -q, 2 for -v |
| interactive = 0 |
| mac = 0 |
| rmok = 0 |
| nologin = 0 |
| skippats = ['.', '..', '.mirrorinfo'] |
| |
| # Main program: parse command line and start processing |
| def main(): |
| global verbose, interactive, mac, rmok, nologin |
| try: |
| opts, args = getopt.getopt(sys.argv[1:], 'a:bil:mnp:qrs:v') |
| except getopt.error, msg: |
| usage(msg) |
| login = '' |
| passwd = '' |
| account = '' |
| if not args: usage('hostname missing') |
| host = args[0] |
| port = 0 |
| if ':' in host: |
| host, port = host.split(':', 1) |
| port = int(port) |
| try: |
| auth = netrc.netrc().authenticators(host) |
| if auth is not None: |
| login, account, passwd = auth |
| except (netrc.NetrcParseError, IOError): |
| pass |
| for o, a in opts: |
| if o == '-l': login = a |
| if o == '-p': passwd = a |
| if o == '-a': account = a |
| if o == '-v': verbose = verbose + 1 |
| if o == '-q': verbose = 0 |
| if o == '-i': interactive = 1 |
| if o == '-m': mac = 1; nologin = 1; skippats.append('*.o') |
| if o == '-n': nologin = 1 |
| if o == '-r': rmok = 1 |
| if o == '-s': skippats.append(a) |
| remotedir = '' |
| localdir = '' |
| if args[1:]: |
| remotedir = args[1] |
| if args[2:]: |
| localdir = args[2] |
| if args[3:]: usage('too many arguments') |
| # |
| f = ftplib.FTP() |
| if verbose: print "Connecting to '%s%s'..." % (host, |
| (port and ":%d"%port or "")) |
| f.connect(host,port) |
| if not nologin: |
| if verbose: |
| print 'Logging in as %r...' % (login or 'anonymous') |
| f.login(login, passwd, account) |
| if verbose: print 'OK.' |
| pwd = f.pwd() |
| if verbose > 1: print 'PWD =', repr(pwd) |
| if remotedir: |
| if verbose > 1: print 'cwd(%s)' % repr(remotedir) |
| f.cwd(remotedir) |
| if verbose > 1: print 'OK.' |
| pwd = f.pwd() |
| if verbose > 1: print 'PWD =', repr(pwd) |
| # |
| mirrorsubdir(f, localdir) |
| |
| # Core logic: mirror one subdirectory (recursively) |
| def mirrorsubdir(f, localdir): |
| pwd = f.pwd() |
| if localdir and not os.path.isdir(localdir): |
| if verbose: print 'Creating local directory', repr(localdir) |
| try: |
| makedir(localdir) |
| except os.error, msg: |
| print "Failed to establish local directory", repr(localdir) |
| return |
| infofilename = os.path.join(localdir, '.mirrorinfo') |
| try: |
| text = open(infofilename, 'r').read() |
| except IOError, msg: |
| text = '{}' |
| try: |
| info = eval(text) |
| except (SyntaxError, NameError): |
| print 'Bad mirror info in', repr(infofilename) |
| info = {} |
| subdirs = [] |
| listing = [] |
| if verbose: print 'Listing remote directory %r...' % (pwd,) |
| f.retrlines('LIST', listing.append) |
| filesfound = [] |
| for line in listing: |
| if verbose > 1: print '-->', repr(line) |
| if mac: |
| # Mac listing has just filenames; |
| # trailing / means subdirectory |
| filename = line.strip() |
| mode = '-' |
| if filename[-1:] == '/': |
| filename = filename[:-1] |
| mode = 'd' |
| infostuff = '' |
| else: |
| # Parse, assuming a UNIX listing |
| words = line.split(None, 8) |
| if len(words) < 6: |
| if verbose > 1: print 'Skipping short line' |
| continue |
| filename = words[-1].lstrip() |
| i = filename.find(" -> ") |
| if i >= 0: |
| # words[0] had better start with 'l'... |
| if verbose > 1: |
| print 'Found symbolic link %r' % (filename,) |
| linkto = filename[i+4:] |
| filename = filename[:i] |
| infostuff = words[-5:-1] |
| mode = words[0] |
| skip = 0 |
| for pat in skippats: |
| if fnmatch(filename, pat): |
| if verbose > 1: |
| print 'Skip pattern', repr(pat), |
| print 'matches', repr(filename) |
| skip = 1 |
| break |
| if skip: |
| continue |
| if mode[0] == 'd': |
| if verbose > 1: |
| print 'Remembering subdirectory', repr(filename) |
| subdirs.append(filename) |
| continue |
| filesfound.append(filename) |
| if info.has_key(filename) and info[filename] == infostuff: |
| if verbose > 1: |
| print 'Already have this version of',repr(filename) |
| continue |
| fullname = os.path.join(localdir, filename) |
| tempname = os.path.join(localdir, '@'+filename) |
| if interactive: |
| doit = askabout('file', filename, pwd) |
| if not doit: |
| if not info.has_key(filename): |
| info[filename] = 'Not retrieved' |
| continue |
| try: |
| os.unlink(tempname) |
| except os.error: |
| pass |
| if mode[0] == 'l': |
| if verbose: |
| print "Creating symlink %r -> %r" % (filename, linkto) |
| try: |
| os.symlink(linkto, tempname) |
| except IOError, msg: |
| print "Can't create %r: %s" % (tempname, msg) |
| continue |
| else: |
| try: |
| fp = open(tempname, 'wb') |
| except IOError, msg: |
| print "Can't create %r: %s" % (tempname, msg) |
| continue |
| if verbose: |
| print 'Retrieving %r from %r as %r...' % (filename, pwd, fullname) |
| if verbose: |
| fp1 = LoggingFile(fp, 1024, sys.stdout) |
| else: |
| fp1 = fp |
| t0 = time.time() |
| try: |
| f.retrbinary('RETR ' + filename, |
| fp1.write, 8*1024) |
| except ftplib.error_perm, msg: |
| print msg |
| t1 = time.time() |
| bytes = fp.tell() |
| fp.close() |
| if fp1 != fp: |
| fp1.close() |
| try: |
| os.unlink(fullname) |
| except os.error: |
| pass # Ignore the error |
| try: |
| os.rename(tempname, fullname) |
| except os.error, msg: |
| print "Can't rename %r to %r: %s" % (tempname, fullname, msg) |
| continue |
| info[filename] = infostuff |
| writedict(info, infofilename) |
| if verbose and mode[0] != 'l': |
| dt = t1 - t0 |
| kbytes = bytes / 1024.0 |
| print int(round(kbytes)), |
| print 'Kbytes in', |
| print int(round(dt)), |
| print 'seconds', |
| if t1 > t0: |
| print '(~%d Kbytes/sec)' % \ |
| int(round(kbytes/dt),) |
| print |
| # |
| # Remove files from info that are no longer remote |
| deletions = 0 |
| for filename in info.keys(): |
| if filename not in filesfound: |
| if verbose: |
| print "Removing obsolete info entry for", |
| print repr(filename), "in", repr(localdir or ".") |
| del info[filename] |
| deletions = deletions + 1 |
| if deletions: |
| writedict(info, infofilename) |
| # |
| # Remove local files that are no longer in the remote directory |
| try: |
| if not localdir: names = os.listdir(os.curdir) |
| else: names = os.listdir(localdir) |
| except os.error: |
| names = [] |
| for name in names: |
| if name[0] == '.' or info.has_key(name) or name in subdirs: |
| continue |
| skip = 0 |
| for pat in skippats: |
| if fnmatch(name, pat): |
| if verbose > 1: |
| print 'Skip pattern', repr(pat), |
| print 'matches', repr(name) |
| skip = 1 |
| break |
| if skip: |
| continue |
| fullname = os.path.join(localdir, name) |
| if not rmok: |
| if verbose: |
| print 'Local file', repr(fullname), |
| print 'is no longer pertinent' |
| continue |
| if verbose: print 'Removing local file/dir', repr(fullname) |
| remove(fullname) |
| # |
| # Recursively mirror subdirectories |
| for subdir in subdirs: |
| if interactive: |
| doit = askabout('subdirectory', subdir, pwd) |
| if not doit: continue |
| if verbose: print 'Processing subdirectory', repr(subdir) |
| localsubdir = os.path.join(localdir, subdir) |
| pwd = f.pwd() |
| if verbose > 1: |
| print 'Remote directory now:', repr(pwd) |
| print 'Remote cwd', repr(subdir) |
| try: |
| f.cwd(subdir) |
| except ftplib.error_perm, msg: |
| print "Can't chdir to", repr(subdir), ":", repr(msg) |
| else: |
| if verbose: print 'Mirroring as', repr(localsubdir) |
| mirrorsubdir(f, localsubdir) |
| if verbose > 1: print 'Remote cwd ..' |
| f.cwd('..') |
| newpwd = f.pwd() |
| if newpwd != pwd: |
| print 'Ended up in wrong directory after cd + cd ..' |
| print 'Giving up now.' |
| break |
| else: |
| if verbose > 1: print 'OK.' |
| |
| # Helper to remove a file or directory tree |
| def remove(fullname): |
| if os.path.isdir(fullname) and not os.path.islink(fullname): |
| try: |
| names = os.listdir(fullname) |
| except os.error: |
| names = [] |
| ok = 1 |
| for name in names: |
| if not remove(os.path.join(fullname, name)): |
| ok = 0 |
| if not ok: |
| return 0 |
| try: |
| os.rmdir(fullname) |
| except os.error, msg: |
| print "Can't remove local directory %r: %s" % (fullname, msg) |
| return 0 |
| else: |
| try: |
| os.unlink(fullname) |
| except os.error, msg: |
| print "Can't remove local file %r: %s" % (fullname, msg) |
| return 0 |
| return 1 |
| |
| # Wrapper around a file for writing to write a hash sign every block. |
| class LoggingFile: |
| def __init__(self, fp, blocksize, outfp): |
| self.fp = fp |
| self.bytes = 0 |
| self.hashes = 0 |
| self.blocksize = blocksize |
| self.outfp = outfp |
| def write(self, data): |
| self.bytes = self.bytes + len(data) |
| hashes = int(self.bytes) / self.blocksize |
| while hashes > self.hashes: |
| self.outfp.write('#') |
| self.outfp.flush() |
| self.hashes = self.hashes + 1 |
| self.fp.write(data) |
| def close(self): |
| self.outfp.write('\n') |
| |
| # Ask permission to download a file. |
| def askabout(filetype, filename, pwd): |
| prompt = 'Retrieve %s %s from %s ? [ny] ' % (filetype, filename, pwd) |
| while 1: |
| reply = raw_input(prompt).strip().lower() |
| if reply in ['y', 'ye', 'yes']: |
| return 1 |
| if reply in ['', 'n', 'no', 'nop', 'nope']: |
| return 0 |
| print 'Please answer yes or no.' |
| |
| # Create a directory if it doesn't exist. Recursively create the |
| # parent directory as well if needed. |
| def makedir(pathname): |
| if os.path.isdir(pathname): |
| return |
| dirname = os.path.dirname(pathname) |
| if dirname: makedir(dirname) |
| os.mkdir(pathname, 0777) |
| |
| # Write a dictionary to a file in a way that can be read back using |
| # rval() but is still somewhat readable (i.e. not a single long line). |
| # Also creates a backup file. |
| def writedict(dict, filename): |
| dir, fname = os.path.split(filename) |
| tempname = os.path.join(dir, '@' + fname) |
| backup = os.path.join(dir, fname + '~') |
| try: |
| os.unlink(backup) |
| except os.error: |
| pass |
| fp = open(tempname, 'w') |
| fp.write('{\n') |
| for key, value in dict.items(): |
| fp.write('%r: %r,\n' % (key, value)) |
| fp.write('}\n') |
| fp.close() |
| try: |
| os.rename(filename, backup) |
| except os.error: |
| pass |
| os.rename(tempname, filename) |
| |
| |
| if __name__ == '__main__': |
| main() |