| #! /usr/bin/env python |
| |
| """A variant on webchecker that creates a mirror copy of a remote site.""" |
| |
| __version__ = "0.1" |
| |
| import os |
| import sys |
| import string |
| import urllib |
| import getopt |
| |
| import webchecker |
| verbose = webchecker.verbose |
| |
| def main(): |
| global verbose |
| try: |
| opts, args = getopt.getopt(sys.argv[1:], "qv") |
| except getopt.error, msg: |
| print msg |
| print "usage:", sys.argv[0], "[-v] ... [rooturl] ..." |
| return 2 |
| for o, a in opts: |
| if o == "-q": |
| webchecker.verbose = verbose = 0 |
| if o == "-v": |
| webchecker.verbose = verbose = verbose + 1 |
| c = Sucker(0) |
| c.urlopener.addheaders = [ |
| ('User-agent', 'websucker/%s' % __version__), |
| ] |
| for arg in args: |
| print "Adding root", arg |
| c.addroot(arg) |
| print "Run..." |
| c.run() |
| |
| class Sucker(webchecker.Checker): |
| |
| # Alas, had to copy this to make one change... |
| def getpage(self, url): |
| if url[:7] == 'mailto:' or url[:5] == 'news:': |
| if verbose > 1: print " Not checking mailto/news URL" |
| return None |
| isint = self.inroots(url) |
| if not isint and not self.checkext: |
| if verbose > 1: print " Not checking ext link" |
| return None |
| path = self.savefilename(url) |
| saved = 0 |
| try: |
| f = open(path, "rb") |
| except IOError: |
| try: |
| f = self.urlopener.open(url) |
| except IOError, msg: |
| msg = webchecker.sanitize(msg) |
| if verbose > 0: |
| print "Error ", msg |
| if verbose > 0: |
| webchecker.show(" HREF ", url, " from", self.todo[url]) |
| self.setbad(url, msg) |
| return None |
| if not isint: |
| if verbose > 1: print " Not gathering links from ext URL" |
| safeclose(f) |
| return None |
| nurl = f.geturl() |
| if nurl != url: |
| path = self.savefilename(nurl) |
| info = f.info() |
| else: |
| if verbose: print "Loading cached URL", url |
| saved = 1 |
| nurl = url |
| info = {} |
| if url[-1:] == "/": |
| info["content-type"] = "text/html" |
| text = f.read() |
| if not saved: self.savefile(text, path) |
| if info.has_key('content-type'): |
| ctype = string.lower(info['content-type']) |
| else: |
| ctype = None |
| if nurl != url: |
| if verbose > 1: |
| print " Redirected to", nurl |
| if not ctype: |
| ctype, encoding = webchecker.mimetypes.guess_type(nurl) |
| if ctype != 'text/html': |
| webchecker.safeclose(f) |
| if verbose > 1: |
| print " Not HTML, mime type", ctype |
| return None |
| f.close() |
| return webchecker.Page(text, nurl) |
| |
| def savefile(self, text, path): |
| dir, base = os.path.split(path) |
| makedirs(dir) |
| f = open(path, "wb") |
| f.write(text) |
| f.close() |
| print "saved", path |
| |
| def savefilename(self, url): |
| type, rest = urllib.splittype(url) |
| host, path = urllib.splithost(rest) |
| while path[:1] == "/": path = path[1:] |
| user, host = urllib.splituser(host) |
| host, port = urllib.splitnport(host) |
| host = string.lower(host) |
| path = os.path.join(host, path) |
| if path[-1] == "/": path = path + "index.html" |
| if os.sep != "/": |
| path = string.join(string.split(path, "/"), os.sep) |
| return path |
| |
| def makedirs(dir): |
| if not dir or os.path.exists(dir): |
| return |
| head, tail = os.path.split(dir) |
| if not tail: |
| print "Huh? Don't know how to make dir", dir |
| return |
| makedirs(head) |
| os.mkdir(dir, 0777) |
| |
| if __name__ == '__main__': |
| sys.exit(main() or 0) |