Added -x option to check external links. Slooooow!
diff --git a/Tools/webchecker/webchecker.py b/Tools/webchecker/webchecker.py
index 2ec9b61..d6c81cc 100755
--- a/Tools/webchecker/webchecker.py
+++ b/Tools/webchecker/webchecker.py
@@ -70,9 +70,11 @@
links in <FORM> or <IMG> or whatever other tags might contain
hyperlinks. It does honor the <BASE> tag.
-- It could be argued that it should also check external links for
-validity. This is true, but is is more error-prone. I think I will
-make this an option in the future.
+- Checking external links is not done by default; use -x to enable
+this feature. This is done because checking external links usually
+takes a lot of time. When enabled, this check is executed during the
+report generation phase (so -x is ignored when -q is specified). Even
+when -x is enabled, only ``http:'' URLs are checked.
Usage: webchecker.py [option] ... [rooturl] ...
@@ -85,6 +87,7 @@
-q -- quiet operation (also suppresses external links report)
-r number -- number of links processed per round (default %(ROUNDSIZE)d)
-v -- verbose operation; repeating -v will increase verbosity
+-x -- check external links (during report phase)
Arguments:
@@ -131,9 +134,10 @@
global verbose, maxpage, roundsize
dumpfile = DUMPFILE
restart = 0
+ checkext = 0
try:
- opts, args = getopt.getopt(sys.argv[1:], 'Rd:m:qr:v')
+ opts, args = getopt.getopt(sys.argv[1:], 'Rd:m:qr:vx')
except getopt.error, msg:
sys.stdout = sys.stderr
print msg
@@ -151,6 +155,8 @@
roundsize = string.atoi(a)
if o == '-v':
verbose = verbose + 1
+ if o == '-x':
+ checkext = 1
if verbose:
print AGENTNAME, "version", __version__
@@ -180,8 +186,12 @@
c.run()
except KeyboardInterrupt:
if verbose > 0:
- print "[interrupted]"
- c.report()
+ print "[run interrupted]"
+ try:
+ c.report(checkext)
+ except KeyboardInterrupt:
+ if verbose > 0:
+ print "[report interrupted]"
if not needsave:
if verbose > 0:
print
@@ -266,7 +276,7 @@
self.done[url] = self.todo[url]
del self.todo[url]
- def report(self):
+ def report(self, checkext=0):
print
if not self.todo: print "Final",
else: print "Interim",
@@ -274,22 +284,34 @@
len(self.todo), len(self.done),
len(self.ext), len(self.bad))
if verbose > 0:
- self.report_extrefs()
+ self.report_extrefs(checkext)
# Report errors last because the output may get truncated
self.report_errors()
- def report_extrefs(self):
+ def report_extrefs(self, checkext=0):
if not self.ext:
print
print "No external URLs"
return
print
- print "External URLs:"
+ if checkext:
+ print "External URLs (checking validity):"
+ else:
+ print "External URLs (not checked):"
print
urls = self.ext.keys()
urls.sort()
for url in urls:
show("HREF ", url, " from", self.ext[url])
+ if not checkext:
+ continue
+ if verbose > 2: print "Checking", url, "..."
+ try:
+ f = self.urlopener.open(url)
+ f.close()
+ if verbose > 3: print "OK"
+ except IOError, msg:
+ print "Error:", msg
def report_errors(self):
if not self.bad: