Restructured Checker class to get rid of 'ext' table.
Links are now either in 'todo' or 'done', and ext links
are hadled more like local links except that no further
links are gathered (and sometimes they aren't checked,
e.g. for mailto and news URLs). The -x option reverses
its meaning: it disables checking of ext links (they are
moved to 'done' without checking). A new 'errors' table
collects pages with bad links as we go -- redundant,
but useful for the GUI version which needs to report
this as we go. Some new methods, including reset().
New checkpoint format.
Adapted the GUI to the changes in the Checker class.
Added Quit and "Start over" buttons, and a checkbox
to disable checking external links. The details
window now also shows bad links emanating from the
selected page. Miscellaneous small chages.
diff --git a/Tools/webchecker/wcgui.py b/Tools/webchecker/wcgui.py
index f5649be..ae08734 100755
--- a/Tools/webchecker/wcgui.py
+++ b/Tools/webchecker/wcgui.py
@@ -6,42 +6,37 @@
<APPLET CODE=wcgui.py NAME=CheckerWindow></APPLET>
-Checkpoints are not (yet?) supported.
+Checkpoints are not (yet??? ever???) supported.
User interface:
Enter a root to check in the text entry box. To enter more than one root,
-enter each root and press <Return>.
+enter them one at a time and press <Return> for each one.
Command buttons Start, Stop and "Check one" govern the checking process in
the obvious way. Start and "Check one" also enter the root from the text
-entry box if one is present.
+entry box if one is present. There's also a check box (enabled by default)
+to decide whether actually to follow external links (since this can slow
+the checking down considerably). Finally there's a Quit button.
A series of checkbuttons determines whether the corresponding output panel
is shown. List panels are also automatically shown or hidden when their
status changes between empty to non-empty. There are six panels:
Log -- raw output from the checker (-v, -q affect this)
-To check -- local links discovered but not yet checked
-Off site -- links discovered that point off site
-Checked -- local links that have been checked
+To check -- links discovered but not yet checked
+Checked -- links that have been checked
Bad links -- links that failed upon checking
+Errors -- pages containing at least one bad link
Details -- details about one URL; double click on a URL in any of
- the aboce list panels (not in Log) will show that URL
-
-XXX There ought to be a list of pages known to contain at least one
-bad link.
-
-XXX The checking of off site links should be made more similar to the
-checking of local links (even if they are checked after all local links are
-checked).
+ the above list panels (not in Log) will show details
+ for that URL
Use your window manager's Close command to quit.
Command line options:
-m bytes -- skip HTML pages larger than this size (default %(MAXPAGE)d)
--n -- reports only, no checking (use with -R)
-q -- quiet operation (also suppresses external links report)
-v -- verbose operation; repeating -v will increase verbosity
@@ -50,11 +45,11 @@
rooturl -- URL to start checking
(default %(DEFROOT)s)
-XXX The command line options should all be GUI accessible.
+XXX The command line options (-m, -q, -v) should be GUI accessible.
XXX The roots should be visible as a list (?).
-XXX The multipanel user interface is bogus.
+XXX The multipanel user interface is clumsy.
"""
@@ -104,39 +99,66 @@
def __init__(self, parent, root=webchecker.DEFROOT):
self.__parent = parent
- self.__controls = Frame(parent)
- self.__controls.pack(side=TOP, fill=X)
- self.__label = Label(self.__controls, text="Root URL:")
+
+ self.__topcontrols = Frame(parent)
+ self.__topcontrols.pack(side=TOP, fill=X)
+ self.__label = Label(self.__topcontrols, text="Root URL:")
self.__label.pack(side=LEFT)
- self.__rootentry = Entry(self.__controls, width=60)
+ self.__rootentry = Entry(self.__topcontrols, width=60)
self.__rootentry.pack(side=LEFT)
self.__rootentry.bind('<Return>', self.enterroot)
self.__rootentry.focus_set()
+
+ self.__controls = Frame(parent)
+ self.__controls.pack(side=TOP, fill=X)
self.__running = 0
self.__start = Button(self.__controls, text="Run", command=self.start)
self.__start.pack(side=LEFT)
self.__stop = Button(self.__controls, text="Stop", command=self.stop,
state=DISABLED)
self.__stop.pack(side=LEFT)
- self.__step = Button(self.__controls, text="Check one", command=self.step)
+ self.__step = Button(self.__controls, text="Check one",
+ command=self.step)
self.__step.pack(side=LEFT)
+ self.__cv = BooleanVar()
+ self.__cv.set(1)
+ self.__checkext = Checkbutton(self.__controls, variable=self.__cv,
+ text="Check nonlocal links")
+ self.__checkext.pack(side=LEFT)
+ self.__reset = Button(self.__controls, text="Start over", command=self.reset)
+ self.__reset.pack(side=LEFT)
+ if __name__ == '__main__': # No Quit button under Grail!
+ self.__quit = Button(self.__controls, text="Quit",
+ command=self.__parent.quit)
+ self.__quit.pack(side=RIGHT)
+
self.__status = Label(parent, text="Status: initial", anchor=W)
self.__status.pack(side=TOP, fill=X)
- self.__checking = Label(parent, text="Checking: none", anchor=W)
+ self.__checking = Label(parent, text="Idle", anchor=W)
self.__checking.pack(side=TOP, fill=X)
self.__mp = mp = MultiPanel(parent)
sys.stdout = self.__log = LogPanel(mp, "Log")
self.__todo = ListPanel(mp, "To check", self.showinfo)
- self.__ext = ListPanel(mp, "Off site", self.showinfo)
self.__done = ListPanel(mp, "Checked", self.showinfo)
self.__bad = ListPanel(mp, "Bad links", self.showinfo)
+ self.__errors = ListPanel(mp, "Pages w/ bad links", self.showinfo)
self.__details = LogPanel(mp, "Details")
- self.__extodo = []
webchecker.Checker.__init__(self)
+ del self.checkext # See __getattr__ below
if root:
root = string.strip(str(root))
if root:
self.suggestroot(root)
+ self.newstatus()
+
+ def __getattr__(self, name):
+ if name != 'checkext': raise AttributeError, name
+ return self.__cv.get()
+
+ def reset(self):
+ webchecker.Checker.reset(self)
+ for p in self.__todo, self.__done, self.__bad, self.__errors:
+ p.clear()
def suggestroot(self, root):
self.__rootentry.delete(0, END)
@@ -147,7 +169,10 @@
root = self.__rootentry.get()
root = string.strip(root)
if root:
+ self.__checking.config(text="Adding root "+root)
+ self.__checking.update_idletasks()
self.addroot(root)
+ self.__checking.config(text="Idle")
try:
i = self.__todo.items.index(root)
except (ValueError, IndexError):
@@ -167,7 +192,7 @@
self.go()
def stop(self):
- self.__stop.config(state=DISABLED)
+ self.__stop.config(state=DISABLED, relief=SUNKEN)
self.__running = 0
def step(self):
@@ -181,8 +206,9 @@
if self.__running:
self.__parent.after_idle(self.dosomething)
else:
- self.__checking.config(text="Checking: none")
+ self.__checking.config(text="Idle")
self.__start.config(state=NORMAL, relief=RAISED)
+ self.__stop.config(state=DISABLED, relief=RAISED)
self.__step.config(state=NORMAL, relief=RAISED)
__busy = 0
@@ -199,18 +225,9 @@
self.__todo.list.select_set(i)
self.__todo.list.yview(i)
url = self.__todo.items[i]
- self.__checking.config(text="Checking: "+url)
+ self.__checking.config(text="Checking "+url)
self.__parent.update()
self.dopage(url)
- elif self.__extodo:
- # XXX Should have an indication of these in the todo window...
- ##i = random.randint(0, len(self.__extodo)-1)
- i = 0
- url = self.__extodo[i]
- del self.__extodo[i]
- self.__checking.config(text="Checking: "+url)
- self.__parent.update()
- self.checkextpage(url)
else:
self.stop()
self.__busy = 0
@@ -219,33 +236,34 @@
def showinfo(self, url):
d = self.__details
d.clear()
- d.write("URL: %s\n" % url)
+ d.put("URL: %s\n" % url)
if self.bad.has_key(url):
- d.write("Error: %s\n" % str(self.bad[url]))
+ d.put("Error: %s\n" % str(self.bad[url]))
if url in self.roots:
- d.write("Note: This is a root URL\n")
+ d.put("Note: This is a root URL\n")
if self.done.has_key(url):
- d.write("Status: checked\n")
+ d.put("Status: checked\n")
o = self.done[url]
elif self.todo.has_key(url):
- d.write("Status: to check\n")
+ d.put("Status: to check\n")
o = self.todo[url]
- elif self.ext.has_key(url):
- d.write("Status: off site\n")
- o = self.ext[url]
else:
- d.write("Status: unknown (!)\n")
+ d.put("Status: unknown (!)\n")
o = []
+ if self.errors.has_key(url):
+ d.put("Bad links from this page:\n")
+ for triple in self.errors[url]:
+ link, rawlink, msg = triple
+ d.put(" HREF %s" % link)
+ if link != rawlink: d.put(" (%s)" %rawlink)
+ d.put("\n")
+ d.put(" error %s\n" % str(msg))
self.__mp.showpanel("Details")
for source, rawlink in o:
- d.write("Origin: %s" % source)
+ d.put("Origin: %s" % source)
if rawlink != url:
- d.write(" (%s)" % rawlink)
- d.write("\n")
-
- def newstatus(self):
- self.__status.config(text="Status: "+self.status()[1:-1])
- self.__parent.update()
+ d.put(" (%s)" % rawlink)
+ d.put("\n")
def setbad(self, url, msg):
webchecker.Checker.setbad(self, url, msg)
@@ -257,14 +275,8 @@
self.__bad.remove(url)
self.newstatus()
- def newextlink(self, url, origin):
- webchecker.Checker.newextlink(self, url, origin)
- self.__extodo.append(url)
- self.__ext.insert(url)
- self.newstatus()
-
- def newintlink(self, url, origin):
- webchecker.Checker.newintlink(self, url, origin)
+ def newlink(self, url, origin):
+ webchecker.Checker.newlink(self, url, origin)
if self.done.has_key(url):
self.__done.insert(url)
elif self.todo.has_key(url):
@@ -277,6 +289,15 @@
self.__todo.remove(url)
self.newstatus()
+ def seterror(self, url, triple):
+ webchecker.Checker.seterror(self, url, triple)
+ self.__errors.insert(url)
+ self.newstatus()
+
+ def newstatus(self):
+ self.__status.config(text="Status: "+self.status())
+ self.__parent.update()
+
class ListPanel:
@@ -292,6 +313,11 @@
self.list.bind('<Double-Button-1>', self.doubleclick)
self.items = []
+ def clear(self):
+ self.items = []
+ self.list.delete(0, END)
+ self.mp.hidepanel(self.name)
+
def doubleclick(self, event):
l = self.selectedindices()
if l:
@@ -309,7 +335,7 @@
# (I tried sorting alphabetically, but the display is too jumpy)
i = len(self.items)
self.list.insert(i, url)
-## self.list.yview(i)
+ self.list.yview(i)
self.items.insert(i, url)
def remove(self, url):
@@ -342,6 +368,11 @@
self.text.delete("1.0", END)
self.text.yview("1.0")
+ def put(self, s):
+ self.text.insert(END, s)
+ if '\n' in s:
+ self.text.yview(END)
+
def write(self, s):
self.text.insert(END, s)
if '\n' in s:
diff --git a/Tools/webchecker/webchecker.py b/Tools/webchecker/webchecker.py
index 69e462a..8b27dd8 100755
--- a/Tools/webchecker/webchecker.py
+++ b/Tools/webchecker/webchecker.py
@@ -19,12 +19,10 @@
web tree is located at /usr/local/etc/httpd/htdpcs (the default for
the NCSA HTTP daemon and probably others).
-Reports printed:
+Report printed:
-When done, it reports links to pages outside the web (unless -q is
-specified), and pages with bad links within the subweb. When
-interrupted, it print those same reports for the pages that it has
-checked already.
+When done, it reports pages with bad links within the subweb. When
+interrupted, it reports for the pages that it has checked already.
In verbose mode, additional messages are printed during the
information gathering phase. By default, it prints a summary of its
@@ -50,31 +48,29 @@
Miscellaneous:
+- You may find the (Tk-based) GUI version easier to use. See wcgui.py.
+
- Webchecker honors the "robots.txt" convention. Thanks to Skip
Montanaro for his robotparser.py module (included in this directory)!
The agent name is hardwired to "webchecker". URLs that are disallowed
by the robots.txt file are reported as external URLs.
-- Because the HTML parser is a bit slow, very large HTML files are
+- Because the SGML parser is a bit slow, very large SGML files are
skipped. The size limit can be set with the -m option.
-- Before fetching a page, it guesses its type based on its extension.
-If it is a known extension and the type is not text/html, the page is
-not fetched. This is a huge optimization but occasionally it means
-links can be missed, and such links aren't checked for validity
-(XXX!). The mimetypes.py module (also in this directory) has a
-built-in table mapping most currently known suffixes, and in addition
-attempts to read the mime.types configuration files in the default
-locations of Netscape and the NCSA HTTP daemon.
+- When the server or protocol does not tell us a file's type, we guess
+it based on the URL's suffix. The mimetypes.py module (also in this
+directory) has a built-in table mapping most currently known suffixes,
+and in addition attempts to read the mime.types configuration files in
+the default locations of Netscape and the NCSA HTTP daemon.
-- It only follows links indicated by <A> tags. It doesn't follow
-links in <FORM> or <IMG> or whatever other tags might contain
-hyperlinks. It does honor the <BASE> tag.
+- We follows links indicated by <A>, <FRAME> and <IMG> tags. We also
+honor the <BASE> tag.
-- Checking external links is not done by default; use -x to enable
-this feature. This is done because checking external links usually
-takes a lot of time. When enabled, this check is executed during the
-report generation phase (even when the report is silent).
+- Checking external links is now done by default; use -x to *disable*
+this feature. External links are now checked during normal
+processing. (XXX The status of a checked link could be categorized
+better. Later...)
Usage: webchecker.py [option] ... [rooturl] ...
@@ -88,7 +84,7 @@
-q -- quiet operation (also suppresses external links report)
-r number -- number of links processed per round (default %(ROUNDSIZE)d)
-v -- verbose operation; repeating -v will increase verbosity
--x -- check external links (during report phase)
+-x -- don't check external links (these are often slow to check)
Arguments:
@@ -100,7 +96,7 @@
# ' Emacs bait
-__version__ = "0.3"
+__version__ = "0.4"
import sys
@@ -137,7 +133,7 @@
global verbose, maxpage, roundsize
dumpfile = DUMPFILE
restart = 0
- checkext = 0
+ checkext = 1
norun = 0
try:
@@ -163,7 +159,7 @@
if o == '-v':
verbose = verbose + 1
if o == '-x':
- checkext = 1
+ checkext = not checkext
if verbose > 0:
print AGENTNAME, "version", __version__
@@ -178,7 +174,7 @@
print "Done."
print "Root:", string.join(c.roots, "\n ")
else:
- c = Checker()
+ c = Checker(checkext)
if not args:
args.append(DEFROOT)
@@ -193,7 +189,7 @@
print "[run interrupted]"
try:
- c.report(checkext)
+ c.report()
except KeyboardInterrupt:
if verbose > 0:
print "[report interrupted]"
@@ -229,33 +225,37 @@
class Checker:
- def __init__(self):
+ def __init__(self, checkext=1):
+ self.reset()
+ self.checkext = checkext
+
+ def reset(self):
self.roots = []
self.todo = {}
self.done = {}
- self.ext = {}
self.bad = {}
self.round = 0
# The following are not pickled:
self.robots = {}
+ self.errors = {}
self.urlopener = MyURLopener()
self.changed = 0
def __getstate__(self):
- return (self.roots, self.todo, self.done,
- self.ext, self.bad, self.round)
+ return (self.roots, self.todo, self.done, self.bad, self.round)
def __setstate__(self, state):
- (self.roots, self.todo, self.done,
- self.ext, self.bad, self.round) = state
+ (self.roots, self.todo, self.done, self.bad, self.round) = state
for root in self.roots:
self.addrobot(root)
+ for url in self.bad.keys():
+ self.markerror(url)
def addroot(self, root):
if root not in self.roots:
self.roots.append(root)
self.addrobot(root)
- self.newintlink(root, ("<root>", root))
+ self.newlink(root, ("<root>", root))
def addrobot(self, root):
url = urlparse.urljoin(root, "/robots.txt")
@@ -275,65 +275,25 @@
self.round = self.round + 1
if verbose > 0:
print
- print "Round", self.round, self.status()
+ print "Round %d (%s)" % (self.round, self.status())
print
urls = self.todo.keys()[:roundsize]
for url in urls:
self.dopage(url)
def status(self):
- return "(%d total, %d to do, %d done, %d external, %d bad)" % (
+ return "%d total, %d to do, %d done, %d bad" % (
len(self.todo)+len(self.done),
len(self.todo), len(self.done),
- len(self.ext), len(self.bad))
+ len(self.bad))
- def report(self, checkext=0):
+ def report(self):
print
if not self.todo: print "Final",
else: print "Interim",
- print "Report", self.status()
- if verbose > 0 or checkext:
- self.report_extrefs(checkext)
- # Report errors last because the output may get truncated
+ print "Report (%s)" % self.status()
self.report_errors()
- def report_extrefs(self, checkext=0):
- if not self.ext:
- if verbose > 0:
- print
- print "No external URLs"
- return
- if verbose > 0:
- print
- if checkext:
- print "External URLs (checking validity):"
- else:
- print "External URLs (not checked):"
- print
- urls = self.ext.keys()
- urls.sort()
- for url in urls:
- if verbose > 0:
- show("HREF ", url, " from", self.ext[url])
- if checkext:
- self.checkextpage(url)
-
- def checkextpage(self, url):
- if url[:7] == 'mailto:' or url[:5] == 'news:':
- if verbose > 2: print "Not checking", url
- return
- if verbose > 2: print "Checking", url, "..."
- try:
- f = self.urlopener.open(url)
- safeclose(f)
- if verbose > 3: print "OK"
- if self.bad.has_key(url):
- self.setgood(url)
- except IOError, msg:
- msg = sanitize(msg)
- if verbose > 0: print "Error", msg
- self.setbad(url, msg)
-
def report_errors(self):
if not self.bad:
print
@@ -341,27 +301,10 @@
return
print
print "Error Report:"
- urls = self.bad.keys()
- urls.sort()
- bysource = {}
- for url in urls:
- try:
- origins = self.done[url]
- except KeyError:
- try:
- origins = self.todo[url]
- except KeyError:
- origins = self.ext[url]
- for source, rawlink in origins:
- triple = url, rawlink, self.bad[url]
- try:
- bysource[source].append(triple)
- except KeyError:
- bysource[source] = [triple]
- sources = bysource.keys()
+ sources = self.errors.keys()
sources.sort()
for source in sources:
- triples = bysource[source]
+ triples = self.errors[source]
print
if len(triples) > 1:
print len(triples), "Errors in", source
@@ -376,31 +319,18 @@
def dopage(self, url):
if verbose > 1:
if verbose > 2:
- show("Page ", url, " from", self.todo[url])
+ show("Check ", url, " from", self.todo[url])
else:
- print "Page ", url
+ print "Check ", url
page = self.getpage(url)
if page:
for info in page.getlinkinfos():
link, rawlink = info
origin = url, rawlink
- if not self.inroots(link):
- self.newextlink(link, origin)
- else:
- self.newintlink(link, origin)
+ self.newlink(link, origin)
self.markdone(url)
- def newextlink(self, url, origin):
- try:
- self.ext[url].append(origin)
- if verbose > 3:
- print " New ext link", url
- except KeyError:
- self.ext[url] = [origin]
- if verbose > 3:
- print " Seen ext link", url
-
- def newintlink(self, url, origin):
+ def newlink(self, url, origin):
if self.done.has_key(url):
self.newdonelink(url, origin)
else:
@@ -433,6 +363,13 @@
return 0
def getpage(self, url):
+ if url[:7] == 'mailto:' or url[:5] == 'news:':
+ if verbose > 1: print " Not checking mailto/news URL"
+ return None
+ isint = self.inroots(url)
+ if not isint and not self.checkext:
+ if verbose > 1: print " Not checking ext link"
+ return None
try:
f = self.urlopener.open(url)
except IOError, msg:
@@ -443,6 +380,10 @@
show(" HREF ", url, " from", self.todo[url])
self.setbad(url, msg)
return None
+ if not isint:
+ if verbose > 1: print " Not gathering links from ext URL"
+ safeclose(f)
+ return None
nurl = f.geturl()
info = f.info()
if info.has_key('content-type'):
@@ -477,6 +418,22 @@
return
self.bad[url] = msg
self.changed = 1
+ self.markerror(url)
+
+ def markerror(self, url):
+ try:
+ origins = self.todo[url]
+ except KeyError:
+ origins = self.done[url]
+ for source, rawlink in origins:
+ triple = url, rawlink, self.bad[url]
+ self.seterror(source, triple)
+
+ def seterror(self, url, triple):
+ try:
+ self.errors[url].append(triple)
+ except KeyError:
+ self.errors[url] = [triple]
class Page: