| # A parser for HTML documents |
| |
| |
| # HTML: HyperText Markup Language; an SGML-like syntax used by WWW to |
| # describe hypertext documents |
| # |
| # SGML: Standard Generalized Markup Language |
| # |
| # WWW: World-Wide Web; a distributed hypertext system develped at CERN |
| # |
| # CERN: European Particle Physics Laboratory in Geneva, Switzerland |
| |
| |
| # This file is only concerned with parsing and formatting HTML |
| # documents, not with the other (hypertext and networking) aspects of |
| # the WWW project. (It does support highlighting of anchors.) |
| |
| |
| import os |
| import sys |
| import regex |
| import string |
| import sgmllib |
| |
| |
| class HTMLParser(sgmllib.SGMLParser): |
| |
| # Copy base class entities and add some |
| entitydefs = {} |
| for key in sgmllib.SGMLParser.entitydefs.keys(): |
| entitydefs[key] = sgmllib.SGMLParser.entitydefs[key] |
| entitydefs['bullet'] = '*' |
| |
| # Provided -- handlers for tags introducing literal text |
| |
| def start_listing(self, attrs): |
| self.setliteral('listing') |
| self.literal_bgn('listing', attrs) |
| |
| def end_listing(self): |
| self.literal_end('listing') |
| |
| def start_xmp(self, attrs): |
| self.setliteral('xmp') |
| self.literal_bgn('xmp', attrs) |
| |
| def end_xmp(self): |
| self.literal_end('xmp') |
| |
| def do_plaintext(self, attrs): |
| self.setnomoretags() |
| self.literal_bgn('plaintext', attrs) |
| |
| # To be overridden -- begin/end literal mode |
| def literal_bgn(self, tag, attrs): pass |
| def literal_end(self, tag): pass |
| |
| |
| # Next level of sophistication -- collect anchors, title, nextid and isindex |
| class CollectingParser(HTMLParser): |
| # |
| def __init__(self): |
| HTMLParser.__init__(self) |
| self.savetext = None |
| self.nextid = '' |
| self.isindex = 0 |
| self.title = '' |
| self.inanchor = 0 |
| self.anchors = [] |
| self.anchornames = [] |
| self.anchortypes = [] |
| # |
| def start_a(self, attrs): |
| self.inanchor = 0 |
| href = '' |
| name = '' |
| type = '' |
| for attrname, value in attrs: |
| if attrname == 'href': |
| href = value |
| if attrname == 'name=': |
| name = value |
| if attrname == 'type=': |
| type = string.lower(value) |
| if not (href or name): |
| return |
| self.anchors.append(href) |
| self.anchornames.append(name) |
| self.anchortypes.append(type) |
| self.inanchor = len(self.anchors) |
| if not href: |
| self.inanchor = -self.inanchor |
| # |
| def end_a(self): |
| if self.inanchor > 0: |
| # Don't show anchors pointing into the current document |
| if self.anchors[self.inanchor-1][:1] <> '#': |
| self.handle_data('[' + `self.inanchor` + ']') |
| self.inanchor = 0 |
| # |
| def start_header(self, attrs): pass |
| def end_header(self): pass |
| # |
| # (head is the same as header) |
| def start_head(self, attrs): pass |
| def end_head(self): pass |
| # |
| def start_body(self, attrs): pass |
| def end_body(self): pass |
| # |
| def do_nextid(self, attrs): |
| self.nextid = attrs |
| # |
| def do_isindex(self, attrs): |
| self.isindex = 1 |
| # |
| def start_title(self, attrs): |
| self.savetext = '' |
| # |
| def end_title(self): |
| if self.savetext <> None: |
| self.title = self.savetext |
| self.savetext = None |
| # |
| def handle_data(self, text): |
| if self.savetext is not None: |
| self.savetext = self.savetext + text |
| |
| |
| # Formatting parser -- takes a formatter and a style sheet as arguments |
| |
| # XXX The use of style sheets should change: for each tag and end tag |
| # there should be a style definition, and a style definition should |
| # encompass many more parameters: font, justification, indentation, |
| # vspace before, vspace after, hanging tag... |
| |
| wordprog = regex.compile('[^ \t\n]*') |
| spaceprog = regex.compile('[ \t\n]*') |
| |
| class FormattingParser(CollectingParser): |
| |
| def __init__(self, formatter, stylesheet): |
| CollectingParser.__init__(self) |
| self.fmt = formatter |
| self.stl = stylesheet |
| self.savetext = None |
| self.compact = 0 |
| self.nofill = 0 |
| self.resetfont() |
| self.setindent(self.stl.stdindent) |
| |
| def resetfont(self): |
| self.fontstack = [] |
| self.stylestack = [] |
| self.fontset = self.stl.stdfontset |
| self.style = ROMAN |
| self.passfont() |
| |
| def passfont(self): |
| font = self.fontset[self.style] |
| self.fmt.setfont(font) |
| |
| def pushstyle(self, style): |
| self.stylestack.append(self.style) |
| self.style = min(style, len(self.fontset)-1) |
| self.passfont() |
| |
| def popstyle(self): |
| self.style = self.stylestack[-1] |
| del self.stylestack[-1] |
| self.passfont() |
| |
| def pushfontset(self, fontset, style): |
| self.fontstack.append(self.fontset) |
| self.fontset = fontset |
| self.pushstyle(style) |
| |
| def popfontset(self): |
| self.fontset = self.fontstack[-1] |
| del self.fontstack[-1] |
| self.popstyle() |
| |
| def flush(self): |
| self.fmt.flush() |
| |
| def setindent(self, n): |
| self.fmt.setleftindent(n) |
| |
| def needvspace(self, n): |
| self.fmt.needvspace(n) |
| |
| def close(self): |
| HTMLParser.close(self) |
| self.fmt.flush() |
| |
| def handle_literal(self, text): |
| lines = string.splitfields(text, '\n') |
| for i in range(1, len(lines)): |
| lines[i] = string.expandtabs(lines[i], 8) |
| for line in lines[:-1]: |
| self.fmt.addword(line, 0) |
| self.fmt.flush() |
| self.fmt.nospace = 0 |
| for line in lines[-1:]: |
| self.fmt.addword(line, 0) |
| |
| def handle_data(self, text): |
| if self.savetext is not None: |
| self.savetext = self.savetext + text |
| return |
| if self.literal: |
| self.handle_literal(text) |
| return |
| i = 0 |
| n = len(text) |
| while i < n: |
| j = i + wordprog.match(text, i) |
| word = text[i:j] |
| i = j + spaceprog.match(text, j) |
| self.fmt.addword(word, i-j) |
| if self.nofill and '\n' in text[j:i]: |
| self.fmt.flush() |
| self.fmt.nospace = 0 |
| i = j+1 |
| while text[i-1] <> '\n': i = i+1 |
| |
| def literal_bgn(self, tag, attrs): |
| if tag == 'plaintext': |
| self.flush() |
| else: |
| self.needvspace(1) |
| self.pushfontset(self.stl.stdfontset, FIXED) |
| self.setindent(self.stl.literalindent) |
| |
| def literal_end(self, tag): |
| self.needvspace(1) |
| self.popfontset() |
| self.setindent(self.stl.stdindent) |
| |
| def start_title(self, attrs): |
| self.flush() |
| self.savetext = '' |
| # NB end_title is unchanged |
| |
| def do_p(self, attrs): |
| if self.compact: |
| self.flush() |
| else: |
| self.needvspace(1) |
| |
| def start_h1(self, attrs): |
| self.needvspace(2) |
| self.setindent(self.stl.h1indent) |
| self.pushfontset(self.stl.h1fontset, BOLD) |
| self.fmt.setjust('c') |
| |
| def end_h1(self): |
| self.popfontset() |
| self.needvspace(2) |
| self.setindent(self.stl.stdindent) |
| self.fmt.setjust('l') |
| |
| def start_h2(self, attrs): |
| self.needvspace(1) |
| self.setindent(self.stl.h2indent) |
| self.pushfontset(self.stl.h2fontset, BOLD) |
| |
| def end_h2(self): |
| self.popfontset() |
| self.needvspace(1) |
| self.setindent(self.stl.stdindent) |
| |
| def start_h3(self, attrs): |
| self.needvspace(1) |
| self.setindent(self.stl.stdindent) |
| self.pushfontset(self.stl.h3fontset, BOLD) |
| |
| def end_h3(self): |
| self.popfontset() |
| self.needvspace(1) |
| self.setindent(self.stl.stdindent) |
| |
| def start_h4(self, attrs): |
| self.needvspace(1) |
| self.setindent(self.stl.stdindent) |
| self.pushfontset(self.stl.stdfontset, BOLD) |
| |
| def end_h4(self): |
| self.popfontset() |
| self.needvspace(1) |
| self.setindent(self.stl.stdindent) |
| |
| start_h5 = start_h4 |
| end_h5 = end_h4 |
| |
| start_h6 = start_h5 |
| end_h6 = end_h5 |
| |
| start_h7 = start_h6 |
| end_h7 = end_h6 |
| |
| def start_ul(self, attrs): |
| self.needvspace(1) |
| for attrname, value in attrs: |
| if attrname == 'compact': |
| self.compact = 1 |
| self.setindent(0) |
| break |
| else: |
| self.setindent(self.stl.ulindent) |
| |
| start_dir = start_menu = start_ol = start_ul |
| |
| do_li = do_p |
| |
| def end_ul(self): |
| self.compact = 0 |
| self.needvspace(1) |
| self.setindent(self.stl.stdindent) |
| |
| end_dir = end_menu = end_ol = end_ul |
| |
| def start_dl(self, attrs): |
| for attrname, value in attrs: |
| if attrname == 'compact': |
| self.compact = 1 |
| self.needvspace(1) |
| |
| def end_dl(self): |
| self.compact = 0 |
| self.needvspace(1) |
| self.setindent(self.stl.stdindent) |
| |
| def do_dt(self, attrs): |
| if self.compact: |
| self.flush() |
| else: |
| self.needvspace(1) |
| self.setindent(self.stl.stdindent) |
| |
| def do_dd(self, attrs): |
| self.fmt.addword('', 1) |
| self.setindent(self.stl.ddindent) |
| |
| def start_address(self, attrs): |
| self.compact = 1 |
| self.needvspace(1) |
| self.fmt.setjust('r') |
| |
| def end_address(self): |
| self.compact = 0 |
| self.needvspace(1) |
| self.setindent(self.stl.stdindent) |
| self.fmt.setjust('l') |
| |
| def start_pre(self, attrs): |
| self.needvspace(1) |
| self.nofill = self.nofill + 1 |
| self.pushstyle(FIXED) |
| |
| def end_pre(self): |
| self.popstyle() |
| self.nofill = self.nofill - 1 |
| self.needvspace(1) |
| |
| start_typewriter = start_pre |
| end_typewriter = end_pre |
| |
| def do_img(self, attrs): |
| self.fmt.addword('(image)', 0) |
| |
| # Physical styles |
| |
| def start_tt(self, attrs): self.pushstyle(FIXED) |
| def end_tt(self): self.popstyle() |
| |
| def start_b(self, attrs): self.pushstyle(BOLD) |
| def end_b(self): self.popstyle() |
| |
| def start_i(self, attrs): self.pushstyle(ITALIC) |
| def end_i(self): self.popstyle() |
| |
| def start_u(self, attrs): self.pushstyle(ITALIC) # Underline??? |
| def end_u(self): self.popstyle() |
| |
| def start_r(self, attrs): self.pushstyle(ROMAN) # Not official |
| def end_r(self): self.popstyle() |
| |
| # Logical styles |
| |
| start_em = start_i |
| end_em = end_i |
| |
| start_strong = start_b |
| end_strong = end_b |
| |
| start_code = start_tt |
| end_code = end_tt |
| |
| start_samp = start_tt |
| end_samp = end_tt |
| |
| start_kbd = start_tt |
| end_kbd = end_tt |
| |
| start_file = start_tt # unofficial |
| end_file = end_tt |
| |
| start_var = start_i |
| end_var = end_i |
| |
| start_dfn = start_i |
| end_dfn = end_i |
| |
| start_cite = start_i |
| end_cite = end_i |
| |
| start_hp1 = start_i |
| end_hp1 = start_i |
| |
| start_hp2 = start_b |
| end_hp2 = end_b |
| |
| def unknown_starttag(self, tag, attrs): |
| print '*** unknown <' + tag + '>' |
| |
| def unknown_endtag(self, tag): |
| print '*** unknown </' + tag + '>' |
| |
| |
| # An extension of the formatting parser which formats anchors differently. |
| class AnchoringParser(FormattingParser): |
| |
| def start_a(self, attrs): |
| FormattingParser.start_a(self, attrs) |
| if self.inanchor: |
| self.fmt.bgn_anchor(self.inanchor) |
| |
| def end_a(self): |
| if self.inanchor: |
| self.fmt.end_anchor(self.inanchor) |
| self.inanchor = 0 |
| |
| |
| # Style sheet -- this is never instantiated, but the attributes |
| # of the class object itself are used to specify fonts to be used |
| # for various paragraph styles. |
| # A font set is a non-empty list of fonts, in the order: |
| # [roman, italic, bold, fixed]. |
| # When a style is not available the nearest lower style is used |
| |
| ROMAN = 0 |
| ITALIC = 1 |
| BOLD = 2 |
| FIXED = 3 |
| |
| class NullStylesheet: |
| # Fonts -- none |
| stdfontset = [None] |
| h1fontset = [None] |
| h2fontset = [None] |
| h3fontset = [None] |
| # Indents |
| stdindent = 2 |
| ddindent = 25 |
| ulindent = 4 |
| h1indent = 0 |
| h2indent = 0 |
| literalindent = 0 |
| |
| |
| class X11Stylesheet(NullStylesheet): |
| stdfontset = [ \ |
| '-*-helvetica-medium-r-normal-*-*-100-100-*-*-*-*-*', \ |
| '-*-helvetica-medium-o-normal-*-*-100-100-*-*-*-*-*', \ |
| '-*-helvetica-bold-r-normal-*-*-100-100-*-*-*-*-*', \ |
| '-*-courier-medium-r-normal-*-*-100-100-*-*-*-*-*', \ |
| ] |
| h1fontset = [ \ |
| '-*-helvetica-medium-r-normal-*-*-180-100-*-*-*-*-*', \ |
| '-*-helvetica-medium-o-normal-*-*-180-100-*-*-*-*-*', \ |
| '-*-helvetica-bold-r-normal-*-*-180-100-*-*-*-*-*', \ |
| ] |
| h2fontset = [ \ |
| '-*-helvetica-medium-r-normal-*-*-140-100-*-*-*-*-*', \ |
| '-*-helvetica-medium-o-normal-*-*-140-100-*-*-*-*-*', \ |
| '-*-helvetica-bold-r-normal-*-*-140-100-*-*-*-*-*', \ |
| ] |
| h3fontset = [ \ |
| '-*-helvetica-medium-r-normal-*-*-120-100-*-*-*-*-*', \ |
| '-*-helvetica-medium-o-normal-*-*-120-100-*-*-*-*-*', \ |
| '-*-helvetica-bold-r-normal-*-*-120-100-*-*-*-*-*', \ |
| ] |
| ddindent = 40 |
| |
| |
| class MacStylesheet(NullStylesheet): |
| stdfontset = [ \ |
| ('Geneva', 'p', 10), \ |
| ('Geneva', 'i', 10), \ |
| ('Geneva', 'b', 10), \ |
| ('Monaco', 'p', 10), \ |
| ] |
| h1fontset = [ \ |
| ('Geneva', 'p', 18), \ |
| ('Geneva', 'i', 18), \ |
| ('Geneva', 'b', 18), \ |
| ('Monaco', 'p', 18), \ |
| ] |
| h3fontset = [ \ |
| ('Geneva', 'p', 14), \ |
| ('Geneva', 'i', 14), \ |
| ('Geneva', 'b', 14), \ |
| ('Monaco', 'p', 14), \ |
| ] |
| h3fontset = [ \ |
| ('Geneva', 'p', 12), \ |
| ('Geneva', 'i', 12), \ |
| ('Geneva', 'b', 12), \ |
| ('Monaco', 'p', 12), \ |
| ] |
| |
| |
| if os.name == 'mac': |
| StdwinStylesheet = MacStylesheet |
| else: |
| StdwinStylesheet = X11Stylesheet |
| |
| |
| class GLStylesheet(NullStylesheet): |
| stdfontset = [ \ |
| 'Helvetica 10', \ |
| 'Helvetica-Italic 10', \ |
| 'Helvetica-Bold 10', \ |
| 'Courier 10', \ |
| ] |
| h1fontset = [ \ |
| 'Helvetica 18', \ |
| 'Helvetica-Italic 18', \ |
| 'Helvetica-Bold 18', \ |
| 'Courier 18', \ |
| ] |
| h2fontset = [ \ |
| 'Helvetica 14', \ |
| 'Helvetica-Italic 14', \ |
| 'Helvetica-Bold 14', \ |
| 'Courier 14', \ |
| ] |
| h3fontset = [ \ |
| 'Helvetica 12', \ |
| 'Helvetica-Italic 12', \ |
| 'Helvetica-Bold 12', \ |
| 'Courier 12', \ |
| ] |
| |
| |
| # Test program -- produces no output but times how long it takes |
| # to send a document to a null formatter, exclusive of I/O |
| |
| def test(): |
| import fmt |
| import time |
| if sys.argv[1:]: file = sys.argv[1] |
| else: file = 'test.html' |
| data = open(file, 'r').read() |
| t0 = time.time() |
| fmtr = fmt.WritingFormatter(sys.stdout, 79) |
| p = FormattingParser(fmtr, NullStylesheet) |
| p.feed(data) |
| p.close() |
| t1 = time.time() |
| print |
| print '*** Formatting time:', round(t1-t0, 3), 'seconds.' |
| |
| |
| # Test program using stdwin |
| |
| def testStdwin(): |
| import stdwin, fmt |
| from stdwinevents import * |
| if sys.argv[1:]: file = sys.argv[1] |
| else: file = 'test.html' |
| data = open(file, 'r').read() |
| window = stdwin.open('testStdwin') |
| b = None |
| while 1: |
| etype, ewin, edetail = stdwin.getevent() |
| if etype == WE_CLOSE: |
| break |
| if etype == WE_SIZE: |
| window.setdocsize(0, 0) |
| window.setorigin(0, 0) |
| window.change((0, 0), (10000, 30000)) # XXX |
| if etype == WE_DRAW: |
| if not b: |
| b = fmt.StdwinBackEnd(window, 1) |
| f = fmt.BaseFormatter(b.d, b) |
| p = FormattingParser(f, \ |
| MacStylesheet) |
| p.feed(data) |
| p.close() |
| b.finish() |
| else: |
| b.redraw(edetail) |
| window.close() |
| |
| |
| # Test program using GL |
| |
| def testGL(): |
| import gl, GL, fmt |
| if sys.argv[1:]: file = sys.argv[1] |
| else: file = 'test.html' |
| data = open(file, 'r').read() |
| W, H = 600, 600 |
| gl.foreground() |
| gl.prefsize(W, H) |
| wid = gl.winopen('testGL') |
| gl.ortho2(0, W, H, 0) |
| gl.color(GL.WHITE) |
| gl.clear() |
| gl.color(GL.BLACK) |
| b = fmt.GLBackEnd(wid) |
| f = fmt.BaseFormatter(b.d, b) |
| p = FormattingParser(f, GLStylesheet) |
| p.feed(data) |
| p.close() |
| b.finish() |
| # |
| import time |
| time.sleep(5) |
| |
| |
| if __name__ == '__main__': |
| test() |