initial source import
diff --git a/doc/tools/sgmlconv/docfixer.py b/doc/tools/sgmlconv/docfixer.py
new file mode 100755
index 0000000..463276b
--- /dev/null
+++ b/doc/tools/sgmlconv/docfixer.py
@@ -0,0 +1,1033 @@
+#! /usr/bin/env python
+
+"""Perform massive transformations on a document tree created from the LaTeX
+of the Python documentation, and dump the ESIS data for the transformed tree.
+"""
+
+
+import errno
+import esistools
+import re
+import string
+import sys
+import xml.dom
+import xml.dom.minidom
+
+ELEMENT = xml.dom.Node.ELEMENT_NODE
+ENTITY_REFERENCE = xml.dom.Node.ENTITY_REFERENCE_NODE
+TEXT = xml.dom.Node.TEXT_NODE
+
+
+class ConversionError(Exception):
+ pass
+
+
+ewrite = sys.stderr.write
+try:
+ # We can only do this trick on Unix (if tput is on $PATH)!
+ if sys.platform != "posix" or not sys.stderr.isatty():
+ raise ImportError
+ import commands
+except ImportError:
+ bwrite = ewrite
+else:
+ def bwrite(s, BOLDON=commands.getoutput("tput bold"),
+ BOLDOFF=commands.getoutput("tput sgr0")):
+ ewrite("%s%s%s" % (BOLDON, s, BOLDOFF))
+
+
+PARA_ELEMENT = "para"
+
+DEBUG_PARA_FIXER = 0
+
+if DEBUG_PARA_FIXER:
+ def para_msg(s):
+ ewrite("*** %s\n" % s)
+else:
+ def para_msg(s):
+ pass
+
+
+def get_first_element(doc, gi):
+ for n in doc.childNodes:
+ if n.nodeName == gi:
+ return n
+
+def extract_first_element(doc, gi):
+ node = get_first_element(doc, gi)
+ if node is not None:
+ doc.removeChild(node)
+ return node
+
+
+def get_documentElement(node):
+ result = None
+ for child in node.childNodes:
+ if child.nodeType == ELEMENT:
+ result = child
+ return result
+
+
+def set_tagName(elem, gi):
+ elem.nodeName = elem.tagName = gi
+
+
+def find_all_elements(doc, gi):
+ nodes = []
+ if doc.nodeName == gi:
+ nodes.append(doc)
+ for child in doc.childNodes:
+ if child.nodeType == ELEMENT:
+ if child.tagName == gi:
+ nodes.append(child)
+ for node in child.getElementsByTagName(gi):
+ nodes.append(node)
+ return nodes
+
+def find_all_child_elements(doc, gi):
+ nodes = []
+ for child in doc.childNodes:
+ if child.nodeName == gi:
+ nodes.append(child)
+ return nodes
+
+
+def find_all_elements_from_set(doc, gi_set):
+ return __find_all_elements_from_set(doc, gi_set, [])
+
+def __find_all_elements_from_set(doc, gi_set, nodes):
+ if doc.nodeName in gi_set:
+ nodes.append(doc)
+ for child in doc.childNodes:
+ if child.nodeType == ELEMENT:
+ __find_all_elements_from_set(child, gi_set, nodes)
+ return nodes
+
+
+def simplify(doc, fragment):
+ # Try to rationalize the document a bit, since these things are simply
+ # not valid SGML/XML documents as they stand, and need a little work.
+ documentclass = "document"
+ inputs = []
+ node = extract_first_element(fragment, "documentclass")
+ if node is not None:
+ documentclass = node.getAttribute("classname")
+ node = extract_first_element(fragment, "title")
+ if node is not None:
+ inputs.append(node)
+ # update the name of the root element
+ node = get_first_element(fragment, "document")
+ if node is not None:
+ set_tagName(node, documentclass)
+ while 1:
+ node = extract_first_element(fragment, "input")
+ if node is None:
+ break
+ inputs.append(node)
+ if inputs:
+ docelem = get_documentElement(fragment)
+ inputs.reverse()
+ for node in inputs:
+ text = doc.createTextNode("\n")
+ docelem.insertBefore(text, docelem.firstChild)
+ docelem.insertBefore(node, text)
+ docelem.insertBefore(doc.createTextNode("\n"), docelem.firstChild)
+ while fragment.firstChild and fragment.firstChild.nodeType == TEXT:
+ fragment.removeChild(fragment.firstChild)
+
+
+def cleanup_root_text(doc):
+ discards = []
+ skip = 0
+ for n in doc.childNodes:
+ prevskip = skip
+ skip = 0
+ if n.nodeType == TEXT and not prevskip:
+ discards.append(n)
+ elif n.nodeName == "COMMENT":
+ skip = 1
+ for node in discards:
+ doc.removeChild(node)
+
+
+DESCRIPTOR_ELEMENTS = (
+ "cfuncdesc", "cvardesc", "ctypedesc",
+ "classdesc", "memberdesc", "memberdescni", "methoddesc", "methoddescni",
+ "excdesc", "funcdesc", "funcdescni", "opcodedesc",
+ "datadesc", "datadescni",
+ )
+
+def fixup_descriptors(doc, fragment):
+ sections = find_all_elements(fragment, "section")
+ for section in sections:
+ find_and_fix_descriptors(doc, section)
+
+
+def find_and_fix_descriptors(doc, container):
+ children = container.childNodes
+ for child in children:
+ if child.nodeType == ELEMENT:
+ tagName = child.tagName
+ if tagName in DESCRIPTOR_ELEMENTS:
+ rewrite_descriptor(doc, child)
+ elif tagName == "subsection":
+ find_and_fix_descriptors(doc, child)
+
+
+def rewrite_descriptor(doc, descriptor):
+ #
+ # Do these things:
+ # 1. Add an "index='no'" attribute to the element if the tagName
+ # ends in 'ni', removing the 'ni' from the name.
+ # 2. Create a <signature> from the name attribute
+ # 2a.Create an <args> if it appears to be available.
+ # 3. Create additional <signature>s from <*line{,ni}> elements,
+ # if found.
+ # 4. If a <versionadded> is found, move it to an attribute on the
+ # descriptor.
+ # 5. Move remaining child nodes to a <description> element.
+ # 6. Put it back together.
+ #
+ # 1.
+ descname = descriptor.tagName
+ index = 1
+ if descname[-2:] == "ni":
+ descname = descname[:-2]
+ descriptor.setAttribute("index", "no")
+ set_tagName(descriptor, descname)
+ index = 0
+ desctype = descname[:-4] # remove 'desc'
+ linename = desctype + "line"
+ if not index:
+ linename = linename + "ni"
+ # 2.
+ signature = doc.createElement("signature")
+ name = doc.createElement("name")
+ signature.appendChild(doc.createTextNode("\n "))
+ signature.appendChild(name)
+ name.appendChild(doc.createTextNode(descriptor.getAttribute("name")))
+ descriptor.removeAttribute("name")
+ # 2a.
+ if descriptor.hasAttribute("var"):
+ if descname != "opcodedesc":
+ raise RuntimeError, \
+ "got 'var' attribute on descriptor other than opcodedesc"
+ variable = descriptor.getAttribute("var")
+ if variable:
+ args = doc.createElement("args")
+ args.appendChild(doc.createTextNode(variable))
+ signature.appendChild(doc.createTextNode("\n "))
+ signature.appendChild(args)
+ descriptor.removeAttribute("var")
+ newchildren = [signature]
+ children = descriptor.childNodes
+ pos = skip_leading_nodes(children)
+ if pos < len(children):
+ child = children[pos]
+ if child.nodeName == "args":
+ # move <args> to <signature>, or remove if empty:
+ child.parentNode.removeChild(child)
+ if len(child.childNodes):
+ signature.appendChild(doc.createTextNode("\n "))
+ signature.appendChild(child)
+ signature.appendChild(doc.createTextNode("\n "))
+ # 3, 4.
+ pos = skip_leading_nodes(children, pos)
+ while pos < len(children) \
+ and children[pos].nodeName in (linename, "versionadded"):
+ if children[pos].tagName == linename:
+ # this is really a supplemental signature, create <signature>
+ oldchild = children[pos].cloneNode(1)
+ try:
+ sig = methodline_to_signature(doc, children[pos])
+ except KeyError:
+ print oldchild.toxml()
+ raise
+ newchildren.append(sig)
+ else:
+ # <versionadded added=...>
+ descriptor.setAttribute(
+ "added", children[pos].getAttribute("version"))
+ pos = skip_leading_nodes(children, pos + 1)
+ # 5.
+ description = doc.createElement("description")
+ description.appendChild(doc.createTextNode("\n"))
+ newchildren.append(description)
+ move_children(descriptor, description, pos)
+ last = description.childNodes[-1]
+ if last.nodeType == TEXT:
+ last.data = string.rstrip(last.data) + "\n "
+ # 6.
+ # should have nothing but whitespace and signature lines in <descriptor>;
+ # discard them
+ while descriptor.childNodes:
+ descriptor.removeChild(descriptor.childNodes[0])
+ for node in newchildren:
+ descriptor.appendChild(doc.createTextNode("\n "))
+ descriptor.appendChild(node)
+ descriptor.appendChild(doc.createTextNode("\n"))
+
+
+def methodline_to_signature(doc, methodline):
+ signature = doc.createElement("signature")
+ signature.appendChild(doc.createTextNode("\n "))
+ name = doc.createElement("name")
+ name.appendChild(doc.createTextNode(methodline.getAttribute("name")))
+ methodline.removeAttribute("name")
+ signature.appendChild(name)
+ if len(methodline.childNodes):
+ args = doc.createElement("args")
+ signature.appendChild(doc.createTextNode("\n "))
+ signature.appendChild(args)
+ move_children(methodline, args)
+ signature.appendChild(doc.createTextNode("\n "))
+ return signature
+
+
+def move_children(origin, dest, start=0):
+ children = origin.childNodes
+ while start < len(children):
+ node = children[start]
+ origin.removeChild(node)
+ dest.appendChild(node)
+
+
+def handle_appendix(doc, fragment):
+ # must be called after simplfy() if document is multi-rooted to begin with
+ docelem = get_documentElement(fragment)
+ toplevel = docelem.tagName == "manual" and "chapter" or "section"
+ appendices = 0
+ nodes = []
+ for node in docelem.childNodes:
+ if appendices:
+ nodes.append(node)
+ elif node.nodeType == ELEMENT:
+ appnodes = node.getElementsByTagName("appendix")
+ if appnodes:
+ appendices = 1
+ parent = appnodes[0].parentNode
+ parent.removeChild(appnodes[0])
+ parent.normalize()
+ if nodes:
+ map(docelem.removeChild, nodes)
+ docelem.appendChild(doc.createTextNode("\n\n\n"))
+ back = doc.createElement("back-matter")
+ docelem.appendChild(back)
+ back.appendChild(doc.createTextNode("\n"))
+ while nodes and nodes[0].nodeType == TEXT \
+ and not string.strip(nodes[0].data):
+ del nodes[0]
+ map(back.appendChild, nodes)
+ docelem.appendChild(doc.createTextNode("\n"))
+
+
+def handle_labels(doc, fragment):
+ for label in find_all_elements(fragment, "label"):
+ id = label.getAttribute("id")
+ if not id:
+ continue
+ parent = label.parentNode
+ parentTagName = parent.tagName
+ if parentTagName == "title":
+ parent.parentNode.setAttribute("id", id)
+ else:
+ parent.setAttribute("id", id)
+ # now, remove <label id="..."/> from parent:
+ parent.removeChild(label)
+ if parentTagName == "title":
+ parent.normalize()
+ children = parent.childNodes
+ if children[-1].nodeType == TEXT:
+ children[-1].data = string.rstrip(children[-1].data)
+
+
+def fixup_trailing_whitespace(doc, wsmap):
+ queue = [doc]
+ while queue:
+ node = queue[0]
+ del queue[0]
+ if wsmap.has_key(node.nodeName):
+ ws = wsmap[node.tagName]
+ children = node.childNodes
+ children.reverse()
+ if children[0].nodeType == TEXT:
+ data = string.rstrip(children[0].data) + ws
+ children[0].data = data
+ children.reverse()
+ # hack to get the title in place:
+ if node.tagName == "title" \
+ and node.parentNode.firstChild.nodeType == ELEMENT:
+ node.parentNode.insertBefore(doc.createText("\n "),
+ node.parentNode.firstChild)
+ for child in node.childNodes:
+ if child.nodeType == ELEMENT:
+ queue.append(child)
+
+
+def normalize(doc):
+ for node in doc.childNodes:
+ if node.nodeType == ELEMENT:
+ node.normalize()
+
+
+def cleanup_trailing_parens(doc, element_names):
+ d = {}
+ for gi in element_names:
+ d[gi] = gi
+ rewrite_element = d.has_key
+ queue = []
+ for node in doc.childNodes:
+ if node.nodeType == ELEMENT:
+ queue.append(node)
+ while queue:
+ node = queue[0]
+ del queue[0]
+ if rewrite_element(node.tagName):
+ children = node.childNodes
+ if len(children) == 1 \
+ and children[0].nodeType == TEXT:
+ data = children[0].data
+ if data[-2:] == "()":
+ children[0].data = data[:-2]
+ else:
+ for child in node.childNodes:
+ if child.nodeType == ELEMENT:
+ queue.append(child)
+
+
+def contents_match(left, right):
+ left_children = left.childNodes
+ right_children = right.childNodes
+ if len(left_children) != len(right_children):
+ return 0
+ for l, r in map(None, left_children, right_children):
+ nodeType = l.nodeType
+ if nodeType != r.nodeType:
+ return 0
+ if nodeType == ELEMENT:
+ if l.tagName != r.tagName:
+ return 0
+ # should check attributes, but that's not a problem here
+ if not contents_match(l, r):
+ return 0
+ elif nodeType == TEXT:
+ if l.data != r.data:
+ return 0
+ else:
+ # not quite right, but good enough
+ return 0
+ return 1
+
+
+def create_module_info(doc, section):
+ # Heavy.
+ node = extract_first_element(section, "modulesynopsis")
+ if node is None:
+ return
+ set_tagName(node, "synopsis")
+ lastchild = node.childNodes[-1]
+ if lastchild.nodeType == TEXT \
+ and lastchild.data[-1:] == ".":
+ lastchild.data = lastchild.data[:-1]
+ modauthor = extract_first_element(section, "moduleauthor")
+ if modauthor:
+ set_tagName(modauthor, "author")
+ modauthor.appendChild(doc.createTextNode(
+ modauthor.getAttribute("name")))
+ modauthor.removeAttribute("name")
+ platform = extract_first_element(section, "platform")
+ if section.tagName == "section":
+ modinfo_pos = 2
+ modinfo = doc.createElement("moduleinfo")
+ moddecl = extract_first_element(section, "declaremodule")
+ name = None
+ if moddecl:
+ modinfo.appendChild(doc.createTextNode("\n "))
+ name = moddecl.attributes["name"].value
+ namenode = doc.createElement("name")
+ namenode.appendChild(doc.createTextNode(name))
+ modinfo.appendChild(namenode)
+ type = moddecl.attributes.get("type")
+ if type:
+ type = type.value
+ modinfo.appendChild(doc.createTextNode("\n "))
+ typenode = doc.createElement("type")
+ typenode.appendChild(doc.createTextNode(type))
+ modinfo.appendChild(typenode)
+ versionadded = extract_first_element(section, "versionadded")
+ if versionadded:
+ modinfo.setAttribute("added", versionadded.getAttribute("version"))
+ title = get_first_element(section, "title")
+ if title:
+ children = title.childNodes
+ if len(children) >= 2 \
+ and children[0].nodeName == "module" \
+ and children[0].childNodes[0].data == name:
+ # this is it; morph the <title> into <short-synopsis>
+ first_data = children[1]
+ if first_data.data[:4] == " ---":
+ first_data.data = string.lstrip(first_data.data[4:])
+ set_tagName(title, "short-synopsis")
+ if children[-1].nodeType == TEXT \
+ and children[-1].data[-1:] == ".":
+ children[-1].data = children[-1].data[:-1]
+ section.removeChild(title)
+ section.removeChild(section.childNodes[0])
+ title.removeChild(children[0])
+ modinfo_pos = 0
+ else:
+ ewrite("module name in title doesn't match"
+ " <declaremodule/>; no <short-synopsis/>\n")
+ else:
+ ewrite("Unexpected condition: <section/> without <title/>\n")
+ modinfo.appendChild(doc.createTextNode("\n "))
+ modinfo.appendChild(node)
+ if title and not contents_match(title, node):
+ # The short synopsis is actually different,
+ # and needs to be stored:
+ modinfo.appendChild(doc.createTextNode("\n "))
+ modinfo.appendChild(title)
+ if modauthor:
+ modinfo.appendChild(doc.createTextNode("\n "))
+ modinfo.appendChild(modauthor)
+ if platform:
+ modinfo.appendChild(doc.createTextNode("\n "))
+ modinfo.appendChild(platform)
+ modinfo.appendChild(doc.createTextNode("\n "))
+ section.insertBefore(modinfo, section.childNodes[modinfo_pos])
+ section.insertBefore(doc.createTextNode("\n "), modinfo)
+ #
+ # The rest of this removes extra newlines from where we cut out
+ # a lot of elements. A lot of code for minimal value, but keeps
+ # keeps the generated *ML from being too funny looking.
+ #
+ section.normalize()
+ children = section.childNodes
+ for i in range(len(children)):
+ node = children[i]
+ if node.nodeName == "moduleinfo":
+ nextnode = children[i+1]
+ if nextnode.nodeType == TEXT:
+ data = nextnode.data
+ if len(string.lstrip(data)) < (len(data) - 4):
+ nextnode.data = "\n\n\n" + string.lstrip(data)
+
+
+def cleanup_synopses(doc, fragment):
+ for node in find_all_elements(fragment, "section"):
+ create_module_info(doc, node)
+
+
+def fixup_table_structures(doc, fragment):
+ for table in find_all_elements(fragment, "table"):
+ fixup_table(doc, table)
+
+
+def fixup_table(doc, table):
+ # create the table head
+ thead = doc.createElement("thead")
+ row = doc.createElement("row")
+ move_elements_by_name(doc, table, row, "entry")
+ thead.appendChild(doc.createTextNode("\n "))
+ thead.appendChild(row)
+ thead.appendChild(doc.createTextNode("\n "))
+ # create the table body
+ tbody = doc.createElement("tbody")
+ prev_row = None
+ last_was_hline = 0
+ children = table.childNodes
+ for child in children:
+ if child.nodeType == ELEMENT:
+ tagName = child.tagName
+ if tagName == "hline" and prev_row is not None:
+ prev_row.setAttribute("rowsep", "1")
+ elif tagName == "row":
+ prev_row = child
+ # save the rows:
+ tbody.appendChild(doc.createTextNode("\n "))
+ move_elements_by_name(doc, table, tbody, "row", sep="\n ")
+ # and toss the rest:
+ while children:
+ child = children[0]
+ nodeType = child.nodeType
+ if nodeType == TEXT:
+ if string.strip(child.data):
+ raise ConversionError("unexpected free data in <%s>: %r"
+ % (table.tagName, child.data))
+ table.removeChild(child)
+ continue
+ if nodeType == ELEMENT:
+ if child.tagName != "hline":
+ raise ConversionError(
+ "unexpected <%s> in table" % child.tagName)
+ table.removeChild(child)
+ continue
+ raise ConversionError(
+ "unexpected %s node in table" % child.__class__.__name__)
+ # nothing left in the <table>; add the <thead> and <tbody>
+ tgroup = doc.createElement("tgroup")
+ tgroup.appendChild(doc.createTextNode("\n "))
+ tgroup.appendChild(thead)
+ tgroup.appendChild(doc.createTextNode("\n "))
+ tgroup.appendChild(tbody)
+ tgroup.appendChild(doc.createTextNode("\n "))
+ table.appendChild(tgroup)
+ # now make the <entry>s look nice:
+ for row in table.getElementsByTagName("row"):
+ fixup_row(doc, row)
+
+
+def fixup_row(doc, row):
+ entries = []
+ map(entries.append, row.childNodes[1:])
+ for entry in entries:
+ row.insertBefore(doc.createTextNode("\n "), entry)
+# row.appendChild(doc.createTextNode("\n "))
+
+
+def move_elements_by_name(doc, source, dest, name, sep=None):
+ nodes = []
+ for child in source.childNodes:
+ if child.nodeName == name:
+ nodes.append(child)
+ for node in nodes:
+ source.removeChild(node)
+ dest.appendChild(node)
+ if sep:
+ dest.appendChild(doc.createTextNode(sep))
+
+
+RECURSE_INTO_PARA_CONTAINERS = (
+ "chapter", "abstract", "enumerate",
+ "section", "subsection", "subsubsection",
+ "paragraph", "subparagraph", "back-matter",
+ "howto", "manual",
+ "item", "itemize", "fulllineitems", "enumeration", "descriptionlist",
+ "definitionlist", "definition",
+ )
+
+PARA_LEVEL_ELEMENTS = (
+ "moduleinfo", "title", "verbatim", "enumerate", "item",
+ "interpreter-session", "back-matter", "interactive-session",
+ "opcodedesc", "classdesc", "datadesc",
+ "funcdesc", "methoddesc", "excdesc", "memberdesc", "membderdescni",
+ "funcdescni", "methoddescni", "excdescni",
+ "tableii", "tableiii", "tableiv", "localmoduletable",
+ "sectionauthor", "seealso", "itemize",
+ # include <para>, so we can just do it again to get subsequent paras:
+ PARA_ELEMENT,
+ )
+
+PARA_LEVEL_PRECEEDERS = (
+ "setindexsubitem", "author",
+ "stindex", "obindex", "COMMENT", "label", "input", "title",
+ "versionadded", "versionchanged", "declaremodule", "modulesynopsis",
+ "moduleauthor", "indexterm", "leader",
+ )
+
+
+def fixup_paras(doc, fragment):
+ for child in fragment.childNodes:
+ if child.nodeName in RECURSE_INTO_PARA_CONTAINERS:
+ fixup_paras_helper(doc, child)
+ descriptions = find_all_elements(fragment, "description")
+ for description in descriptions:
+ fixup_paras_helper(doc, description)
+
+
+def fixup_paras_helper(doc, container, depth=0):
+ # document is already normalized
+ children = container.childNodes
+ start = skip_leading_nodes(children)
+ while len(children) > start:
+ if children[start].nodeName in RECURSE_INTO_PARA_CONTAINERS:
+ # Something to recurse into:
+ fixup_paras_helper(doc, children[start])
+ else:
+ # Paragraph material:
+ build_para(doc, container, start, len(children))
+ if DEBUG_PARA_FIXER and depth == 10:
+ sys.exit(1)
+ start = skip_leading_nodes(children, start + 1)
+
+
+def build_para(doc, parent, start, i):
+ children = parent.childNodes
+ after = start + 1
+ have_last = 0
+ BREAK_ELEMENTS = PARA_LEVEL_ELEMENTS + RECURSE_INTO_PARA_CONTAINERS
+ # Collect all children until \n\n+ is found in a text node or a
+ # member of BREAK_ELEMENTS is found.
+ for j in range(start, i):
+ after = j + 1
+ child = children[j]
+ nodeType = child.nodeType
+ if nodeType == ELEMENT:
+ if child.tagName in BREAK_ELEMENTS:
+ after = j
+ break
+ elif nodeType == TEXT:
+ pos = string.find(child.data, "\n\n")
+ if pos == 0:
+ after = j
+ break
+ if pos >= 1:
+ child.splitText(pos)
+ break
+ else:
+ have_last = 1
+ if (start + 1) > after:
+ raise ConversionError(
+ "build_para() could not identify content to turn into a paragraph")
+ if children[after - 1].nodeType == TEXT:
+ # we may need to split off trailing white space:
+ child = children[after - 1]
+ data = child.data
+ if string.rstrip(data) != data:
+ have_last = 0
+ child.splitText(len(string.rstrip(data)))
+ para = doc.createElement(PARA_ELEMENT)
+ prev = None
+ indexes = range(start, after)
+ indexes.reverse()
+ for j in indexes:
+ node = parent.childNodes[j]
+ parent.removeChild(node)
+ para.insertBefore(node, prev)
+ prev = node
+ if have_last:
+ parent.appendChild(para)
+ parent.appendChild(doc.createTextNode("\n\n"))
+ return len(parent.childNodes)
+ else:
+ nextnode = parent.childNodes[start]
+ if nextnode.nodeType == TEXT:
+ if nextnode.data and nextnode.data[0] != "\n":
+ nextnode.data = "\n" + nextnode.data
+ else:
+ newnode = doc.createTextNode("\n")
+ parent.insertBefore(newnode, nextnode)
+ nextnode = newnode
+ start = start + 1
+ parent.insertBefore(para, nextnode)
+ return start + 1
+
+
+def skip_leading_nodes(children, start=0):
+ """Return index into children of a node at which paragraph building should
+ begin or a recursive call to fixup_paras_helper() should be made (for
+ subsections, etc.).
+
+ When the return value >= len(children), we've built all the paras we can
+ from this list of children.
+ """
+ i = len(children)
+ while i > start:
+ # skip over leading comments and whitespace:
+ child = children[start]
+ nodeType = child.nodeType
+ if nodeType == TEXT:
+ data = child.data
+ shortened = string.lstrip(data)
+ if shortened:
+ if data != shortened:
+ # break into two nodes: whitespace and non-whitespace
+ child.splitText(len(data) - len(shortened))
+ return start + 1
+ return start
+ # all whitespace, just skip
+ elif nodeType == ELEMENT:
+ tagName = child.tagName
+ if tagName in RECURSE_INTO_PARA_CONTAINERS:
+ return start
+ if tagName not in PARA_LEVEL_ELEMENTS + PARA_LEVEL_PRECEEDERS:
+ return start
+ start = start + 1
+ return start
+
+
+def fixup_rfc_references(doc, fragment):
+ for rfcnode in find_all_elements(fragment, "rfc"):
+ rfcnode.appendChild(doc.createTextNode(
+ "RFC " + rfcnode.getAttribute("num")))
+
+
+def fixup_signatures(doc, fragment):
+ for child in fragment.childNodes:
+ if child.nodeType == ELEMENT:
+ args = child.getElementsByTagName("args")
+ for arg in args:
+ fixup_args(doc, arg)
+ arg.normalize()
+ args = child.getElementsByTagName("constructor-args")
+ for arg in args:
+ fixup_args(doc, arg)
+ arg.normalize()
+
+
+def fixup_args(doc, arglist):
+ for child in arglist.childNodes:
+ if child.nodeName == "optional":
+ # found it; fix and return
+ arglist.insertBefore(doc.createTextNode("["), child)
+ optkids = child.childNodes
+ while optkids:
+ k = optkids[0]
+ child.removeChild(k)
+ arglist.insertBefore(k, child)
+ arglist.insertBefore(doc.createTextNode("]"), child)
+ arglist.removeChild(child)
+ return fixup_args(doc, arglist)
+
+
+def fixup_sectionauthors(doc, fragment):
+ for sectauth in find_all_elements(fragment, "sectionauthor"):
+ section = sectauth.parentNode
+ section.removeChild(sectauth)
+ set_tagName(sectauth, "author")
+ sectauth.appendChild(doc.createTextNode(
+ sectauth.getAttribute("name")))
+ sectauth.removeAttribute("name")
+ after = section.childNodes[2]
+ title = section.childNodes[1]
+ if title.nodeName != "title":
+ after = section.childNodes[0]
+ section.insertBefore(doc.createTextNode("\n "), after)
+ section.insertBefore(sectauth, after)
+
+
+def fixup_verbatims(doc):
+ for verbatim in find_all_elements(doc, "verbatim"):
+ child = verbatim.childNodes[0]
+ if child.nodeType == TEXT \
+ and string.lstrip(child.data)[:3] == ">>>":
+ set_tagName(verbatim, "interactive-session")
+
+
+def add_node_ids(fragment, counter=0):
+ fragment.node_id = counter
+ for node in fragment.childNodes:
+ counter = counter + 1
+ if node.nodeType == ELEMENT:
+ counter = add_node_ids(node, counter)
+ else:
+ node.node_id = counter
+ return counter + 1
+
+
+REFMODINDEX_ELEMENTS = ('refmodindex', 'refbimodindex',
+ 'refexmodindex', 'refstmodindex')
+
+def fixup_refmodindexes(fragment):
+ # Locate <ref*modindex>...</> co-located with <module>...</>, and
+ # remove the <ref*modindex>, replacing it with index=index on the
+ # <module> element.
+ nodes = find_all_elements_from_set(fragment, REFMODINDEX_ELEMENTS)
+ d = {}
+ for node in nodes:
+ parent = node.parentNode
+ d[parent.node_id] = parent
+ del nodes
+ map(fixup_refmodindexes_chunk, d.values())
+
+
+def fixup_refmodindexes_chunk(container):
+ # node is probably a <para>; let's see how often it isn't:
+ if container.tagName != PARA_ELEMENT:
+ bwrite("--- fixup_refmodindexes_chunk(%s)\n" % container)
+ module_entries = find_all_elements(container, "module")
+ if not module_entries:
+ return
+ index_entries = find_all_elements_from_set(container, REFMODINDEX_ELEMENTS)
+ removes = []
+ for entry in index_entries:
+ children = entry.childNodes
+ if len(children) != 0:
+ bwrite("--- unexpected number of children for %s node:\n"
+ % entry.tagName)
+ ewrite(entry.toxml() + "\n")
+ continue
+ found = 0
+ module_name = entry.getAttribute("module")
+ for node in module_entries:
+ if len(node.childNodes) != 1:
+ continue
+ this_name = node.childNodes[0].data
+ if this_name == module_name:
+ found = 1
+ node.setAttribute("index", "yes")
+ if found:
+ removes.append(entry)
+ for node in removes:
+ container.removeChild(node)
+
+
+def fixup_bifuncindexes(fragment):
+ nodes = find_all_elements(fragment, 'bifuncindex')
+ d = {}
+ # make sure that each parent is only processed once:
+ for node in nodes:
+ parent = node.parentNode
+ d[parent.node_id] = parent
+ del nodes
+ map(fixup_bifuncindexes_chunk, d.values())
+
+
+def fixup_bifuncindexes_chunk(container):
+ removes = []
+ entries = find_all_child_elements(container, "bifuncindex")
+ function_entries = find_all_child_elements(container, "function")
+ for entry in entries:
+ function_name = entry.getAttribute("name")
+ found = 0
+ for func_entry in function_entries:
+ t2 = func_entry.childNodes[0].data
+ if t2[-2:] != "()":
+ continue
+ t2 = t2[:-2]
+ if t2 == function_name:
+ func_entry.setAttribute("index", "yes")
+ func_entry.setAttribute("module", "__builtin__")
+ if not found:
+ found = 1
+ removes.append(entry)
+ for entry in removes:
+ container.removeChild(entry)
+
+
+def join_adjacent_elements(container, gi):
+ queue = [container]
+ while queue:
+ parent = queue.pop()
+ i = 0
+ children = parent.childNodes
+ nchildren = len(children)
+ while i < (nchildren - 1):
+ child = children[i]
+ if child.nodeName == gi:
+ if children[i+1].nodeName == gi:
+ ewrite("--- merging two <%s/> elements\n" % gi)
+ child = children[i]
+ nextchild = children[i+1]
+ nextchildren = nextchild.childNodes
+ while len(nextchildren):
+ node = nextchildren[0]
+ nextchild.removeChild(node)
+ child.appendChild(node)
+ parent.removeChild(nextchild)
+ continue
+ if child.nodeType == ELEMENT:
+ queue.append(child)
+ i = i + 1
+
+
+_token_rx = re.compile(r"[a-zA-Z][a-zA-Z0-9.-]*$")
+
+def write_esis(doc, ofp, knownempty):
+ for node in doc.childNodes:
+ nodeType = node.nodeType
+ if nodeType == ELEMENT:
+ gi = node.tagName
+ if knownempty(gi):
+ if node.hasChildNodes():
+ raise ValueError, \
+ "declared-empty node <%s> has children" % gi
+ ofp.write("e\n")
+ for k, value in node.attributes.items():
+ if _token_rx.match(value):
+ dtype = "TOKEN"
+ else:
+ dtype = "CDATA"
+ ofp.write("A%s %s %s\n" % (k, dtype, esistools.encode(value)))
+ ofp.write("(%s\n" % gi)
+ write_esis(node, ofp, knownempty)
+ ofp.write(")%s\n" % gi)
+ elif nodeType == TEXT:
+ ofp.write("-%s\n" % esistools.encode(node.data))
+ elif nodeType == ENTITY_REFERENCE:
+ ofp.write("&%s\n" % node.nodeName)
+ else:
+ raise RuntimeError, "unsupported node type: %s" % nodeType
+
+
+def convert(ifp, ofp):
+ events = esistools.parse(ifp)
+ toktype, doc = events.getEvent()
+ fragment = doc.createDocumentFragment()
+ events.expandNode(fragment)
+
+ normalize(fragment)
+ simplify(doc, fragment)
+ handle_labels(doc, fragment)
+ handle_appendix(doc, fragment)
+ fixup_trailing_whitespace(doc, {
+ "abstract": "\n",
+ "title": "",
+ "chapter": "\n\n",
+ "section": "\n\n",
+ "subsection": "\n\n",
+ "subsubsection": "\n\n",
+ "paragraph": "\n\n",
+ "subparagraph": "\n\n",
+ })
+ cleanup_root_text(doc)
+ cleanup_trailing_parens(fragment, ["function", "method", "cfunction"])
+ cleanup_synopses(doc, fragment)
+ fixup_descriptors(doc, fragment)
+ fixup_verbatims(fragment)
+ normalize(fragment)
+ fixup_paras(doc, fragment)
+ fixup_sectionauthors(doc, fragment)
+ fixup_table_structures(doc, fragment)
+ fixup_rfc_references(doc, fragment)
+ fixup_signatures(doc, fragment)
+ add_node_ids(fragment)
+ fixup_refmodindexes(fragment)
+ fixup_bifuncindexes(fragment)
+ # Take care of ugly hacks in the LaTeX markup to avoid LaTeX and
+ # LaTeX2HTML screwing with GNU-style long options (the '--' problem).
+ join_adjacent_elements(fragment, "option")
+ #
+ d = {}
+ for gi in events.parser.get_empties():
+ d[gi] = gi
+ if d.has_key("author"):
+ del d["author"]
+ if d.has_key("rfc"):
+ del d["rfc"]
+ knownempty = d.has_key
+ #
+ try:
+ write_esis(fragment, ofp, knownempty)
+ except IOError, (err, msg):
+ # Ignore EPIPE; it just means that whoever we're writing to stopped
+ # reading. The rest of the output would be ignored. All other errors
+ # should still be reported,
+ if err != errno.EPIPE:
+ raise
+
+
+def main():
+ if len(sys.argv) == 1:
+ ifp = sys.stdin
+ ofp = sys.stdout
+ elif len(sys.argv) == 2:
+ ifp = open(sys.argv[1])
+ ofp = sys.stdout
+ elif len(sys.argv) == 3:
+ ifp = open(sys.argv[1])
+ import StringIO
+ ofp = StringIO.StringIO()
+ else:
+ usage()
+ sys.exit(2)
+ convert(ifp, ofp)
+ if len(sys.argv) == 3:
+ fp = open(sys.argv[2], "w")
+ fp.write(ofp.getvalue())
+ fp.close()
+ ofp.close()
+
+
+if __name__ == "__main__":
+ main()