Gitiles
Code Review Sign In
gerrit-public.fairphone.software / platform / external / python / cpython3 / e34bbfd61f405eef89e8aa50672b0b25022de320 / . / Lib / _markupbase.py
blob: 3ad7e279960f7e1f2bf79d89fe9b905e53f6a12b [file] [log] [blame]
Fred Drake7ed44e52005-08-23 04:06:46 +0000[diff] [blame]1"""Shared support for scanning document type declarations in HTML and XHTML.
2
Georg Brandl877b10a2008-06-01 21:25:55 +0000[diff] [blame]3This module is used as a foundation for the html.parser module. It has no
4documented public API and should not be used directly.
Fred Drake7ed44e52005-08-23 04:06:46 +0000[diff] [blame]5
6"""
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]7
8import re
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]9
10_declname_match = re.compile(r'[a-zA-Z][-_.a-zA-Z0-9]*\s*').match
11_declstringlit_match = re.compile(r'(\'[^\']*\'|"[^"]*")\s*').match
Martin v. Löwis3163a3b2003-03-30 14:25:40 +0000[diff] [blame]12_commentclose = re.compile(r'--\s*>')
13_markedsectionclose = re.compile(r']\s*]\s*>')
14
15# An analysis of the MS-Word extensions is available at
16# http://www.planetpublish.com/xmlarena/xap/Thursday/WordtoXML.pdf
17
18_msmarkedsectionclose = re.compile(r']\s*>')
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]19
20del re
21
22
23class ParserBase:
24 """Parser base class which provides some common support methods used
25 by the SGML/HTML and XHTML parsers."""
26
Fred Drake5445f072001-10-26 18:02:28 +0000[diff] [blame]27 def __init__(self):
28 if self.__class__ is ParserBase:
29 raise RuntimeError(
Fred Drakecb5c80f2007-12-07 11:10:11 +0000[diff] [blame]30 "_markupbase.ParserBase must be subclassed")
Fred Drake5445f072001-10-26 18:02:28 +0000[diff] [blame]31
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]32 def reset(self):
33 self.lineno = 1
34 self.offset = 0
35
36 def getpos(self):
37 """Return current line number and offset."""
38 return self.lineno, self.offset
39
40 # Internal -- update line number and offset. This should be
41 # called for each piece of data exactly once, in order -- in other
42 # words the concatenation of all the input strings to this
43 # function should be exactly the entire input.
44 def updatepos(self, i, j):
45 if i >= j:
46 return j
47 rawdata = self.rawdata
Neal Norwitz7ce734c2002-05-31 14:13:04 +0000[diff] [blame]48 nlines = rawdata.count("\n", i, j)
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]49 if nlines:
50 self.lineno = self.lineno + nlines
Neal Norwitz7ce734c2002-05-31 14:13:04 +0000[diff] [blame]51 pos = rawdata.rindex("\n", i, j) # Should not fail
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]52 self.offset = j-(pos+1)
53 else:
54 self.offset = self.offset + j-i
55 return j
56
57 _decl_otherchars = ''
58
59 # Internal -- parse declaration (for use by subclasses).
60 def parse_declaration(self, i):
61 # This is some sort of declaration; in "HTML as
62 # deployed," this should only be the document type
63 # declaration ("<!DOCTYPE html...>").
Tim Peters0eadaac2003-04-24 16:02:54 +0000[diff] [blame]64 # ISO 8879:1986, however, has more complex
Martin v. Löwis3163a3b2003-03-30 14:25:40 +0000[diff] [blame]65 # declaration syntax for elements in <!...>, including:
66 # --comment--
67 # [marked section]
Tim Peters0eadaac2003-04-24 16:02:54 +0000[diff] [blame]68 # name in the following list: ENTITY, DOCTYPE, ELEMENT,
69 # ATTLIST, NOTATION, SHORTREF, USEMAP,
Martin v. Löwis3163a3b2003-03-30 14:25:40 +0000[diff] [blame]70 # LINKTYPE, LINK, IDLINK, USELINK, SYSTEM
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]71 rawdata = self.rawdata
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]72 j = i + 2
73 assert rawdata[i:j] == "<!", "unexpected call to parse_declaration"
Georg Brandld09def32006-03-09 13:27:14 +0000[diff] [blame]74 if rawdata[j:j+1] == ">":
75 # the empty comment <!>
76 return j + 1
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]77 if rawdata[j:j+1] in ("-", ""):
78 # Start of comment followed by buffer boundary,
79 # or just a buffer boundary.
80 return -1
Martin v. Löwis3163a3b2003-03-30 14:25:40 +0000[diff] [blame]81 # A simple, practical version could look like: ((name|stringlit) S*) + '>'
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]82 n = len(rawdata)
Georg Brandld09def32006-03-09 13:27:14 +0000[diff] [blame]83 if rawdata[j:j+2] == '--': #comment
Martin v. Löwis3163a3b2003-03-30 14:25:40 +0000[diff] [blame]84 # Locate --.*-- as the body of the comment
85 return self.parse_comment(i)
86 elif rawdata[j] == '[': #marked section
87 # Locate [statusWord [...arbitrary SGML...]] as the body of the marked section
88 # Where statusWord is one of TEMP, CDATA, IGNORE, INCLUDE, RCDATA
89 # Note that this is extended by Microsoft Office "Save as Web" function
90 # to include [if...] and [endif].
91 return self.parse_marked_section(i)
92 else: #all other declaration elements
93 decltype, j = self._scan_name(j, i)
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]94 if j < 0:
95 return j
96 if decltype == "doctype":
97 self._decl_otherchars = ''
98 while j < n:
99 c = rawdata[j]
100 if c == ">":
101 # end of declaration syntax
102 data = rawdata[i+2:j]
103 if decltype == "doctype":
104 self.handle_decl(data)
105 else:
Ezio Melotti62f3d032011-12-19 07:29:03 +0200[diff] [blame]106 # According to the HTML5 specs sections "8.2.4.44 Bogus
107 # comment state" and "8.2.4.45 Markup declaration open
108 # state", a comment token should be emitted.
109 # Calling unknown_decl provides more flexibility though.
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]110 self.unknown_decl(data)
111 return j + 1
112 if c in "\"'":
113 m = _declstringlit_match(rawdata, j)
114 if not m:
115 return -1 # incomplete
116 j = m.end()
117 elif c in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ":
118 name, j = self._scan_name(j, i)
119 elif c in self._decl_otherchars:
120 j = j + 1
121 elif c == "[":
Martin v. Löwis3163a3b2003-03-30 14:25:40 +0000[diff] [blame]122 # this could be handled in a separate doctype parser
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]123 if decltype == "doctype":
124 j = self._parse_doctype_subset(j + 1, i)
Raymond Hettingerd5825cc2010-09-05 23:15:06 +0000[diff] [blame]125 elif decltype in {"attlist", "linktype", "link", "element"}:
Martin v. Löwis3163a3b2003-03-30 14:25:40 +0000[diff] [blame]126 # must tolerate []'d groups in a content model in an element declaration
127 # also in data attribute specifications of attlist declaration
128 # also link type declaration subsets in linktype declarations
129 # also link attribute specification lists in link declarations
Berker Peksage34bbfd2020-07-16 09:13:05 +0300[diff] [blame^]130 raise AssertionError("unsupported '[' char in %s declaration" % decltype)
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]131 else:
Berker Peksage34bbfd2020-07-16 09:13:05 +0300[diff] [blame^]132 raise AssertionError("unexpected '[' char in declaration")
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]133 else:
Berker Peksage34bbfd2020-07-16 09:13:05 +0300[diff] [blame^]134 raise AssertionError("unexpected %r char in declaration" % rawdata[j])
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]135 if j < 0:
136 return j
137 return -1 # incomplete
138
Martin v. Löwis3163a3b2003-03-30 14:25:40 +0000[diff] [blame]139 # Internal -- parse a marked section
140 # Override this to handle MS-word extension syntax <![if word]>content<![endif]>
Thomas Wouters73e5a5b2006-06-08 15:35:45 +0000[diff] [blame]141 def parse_marked_section(self, i, report=1):
Martin v. Löwis3163a3b2003-03-30 14:25:40 +0000[diff] [blame]142 rawdata= self.rawdata
143 assert rawdata[i:i+3] == '<![', "unexpected call to parse_marked_section()"
144 sectName, j = self._scan_name( i+3, i )
145 if j < 0:
146 return j
Raymond Hettingerd5825cc2010-09-05 23:15:06 +0000[diff] [blame]147 if sectName in {"temp", "cdata", "ignore", "include", "rcdata"}:
Martin v. Löwis3163a3b2003-03-30 14:25:40 +0000[diff] [blame]148 # look for standard ]]> ending
149 match= _markedsectionclose.search(rawdata, i+3)
Raymond Hettingerd5825cc2010-09-05 23:15:06 +0000[diff] [blame]150 elif sectName in {"if", "else", "endif"}:
Martin v. Löwis3163a3b2003-03-30 14:25:40 +0000[diff] [blame]151 # look for MS Office ]> ending
152 match= _msmarkedsectionclose.search(rawdata, i+3)
153 else:
Berker Peksage34bbfd2020-07-16 09:13:05 +0300[diff] [blame^]154 raise AssertionError(
155 'unknown status keyword %r in marked section' % rawdata[i+3:j]
156 )
Martin v. Löwis3163a3b2003-03-30 14:25:40 +0000[diff] [blame]157 if not match:
158 return -1
159 if report:
160 j = match.start(0)
161 self.unknown_decl(rawdata[i+3: j])
162 return match.end(0)
Tim Peters0eadaac2003-04-24 16:02:54 +0000[diff] [blame]163
Martin v. Löwis3163a3b2003-03-30 14:25:40 +0000[diff] [blame]164 # Internal -- parse comment, return length or -1 if not terminated
165 def parse_comment(self, i, report=1):
166 rawdata = self.rawdata
167 if rawdata[i:i+4] != '<!--':
Berker Peksage34bbfd2020-07-16 09:13:05 +0300[diff] [blame^]168 raise AssertionError('unexpected call to parse_comment()')
Martin v. Löwis3163a3b2003-03-30 14:25:40 +0000[diff] [blame]169 match = _commentclose.search(rawdata, i+4)
170 if not match:
171 return -1
172 if report:
173 j = match.start(0)
174 self.handle_comment(rawdata[i+4: j])
175 return match.end(0)
176
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]177 # Internal -- scan past the internal subset in a <!DOCTYPE declaration,
178 # returning the index just past any whitespace following the trailing ']'.
179 def _parse_doctype_subset(self, i, declstartpos):
180 rawdata = self.rawdata
181 n = len(rawdata)
182 j = i
183 while j < n:
184 c = rawdata[j]
185 if c == "<":
186 s = rawdata[j:j+2]
187 if s == "<":
188 # end of buffer; incomplete
189 return -1
190 if s != "<!":
191 self.updatepos(declstartpos, j + 1)
Berker Peksage34bbfd2020-07-16 09:13:05 +0300[diff] [blame^]192 raise AssertionError(
193 "unexpected char in internal subset (in %r)" % s
194 )
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]195 if (j + 2) == n:
196 # end of buffer; incomplete
197 return -1
198 if (j + 4) > n:
199 # end of buffer; incomplete
200 return -1
201 if rawdata[j:j+4] == "<!--":
202 j = self.parse_comment(j, report=0)
203 if j < 0:
204 return j
205 continue
206 name, j = self._scan_name(j + 2, declstartpos)
207 if j == -1:
208 return -1
Raymond Hettingerd5825cc2010-09-05 23:15:06 +0000[diff] [blame]209 if name not in {"attlist", "element", "entity", "notation"}:
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]210 self.updatepos(declstartpos, j + 2)
Berker Peksage34bbfd2020-07-16 09:13:05 +0300[diff] [blame^]211 raise AssertionError(
212 "unknown declaration %r in internal subset" % name
213 )
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]214 # handle the individual names
215 meth = getattr(self, "_parse_doctype_" + name)
216 j = meth(j, declstartpos)
217 if j < 0:
218 return j
219 elif c == "%":
220 # parameter entity reference
221 if (j + 1) == n:
222 # end of buffer; incomplete
223 return -1
224 s, j = self._scan_name(j + 1, declstartpos)
225 if j < 0:
226 return j
227 if rawdata[j] == ";":
228 j = j + 1
229 elif c == "]":
230 j = j + 1
Walter Dörwald65230a22002-06-03 15:58:32 +0000[diff] [blame]231 while j < n and rawdata[j].isspace():
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]232 j = j + 1
233 if j < n:
234 if rawdata[j] == ">":
235 return j
236 self.updatepos(declstartpos, j)
Berker Peksage34bbfd2020-07-16 09:13:05 +0300[diff] [blame^]237 raise AssertionError("unexpected char after internal subset")
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]238 else:
239 return -1
Walter Dörwald65230a22002-06-03 15:58:32 +0000[diff] [blame]240 elif c.isspace():
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]241 j = j + 1
242 else:
243 self.updatepos(declstartpos, j)
Berker Peksage34bbfd2020-07-16 09:13:05 +0300[diff] [blame^]244 raise AssertionError("unexpected char %r in internal subset" % c)
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]245 # end of buffer reached
246 return -1
247
248 # Internal -- scan past <!ELEMENT declarations
249 def _parse_doctype_element(self, i, declstartpos):
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]250 name, j = self._scan_name(i, declstartpos)
251 if j == -1:
252 return -1
253 # style content model; just skip until '>'
Fred Drake5445f072001-10-26 18:02:28 +0000[diff] [blame]254 rawdata = self.rawdata
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]255 if '>' in rawdata[j:]:
Neal Norwitz7ce734c2002-05-31 14:13:04 +0000[diff] [blame]256 return rawdata.find(">", j) + 1
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]257 return -1
258
259 # Internal -- scan past <!ATTLIST declarations
260 def _parse_doctype_attlist(self, i, declstartpos):
261 rawdata = self.rawdata
262 name, j = self._scan_name(i, declstartpos)
263 c = rawdata[j:j+1]
264 if c == "":
265 return -1
266 if c == ">":
267 return j + 1
268 while 1:
269 # scan a series of attribute descriptions; simplified:
270 # name type [value] [#constraint]
271 name, j = self._scan_name(j, declstartpos)
272 if j < 0:
273 return j
274 c = rawdata[j:j+1]
275 if c == "":
276 return -1
277 if c == "(":
278 # an enumerated type; look for ')'
279 if ")" in rawdata[j:]:
Neal Norwitz7ce734c2002-05-31 14:13:04 +0000[diff] [blame]280 j = rawdata.find(")", j) + 1
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]281 else:
282 return -1
Walter Dörwald65230a22002-06-03 15:58:32 +0000[diff] [blame]283 while rawdata[j:j+1].isspace():
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]284 j = j + 1
285 if not rawdata[j:]:
286 # end of buffer, incomplete
287 return -1
288 else:
289 name, j = self._scan_name(j, declstartpos)
290 c = rawdata[j:j+1]
291 if not c:
292 return -1
293 if c in "'\"":
294 m = _declstringlit_match(rawdata, j)
295 if m:
296 j = m.end()
297 else:
298 return -1
299 c = rawdata[j:j+1]
300 if not c:
301 return -1
302 if c == "#":
303 if rawdata[j:] == "#":
304 # end of buffer
305 return -1
306 name, j = self._scan_name(j + 1, declstartpos)
307 if j < 0:
308 return j
309 c = rawdata[j:j+1]
310 if not c:
311 return -1
312 if c == '>':
313 # all done
314 return j + 1
315
316 # Internal -- scan past <!NOTATION declarations
317 def _parse_doctype_notation(self, i, declstartpos):
318 name, j = self._scan_name(i, declstartpos)
319 if j < 0:
320 return j
321 rawdata = self.rawdata
322 while 1:
323 c = rawdata[j:j+1]
324 if not c:
325 # end of buffer; incomplete
326 return -1
327 if c == '>':
328 return j + 1
329 if c in "'\"":
330 m = _declstringlit_match(rawdata, j)
331 if not m:
332 return -1
333 j = m.end()
334 else:
335 name, j = self._scan_name(j, declstartpos)
336 if j < 0:
337 return j
338
339 # Internal -- scan past <!ENTITY declarations
340 def _parse_doctype_entity(self, i, declstartpos):
341 rawdata = self.rawdata
342 if rawdata[i:i+1] == "%":
343 j = i + 1
344 while 1:
345 c = rawdata[j:j+1]
346 if not c:
347 return -1
Walter Dörwald65230a22002-06-03 15:58:32 +0000[diff] [blame]348 if c.isspace():
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]349 j = j + 1
350 else:
351 break
352 else:
353 j = i
354 name, j = self._scan_name(j, declstartpos)
355 if j < 0:
356 return j
357 while 1:
358 c = self.rawdata[j:j+1]
359 if not c:
360 return -1
361 if c in "'\"":
362 m = _declstringlit_match(rawdata, j)
363 if m:
364 j = m.end()
365 else:
366 return -1 # incomplete
367 elif c == ">":
368 return j + 1
369 else:
370 name, j = self._scan_name(j, declstartpos)
371 if j < 0:
372 return j
373
374 # Internal -- scan a name token and the new position and the token, or
375 # return -1 if we've reached the end of the buffer.
376 def _scan_name(self, i, declstartpos):
377 rawdata = self.rawdata
378 n = len(rawdata)
379 if i == n:
380 return None, -1
381 m = _declname_match(rawdata, i)
382 if m:
383 s = m.group()
384 name = s.strip()
385 if (i + len(s)) == n:
386 return None, -1 # end of buffer
Neal Norwitz7ce734c2002-05-31 14:13:04 +0000[diff] [blame]387 return name.lower(), m.end()
Fred Drake68f8a802001-09-24 20:01:28 +0000[diff] [blame]388 else:
389 self.updatepos(declstartpos, i)
Berker Peksage34bbfd2020-07-16 09:13:05 +0300[diff] [blame^]390 raise AssertionError(
391 "expected name token at %r" % rawdata[declstartpos:declstartpos+20]
392 )
Fred Drake5445f072001-10-26 18:02:28 +0000[diff] [blame]393
394 # To be overridden -- handlers for unknown objects
395 def unknown_decl(self, data):
396 pass