blob: 84c912573e7f5443bdb63ffaecc1a645f4fc4c43 [file] [log] [blame]
Guido van Rossum7627c0d2000-03-31 14:58:54 +00001#
2# Secret Labs' Regular Expression Engine
Guido van Rossum7627c0d2000-03-31 14:58:54 +00003#
Andrew M. Kuchling815d5b92000-06-09 14:08:07 +00004# convert re-style regular expression to sre pattern
Guido van Rossum7627c0d2000-03-31 14:58:54 +00005#
Fredrik Lundh770617b2001-01-14 15:06:11 +00006# Copyright (c) 1998-2001 by Secret Labs AB. All rights reserved.
Guido van Rossum7627c0d2000-03-31 14:58:54 +00007#
Fredrik Lundh29c4ba92000-08-01 18:20:07 +00008# See the sre.py file for information on usage and redistribution.
Guido van Rossum7627c0d2000-03-31 14:58:54 +00009#
10
Fred Drakeb8f22742001-09-04 19:10:20 +000011"""Internal support module for sre"""
12
Fredrik Lundh470ea5a2001-01-14 21:00:44 +000013# XXX: show string offset and offending character for all errors
14
Guido van Rossum7627c0d2000-03-31 14:58:54 +000015from sre_constants import *
16
17SPECIAL_CHARS = ".\\[{()*+?^$|"
Fredrik Lundh143328b2000-09-02 11:03:34 +000018REPEAT_CHARS = "*+?{"
Guido van Rossum7627c0d2000-03-31 14:58:54 +000019
Serhiy Storchakae2ccf562014-10-10 11:14:49 +030020DIGITS = frozenset("0123456789")
Guido van Rossumb81e70e2000-04-10 17:10:48 +000021
Serhiy Storchakae2ccf562014-10-10 11:14:49 +030022OCTDIGITS = frozenset("01234567")
23HEXDIGITS = frozenset("0123456789abcdefABCDEF")
Serhiy Storchakaa54aae02015-03-24 22:58:14 +020024ASCIILETTERS = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
Guido van Rossum7627c0d2000-03-31 14:58:54 +000025
Serhiy Storchakae2ccf562014-10-10 11:14:49 +030026WHITESPACE = frozenset(" \t\n\r\v\f")
27
Raymond Hettingerdf1b6992014-11-09 15:56:33 -080028_REPEATCODES = frozenset({MIN_REPEAT, MAX_REPEAT})
29_UNITCODES = frozenset({ANY, RANGE, IN, LITERAL, NOT_LITERAL, CATEGORY})
Andrew M. Kuchling815d5b92000-06-09 14:08:07 +000030
Guido van Rossum7627c0d2000-03-31 14:58:54 +000031ESCAPES = {
Fredrik Lundhf2989b22001-02-18 12:05:16 +000032 r"\a": (LITERAL, ord("\a")),
33 r"\b": (LITERAL, ord("\b")),
34 r"\f": (LITERAL, ord("\f")),
35 r"\n": (LITERAL, ord("\n")),
36 r"\r": (LITERAL, ord("\r")),
37 r"\t": (LITERAL, ord("\t")),
38 r"\v": (LITERAL, ord("\v")),
Fredrik Lundh0640e112000-06-30 13:55:15 +000039 r"\\": (LITERAL, ord("\\"))
Guido van Rossum7627c0d2000-03-31 14:58:54 +000040}
41
42CATEGORIES = {
Fredrik Lundh770617b2001-01-14 15:06:11 +000043 r"\A": (AT, AT_BEGINNING_STRING), # start of string
Fredrik Lundh01016fe2000-06-30 00:27:46 +000044 r"\b": (AT, AT_BOUNDARY),
45 r"\B": (AT, AT_NON_BOUNDARY),
46 r"\d": (IN, [(CATEGORY, CATEGORY_DIGIT)]),
47 r"\D": (IN, [(CATEGORY, CATEGORY_NOT_DIGIT)]),
48 r"\s": (IN, [(CATEGORY, CATEGORY_SPACE)]),
49 r"\S": (IN, [(CATEGORY, CATEGORY_NOT_SPACE)]),
50 r"\w": (IN, [(CATEGORY, CATEGORY_WORD)]),
51 r"\W": (IN, [(CATEGORY, CATEGORY_NOT_WORD)]),
Fredrik Lundh770617b2001-01-14 15:06:11 +000052 r"\Z": (AT, AT_END_STRING), # end of string
Guido van Rossum7627c0d2000-03-31 14:58:54 +000053}
54
Andrew M. Kuchling815d5b92000-06-09 14:08:07 +000055FLAGS = {
Fredrik Lundh436c3d582000-06-29 08:58:44 +000056 # standard flags
Andrew M. Kuchling815d5b92000-06-09 14:08:07 +000057 "i": SRE_FLAG_IGNORECASE,
58 "L": SRE_FLAG_LOCALE,
59 "m": SRE_FLAG_MULTILINE,
60 "s": SRE_FLAG_DOTALL,
Andrew M. Kuchling815d5b92000-06-09 14:08:07 +000061 "x": SRE_FLAG_VERBOSE,
Fredrik Lundh436c3d582000-06-29 08:58:44 +000062 # extensions
Antoine Pitroufd036452008-08-19 17:56:33 +000063 "a": SRE_FLAG_ASCII,
Fredrik Lundh436c3d582000-06-29 08:58:44 +000064 "t": SRE_FLAG_TEMPLATE,
65 "u": SRE_FLAG_UNICODE,
Andrew M. Kuchling815d5b92000-06-09 14:08:07 +000066}
67
Serhiy Storchaka3557b052017-10-24 23:31:42 +030068TYPE_FLAGS = SRE_FLAG_ASCII | SRE_FLAG_LOCALE | SRE_FLAG_UNICODE
69GLOBAL_FLAGS = SRE_FLAG_DEBUG | SRE_FLAG_TEMPLATE
Serhiy Storchakabe9a4e52016-09-10 00:57:55 +030070
71class Verbose(Exception):
72 pass
73
Serhiy Storchakae0c19dd2018-09-18 09:16:26 +030074class State:
75 # keeps track of state for parsing
Guido van Rossum7627c0d2000-03-31 14:58:54 +000076 def __init__(self):
Fredrik Lundh90a07912000-06-30 07:50:59 +000077 self.flags = 0
Fredrik Lundh90a07912000-06-30 07:50:59 +000078 self.groupdict = {}
Serhiy Storchakab5d0a212015-11-05 17:49:26 +020079 self.groupwidths = [None] # group 0
Serhiy Storchaka4eea62f2015-02-21 10:07:35 +020080 self.lookbehindgroups = None
81 @property
82 def groups(self):
Serhiy Storchakab5d0a212015-11-05 17:49:26 +020083 return len(self.groupwidths)
Fredrik Lundhebc37b22000-10-28 19:30:41 +000084 def opengroup(self, name=None):
Fredrik Lundh90a07912000-06-30 07:50:59 +000085 gid = self.groups
Serhiy Storchakab5d0a212015-11-05 17:49:26 +020086 self.groupwidths.append(None)
Serhiy Storchaka9baa5b22014-09-29 22:49:23 +030087 if self.groups > MAXGROUPS:
Serhiy Storchaka632a77e2015-03-25 21:03:47 +020088 raise error("too many groups")
Raymond Hettingerf13eb552002-06-02 00:40:05 +000089 if name is not None:
Tim Peters75335872001-11-03 19:35:43 +000090 ogid = self.groupdict.get(name, None)
91 if ogid is not None:
Serhiy Storchakaad446d52014-11-10 13:49:00 +020092 raise error("redefinition of group name %r as group %d; "
93 "was group %d" % (name, gid, ogid))
Fredrik Lundh90a07912000-06-30 07:50:59 +000094 self.groupdict[name] = gid
95 return gid
Serhiy Storchaka4eea62f2015-02-21 10:07:35 +020096 def closegroup(self, gid, p):
Serhiy Storchakab5d0a212015-11-05 17:49:26 +020097 self.groupwidths[gid] = p.getwidth()
Fredrik Lundhebc37b22000-10-28 19:30:41 +000098 def checkgroup(self, gid):
Serhiy Storchakab5d0a212015-11-05 17:49:26 +020099 return gid < self.groups and self.groupwidths[gid] is not None
Serhiy Storchaka4eea62f2015-02-21 10:07:35 +0200100
101 def checklookbehindgroup(self, gid, source):
102 if self.lookbehindgroups is not None:
103 if not self.checkgroup(gid):
104 raise source.error('cannot refer to an open group')
105 if gid >= self.lookbehindgroups:
106 raise source.error('cannot refer to group defined in the same '
107 'lookbehind subpattern')
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000108
109class SubPattern:
110 # a subpattern, in intermediate form
Serhiy Storchakae0c19dd2018-09-18 09:16:26 +0300111 def __init__(self, state, data=None):
112 self.state = state
Raymond Hettingerf13eb552002-06-02 00:40:05 +0000113 if data is None:
Fredrik Lundh90a07912000-06-30 07:50:59 +0000114 data = []
115 self.data = data
116 self.width = None
Serhiy Storchaka821a9d12017-05-14 08:32:33 +0300117
Fredrik Lundh8a3ebf82000-07-23 21:46:17 +0000118 def dump(self, level=0):
Serhiy Storchaka44dae8b2014-09-21 22:47:55 +0300119 nl = True
Guido van Rossum13257902007-06-07 23:15:56 +0000120 seqtypes = (tuple, list)
Fredrik Lundh8a3ebf82000-07-23 21:46:17 +0000121 for op, av in self.data:
Serhiy Storchakac7f7d382014-11-09 20:48:36 +0200122 print(level*" " + str(op), end='')
Serhiy Storchakaab140882014-11-11 21:13:28 +0200123 if op is IN:
Fredrik Lundh8a3ebf82000-07-23 21:46:17 +0000124 # member sublanguage
Serhiy Storchaka44dae8b2014-09-21 22:47:55 +0300125 print()
Fredrik Lundh8a3ebf82000-07-23 21:46:17 +0000126 for op, a in av:
Serhiy Storchakac7f7d382014-11-09 20:48:36 +0200127 print((level+1)*" " + str(op), a)
Serhiy Storchakaab140882014-11-11 21:13:28 +0200128 elif op is BRANCH:
Serhiy Storchaka44dae8b2014-09-21 22:47:55 +0300129 print()
130 for i, a in enumerate(av[1]):
131 if i:
Serhiy Storchakac7f7d382014-11-09 20:48:36 +0200132 print(level*" " + "OR")
Serhiy Storchaka44dae8b2014-09-21 22:47:55 +0300133 a.dump(level+1)
Serhiy Storchakaab140882014-11-11 21:13:28 +0200134 elif op is GROUPREF_EXISTS:
Serhiy Storchaka44dae8b2014-09-21 22:47:55 +0300135 condgroup, item_yes, item_no = av
136 print('', condgroup)
137 item_yes.dump(level+1)
138 if item_no:
Serhiy Storchakac7f7d382014-11-09 20:48:36 +0200139 print(level*" " + "ELSE")
Serhiy Storchaka44dae8b2014-09-21 22:47:55 +0300140 item_no.dump(level+1)
Guido van Rossum13257902007-06-07 23:15:56 +0000141 elif isinstance(av, seqtypes):
Serhiy Storchaka44dae8b2014-09-21 22:47:55 +0300142 nl = False
Fredrik Lundh8a3ebf82000-07-23 21:46:17 +0000143 for a in av:
144 if isinstance(a, SubPattern):
Serhiy Storchaka44dae8b2014-09-21 22:47:55 +0300145 if not nl:
146 print()
147 a.dump(level+1)
148 nl = True
Fredrik Lundh8a3ebf82000-07-23 21:46:17 +0000149 else:
Serhiy Storchaka44dae8b2014-09-21 22:47:55 +0300150 if not nl:
151 print(' ', end='')
152 print(a, end='')
153 nl = False
154 if not nl:
155 print()
Fredrik Lundh8a3ebf82000-07-23 21:46:17 +0000156 else:
Serhiy Storchaka44dae8b2014-09-21 22:47:55 +0300157 print('', av)
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000158 def __repr__(self):
Fredrik Lundh90a07912000-06-30 07:50:59 +0000159 return repr(self.data)
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000160 def __len__(self):
Fredrik Lundh90a07912000-06-30 07:50:59 +0000161 return len(self.data)
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000162 def __delitem__(self, index):
Fredrik Lundh90a07912000-06-30 07:50:59 +0000163 del self.data[index]
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000164 def __getitem__(self, index):
Thomas Wouters902d6eb2007-01-09 23:18:33 +0000165 if isinstance(index, slice):
Serhiy Storchakae0c19dd2018-09-18 09:16:26 +0300166 return SubPattern(self.state, self.data[index])
Fredrik Lundh90a07912000-06-30 07:50:59 +0000167 return self.data[index]
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000168 def __setitem__(self, index, code):
Fredrik Lundh90a07912000-06-30 07:50:59 +0000169 self.data[index] = code
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000170 def insert(self, index, code):
Fredrik Lundh90a07912000-06-30 07:50:59 +0000171 self.data.insert(index, code)
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000172 def append(self, code):
Fredrik Lundh90a07912000-06-30 07:50:59 +0000173 self.data.append(code)
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000174 def getwidth(self):
Fredrik Lundh90a07912000-06-30 07:50:59 +0000175 # determine the width (min, max) for this subpattern
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300176 if self.width is not None:
Fredrik Lundh90a07912000-06-30 07:50:59 +0000177 return self.width
Guido van Rossume2a383d2007-01-15 16:59:06 +0000178 lo = hi = 0
Fredrik Lundh90a07912000-06-30 07:50:59 +0000179 for op, av in self.data:
180 if op is BRANCH:
Serhiy Storchaka9d965422013-08-19 22:50:54 +0300181 i = MAXREPEAT - 1
Fredrik Lundh2f2c67d2000-08-01 21:05:41 +0000182 j = 0
Fredrik Lundh90a07912000-06-30 07:50:59 +0000183 for av in av[1]:
Fredrik Lundh2f2c67d2000-08-01 21:05:41 +0000184 l, h = av.getwidth()
185 i = min(i, l)
Fredrik Lundhe1869832000-08-01 22:47:49 +0000186 j = max(j, h)
Fredrik Lundh90a07912000-06-30 07:50:59 +0000187 lo = lo + i
188 hi = hi + j
189 elif op is CALL:
190 i, j = av.getwidth()
191 lo = lo + i
192 hi = hi + j
193 elif op is SUBPATTERN:
Serhiy Storchakabe9a4e52016-09-10 00:57:55 +0300194 i, j = av[-1].getwidth()
Fredrik Lundh90a07912000-06-30 07:50:59 +0000195 lo = lo + i
196 hi = hi + j
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300197 elif op in _REPEATCODES:
Fredrik Lundh90a07912000-06-30 07:50:59 +0000198 i, j = av[2].getwidth()
Serhiy Storchaka9d965422013-08-19 22:50:54 +0300199 lo = lo + i * av[0]
200 hi = hi + j * av[1]
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300201 elif op in _UNITCODES:
Fredrik Lundh90a07912000-06-30 07:50:59 +0000202 lo = lo + 1
203 hi = hi + 1
Serhiy Storchaka4eea62f2015-02-21 10:07:35 +0200204 elif op is GROUPREF:
Serhiy Storchakae0c19dd2018-09-18 09:16:26 +0300205 i, j = self.state.groupwidths[av]
Serhiy Storchaka4eea62f2015-02-21 10:07:35 +0200206 lo = lo + i
207 hi = hi + j
208 elif op is GROUPREF_EXISTS:
209 i, j = av[1].getwidth()
210 if av[2] is not None:
211 l, h = av[2].getwidth()
212 i = min(i, l)
213 j = max(j, h)
214 else:
215 i = 0
216 lo = lo + i
217 hi = hi + j
218 elif op is SUCCESS:
Fredrik Lundh90a07912000-06-30 07:50:59 +0000219 break
Serhiy Storchaka9d965422013-08-19 22:50:54 +0300220 self.width = min(lo, MAXREPEAT - 1), min(hi, MAXREPEAT)
Fredrik Lundh90a07912000-06-30 07:50:59 +0000221 return self.width
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000222
223class Tokenizer:
224 def __init__(self, string):
Antoine Pitrou463badf2012-06-23 13:29:19 +0200225 self.istext = isinstance(string, str)
Serhiy Storchakaad446d52014-11-10 13:49:00 +0200226 self.string = string
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300227 if not self.istext:
228 string = str(string, 'latin1')
Serhiy Storchakaad446d52014-11-10 13:49:00 +0200229 self.decoded_string = string
Fredrik Lundhc13222c2000-07-01 23:49:14 +0000230 self.index = 0
Serhiy Storchakab99c1322014-11-10 14:38:16 +0200231 self.next = None
Fredrik Lundhc13222c2000-07-01 23:49:14 +0000232 self.__next()
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000233 def __next(self):
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300234 index = self.index
235 try:
Serhiy Storchakaad446d52014-11-10 13:49:00 +0200236 char = self.decoded_string[index]
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300237 except IndexError:
Fredrik Lundhc13222c2000-07-01 23:49:14 +0000238 self.next = None
239 return
Guido van Rossum75a902d2007-10-19 22:06:24 +0000240 if char == "\\":
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300241 index += 1
Fredrik Lundh90a07912000-06-30 07:50:59 +0000242 try:
Serhiy Storchakaad446d52014-11-10 13:49:00 +0200243 char += self.decoded_string[index]
Fredrik Lundh90a07912000-06-30 07:50:59 +0000244 except IndexError:
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200245 raise error("bad escape (end of pattern)",
Serhiy Storchaka1b2004f2014-11-10 18:28:53 +0200246 self.string, len(self.string) - 1) from None
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300247 self.index = index + 1
Fredrik Lundhc13222c2000-07-01 23:49:14 +0000248 self.next = char
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300249 def match(self, char):
Fredrik Lundh90a07912000-06-30 07:50:59 +0000250 if char == self.next:
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300251 self.__next()
252 return True
253 return False
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000254 def get(self):
Fredrik Lundh90a07912000-06-30 07:50:59 +0000255 this = self.next
Fredrik Lundhc13222c2000-07-01 23:49:14 +0000256 self.__next()
Fredrik Lundh90a07912000-06-30 07:50:59 +0000257 return this
Antoine Pitrou463badf2012-06-23 13:29:19 +0200258 def getwhile(self, n, charset):
259 result = ''
260 for _ in range(n):
261 c = self.next
262 if c not in charset:
263 break
264 result += c
265 self.__next()
266 return result
Serhiy Storchakaa445feb2018-02-10 00:08:17 +0200267 def getuntil(self, terminator, name):
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300268 result = ''
269 while True:
270 c = self.next
271 self.__next()
272 if c is None:
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200273 if not result:
Serhiy Storchakaa445feb2018-02-10 00:08:17 +0200274 raise self.error("missing " + name)
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200275 raise self.error("missing %s, unterminated name" % terminator,
276 len(result))
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300277 if c == terminator:
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200278 if not result:
Serhiy Storchakaa445feb2018-02-10 00:08:17 +0200279 raise self.error("missing " + name, 1)
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300280 break
281 result += c
282 return result
Serhiy Storchakabd48d272016-09-11 12:50:02 +0300283 @property
284 def pos(self):
285 return self.index - len(self.next or '')
Fredrik Lundhc13222c2000-07-01 23:49:14 +0000286 def tell(self):
Serhiy Storchakaad446d52014-11-10 13:49:00 +0200287 return self.index - len(self.next or '')
Fredrik Lundhc13222c2000-07-01 23:49:14 +0000288 def seek(self, index):
Serhiy Storchakaad446d52014-11-10 13:49:00 +0200289 self.index = index
290 self.__next()
291
292 def error(self, msg, offset=0):
293 return error(msg, self.string, self.tell() - offset)
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000294
Andrew M. Kuchling815d5b92000-06-09 14:08:07 +0000295def _class_escape(source, escape):
296 # handle escape code inside character class
297 code = ESCAPES.get(escape)
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000298 if code:
Fredrik Lundh90a07912000-06-30 07:50:59 +0000299 return code
Andrew M. Kuchling815d5b92000-06-09 14:08:07 +0000300 code = CATEGORIES.get(escape)
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300301 if code and code[0] is IN:
Fredrik Lundh90a07912000-06-30 07:50:59 +0000302 return code
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000303 try:
Gustavo Niemeyera01a2ee2004-09-03 17:06:10 +0000304 c = escape[1:2]
305 if c == "x":
Fredrik Lundh0c4fdba2000-08-31 22:57:55 +0000306 # hexadecimal escape (exactly two digits)
Antoine Pitrou463badf2012-06-23 13:29:19 +0200307 escape += source.getwhile(2, HEXDIGITS)
308 if len(escape) != 4:
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200309 raise source.error("incomplete escape %s" % escape, len(escape))
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300310 return LITERAL, int(escape[2:], 16)
Antoine Pitrou463badf2012-06-23 13:29:19 +0200311 elif c == "u" and source.istext:
312 # unicode escape (exactly four digits)
313 escape += source.getwhile(4, HEXDIGITS)
314 if len(escape) != 6:
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200315 raise source.error("incomplete escape %s" % escape, len(escape))
Antoine Pitrou463badf2012-06-23 13:29:19 +0200316 return LITERAL, int(escape[2:], 16)
317 elif c == "U" and source.istext:
318 # unicode escape (exactly eight digits)
319 escape += source.getwhile(8, HEXDIGITS)
320 if len(escape) != 10:
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200321 raise source.error("incomplete escape %s" % escape, len(escape))
Antoine Pitrou463badf2012-06-23 13:29:19 +0200322 c = int(escape[2:], 16)
323 chr(c) # raise ValueError for invalid code
324 return LITERAL, c
Serhiy Storchakaa445feb2018-02-10 00:08:17 +0200325 elif c == "N" and source.istext:
Zhou Fangyi5df52862018-02-10 06:59:29 +0000326 import unicodedata
Serhiy Storchakaa445feb2018-02-10 00:08:17 +0200327 # named unicode escape e.g. \N{EM DASH}
328 if not source.match('{'):
329 raise source.error("missing {")
330 charname = source.getuntil('}', 'character name')
331 try:
332 c = ord(unicodedata.lookup(charname))
333 except KeyError:
334 raise source.error("undefined character name %r" % charname,
335 len(charname) + len(r'\N{}'))
336 return LITERAL, c
Gustavo Niemeyera01a2ee2004-09-03 17:06:10 +0000337 elif c in OCTDIGITS:
Fredrik Lundh0c4fdba2000-08-31 22:57:55 +0000338 # octal escape (up to three digits)
Antoine Pitrou463badf2012-06-23 13:29:19 +0200339 escape += source.getwhile(2, OCTDIGITS)
Serhiy Storchakac563caf2014-09-23 23:22:41 +0300340 c = int(escape[1:], 8)
341 if c > 0o377:
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200342 raise source.error('octal escape value %s outside of '
Serhiy Storchakaad446d52014-11-10 13:49:00 +0200343 'range 0-0o377' % escape, len(escape))
Serhiy Storchakac563caf2014-09-23 23:22:41 +0300344 return LITERAL, c
Gustavo Niemeyera01a2ee2004-09-03 17:06:10 +0000345 elif c in DIGITS:
Antoine Pitrou463badf2012-06-23 13:29:19 +0200346 raise ValueError
Fredrik Lundh90a07912000-06-30 07:50:59 +0000347 if len(escape) == 2:
Serhiy Storchakaa54aae02015-03-24 22:58:14 +0200348 if c in ASCIILETTERS:
Serhiy Storchaka9bd85b82016-06-11 19:15:00 +0300349 raise source.error('bad escape %s' % escape, len(escape))
Fredrik Lundh0640e112000-06-30 13:55:15 +0000350 return LITERAL, ord(escape[1])
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000351 except ValueError:
Fredrik Lundh90a07912000-06-30 07:50:59 +0000352 pass
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200353 raise source.error("bad escape %s" % escape, len(escape))
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000354
Andrew M. Kuchling815d5b92000-06-09 14:08:07 +0000355def _escape(source, escape, state):
356 # handle escape code in expression
357 code = CATEGORIES.get(escape)
358 if code:
Fredrik Lundh90a07912000-06-30 07:50:59 +0000359 return code
Andrew M. Kuchling815d5b92000-06-09 14:08:07 +0000360 code = ESCAPES.get(escape)
361 if code:
Fredrik Lundh90a07912000-06-30 07:50:59 +0000362 return code
Andrew M. Kuchling815d5b92000-06-09 14:08:07 +0000363 try:
Gustavo Niemeyera01a2ee2004-09-03 17:06:10 +0000364 c = escape[1:2]
365 if c == "x":
Fredrik Lundh0c4fdba2000-08-31 22:57:55 +0000366 # hexadecimal escape
Antoine Pitrou463badf2012-06-23 13:29:19 +0200367 escape += source.getwhile(2, HEXDIGITS)
Fredrik Lundh143328b2000-09-02 11:03:34 +0000368 if len(escape) != 4:
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200369 raise source.error("incomplete escape %s" % escape, len(escape))
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300370 return LITERAL, int(escape[2:], 16)
Antoine Pitrou463badf2012-06-23 13:29:19 +0200371 elif c == "u" and source.istext:
372 # unicode escape (exactly four digits)
373 escape += source.getwhile(4, HEXDIGITS)
374 if len(escape) != 6:
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200375 raise source.error("incomplete escape %s" % escape, len(escape))
Antoine Pitrou463badf2012-06-23 13:29:19 +0200376 return LITERAL, int(escape[2:], 16)
377 elif c == "U" and source.istext:
378 # unicode escape (exactly eight digits)
379 escape += source.getwhile(8, HEXDIGITS)
380 if len(escape) != 10:
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200381 raise source.error("incomplete escape %s" % escape, len(escape))
Antoine Pitrou463badf2012-06-23 13:29:19 +0200382 c = int(escape[2:], 16)
383 chr(c) # raise ValueError for invalid code
384 return LITERAL, c
Serhiy Storchakaa445feb2018-02-10 00:08:17 +0200385 elif c == "N" and source.istext:
Zhou Fangyi5df52862018-02-10 06:59:29 +0000386 import unicodedata
Serhiy Storchakaa445feb2018-02-10 00:08:17 +0200387 # named unicode escape e.g. \N{EM DASH}
388 if not source.match('{'):
389 raise source.error("missing {")
390 charname = source.getuntil('}', 'character name')
391 try:
392 c = ord(unicodedata.lookup(charname))
393 except KeyError:
394 raise source.error("undefined character name %r" % charname,
395 len(charname) + len(r'\N{}'))
396 return LITERAL, c
Gustavo Niemeyera01a2ee2004-09-03 17:06:10 +0000397 elif c == "0":
Fredrik Lundh0c4fdba2000-08-31 22:57:55 +0000398 # octal escape
Antoine Pitrou463badf2012-06-23 13:29:19 +0200399 escape += source.getwhile(2, OCTDIGITS)
Serhiy Storchakac563caf2014-09-23 23:22:41 +0300400 return LITERAL, int(escape[1:], 8)
Gustavo Niemeyera01a2ee2004-09-03 17:06:10 +0000401 elif c in DIGITS:
Fredrik Lundh0c4fdba2000-08-31 22:57:55 +0000402 # octal escape *or* decimal group reference (sigh)
Fredrik Lundh0c4fdba2000-08-31 22:57:55 +0000403 if source.next in DIGITS:
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300404 escape += source.get()
Fredrik Lundh143328b2000-09-02 11:03:34 +0000405 if (escape[1] in OCTDIGITS and escape[2] in OCTDIGITS and
406 source.next in OCTDIGITS):
Fredrik Lundh0c4fdba2000-08-31 22:57:55 +0000407 # got three octal digits; this is an octal escape
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300408 escape += source.get()
Serhiy Storchakac563caf2014-09-23 23:22:41 +0300409 c = int(escape[1:], 8)
410 if c > 0o377:
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200411 raise source.error('octal escape value %s outside of '
Serhiy Storchakaad446d52014-11-10 13:49:00 +0200412 'range 0-0o377' % escape,
413 len(escape))
Serhiy Storchakac563caf2014-09-23 23:22:41 +0300414 return LITERAL, c
Gustavo Niemeyera01a2ee2004-09-03 17:06:10 +0000415 # not an octal escape, so this is a group reference
416 group = int(escape[1:])
417 if group < state.groups:
Fredrik Lundhebc37b22000-10-28 19:30:41 +0000418 if not state.checkgroup(group):
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200419 raise source.error("cannot refer to an open group",
Serhiy Storchakaad446d52014-11-10 13:49:00 +0200420 len(escape))
Serhiy Storchaka4eea62f2015-02-21 10:07:35 +0200421 state.checklookbehindgroup(group, source)
Fredrik Lundh0c4fdba2000-08-31 22:57:55 +0000422 return GROUPREF, group
Serhiy Storchaka662cef62016-10-23 12:11:19 +0300423 raise source.error("invalid group reference %d" % group, len(escape) - 1)
Fredrik Lundh90a07912000-06-30 07:50:59 +0000424 if len(escape) == 2:
Serhiy Storchakaa54aae02015-03-24 22:58:14 +0200425 if c in ASCIILETTERS:
Serhiy Storchaka9bd85b82016-06-11 19:15:00 +0300426 raise source.error("bad escape %s" % escape, len(escape))
Fredrik Lundh0640e112000-06-30 13:55:15 +0000427 return LITERAL, ord(escape[1])
Andrew M. Kuchling815d5b92000-06-09 14:08:07 +0000428 except ValueError:
Fredrik Lundh90a07912000-06-30 07:50:59 +0000429 pass
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200430 raise source.error("bad escape %s" % escape, len(escape))
Andrew M. Kuchling815d5b92000-06-09 14:08:07 +0000431
Serhiy Storchaka821a9d12017-05-14 08:32:33 +0300432def _uniq(items):
433 if len(set(items)) == len(items):
434 return items
435 newitems = []
436 for item in items:
437 if item not in newitems:
438 newitems.append(item)
439 return newitems
440
Serhiy Storchakac7ac7282017-05-16 15:16:15 +0300441def _parse_sub(source, state, verbose, nested):
Fredrik Lundh8a3ebf82000-07-23 21:46:17 +0000442 # parse an alternation: a|b|c
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000443
Fredrik Lundh8a3ebf82000-07-23 21:46:17 +0000444 items = []
Raymond Hettinger968c56a2004-03-26 23:24:00 +0000445 itemsappend = items.append
446 sourcematch = source.match
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200447 start = source.tell()
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300448 while True:
Serhiy Storchakac7ac7282017-05-16 15:16:15 +0300449 itemsappend(_parse(source, state, verbose, nested + 1,
450 not nested and not items))
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300451 if not sourcematch("|"):
Fredrik Lundh8a3ebf82000-07-23 21:46:17 +0000452 break
Fredrik Lundh8a3ebf82000-07-23 21:46:17 +0000453
454 if len(items) == 1:
455 return items[0]
456
457 subpattern = SubPattern(state)
Andrew M. Kuchling815d5b92000-06-09 14:08:07 +0000458
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000459 # check if all items share a common prefix
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300460 while True:
Fredrik Lundh90a07912000-06-30 07:50:59 +0000461 prefix = None
462 for item in items:
463 if not item:
464 break
465 if prefix is None:
466 prefix = item[0]
467 elif item[0] != prefix:
468 break
469 else:
470 # all subitems start with a common "prefix".
471 # move it out of the branch
472 for item in items:
473 del item[0]
Serhiy Storchaka821a9d12017-05-14 08:32:33 +0300474 subpattern.append(prefix)
Fredrik Lundh90a07912000-06-30 07:50:59 +0000475 continue # check next one
476 break
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000477
478 # check if the branch can be replaced by a character set
Serhiy Storchaka821a9d12017-05-14 08:32:33 +0300479 set = []
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000480 for item in items:
Serhiy Storchaka821a9d12017-05-14 08:32:33 +0300481 if len(item) != 1:
482 break
483 op, av = item[0]
484 if op is LITERAL:
485 set.append((op, av))
486 elif op is IN and av[0][0] is not NEGATE:
487 set.extend(av)
488 else:
Fredrik Lundh90a07912000-06-30 07:50:59 +0000489 break
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000490 else:
Fredrik Lundh90a07912000-06-30 07:50:59 +0000491 # we can store this as a character set instead of a
Fredrik Lundh8a3ebf82000-07-23 21:46:17 +0000492 # branch (the compiler may optimize this even more)
Serhiy Storchaka821a9d12017-05-14 08:32:33 +0300493 subpattern.append((IN, _uniq(set)))
Fredrik Lundh90a07912000-06-30 07:50:59 +0000494 return subpattern
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000495
496 subpattern.append((BRANCH, (None, items)))
Andrew M. Kuchling815d5b92000-06-09 14:08:07 +0000497 return subpattern
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000498
Serhiy Storchakac7ac7282017-05-16 15:16:15 +0300499def _parse(source, state, verbose, nested, first=False):
Fredrik Lundh8a3ebf82000-07-23 21:46:17 +0000500 # parse a simple pattern
Andrew M. Kuchling815d5b92000-06-09 14:08:07 +0000501 subpattern = SubPattern(state)
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000502
Raymond Hettinger968c56a2004-03-26 23:24:00 +0000503 # precompute constants into local variables
504 subpatternappend = subpattern.append
505 sourceget = source.get
506 sourcematch = source.match
507 _len = len
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300508 _ord = ord
Raymond Hettinger968c56a2004-03-26 23:24:00 +0000509
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300510 while True:
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000511
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300512 this = source.next
Fredrik Lundh90a07912000-06-30 07:50:59 +0000513 if this is None:
514 break # end of pattern
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300515 if this in "|)":
516 break # end of subpattern
517 sourceget()
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000518
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300519 if verbose:
Fredrik Lundh90a07912000-06-30 07:50:59 +0000520 # skip whitespace and comments
521 if this in WHITESPACE:
522 continue
523 if this == "#":
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300524 while True:
Raymond Hettinger968c56a2004-03-26 23:24:00 +0000525 this = sourceget()
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300526 if this is None or this == "\n":
Fredrik Lundh90a07912000-06-30 07:50:59 +0000527 break
528 continue
Andrew M. Kuchling815d5b92000-06-09 14:08:07 +0000529
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300530 if this[0] == "\\":
531 code = _escape(source, this, state)
532 subpatternappend(code)
533
534 elif this not in SPECIAL_CHARS:
535 subpatternappend((LITERAL, _ord(this)))
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000536
Fredrik Lundh90a07912000-06-30 07:50:59 +0000537 elif this == "[":
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200538 here = source.tell() - 1
Fredrik Lundh90a07912000-06-30 07:50:59 +0000539 # character set
540 set = []
Raymond Hettinger968c56a2004-03-26 23:24:00 +0000541 setappend = set.append
542## if sourcematch(":"):
Fredrik Lundh90a07912000-06-30 07:50:59 +0000543## pass # handle character classes
Serhiy Storchaka05cb7282017-11-16 12:38:26 +0200544 if source.next == '[':
545 import warnings
546 warnings.warn(
547 'Possible nested set at position %d' % source.tell(),
548 FutureWarning, stacklevel=nested + 6
549 )
Serhiy Storchaka821a9d12017-05-14 08:32:33 +0300550 negate = sourcematch("^")
Fredrik Lundh90a07912000-06-30 07:50:59 +0000551 # check remaining characters
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300552 while True:
Raymond Hettinger968c56a2004-03-26 23:24:00 +0000553 this = sourceget()
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300554 if this is None:
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200555 raise source.error("unterminated character set",
556 source.tell() - here)
Serhiy Storchaka821a9d12017-05-14 08:32:33 +0300557 if this == "]" and set:
Fredrik Lundh90a07912000-06-30 07:50:59 +0000558 break
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300559 elif this[0] == "\\":
Fredrik Lundh90a07912000-06-30 07:50:59 +0000560 code1 = _class_escape(source, this)
Fredrik Lundh90a07912000-06-30 07:50:59 +0000561 else:
Serhiy Storchaka05cb7282017-11-16 12:38:26 +0200562 if set and this in '-&~|' and source.next == this:
563 import warnings
564 warnings.warn(
565 'Possible set %s at position %d' % (
566 'difference' if this == '-' else
567 'intersection' if this == '&' else
568 'symmetric difference' if this == '~' else
569 'union',
570 source.tell() - 1),
571 FutureWarning, stacklevel=nested + 6
572 )
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300573 code1 = LITERAL, _ord(this)
Raymond Hettinger968c56a2004-03-26 23:24:00 +0000574 if sourcematch("-"):
Fredrik Lundh90a07912000-06-30 07:50:59 +0000575 # potential range
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200576 that = sourceget()
577 if that is None:
578 raise source.error("unterminated character set",
579 source.tell() - here)
580 if that == "]":
Fredrik Lundh025468d2000-10-07 10:16:19 +0000581 if code1[0] is IN:
582 code1 = code1[1][0]
Raymond Hettinger968c56a2004-03-26 23:24:00 +0000583 setappend(code1)
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300584 setappend((LITERAL, _ord("-")))
Fredrik Lundh90a07912000-06-30 07:50:59 +0000585 break
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200586 if that[0] == "\\":
587 code2 = _class_escape(source, that)
Guido van Rossum41c99e72003-04-14 17:59:34 +0000588 else:
Serhiy Storchaka05cb7282017-11-16 12:38:26 +0200589 if that == '-':
590 import warnings
591 warnings.warn(
592 'Possible set difference at position %d' % (
593 source.tell() - 2),
594 FutureWarning, stacklevel=nested + 6
595 )
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200596 code2 = LITERAL, _ord(that)
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300597 if code1[0] != LITERAL or code2[0] != LITERAL:
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200598 msg = "bad character range %s-%s" % (this, that)
599 raise source.error(msg, len(this) + 1 + len(that))
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300600 lo = code1[1]
601 hi = code2[1]
602 if hi < lo:
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200603 msg = "bad character range %s-%s" % (this, that)
604 raise source.error(msg, len(this) + 1 + len(that))
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300605 setappend((RANGE, (lo, hi)))
Fredrik Lundh90a07912000-06-30 07:50:59 +0000606 else:
607 if code1[0] is IN:
608 code1 = code1[1][0]
Raymond Hettinger968c56a2004-03-26 23:24:00 +0000609 setappend(code1)
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000610
Serhiy Storchaka821a9d12017-05-14 08:32:33 +0300611 set = _uniq(set)
Fredrik Lundh770617b2001-01-14 15:06:11 +0000612 # XXX: <fl> should move set optimization to compiler!
Serhiy Storchaka821a9d12017-05-14 08:32:33 +0300613 if _len(set) == 1 and set[0][0] is LITERAL:
614 # optimization
615 if negate:
616 subpatternappend((NOT_LITERAL, set[0][1]))
617 else:
618 subpatternappend(set[0])
Fredrik Lundh90a07912000-06-30 07:50:59 +0000619 else:
Serhiy Storchaka821a9d12017-05-14 08:32:33 +0300620 if negate:
621 set.insert(0, (NEGATE, None))
622 # charmap optimization can't be added here because
623 # global flags still are not known
Raymond Hettinger968c56a2004-03-26 23:24:00 +0000624 subpatternappend((IN, set))
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000625
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300626 elif this in REPEAT_CHARS:
Fredrik Lundh90a07912000-06-30 07:50:59 +0000627 # repeat previous item
Serhiy Storchakaad446d52014-11-10 13:49:00 +0200628 here = source.tell()
Fredrik Lundh90a07912000-06-30 07:50:59 +0000629 if this == "?":
630 min, max = 0, 1
631 elif this == "*":
632 min, max = 0, MAXREPEAT
Fredrik Lundhc0c7ee32001-02-18 21:04:48 +0000633
Fredrik Lundh90a07912000-06-30 07:50:59 +0000634 elif this == "+":
635 min, max = 1, MAXREPEAT
636 elif this == "{":
Gustavo Niemeyer6fa0c5a2005-09-14 08:54:39 +0000637 if source.next == "}":
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300638 subpatternappend((LITERAL, _ord(this)))
Gustavo Niemeyer6fa0c5a2005-09-14 08:54:39 +0000639 continue
Serhiy Storchaka821a9d12017-05-14 08:32:33 +0300640
Fredrik Lundh90a07912000-06-30 07:50:59 +0000641 min, max = 0, MAXREPEAT
642 lo = hi = ""
643 while source.next in DIGITS:
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300644 lo += sourceget()
Raymond Hettinger968c56a2004-03-26 23:24:00 +0000645 if sourcematch(","):
Fredrik Lundh90a07912000-06-30 07:50:59 +0000646 while source.next in DIGITS:
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300647 hi += sourceget()
Fredrik Lundh90a07912000-06-30 07:50:59 +0000648 else:
649 hi = lo
Raymond Hettinger968c56a2004-03-26 23:24:00 +0000650 if not sourcematch("}"):
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300651 subpatternappend((LITERAL, _ord(this)))
Fredrik Lundhc13222c2000-07-01 23:49:14 +0000652 source.seek(here)
653 continue
Serhiy Storchaka821a9d12017-05-14 08:32:33 +0300654
Fredrik Lundh90a07912000-06-30 07:50:59 +0000655 if lo:
Barry Warsaw8bee7612004-08-25 02:22:30 +0000656 min = int(lo)
Serhiy Storchaka70ca0212013-02-16 16:47:47 +0200657 if min >= MAXREPEAT:
658 raise OverflowError("the repetition number is too large")
Fredrik Lundh90a07912000-06-30 07:50:59 +0000659 if hi:
Barry Warsaw8bee7612004-08-25 02:22:30 +0000660 max = int(hi)
Serhiy Storchaka70ca0212013-02-16 16:47:47 +0200661 if max >= MAXREPEAT:
662 raise OverflowError("the repetition number is too large")
663 if max < min:
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200664 raise source.error("min repeat greater than max repeat",
Serhiy Storchakaad446d52014-11-10 13:49:00 +0200665 source.tell() - here)
Fredrik Lundh90a07912000-06-30 07:50:59 +0000666 else:
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200667 raise AssertionError("unsupported quantifier %r" % (char,))
Fredrik Lundh90a07912000-06-30 07:50:59 +0000668 # figure out which item to repeat
669 if subpattern:
670 item = subpattern[-1:]
671 else:
Fredrik Lundhc0c7ee32001-02-18 21:04:48 +0000672 item = None
Serhiy Storchaka821a9d12017-05-14 08:32:33 +0300673 if not item or item[0][0] is AT:
Serhiy Storchakaad446d52014-11-10 13:49:00 +0200674 raise source.error("nothing to repeat",
675 source.tell() - here + len(this))
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300676 if item[0][0] in _REPEATCODES:
Serhiy Storchakaad446d52014-11-10 13:49:00 +0200677 raise source.error("multiple repeat",
678 source.tell() - here + len(this))
Serhiy Storchaka821a9d12017-05-14 08:32:33 +0300679 if item[0][0] is SUBPATTERN:
680 group, add_flags, del_flags, p = item[0][1]
681 if group is None and not add_flags and not del_flags:
682 item = p
Raymond Hettinger968c56a2004-03-26 23:24:00 +0000683 if sourcematch("?"):
Fredrik Lundh90a07912000-06-30 07:50:59 +0000684 subpattern[-1] = (MIN_REPEAT, (min, max, item))
685 else:
686 subpattern[-1] = (MAX_REPEAT, (min, max, item))
Andrew M. Kuchling815d5b92000-06-09 14:08:07 +0000687
Fredrik Lundh90a07912000-06-30 07:50:59 +0000688 elif this == ".":
Raymond Hettinger968c56a2004-03-26 23:24:00 +0000689 subpatternappend((ANY, None))
Andrew M. Kuchling815d5b92000-06-09 14:08:07 +0000690
Fredrik Lundh90a07912000-06-30 07:50:59 +0000691 elif this == "(":
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200692 start = source.tell() - 1
693 group = True
Fredrik Lundh90a07912000-06-30 07:50:59 +0000694 name = None
Serhiy Storchakabe9a4e52016-09-10 00:57:55 +0300695 add_flags = 0
696 del_flags = 0
Raymond Hettinger968c56a2004-03-26 23:24:00 +0000697 if sourcematch("?"):
Fredrik Lundh90a07912000-06-30 07:50:59 +0000698 # options
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300699 char = sourceget()
700 if char is None:
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200701 raise source.error("unexpected end of pattern")
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300702 if char == "P":
Fredrik Lundh90a07912000-06-30 07:50:59 +0000703 # python extensions
Raymond Hettinger968c56a2004-03-26 23:24:00 +0000704 if sourcematch("<"):
Fredrik Lundh90a07912000-06-30 07:50:59 +0000705 # named group: skip forward to end of name
Serhiy Storchakaa445feb2018-02-10 00:08:17 +0200706 name = source.getuntil(">", "group name")
Georg Brandl1d472b72013-04-14 11:40:00 +0200707 if not name.isidentifier():
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200708 msg = "bad character in group name %r" % name
709 raise source.error(msg, len(name) + 1)
Raymond Hettinger968c56a2004-03-26 23:24:00 +0000710 elif sourcematch("="):
Fredrik Lundh90a07912000-06-30 07:50:59 +0000711 # named backreference
Serhiy Storchakaa445feb2018-02-10 00:08:17 +0200712 name = source.getuntil(")", "group name")
Georg Brandl1d472b72013-04-14 11:40:00 +0200713 if not name.isidentifier():
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200714 msg = "bad character in group name %r" % name
715 raise source.error(msg, len(name) + 1)
Fredrik Lundhb71624e2000-06-30 09:13:06 +0000716 gid = state.groupdict.get(name)
717 if gid is None:
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200718 msg = "unknown group name %r" % name
Serhiy Storchakaad446d52014-11-10 13:49:00 +0200719 raise source.error(msg, len(name) + 1)
Serhiy Storchaka485407c2015-07-18 23:27:00 +0300720 if not state.checkgroup(gid):
721 raise source.error("cannot refer to an open group",
722 len(name) + 1)
Serhiy Storchaka4eea62f2015-02-21 10:07:35 +0200723 state.checklookbehindgroup(gid, source)
Raymond Hettinger968c56a2004-03-26 23:24:00 +0000724 subpatternappend((GROUPREF, gid))
Fredrik Lundh7cafe4d2000-07-02 17:33:27 +0000725 continue
Serhiy Storchaka821a9d12017-05-14 08:32:33 +0300726
Fredrik Lundh90a07912000-06-30 07:50:59 +0000727 else:
Raymond Hettinger968c56a2004-03-26 23:24:00 +0000728 char = sourceget()
Fredrik Lundh90a07912000-06-30 07:50:59 +0000729 if char is None:
Serhiy Storchakaad446d52014-11-10 13:49:00 +0200730 raise source.error("unexpected end of pattern")
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200731 raise source.error("unknown extension ?P" + char,
732 len(char) + 2)
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300733 elif char == ":":
Fredrik Lundh90a07912000-06-30 07:50:59 +0000734 # non-capturing group
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200735 group = None
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300736 elif char == "#":
Fredrik Lundh90a07912000-06-30 07:50:59 +0000737 # comment
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300738 while True:
739 if source.next is None:
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200740 raise source.error("missing ), unterminated comment",
741 source.tell() - start)
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300742 if sourceget() == ")":
Fredrik Lundh90a07912000-06-30 07:50:59 +0000743 break
Fredrik Lundh0c4fdba2000-08-31 22:57:55 +0000744 continue
Serhiy Storchaka821a9d12017-05-14 08:32:33 +0300745
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300746 elif char in "=!<":
Fredrik Lundh43b3b492000-06-30 10:41:31 +0000747 # lookahead assertions
Fredrik Lundh6f013982000-07-03 18:44:21 +0000748 dir = 1
749 if char == "<":
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300750 char = sourceget()
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200751 if char is None:
752 raise source.error("unexpected end of pattern")
753 if char not in "=!":
754 raise source.error("unknown extension ?<" + char,
755 len(char) + 2)
Fredrik Lundh6f013982000-07-03 18:44:21 +0000756 dir = -1 # lookbehind
Serhiy Storchaka4eea62f2015-02-21 10:07:35 +0200757 lookbehindgroups = state.lookbehindgroups
758 if lookbehindgroups is None:
759 state.lookbehindgroups = state.groups
Serhiy Storchakac7ac7282017-05-16 15:16:15 +0300760 p = _parse_sub(source, state, verbose, nested + 1)
Serhiy Storchaka4eea62f2015-02-21 10:07:35 +0200761 if dir < 0:
762 if lookbehindgroups is None:
763 state.lookbehindgroups = None
Raymond Hettinger968c56a2004-03-26 23:24:00 +0000764 if not sourcematch(")"):
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200765 raise source.error("missing ), unterminated subpattern",
766 source.tell() - start)
Fredrik Lundh8a3ebf82000-07-23 21:46:17 +0000767 if char == "=":
Raymond Hettinger968c56a2004-03-26 23:24:00 +0000768 subpatternappend((ASSERT, (dir, p)))
Fredrik Lundh8a3ebf82000-07-23 21:46:17 +0000769 else:
Raymond Hettinger968c56a2004-03-26 23:24:00 +0000770 subpatternappend((ASSERT_NOT, (dir, p)))
Fredrik Lundh8a3ebf82000-07-23 21:46:17 +0000771 continue
Serhiy Storchaka821a9d12017-05-14 08:32:33 +0300772
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300773 elif char == "(":
Gustavo Niemeyerad3fc442003-10-17 22:13:16 +0000774 # conditional backreference group
Serhiy Storchakaa445feb2018-02-10 00:08:17 +0200775 condname = source.getuntil(")", "group name")
Georg Brandl1d472b72013-04-14 11:40:00 +0200776 if condname.isidentifier():
Gustavo Niemeyerad3fc442003-10-17 22:13:16 +0000777 condgroup = state.groupdict.get(condname)
778 if condgroup is None:
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200779 msg = "unknown group name %r" % condname
Serhiy Storchakaad446d52014-11-10 13:49:00 +0200780 raise source.error(msg, len(condname) + 1)
Gustavo Niemeyerad3fc442003-10-17 22:13:16 +0000781 else:
782 try:
Barry Warsaw8bee7612004-08-25 02:22:30 +0000783 condgroup = int(condname)
Serhiy Storchaka9baa5b22014-09-29 22:49:23 +0300784 if condgroup < 0:
785 raise ValueError
Gustavo Niemeyerad3fc442003-10-17 22:13:16 +0000786 except ValueError:
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200787 msg = "bad character in group name %r" % condname
788 raise source.error(msg, len(condname) + 1) from None
Serhiy Storchaka9baa5b22014-09-29 22:49:23 +0300789 if not condgroup:
Serhiy Storchakaad446d52014-11-10 13:49:00 +0200790 raise source.error("bad group number",
791 len(condname) + 1)
Serhiy Storchaka9baa5b22014-09-29 22:49:23 +0300792 if condgroup >= MAXGROUPS:
Serhiy Storchaka662cef62016-10-23 12:11:19 +0300793 msg = "invalid group reference %d" % condgroup
794 raise source.error(msg, len(condname) + 1)
Serhiy Storchaka4eea62f2015-02-21 10:07:35 +0200795 state.checklookbehindgroup(condgroup, source)
Serhiy Storchakac7ac7282017-05-16 15:16:15 +0300796 item_yes = _parse(source, state, verbose, nested + 1)
Serhiy Storchaka821a9d12017-05-14 08:32:33 +0300797 if source.match("|"):
Serhiy Storchakac7ac7282017-05-16 15:16:15 +0300798 item_no = _parse(source, state, verbose, nested + 1)
Serhiy Storchaka821a9d12017-05-14 08:32:33 +0300799 if source.next == "|":
800 raise source.error("conditional backref with more than two branches")
801 else:
802 item_no = None
803 if not source.match(")"):
804 raise source.error("missing ), unterminated subpattern",
805 source.tell() - start)
806 subpatternappend((GROUPREF_EXISTS, (condgroup, item_yes, item_no)))
807 continue
808
Serhiy Storchakabe9a4e52016-09-10 00:57:55 +0300809 elif char in FLAGS or char == "-":
Fredrik Lundh90a07912000-06-30 07:50:59 +0000810 # flags
Serhiy Storchakabe9a4e52016-09-10 00:57:55 +0300811 flags = _parse_flags(source, state, char)
812 if flags is None: # global flags
Serhiy Storchaka305ccbe2017-05-10 06:05:20 +0300813 if not first or subpattern:
Serhiy Storchakabd48d272016-09-11 12:50:02 +0300814 import warnings
Serhiy Storchakaabf275a2016-09-17 01:29:58 +0300815 warnings.warn(
Roy Williams171b9a32017-06-09 22:01:16 -0700816 'Flags not at the start of the expression %r%s' % (
Serhiy Storchakaabf275a2016-09-17 01:29:58 +0300817 source.string[:20], # truncate long regexes
818 ' (truncated)' if len(source.string) > 20 else '',
819 ),
Serhiy Storchakac7ac7282017-05-16 15:16:15 +0300820 DeprecationWarning, stacklevel=nested + 6
Serhiy Storchakaabf275a2016-09-17 01:29:58 +0300821 )
Serhiy Storchaka305ccbe2017-05-10 06:05:20 +0300822 if (state.flags & SRE_FLAG_VERBOSE) and not verbose:
823 raise Verbose
Serhiy Storchakabe9a4e52016-09-10 00:57:55 +0300824 continue
Serhiy Storchaka821a9d12017-05-14 08:32:33 +0300825
Serhiy Storchakabe9a4e52016-09-10 00:57:55 +0300826 add_flags, del_flags = flags
827 group = None
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300828 else:
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200829 raise source.error("unknown extension ?" + char,
830 len(char) + 1)
831
832 # parse group contents
833 if group is not None:
834 try:
835 group = state.opengroup(name)
836 except error as err:
837 raise source.error(err.msg, len(name) + 1) from None
Serhiy Storchaka821a9d12017-05-14 08:32:33 +0300838 sub_verbose = ((verbose or (add_flags & SRE_FLAG_VERBOSE)) and
839 not (del_flags & SRE_FLAG_VERBOSE))
Serhiy Storchakac7ac7282017-05-16 15:16:15 +0300840 p = _parse_sub(source, state, sub_verbose, nested + 1)
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200841 if not source.match(")"):
842 raise source.error("missing ), unterminated subpattern",
843 source.tell() - start)
844 if group is not None:
845 state.closegroup(group, p)
Serhiy Storchakabe9a4e52016-09-10 00:57:55 +0300846 subpatternappend((SUBPATTERN, (group, add_flags, del_flags, p)))
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000847
Fredrik Lundh90a07912000-06-30 07:50:59 +0000848 elif this == "^":
Raymond Hettinger968c56a2004-03-26 23:24:00 +0000849 subpatternappend((AT, AT_BEGINNING))
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000850
Fredrik Lundh90a07912000-06-30 07:50:59 +0000851 elif this == "$":
Serhiy Storchaka821a9d12017-05-14 08:32:33 +0300852 subpatternappend((AT, AT_END))
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000853
Fredrik Lundh90a07912000-06-30 07:50:59 +0000854 else:
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200855 raise AssertionError("unsupported special character %r" % (char,))
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000856
Serhiy Storchaka821a9d12017-05-14 08:32:33 +0300857 # unpack non-capturing groups
858 for i in range(len(subpattern))[::-1]:
859 op, av = subpattern[i]
860 if op is SUBPATTERN:
861 group, add_flags, del_flags, p = av
862 if group is None and not add_flags and not del_flags:
863 subpattern[i: i+1] = p
864
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000865 return subpattern
866
Serhiy Storchakabe9a4e52016-09-10 00:57:55 +0300867def _parse_flags(source, state, char):
868 sourceget = source.get
869 add_flags = 0
870 del_flags = 0
871 if char != "-":
872 while True:
Serhiy Storchaka3557b052017-10-24 23:31:42 +0300873 flag = FLAGS[char]
874 if source.istext:
875 if char == 'L':
876 msg = "bad inline flags: cannot use 'L' flag with a str pattern"
877 raise source.error(msg)
878 else:
879 if char == 'u':
880 msg = "bad inline flags: cannot use 'u' flag with a bytes pattern"
881 raise source.error(msg)
882 add_flags |= flag
883 if (flag & TYPE_FLAGS) and (add_flags & TYPE_FLAGS) != flag:
884 msg = "bad inline flags: flags 'a', 'u' and 'L' are incompatible"
885 raise source.error(msg)
Serhiy Storchakabe9a4e52016-09-10 00:57:55 +0300886 char = sourceget()
887 if char is None:
888 raise source.error("missing -, : or )")
889 if char in ")-:":
890 break
891 if char not in FLAGS:
892 msg = "unknown flag" if char.isalpha() else "missing -, : or )"
893 raise source.error(msg, len(char))
894 if char == ")":
Serhiy Storchakabe9a4e52016-09-10 00:57:55 +0300895 state.flags |= add_flags
896 return None
897 if add_flags & GLOBAL_FLAGS:
898 raise source.error("bad inline flags: cannot turn on global flag", 1)
899 if char == "-":
900 char = sourceget()
901 if char is None:
902 raise source.error("missing flag")
903 if char not in FLAGS:
904 msg = "unknown flag" if char.isalpha() else "missing flag"
905 raise source.error(msg, len(char))
906 while True:
Serhiy Storchaka3557b052017-10-24 23:31:42 +0300907 flag = FLAGS[char]
908 if flag & TYPE_FLAGS:
909 msg = "bad inline flags: cannot turn off flags 'a', 'u' and 'L'"
910 raise source.error(msg)
911 del_flags |= flag
Serhiy Storchakabe9a4e52016-09-10 00:57:55 +0300912 char = sourceget()
913 if char is None:
914 raise source.error("missing :")
915 if char == ":":
916 break
917 if char not in FLAGS:
918 msg = "unknown flag" if char.isalpha() else "missing :"
919 raise source.error(msg, len(char))
920 assert char == ":"
921 if del_flags & GLOBAL_FLAGS:
922 raise source.error("bad inline flags: cannot turn off global flag", 1)
923 if add_flags & del_flags:
924 raise source.error("bad inline flags: flag turned on and off", 1)
925 return add_flags, del_flags
926
Antoine Pitroufd036452008-08-19 17:56:33 +0000927def fix_flags(src, flags):
928 # Check and fix flags according to the type of pattern (str or bytes)
929 if isinstance(src, str):
Serhiy Storchaka22a309a2014-12-01 11:50:07 +0200930 if flags & SRE_FLAG_LOCALE:
Serhiy Storchaka9bd85b82016-06-11 19:15:00 +0300931 raise ValueError("cannot use LOCALE flag with a str pattern")
Antoine Pitroufd036452008-08-19 17:56:33 +0000932 if not flags & SRE_FLAG_ASCII:
933 flags |= SRE_FLAG_UNICODE
934 elif flags & SRE_FLAG_UNICODE:
935 raise ValueError("ASCII and UNICODE flags are incompatible")
936 else:
937 if flags & SRE_FLAG_UNICODE:
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200938 raise ValueError("cannot use UNICODE flag with a bytes pattern")
Serhiy Storchaka22a309a2014-12-01 11:50:07 +0200939 if flags & SRE_FLAG_LOCALE and flags & SRE_FLAG_ASCII:
Serhiy Storchaka9bd85b82016-06-11 19:15:00 +0300940 raise ValueError("ASCII and LOCALE flags are incompatible")
Antoine Pitroufd036452008-08-19 17:56:33 +0000941 return flags
942
Serhiy Storchakae0c19dd2018-09-18 09:16:26 +0300943def parse(str, flags=0, state=None):
Andrew M. Kuchling815d5b92000-06-09 14:08:07 +0000944 # parse 're' pattern into list of (opcode, argument) tuples
Fredrik Lundh8a3ebf82000-07-23 21:46:17 +0000945
946 source = Tokenizer(str)
947
Serhiy Storchakae0c19dd2018-09-18 09:16:26 +0300948 if state is None:
949 state = State()
950 state.flags = flags
951 state.str = str
Fredrik Lundh8a3ebf82000-07-23 21:46:17 +0000952
Serhiy Storchakabe9a4e52016-09-10 00:57:55 +0300953 try:
Serhiy Storchakae0c19dd2018-09-18 09:16:26 +0300954 p = _parse_sub(source, state, flags & SRE_FLAG_VERBOSE, 0)
Serhiy Storchakabe9a4e52016-09-10 00:57:55 +0300955 except Verbose:
956 # the VERBOSE flag was switched on inside the pattern. to be
957 # on the safe side, we'll parse the whole thing again...
Serhiy Storchakae0c19dd2018-09-18 09:16:26 +0300958 state = State()
959 state.flags = flags | SRE_FLAG_VERBOSE
960 state.str = str
Serhiy Storchakad65cd092016-09-11 01:39:01 +0300961 source.seek(0)
Serhiy Storchakae0c19dd2018-09-18 09:16:26 +0300962 p = _parse_sub(source, state, True, 0)
Serhiy Storchakabe9a4e52016-09-10 00:57:55 +0300963
Serhiy Storchakae0c19dd2018-09-18 09:16:26 +0300964 p.state.flags = fix_flags(str, p.state.flags)
Fredrik Lundh8a3ebf82000-07-23 21:46:17 +0000965
Serhiy Storchakae2ccf562014-10-10 11:14:49 +0300966 if source.next is not None:
Serhiy Storchaka632a77e2015-03-25 21:03:47 +0200967 assert source.next == ")"
968 raise source.error("unbalanced parenthesis")
Fredrik Lundh8a3ebf82000-07-23 21:46:17 +0000969
Serhiy Storchakaa01a1442016-03-06 09:15:47 +0200970 if flags & SRE_FLAG_DEBUG:
971 p.dump()
972
Guido van Rossum7627c0d2000-03-31 14:58:54 +0000973 return p
974
Serhiy Storchakae0c19dd2018-09-18 09:16:26 +0300975def parse_template(source, state):
Andrew M. Kuchling815d5b92000-06-09 14:08:07 +0000976 # parse 're' replacement string into list of literals and
977 # group references
978 s = Tokenizer(source)
Raymond Hettinger968c56a2004-03-26 23:24:00 +0000979 sget = s.get
Serhiy Storchaka9c15ec12013-10-23 22:27:52 +0300980 groups = []
981 literals = []
982 literal = []
983 lappend = literal.append
Serhiy Storchaka662cef62016-10-23 12:11:19 +0300984 def addgroup(index, pos):
Serhiy Storchakae0c19dd2018-09-18 09:16:26 +0300985 if index > state.groups:
Serhiy Storchaka662cef62016-10-23 12:11:19 +0300986 raise s.error("invalid group reference %d" % index, pos)
Serhiy Storchaka9c15ec12013-10-23 22:27:52 +0300987 if literal:
988 literals.append(''.join(literal))
989 del literal[:]
990 groups.append((len(literals), index))
991 literals.append(None)
Serhiy Storchakae0c19dd2018-09-18 09:16:26 +0300992 groupindex = state.groupindex
Serhiy Storchaka9c15ec12013-10-23 22:27:52 +0300993 while True:
Raymond Hettinger968c56a2004-03-26 23:24:00 +0000994 this = sget()
Fredrik Lundh90a07912000-06-30 07:50:59 +0000995 if this is None:
996 break # end of replacement string
Serhiy Storchaka9c15ec12013-10-23 22:27:52 +0300997 if this[0] == "\\":
Fredrik Lundh90a07912000-06-30 07:50:59 +0000998 # group
Serhiy Storchaka9c15ec12013-10-23 22:27:52 +0300999 c = this[1]
Gustavo Niemeyera01a2ee2004-09-03 17:06:10 +00001000 if c == "g":
Fredrik Lundh90a07912000-06-30 07:50:59 +00001001 name = ""
Serhiy Storchaka632a77e2015-03-25 21:03:47 +02001002 if not s.match("<"):
1003 raise s.error("missing <")
Serhiy Storchakaa445feb2018-02-10 00:08:17 +02001004 name = s.getuntil(">", "group name")
Serhiy Storchaka632a77e2015-03-25 21:03:47 +02001005 if name.isidentifier():
Fredrik Lundh90a07912000-06-30 07:50:59 +00001006 try:
Serhiy Storchaka07360df2015-03-30 01:01:48 +03001007 index = groupindex[name]
Fredrik Lundh90a07912000-06-30 07:50:59 +00001008 except KeyError:
Serhiy Storchaka632a77e2015-03-25 21:03:47 +02001009 raise IndexError("unknown group name %r" % name)
1010 else:
1011 try:
1012 index = int(name)
1013 if index < 0:
1014 raise ValueError
1015 except ValueError:
1016 raise s.error("bad character in group name %r" % name,
1017 len(name) + 1) from None
1018 if index >= MAXGROUPS:
Serhiy Storchaka662cef62016-10-23 12:11:19 +03001019 raise s.error("invalid group reference %d" % index,
Serhiy Storchaka632a77e2015-03-25 21:03:47 +02001020 len(name) + 1)
Serhiy Storchaka662cef62016-10-23 12:11:19 +03001021 addgroup(index, len(name) + 1)
Gustavo Niemeyera01a2ee2004-09-03 17:06:10 +00001022 elif c == "0":
1023 if s.next in OCTDIGITS:
Serhiy Storchaka9c15ec12013-10-23 22:27:52 +03001024 this += sget()
Gustavo Niemeyera01a2ee2004-09-03 17:06:10 +00001025 if s.next in OCTDIGITS:
Serhiy Storchaka9c15ec12013-10-23 22:27:52 +03001026 this += sget()
1027 lappend(chr(int(this[1:], 8) & 0xff))
Gustavo Niemeyera01a2ee2004-09-03 17:06:10 +00001028 elif c in DIGITS:
1029 isoctal = False
1030 if s.next in DIGITS:
Serhiy Storchaka9c15ec12013-10-23 22:27:52 +03001031 this += sget()
Gustavo Niemeyerf5a15992004-09-03 20:15:56 +00001032 if (c in OCTDIGITS and this[2] in OCTDIGITS and
1033 s.next in OCTDIGITS):
Serhiy Storchaka9c15ec12013-10-23 22:27:52 +03001034 this += sget()
Gustavo Niemeyera01a2ee2004-09-03 17:06:10 +00001035 isoctal = True
Serhiy Storchakac563caf2014-09-23 23:22:41 +03001036 c = int(this[1:], 8)
1037 if c > 0o377:
Serhiy Storchaka632a77e2015-03-25 21:03:47 +02001038 raise s.error('octal escape value %s outside of '
Serhiy Storchakaad446d52014-11-10 13:49:00 +02001039 'range 0-0o377' % this, len(this))
Serhiy Storchakac563caf2014-09-23 23:22:41 +03001040 lappend(chr(c))
Gustavo Niemeyera01a2ee2004-09-03 17:06:10 +00001041 if not isoctal:
Serhiy Storchaka662cef62016-10-23 12:11:19 +03001042 addgroup(int(this[1:]), len(this) - 1)
Fredrik Lundh90a07912000-06-30 07:50:59 +00001043 else:
1044 try:
Serhiy Storchaka9c15ec12013-10-23 22:27:52 +03001045 this = chr(ESCAPES[this][1])
Fredrik Lundh90a07912000-06-30 07:50:59 +00001046 except KeyError:
Serhiy Storchakaa54aae02015-03-24 22:58:14 +02001047 if c in ASCIILETTERS:
Serhiy Storchaka9bd85b82016-06-11 19:15:00 +03001048 raise s.error('bad escape %s' % this, len(this))
Serhiy Storchaka9c15ec12013-10-23 22:27:52 +03001049 lappend(this)
Fredrik Lundh90a07912000-06-30 07:50:59 +00001050 else:
Serhiy Storchaka9c15ec12013-10-23 22:27:52 +03001051 lappend(this)
1052 if literal:
1053 literals.append(''.join(literal))
1054 if not isinstance(source, str):
Ezio Melottib92ed7c2010-03-06 15:24:08 +00001055 # The tokenizer implicitly decodes bytes objects as latin-1, we must
1056 # therefore re-encode the final representation.
Serhiy Storchaka9c15ec12013-10-23 22:27:52 +03001057 literals = [None if s is None else s.encode('latin-1') for s in literals]
Fredrik Lundhb25e1ad2001-03-22 15:50:10 +00001058 return groups, literals
Andrew M. Kuchling815d5b92000-06-09 14:08:07 +00001059
Fredrik Lundh436c3d582000-06-29 08:58:44 +00001060def expand_template(template, match):
Fredrik Lundhb25e1ad2001-03-22 15:50:10 +00001061 g = match.group
Serhiy Storchaka7438e4b2014-10-10 11:06:31 +03001062 empty = match.string[:0]
Fredrik Lundhb25e1ad2001-03-22 15:50:10 +00001063 groups, literals = template
1064 literals = literals[:]
1065 try:
1066 for index, group in groups:
Serhiy Storchaka7438e4b2014-10-10 11:06:31 +03001067 literals[index] = g(group) or empty
Fredrik Lundhb25e1ad2001-03-22 15:50:10 +00001068 except IndexError:
Serhiy Storchaka662cef62016-10-23 12:11:19 +03001069 raise error("invalid group reference %d" % index)
Serhiy Storchaka7438e4b2014-10-10 11:06:31 +03001070 return empty.join(literals)