blob: f575e9bc237a9258254c29391889cae3e4137572 [file] [log] [blame]
Guido van Rossumb51eaa11997-03-07 00:21:55 +00001"""Tokenization help for Python programs.
Guido van Rossum4d8e8591992-01-01 19:34:47 +00002
Florent Xicluna43e4ea12010-09-03 19:54:02 +00003tokenize(readline) is a generator that breaks a stream of bytes into
4Python tokens. It decodes the bytes according to PEP-0263 for
5determining source file encoding.
Trent Nelson428de652008-03-18 22:41:35 +00006
Florent Xicluna43e4ea12010-09-03 19:54:02 +00007It accepts a readline-like method which is called repeatedly to get the
8next line of input (or b"" for EOF). It generates 5-tuples with these
9members:
Tim Peters4efb6e92001-06-29 23:51:08 +000010
11 the token type (see token.py)
12 the token (a string)
13 the starting (row, column) indices of the token (a 2-tuple of ints)
14 the ending (row, column) indices of the token (a 2-tuple of ints)
15 the original line (string)
16
17It is designed to match the working of the Python tokenizer exactly, except
18that it produces COMMENT tokens for comments and gives type OP for all
Florent Xicluna43e4ea12010-09-03 19:54:02 +000019operators. Additionally, all token lists start with an ENCODING token
20which tells you which encoding was used to decode the bytes stream.
21"""
Guido van Rossumb51eaa11997-03-07 00:21:55 +000022
Ka-Ping Yee244c5932001-03-01 13:56:40 +000023__author__ = 'Ka-Ping Yee <ping@lfw.org>'
Trent Nelson428de652008-03-18 22:41:35 +000024__credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
25 'Skip Montanaro, Raymond Hettinger, Trent Nelson, '
26 'Michael Foord')
Brett Cannon45b96d32011-02-22 03:35:18 +000027import builtins
Florent Xicluna43e4ea12010-09-03 19:54:02 +000028import re
29import sys
Guido van Rossumfc6f5331997-03-07 00:21:12 +000030from token import *
Benjamin Peterson433f32c2008-12-12 01:25:05 +000031from codecs import lookup, BOM_UTF8
Raymond Hettinger3fb79c72010-09-09 07:15:18 +000032import collections
Victor Stinner58c07522010-11-09 01:08:59 +000033from io import TextIOWrapper
Trent Nelson428de652008-03-18 22:41:35 +000034cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
Guido van Rossum4d8e8591992-01-01 19:34:47 +000035
Skip Montanaro40fc1602001-03-01 04:27:19 +000036import token
Alexander Belopolskyb9d10d02010-11-11 14:07:41 +000037__all__ = token.__all__ + ["COMMENT", "tokenize", "detect_encoding",
38 "NL", "untokenize", "ENCODING", "TokenInfo"]
Skip Montanaro40fc1602001-03-01 04:27:19 +000039del token
40
Guido van Rossum1aec3231997-04-08 14:24:39 +000041COMMENT = N_TOKENS
42tok_name[COMMENT] = 'COMMENT'
Guido van Rossuma90c78b1998-04-03 16:05:38 +000043NL = N_TOKENS + 1
44tok_name[NL] = 'NL'
Trent Nelson428de652008-03-18 22:41:35 +000045ENCODING = N_TOKENS + 2
46tok_name[ENCODING] = 'ENCODING'
47N_TOKENS += 3
Guido van Rossum1aec3231997-04-08 14:24:39 +000048
Raymond Hettinger3fb79c72010-09-09 07:15:18 +000049class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')):
Raymond Hettingeraa17a7f2009-04-29 14:21:25 +000050 def __repr__(self):
Raymond Hettingera0e79402010-09-09 08:29:05 +000051 annotated_type = '%d (%s)' % (self.type, tok_name[self.type])
52 return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' %
53 self._replace(type=annotated_type))
Raymond Hettingeraa17a7f2009-04-29 14:21:25 +000054
Eric S. Raymondb08b2d32001-02-09 11:10:16 +000055def group(*choices): return '(' + '|'.join(choices) + ')'
Guido van Rossum68468eb2003-02-27 20:14:51 +000056def any(*choices): return group(*choices) + '*'
57def maybe(*choices): return group(*choices) + '?'
Guido van Rossum4d8e8591992-01-01 19:34:47 +000058
Antoine Pitroufd036452008-08-19 17:56:33 +000059# Note: we use unicode matching for names ("\w") but ascii matching for
60# number literals.
Guido van Rossum3b631771997-10-27 20:44:15 +000061Whitespace = r'[ \f\t]*'
62Comment = r'#[^\r\n]*'
63Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
Benjamin Peterson33856de2010-08-30 14:41:20 +000064Name = r'\w+'
Guido van Rossum4d8e8591992-01-01 19:34:47 +000065
Antoine Pitroufd036452008-08-19 17:56:33 +000066Hexnumber = r'0[xX][0-9a-fA-F]+'
Georg Brandlfceab5a2008-01-19 20:08:23 +000067Binnumber = r'0[bB][01]+'
68Octnumber = r'0[oO][0-7]+'
Antoine Pitroufd036452008-08-19 17:56:33 +000069Decnumber = r'(?:0+|[1-9][0-9]*)'
Guido van Rossumcd16bf62007-06-13 18:07:49 +000070Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
Antoine Pitroufd036452008-08-19 17:56:33 +000071Exponent = r'[eE][-+]?[0-9]+'
72Pointfloat = group(r'[0-9]+\.[0-9]*', r'\.[0-9]+') + maybe(Exponent)
73Expfloat = r'[0-9]+' + Exponent
Guido van Rossum1aec3231997-04-08 14:24:39 +000074Floatnumber = group(Pointfloat, Expfloat)
Antoine Pitroufd036452008-08-19 17:56:33 +000075Imagnumber = group(r'[0-9]+[jJ]', Floatnumber + r'[jJ]')
Guido van Rossum1aec3231997-04-08 14:24:39 +000076Number = group(Imagnumber, Floatnumber, Intnumber)
Guido van Rossum4d8e8591992-01-01 19:34:47 +000077
Tim Petersde495832000-10-07 05:09:39 +000078# Tail end of ' string.
79Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
80# Tail end of " string.
81Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
82# Tail end of ''' string.
83Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
84# Tail end of """ string.
85Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
Guido van Rossum4fe72f92007-11-12 17:40:10 +000086Triple = group("[bB]?[rR]?'''", '[bB]?[rR]?"""')
Tim Petersde495832000-10-07 05:09:39 +000087# Single-line ' or " string.
Guido van Rossum4fe72f92007-11-12 17:40:10 +000088String = group(r"[bB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
89 r'[bB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
Guido van Rossum4d8e8591992-01-01 19:34:47 +000090
Tim Petersde495832000-10-07 05:09:39 +000091# Because of leftmost-then-longest match semantics, be sure to put the
92# longest operators first (e.g., if = came before ==, == would get
93# recognized as two instances of =).
Guido van Rossumb053cd82006-08-24 03:53:23 +000094Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=",
Neal Norwitzc1505362006-12-28 06:47:50 +000095 r"//=?", r"->",
Tim Petersde495832000-10-07 05:09:39 +000096 r"[+\-*/%&|^=<>]=?",
97 r"~")
Thomas Wouterse1519a12000-08-24 21:44:52 +000098
Guido van Rossum4d8e8591992-01-01 19:34:47 +000099Bracket = '[][(){}]'
Georg Brandldde00282007-03-18 19:01:53 +0000100Special = group(r'\r?\n', r'\.\.\.', r'[:;.,@]')
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000101Funny = group(Operator, Bracket, Special)
Guido van Rossum4d8e8591992-01-01 19:34:47 +0000102
Guido van Rossum3b631771997-10-27 20:44:15 +0000103PlainToken = group(Number, Funny, String, Name)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000104Token = Ignore + PlainToken
Guido van Rossum4d8e8591992-01-01 19:34:47 +0000105
Tim Petersde495832000-10-07 05:09:39 +0000106# First (or only) line of ' or " string.
Guido van Rossum4fe72f92007-11-12 17:40:10 +0000107ContStr = group(r"[bB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
Ka-Ping Yee1ff08b12001-01-15 22:04:30 +0000108 group("'", r'\\\r?\n'),
Guido van Rossum4fe72f92007-11-12 17:40:10 +0000109 r'[bB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
Ka-Ping Yee1ff08b12001-01-15 22:04:30 +0000110 group('"', r'\\\r?\n'))
Guido van Rossum3b631771997-10-27 20:44:15 +0000111PseudoExtras = group(r'\\\r?\n', Comment, Triple)
112PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
Guido van Rossum1aec3231997-04-08 14:24:39 +0000113
Benjamin Peterson33856de2010-08-30 14:41:20 +0000114def _compile(expr):
115 return re.compile(expr, re.UNICODE)
116
Guido van Rossum3b631771997-10-27 20:44:15 +0000117tokenprog, pseudoprog, single3prog, double3prog = map(
Benjamin Peterson33856de2010-08-30 14:41:20 +0000118 _compile, (Token, PseudoToken, Single3, Double3))
119endprogs = {"'": _compile(Single), '"': _compile(Double),
Guido van Rossum3b631771997-10-27 20:44:15 +0000120 "'''": single3prog, '"""': double3prog,
Guido van Rossumfefc9221997-10-27 21:17:24 +0000121 "r'''": single3prog, 'r"""': double3prog,
Guido van Rossum4fe72f92007-11-12 17:40:10 +0000122 "b'''": single3prog, 'b"""': double3prog,
123 "br'''": single3prog, 'br"""': double3prog,
Ka-Ping Yee1ff08b12001-01-15 22:04:30 +0000124 "R'''": single3prog, 'R"""': double3prog,
Guido van Rossum4fe72f92007-11-12 17:40:10 +0000125 "B'''": single3prog, 'B"""': double3prog,
126 "bR'''": single3prog, 'bR"""': double3prog,
127 "Br'''": single3prog, 'Br"""': double3prog,
128 "BR'''": single3prog, 'BR"""': double3prog,
129 'r': None, 'R': None, 'b': None, 'B': None}
Guido van Rossum4d8e8591992-01-01 19:34:47 +0000130
Guido van Rossum9d6897a2002-08-24 06:54:19 +0000131triple_quoted = {}
132for t in ("'''", '"""',
133 "r'''", 'r"""', "R'''", 'R"""',
Guido van Rossum4fe72f92007-11-12 17:40:10 +0000134 "b'''", 'b"""', "B'''", 'B"""',
135 "br'''", 'br"""', "Br'''", 'Br"""',
136 "bR'''", 'bR"""', "BR'''", 'BR"""'):
Guido van Rossum9d6897a2002-08-24 06:54:19 +0000137 triple_quoted[t] = t
138single_quoted = {}
139for t in ("'", '"',
140 "r'", 'r"', "R'", 'R"',
Guido van Rossum4fe72f92007-11-12 17:40:10 +0000141 "b'", 'b"', "B'", 'B"',
142 "br'", 'br"', "Br'", 'Br"',
143 "bR'", 'bR"', "BR'", 'BR"' ):
Guido van Rossum9d6897a2002-08-24 06:54:19 +0000144 single_quoted[t] = t
145
Benjamin Peterson33856de2010-08-30 14:41:20 +0000146del _compile
147
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000148tabsize = 8
Fred Drake9b8d8012000-08-17 04:45:13 +0000149
Ka-Ping Yee28c62bb2001-03-23 05:22:49 +0000150class TokenError(Exception): pass
151
152class StopTokenizing(Exception): pass
Fred Drake9b8d8012000-08-17 04:45:13 +0000153
Tim Peters5ca576e2001-06-18 22:08:13 +0000154
Thomas Wouters89f507f2006-12-13 04:49:30 +0000155class Untokenizer:
156
157 def __init__(self):
158 self.tokens = []
159 self.prev_row = 1
160 self.prev_col = 0
Trent Nelson428de652008-03-18 22:41:35 +0000161 self.encoding = None
Thomas Wouters89f507f2006-12-13 04:49:30 +0000162
163 def add_whitespace(self, start):
164 row, col = start
165 assert row <= self.prev_row
166 col_offset = col - self.prev_col
167 if col_offset:
168 self.tokens.append(" " * col_offset)
169
170 def untokenize(self, iterable):
171 for t in iterable:
172 if len(t) == 2:
173 self.compat(t, iterable)
174 break
175 tok_type, token, start, end, line = t
Trent Nelson428de652008-03-18 22:41:35 +0000176 if tok_type == ENCODING:
177 self.encoding = token
178 continue
Thomas Wouters89f507f2006-12-13 04:49:30 +0000179 self.add_whitespace(start)
180 self.tokens.append(token)
181 self.prev_row, self.prev_col = end
182 if tok_type in (NEWLINE, NL):
183 self.prev_row += 1
184 self.prev_col = 0
185 return "".join(self.tokens)
186
187 def compat(self, token, iterable):
188 startline = False
189 indents = []
190 toks_append = self.tokens.append
191 toknum, tokval = token
Trent Nelson428de652008-03-18 22:41:35 +0000192
Thomas Wouters89f507f2006-12-13 04:49:30 +0000193 if toknum in (NAME, NUMBER):
194 tokval += ' '
195 if toknum in (NEWLINE, NL):
196 startline = True
Christian Heimesba4af492008-03-28 00:55:15 +0000197 prevstring = False
Thomas Wouters89f507f2006-12-13 04:49:30 +0000198 for tok in iterable:
199 toknum, tokval = tok[:2]
Trent Nelson428de652008-03-18 22:41:35 +0000200 if toknum == ENCODING:
201 self.encoding = tokval
202 continue
Thomas Wouters89f507f2006-12-13 04:49:30 +0000203
204 if toknum in (NAME, NUMBER):
205 tokval += ' '
206
Christian Heimesba4af492008-03-28 00:55:15 +0000207 # Insert a space between two consecutive strings
208 if toknum == STRING:
209 if prevstring:
210 tokval = ' ' + tokval
211 prevstring = True
212 else:
213 prevstring = False
214
Thomas Wouters89f507f2006-12-13 04:49:30 +0000215 if toknum == INDENT:
216 indents.append(tokval)
217 continue
218 elif toknum == DEDENT:
219 indents.pop()
220 continue
221 elif toknum in (NEWLINE, NL):
222 startline = True
223 elif startline and indents:
224 toks_append(indents[-1])
225 startline = False
226 toks_append(tokval)
Raymond Hettinger68c04532005-06-10 11:05:19 +0000227
Trent Nelson428de652008-03-18 22:41:35 +0000228
Raymond Hettinger68c04532005-06-10 11:05:19 +0000229def untokenize(iterable):
230 """Transform tokens back into Python source code.
Trent Nelson428de652008-03-18 22:41:35 +0000231 It returns a bytes object, encoded using the ENCODING
232 token, which is the first token sequence output by tokenize.
Raymond Hettinger68c04532005-06-10 11:05:19 +0000233
234 Each element returned by the iterable must be a token sequence
Thomas Wouters89f507f2006-12-13 04:49:30 +0000235 with at least two elements, a token number and token value. If
236 only two tokens are passed, the resulting output is poor.
Raymond Hettinger68c04532005-06-10 11:05:19 +0000237
Thomas Wouters89f507f2006-12-13 04:49:30 +0000238 Round-trip invariant for full input:
239 Untokenized source will match input source exactly
240
241 Round-trip invariant for limited intput:
Trent Nelson428de652008-03-18 22:41:35 +0000242 # Output bytes will tokenize the back to the input
243 t1 = [tok[:2] for tok in tokenize(f.readline)]
Raymond Hettinger68c04532005-06-10 11:05:19 +0000244 newcode = untokenize(t1)
Trent Nelson428de652008-03-18 22:41:35 +0000245 readline = BytesIO(newcode).readline
246 t2 = [tok[:2] for tok in tokenize(readline)]
Raymond Hettinger68c04532005-06-10 11:05:19 +0000247 assert t1 == t2
248 """
Thomas Wouters89f507f2006-12-13 04:49:30 +0000249 ut = Untokenizer()
Trent Nelson428de652008-03-18 22:41:35 +0000250 out = ut.untokenize(iterable)
251 if ut.encoding is not None:
252 out = out.encode(ut.encoding)
253 return out
Raymond Hettinger68c04532005-06-10 11:05:19 +0000254
Trent Nelson428de652008-03-18 22:41:35 +0000255
Benjamin Petersond3afada2009-10-09 21:43:09 +0000256def _get_normal_name(orig_enc):
257 """Imitates get_normal_name in tokenizer.c."""
258 # Only care about the first 12 characters.
259 enc = orig_enc[:12].lower().replace("_", "-")
260 if enc == "utf-8" or enc.startswith("utf-8-"):
261 return "utf-8"
262 if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \
263 enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")):
264 return "iso-8859-1"
265 return orig_enc
266
Trent Nelson428de652008-03-18 22:41:35 +0000267def detect_encoding(readline):
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000268 """
Trent Nelson428de652008-03-18 22:41:35 +0000269 The detect_encoding() function is used to detect the encoding that should
Florent Xicluna43e4ea12010-09-03 19:54:02 +0000270 be used to decode a Python source file. It requires one argment, readline,
Trent Nelson428de652008-03-18 22:41:35 +0000271 in the same way as the tokenize() generator.
272
273 It will call readline a maximum of twice, and return the encoding used
Florent Xicluna43e4ea12010-09-03 19:54:02 +0000274 (as a string) and a list of any lines (left as bytes) it has read in.
Trent Nelson428de652008-03-18 22:41:35 +0000275
276 It detects the encoding from the presence of a utf-8 bom or an encoding
Florent Xicluna43e4ea12010-09-03 19:54:02 +0000277 cookie as specified in pep-0263. If both a bom and a cookie are present,
278 but disagree, a SyntaxError will be raised. If the encoding cookie is an
279 invalid charset, raise a SyntaxError. Note that if a utf-8 bom is found,
Benjamin Peterson689a5582010-03-18 22:29:52 +0000280 'utf-8-sig' is returned.
Trent Nelson428de652008-03-18 22:41:35 +0000281
282 If no encoding is specified, then the default of 'utf-8' will be returned.
283 """
Trent Nelson428de652008-03-18 22:41:35 +0000284 bom_found = False
285 encoding = None
Benjamin Peterson689a5582010-03-18 22:29:52 +0000286 default = 'utf-8'
Trent Nelson428de652008-03-18 22:41:35 +0000287 def read_or_stop():
288 try:
289 return readline()
290 except StopIteration:
291 return b''
292
293 def find_cookie(line):
294 try:
295 line_string = line.decode('ascii')
296 except UnicodeDecodeError:
Benjamin Peterson433f32c2008-12-12 01:25:05 +0000297 return None
298
299 matches = cookie_re.findall(line_string)
300 if not matches:
301 return None
Benjamin Petersond3afada2009-10-09 21:43:09 +0000302 encoding = _get_normal_name(matches[0])
Benjamin Peterson433f32c2008-12-12 01:25:05 +0000303 try:
304 codec = lookup(encoding)
305 except LookupError:
306 # This behaviour mimics the Python interpreter
307 raise SyntaxError("unknown encoding: " + encoding)
308
Benjamin Peterson1613ed82010-03-18 22:34:15 +0000309 if bom_found:
310 if codec.name != 'utf-8':
311 # This behaviour mimics the Python interpreter
312 raise SyntaxError('encoding problem: utf-8')
313 encoding += '-sig'
Benjamin Peterson433f32c2008-12-12 01:25:05 +0000314 return encoding
Trent Nelson428de652008-03-18 22:41:35 +0000315
316 first = read_or_stop()
Benjamin Peterson433f32c2008-12-12 01:25:05 +0000317 if first.startswith(BOM_UTF8):
Trent Nelson428de652008-03-18 22:41:35 +0000318 bom_found = True
319 first = first[3:]
Benjamin Peterson689a5582010-03-18 22:29:52 +0000320 default = 'utf-8-sig'
Trent Nelson428de652008-03-18 22:41:35 +0000321 if not first:
Benjamin Peterson689a5582010-03-18 22:29:52 +0000322 return default, []
Trent Nelson428de652008-03-18 22:41:35 +0000323
324 encoding = find_cookie(first)
325 if encoding:
326 return encoding, [first]
327
328 second = read_or_stop()
329 if not second:
Benjamin Peterson689a5582010-03-18 22:29:52 +0000330 return default, [first]
Trent Nelson428de652008-03-18 22:41:35 +0000331
332 encoding = find_cookie(second)
333 if encoding:
334 return encoding, [first, second]
335
Benjamin Peterson689a5582010-03-18 22:29:52 +0000336 return default, [first, second]
Trent Nelson428de652008-03-18 22:41:35 +0000337
338
Victor Stinner58c07522010-11-09 01:08:59 +0000339def open(filename):
340 """Open a file in read only mode using the encoding detected by
341 detect_encoding().
342 """
Brett Cannon45b96d32011-02-22 03:35:18 +0000343 buffer = builtins.open(filename, 'rb')
Victor Stinner58c07522010-11-09 01:08:59 +0000344 encoding, lines = detect_encoding(buffer.readline)
345 buffer.seek(0)
346 text = TextIOWrapper(buffer, encoding, line_buffering=True)
347 text.mode = 'r'
348 return text
349
350
Trent Nelson428de652008-03-18 22:41:35 +0000351def tokenize(readline):
352 """
353 The tokenize() generator requires one argment, readline, which
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000354 must be a callable object which provides the same interface as the
Florent Xicluna43e4ea12010-09-03 19:54:02 +0000355 readline() method of built-in file objects. Each call to the function
Trent Nelson428de652008-03-18 22:41:35 +0000356 should return one line of input as bytes. Alternately, readline
Raymond Hettinger68c04532005-06-10 11:05:19 +0000357 can be a callable function terminating with StopIteration:
Trent Nelson428de652008-03-18 22:41:35 +0000358 readline = open(myfile, 'rb').__next__ # Example of alternate readline
Tim Peters8ac14952002-05-23 15:15:30 +0000359
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000360 The generator produces 5-tuples with these members: the token type; the
361 token string; a 2-tuple (srow, scol) of ints specifying the row and
362 column where the token begins in the source; a 2-tuple (erow, ecol) of
363 ints specifying the row and column where the token ends in the source;
Florent Xicluna43e4ea12010-09-03 19:54:02 +0000364 and the line on which the token was found. The line passed is the
Tim Peters8ac14952002-05-23 15:15:30 +0000365 logical line; continuation lines are included.
Trent Nelson428de652008-03-18 22:41:35 +0000366
367 The first token sequence will always be an ENCODING token
368 which tells you which encoding was used to decode the bytes stream.
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000369 """
Benjamin Peterson21db77e2009-11-14 16:27:26 +0000370 # This import is here to avoid problems when the itertools module is not
371 # built yet and tokenize is imported.
Benjamin Peterson81dd8b92009-11-14 18:09:17 +0000372 from itertools import chain, repeat
Trent Nelson428de652008-03-18 22:41:35 +0000373 encoding, consumed = detect_encoding(readline)
Benjamin Peterson81dd8b92009-11-14 18:09:17 +0000374 rl_gen = iter(readline, b"")
375 empty = repeat(b"")
376 return _tokenize(chain(consumed, rl_gen, empty).__next__, encoding)
Trent Nelson428de652008-03-18 22:41:35 +0000377
378
379def _tokenize(readline, encoding):
Guido van Rossum1aec3231997-04-08 14:24:39 +0000380 lnum = parenlev = continued = 0
Benjamin Peterson33856de2010-08-30 14:41:20 +0000381 numchars = '0123456789'
Guido van Rossumde655271997-04-09 17:15:54 +0000382 contstr, needcont = '', 0
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000383 contline = None
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000384 indents = [0]
Guido van Rossum1aec3231997-04-08 14:24:39 +0000385
Trent Nelson428de652008-03-18 22:41:35 +0000386 if encoding is not None:
Benjamin Peterson689a5582010-03-18 22:29:52 +0000387 if encoding == "utf-8-sig":
388 # BOM will already have been stripped.
389 encoding = "utf-8"
Raymond Hettingera48db392009-04-29 00:34:27 +0000390 yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '')
Benjamin Peterson0fe14382008-06-05 23:07:42 +0000391 while True: # loop over lines in stream
Raymond Hettinger68c04532005-06-10 11:05:19 +0000392 try:
393 line = readline()
394 except StopIteration:
Trent Nelson428de652008-03-18 22:41:35 +0000395 line = b''
396
397 if encoding is not None:
398 line = line.decode(encoding)
Benjamin Petersona0dfa822009-11-13 02:25:08 +0000399 lnum += 1
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000400 pos, max = 0, len(line)
401
402 if contstr: # continued string
Guido van Rossumde655271997-04-09 17:15:54 +0000403 if not line:
Collin Winterce36ad82007-08-30 01:19:48 +0000404 raise TokenError("EOF in multi-line string", strstart)
Guido van Rossum3b631771997-10-27 20:44:15 +0000405 endmatch = endprog.match(line)
406 if endmatch:
407 pos = end = endmatch.end(0)
Raymond Hettingera48db392009-04-29 00:34:27 +0000408 yield TokenInfo(STRING, contstr + line[:end],
Thomas Wouters89f507f2006-12-13 04:49:30 +0000409 strstart, (lnum, end), contline + line)
Guido van Rossumde655271997-04-09 17:15:54 +0000410 contstr, needcont = '', 0
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000411 contline = None
Guido van Rossumde655271997-04-09 17:15:54 +0000412 elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
Raymond Hettingera48db392009-04-29 00:34:27 +0000413 yield TokenInfo(ERRORTOKEN, contstr + line,
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000414 strstart, (lnum, len(line)), contline)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000415 contstr = ''
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000416 contline = None
Guido van Rossumde655271997-04-09 17:15:54 +0000417 continue
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000418 else:
419 contstr = contstr + line
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000420 contline = contline + line
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000421 continue
422
Guido van Rossum1aec3231997-04-08 14:24:39 +0000423 elif parenlev == 0 and not continued: # new statement
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000424 if not line: break
425 column = 0
Guido van Rossum1aec3231997-04-08 14:24:39 +0000426 while pos < max: # measure leading whitespace
Benjamin Petersona0dfa822009-11-13 02:25:08 +0000427 if line[pos] == ' ':
428 column += 1
429 elif line[pos] == '\t':
430 column = (column//tabsize + 1)*tabsize
431 elif line[pos] == '\f':
432 column = 0
433 else:
434 break
435 pos += 1
436 if pos == max:
437 break
Guido van Rossum1aec3231997-04-08 14:24:39 +0000438
439 if line[pos] in '#\r\n': # skip comments or blank lines
Thomas Wouters89f507f2006-12-13 04:49:30 +0000440 if line[pos] == '#':
441 comment_token = line[pos:].rstrip('\r\n')
442 nl_pos = pos + len(comment_token)
Raymond Hettingera48db392009-04-29 00:34:27 +0000443 yield TokenInfo(COMMENT, comment_token,
Thomas Wouters89f507f2006-12-13 04:49:30 +0000444 (lnum, pos), (lnum, pos + len(comment_token)), line)
Raymond Hettingera48db392009-04-29 00:34:27 +0000445 yield TokenInfo(NL, line[nl_pos:],
Thomas Wouters89f507f2006-12-13 04:49:30 +0000446 (lnum, nl_pos), (lnum, len(line)), line)
447 else:
Raymond Hettingera48db392009-04-29 00:34:27 +0000448 yield TokenInfo((NL, COMMENT)[line[pos] == '#'], line[pos:],
Guido van Rossum1aec3231997-04-08 14:24:39 +0000449 (lnum, pos), (lnum, len(line)), line)
450 continue
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000451
452 if column > indents[-1]: # count indents or dedents
453 indents.append(column)
Raymond Hettingera48db392009-04-29 00:34:27 +0000454 yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000455 while column < indents[-1]:
Raymond Hettingerda99d1c2005-06-21 07:43:58 +0000456 if column not in indents:
457 raise IndentationError(
Thomas Wouters00ee7ba2006-08-21 19:07:27 +0000458 "unindent does not match any outer indentation level",
459 ("<tokenize>", lnum, pos, line))
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000460 indents = indents[:-1]
Raymond Hettingera48db392009-04-29 00:34:27 +0000461 yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000462
463 else: # continued statement
Guido van Rossumde655271997-04-09 17:15:54 +0000464 if not line:
Collin Winterce36ad82007-08-30 01:19:48 +0000465 raise TokenError("EOF in multi-line statement", (lnum, 0))
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000466 continued = 0
467
468 while pos < max:
Guido van Rossum3b631771997-10-27 20:44:15 +0000469 pseudomatch = pseudoprog.match(line, pos)
470 if pseudomatch: # scan for tokens
471 start, end = pseudomatch.span(1)
Guido van Rossumde655271997-04-09 17:15:54 +0000472 spos, epos, pos = (lnum, start), (lnum, end), end
Guido van Rossum1aec3231997-04-08 14:24:39 +0000473 token, initial = line[start:end], line[start]
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000474
Georg Brandldde00282007-03-18 19:01:53 +0000475 if (initial in numchars or # ordinary number
476 (initial == '.' and token != '.' and token != '...')):
Raymond Hettingera48db392009-04-29 00:34:27 +0000477 yield TokenInfo(NUMBER, token, spos, epos, line)
Guido van Rossum1aec3231997-04-08 14:24:39 +0000478 elif initial in '\r\n':
Raymond Hettingera48db392009-04-29 00:34:27 +0000479 yield TokenInfo(NL if parenlev > 0 else NEWLINE,
Thomas Wouters89f507f2006-12-13 04:49:30 +0000480 token, spos, epos, line)
Guido van Rossum1aec3231997-04-08 14:24:39 +0000481 elif initial == '#':
Thomas Wouters89f507f2006-12-13 04:49:30 +0000482 assert not token.endswith("\n")
Raymond Hettingera48db392009-04-29 00:34:27 +0000483 yield TokenInfo(COMMENT, token, spos, epos, line)
Guido van Rossum9d6897a2002-08-24 06:54:19 +0000484 elif token in triple_quoted:
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000485 endprog = endprogs[token]
Guido van Rossum3b631771997-10-27 20:44:15 +0000486 endmatch = endprog.match(line, pos)
487 if endmatch: # all on one line
488 pos = endmatch.end(0)
Guido van Rossum1aec3231997-04-08 14:24:39 +0000489 token = line[start:pos]
Raymond Hettingera48db392009-04-29 00:34:27 +0000490 yield TokenInfo(STRING, token, spos, (lnum, pos), line)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000491 else:
Guido van Rossum1aec3231997-04-08 14:24:39 +0000492 strstart = (lnum, start) # multiple lines
493 contstr = line[start:]
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000494 contline = line
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000495 break
Guido van Rossum9d6897a2002-08-24 06:54:19 +0000496 elif initial in single_quoted or \
497 token[:2] in single_quoted or \
498 token[:3] in single_quoted:
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000499 if token[-1] == '\n': # continued string
Guido van Rossum1aec3231997-04-08 14:24:39 +0000500 strstart = (lnum, start)
Ka-Ping Yee1ff08b12001-01-15 22:04:30 +0000501 endprog = (endprogs[initial] or endprogs[token[1]] or
502 endprogs[token[2]])
Guido van Rossumde655271997-04-09 17:15:54 +0000503 contstr, needcont = line[start:], 1
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000504 contline = line
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000505 break
506 else: # ordinary string
Raymond Hettingera48db392009-04-29 00:34:27 +0000507 yield TokenInfo(STRING, token, spos, epos, line)
Benjamin Peterson33856de2010-08-30 14:41:20 +0000508 elif initial.isidentifier(): # ordinary name
Raymond Hettingera48db392009-04-29 00:34:27 +0000509 yield TokenInfo(NAME, token, spos, epos, line)
Guido van Rossum3b631771997-10-27 20:44:15 +0000510 elif initial == '\\': # continued stmt
511 continued = 1
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000512 else:
Benjamin Petersona0dfa822009-11-13 02:25:08 +0000513 if initial in '([{':
514 parenlev += 1
515 elif initial in ')]}':
516 parenlev -= 1
Raymond Hettingera48db392009-04-29 00:34:27 +0000517 yield TokenInfo(OP, token, spos, epos, line)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000518 else:
Raymond Hettingera48db392009-04-29 00:34:27 +0000519 yield TokenInfo(ERRORTOKEN, line[pos],
Guido van Rossumde655271997-04-09 17:15:54 +0000520 (lnum, pos), (lnum, pos+1), line)
Benjamin Petersona0dfa822009-11-13 02:25:08 +0000521 pos += 1
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000522
523 for indent in indents[1:]: # pop remaining indent levels
Raymond Hettingera48db392009-04-29 00:34:27 +0000524 yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '')
525 yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '')
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000526
Trent Nelson428de652008-03-18 22:41:35 +0000527
528# An undocumented, backwards compatible, API for all the places in the standard
529# library that expect to be able to use tokenize with strings
530def generate_tokens(readline):
531 return _tokenize(readline, None)
Raymond Hettinger6c60d092010-09-09 04:32:39 +0000532
533if __name__ == "__main__":
534 # Quick sanity check
535 s = b'''def parseline(self, line):
536 """Parse the line into a command name and a string containing
537 the arguments. Returns a tuple containing (command, args, line).
538 'command' and 'args' may be None if the line couldn't be parsed.
539 """
540 line = line.strip()
541 if not line:
542 return None, None, line
543 elif line[0] == '?':
544 line = 'help ' + line[1:]
545 elif line[0] == '!':
546 if hasattr(self, 'do_shell'):
547 line = 'shell ' + line[1:]
548 else:
549 return None, None, line
550 i, n = 0, len(line)
551 while i < n and line[i] in self.identchars: i = i+1
552 cmd, arg = line[:i], line[i:].strip()
553 return cmd, arg, line
554 '''
555 for tok in tokenize(iter(s.splitlines()).__next__):
556 print(tok)