blob: 2fbde0fa9b4005800e4d5786ede900e96ac14e7e [file] [log] [blame]
Guido van Rossumb51eaa11997-03-07 00:21:55 +00001"""Tokenization help for Python programs.
Guido van Rossum4d8e8591992-01-01 19:34:47 +00002
Florent Xicluna43e4ea12010-09-03 19:54:02 +00003tokenize(readline) is a generator that breaks a stream of bytes into
4Python tokens. It decodes the bytes according to PEP-0263 for
5determining source file encoding.
Trent Nelson428de652008-03-18 22:41:35 +00006
Florent Xicluna43e4ea12010-09-03 19:54:02 +00007It accepts a readline-like method which is called repeatedly to get the
8next line of input (or b"" for EOF). It generates 5-tuples with these
9members:
Tim Peters4efb6e92001-06-29 23:51:08 +000010
11 the token type (see token.py)
12 the token (a string)
13 the starting (row, column) indices of the token (a 2-tuple of ints)
14 the ending (row, column) indices of the token (a 2-tuple of ints)
15 the original line (string)
16
17It is designed to match the working of the Python tokenizer exactly, except
18that it produces COMMENT tokens for comments and gives type OP for all
Florent Xicluna43e4ea12010-09-03 19:54:02 +000019operators. Additionally, all token lists start with an ENCODING token
20which tells you which encoding was used to decode the bytes stream.
21"""
Guido van Rossumb51eaa11997-03-07 00:21:55 +000022
Ka-Ping Yee244c5932001-03-01 13:56:40 +000023__author__ = 'Ka-Ping Yee <ping@lfw.org>'
Trent Nelson428de652008-03-18 22:41:35 +000024__credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
25 'Skip Montanaro, Raymond Hettinger, Trent Nelson, '
26 'Michael Foord')
Brett Cannonf3042782011-02-22 03:25:12 +000027import builtins
Florent Xicluna43e4ea12010-09-03 19:54:02 +000028import re
29import sys
Guido van Rossumfc6f5331997-03-07 00:21:12 +000030from token import *
Benjamin Peterson433f32c2008-12-12 01:25:05 +000031from codecs import lookup, BOM_UTF8
Raymond Hettinger3fb79c72010-09-09 07:15:18 +000032import collections
Victor Stinner58c07522010-11-09 01:08:59 +000033from io import TextIOWrapper
Trent Nelson428de652008-03-18 22:41:35 +000034cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
Guido van Rossum4d8e8591992-01-01 19:34:47 +000035
Skip Montanaro40fc1602001-03-01 04:27:19 +000036import token
Alexander Belopolskyb9d10d02010-11-11 14:07:41 +000037__all__ = token.__all__ + ["COMMENT", "tokenize", "detect_encoding",
38 "NL", "untokenize", "ENCODING", "TokenInfo"]
Skip Montanaro40fc1602001-03-01 04:27:19 +000039del token
40
Guido van Rossum1aec3231997-04-08 14:24:39 +000041COMMENT = N_TOKENS
42tok_name[COMMENT] = 'COMMENT'
Guido van Rossuma90c78b1998-04-03 16:05:38 +000043NL = N_TOKENS + 1
44tok_name[NL] = 'NL'
Trent Nelson428de652008-03-18 22:41:35 +000045ENCODING = N_TOKENS + 2
46tok_name[ENCODING] = 'ENCODING'
47N_TOKENS += 3
Meador Inge00c7f852012-01-19 00:44:45 -060048EXACT_TOKEN_TYPES = {
49 '(': LPAR,
50 ')': RPAR,
51 '[': LSQB,
52 ']': RSQB,
53 ':': COLON,
54 ',': COMMA,
55 ';': SEMI,
56 '+': PLUS,
57 '-': MINUS,
58 '*': STAR,
59 '/': SLASH,
60 '|': VBAR,
61 '&': AMPER,
62 '<': LESS,
63 '>': GREATER,
64 '=': EQUAL,
65 '.': DOT,
66 '%': PERCENT,
67 '{': LBRACE,
68 '}': RBRACE,
69 '==': EQEQUAL,
70 '!=': NOTEQUAL,
71 '<=': LESSEQUAL,
72 '>=': GREATEREQUAL,
73 '~': TILDE,
74 '^': CIRCUMFLEX,
75 '<<': LEFTSHIFT,
76 '>>': RIGHTSHIFT,
77 '**': DOUBLESTAR,
78 '+=': PLUSEQUAL,
79 '-=': MINEQUAL,
80 '*=': STAREQUAL,
81 '/=': SLASHEQUAL,
82 '%=': PERCENTEQUAL,
83 '&=': AMPEREQUAL,
84 '|=': VBAREQUAL,
85 '^=': CIRCUMFLEXEQUAL,
86 '<<=': LEFTSHIFTEQUAL,
87 '>>=': RIGHTSHIFTEQUAL,
88 '**=': DOUBLESTAREQUAL,
89 '//': DOUBLESLASH,
90 '//=': DOUBLESLASHEQUAL,
91 '@': AT
92}
Guido van Rossum1aec3231997-04-08 14:24:39 +000093
Raymond Hettinger3fb79c72010-09-09 07:15:18 +000094class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')):
Raymond Hettingeraa17a7f2009-04-29 14:21:25 +000095 def __repr__(self):
Raymond Hettingera0e79402010-09-09 08:29:05 +000096 annotated_type = '%d (%s)' % (self.type, tok_name[self.type])
97 return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' %
98 self._replace(type=annotated_type))
Raymond Hettingeraa17a7f2009-04-29 14:21:25 +000099
Meador Inge00c7f852012-01-19 00:44:45 -0600100 @property
101 def exact_type(self):
102 if self.type == OP and self.string in EXACT_TOKEN_TYPES:
103 return EXACT_TOKEN_TYPES[self.string]
104 else:
105 return self.type
106
Eric S. Raymondb08b2d32001-02-09 11:10:16 +0000107def group(*choices): return '(' + '|'.join(choices) + ')'
Guido van Rossum68468eb2003-02-27 20:14:51 +0000108def any(*choices): return group(*choices) + '*'
109def maybe(*choices): return group(*choices) + '?'
Guido van Rossum4d8e8591992-01-01 19:34:47 +0000110
Antoine Pitroufd036452008-08-19 17:56:33 +0000111# Note: we use unicode matching for names ("\w") but ascii matching for
112# number literals.
Guido van Rossum3b631771997-10-27 20:44:15 +0000113Whitespace = r'[ \f\t]*'
114Comment = r'#[^\r\n]*'
115Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
Benjamin Peterson33856de2010-08-30 14:41:20 +0000116Name = r'\w+'
Guido van Rossum4d8e8591992-01-01 19:34:47 +0000117
Antoine Pitroufd036452008-08-19 17:56:33 +0000118Hexnumber = r'0[xX][0-9a-fA-F]+'
Georg Brandlfceab5a2008-01-19 20:08:23 +0000119Binnumber = r'0[bB][01]+'
120Octnumber = r'0[oO][0-7]+'
Antoine Pitroufd036452008-08-19 17:56:33 +0000121Decnumber = r'(?:0+|[1-9][0-9]*)'
Guido van Rossumcd16bf62007-06-13 18:07:49 +0000122Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
Antoine Pitroufd036452008-08-19 17:56:33 +0000123Exponent = r'[eE][-+]?[0-9]+'
124Pointfloat = group(r'[0-9]+\.[0-9]*', r'\.[0-9]+') + maybe(Exponent)
125Expfloat = r'[0-9]+' + Exponent
Guido van Rossum1aec3231997-04-08 14:24:39 +0000126Floatnumber = group(Pointfloat, Expfloat)
Antoine Pitroufd036452008-08-19 17:56:33 +0000127Imagnumber = group(r'[0-9]+[jJ]', Floatnumber + r'[jJ]')
Guido van Rossum1aec3231997-04-08 14:24:39 +0000128Number = group(Imagnumber, Floatnumber, Intnumber)
Guido van Rossum4d8e8591992-01-01 19:34:47 +0000129
Christian Heimes0b3847d2012-06-20 11:17:58 +0200130StringPrefix = r'(?:[bB][rR]?|[rR][bB]?|[uU])?'
Armin Ronacherc0eaeca2012-03-04 13:07:57 +0000131
Tim Petersde495832000-10-07 05:09:39 +0000132# Tail end of ' string.
133Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
134# Tail end of " string.
135Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
136# Tail end of ''' string.
137Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
138# Tail end of """ string.
139Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
Armin Ronacherc0eaeca2012-03-04 13:07:57 +0000140Triple = group(StringPrefix + "'''", StringPrefix + '"""')
Tim Petersde495832000-10-07 05:09:39 +0000141# Single-line ' or " string.
Armin Ronacherc0eaeca2012-03-04 13:07:57 +0000142String = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
143 StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
Guido van Rossum4d8e8591992-01-01 19:34:47 +0000144
Tim Petersde495832000-10-07 05:09:39 +0000145# Because of leftmost-then-longest match semantics, be sure to put the
146# longest operators first (e.g., if = came before ==, == would get
147# recognized as two instances of =).
Guido van Rossumb053cd82006-08-24 03:53:23 +0000148Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=",
Neal Norwitzc1505362006-12-28 06:47:50 +0000149 r"//=?", r"->",
Tim Petersde495832000-10-07 05:09:39 +0000150 r"[+\-*/%&|^=<>]=?",
151 r"~")
Thomas Wouterse1519a12000-08-24 21:44:52 +0000152
Guido van Rossum4d8e8591992-01-01 19:34:47 +0000153Bracket = '[][(){}]'
Georg Brandldde00282007-03-18 19:01:53 +0000154Special = group(r'\r?\n', r'\.\.\.', r'[:;.,@]')
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000155Funny = group(Operator, Bracket, Special)
Guido van Rossum4d8e8591992-01-01 19:34:47 +0000156
Guido van Rossum3b631771997-10-27 20:44:15 +0000157PlainToken = group(Number, Funny, String, Name)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000158Token = Ignore + PlainToken
Guido van Rossum4d8e8591992-01-01 19:34:47 +0000159
Tim Petersde495832000-10-07 05:09:39 +0000160# First (or only) line of ' or " string.
Armin Ronacherc0eaeca2012-03-04 13:07:57 +0000161ContStr = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
Ka-Ping Yee1ff08b12001-01-15 22:04:30 +0000162 group("'", r'\\\r?\n'),
Armin Ronacherc0eaeca2012-03-04 13:07:57 +0000163 StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
Ka-Ping Yee1ff08b12001-01-15 22:04:30 +0000164 group('"', r'\\\r?\n'))
Ezio Melotti2cc3b4b2012-11-03 17:38:43 +0200165PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple)
Guido van Rossum3b631771997-10-27 20:44:15 +0000166PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
Guido van Rossum1aec3231997-04-08 14:24:39 +0000167
Benjamin Peterson33856de2010-08-30 14:41:20 +0000168def _compile(expr):
169 return re.compile(expr, re.UNICODE)
170
Antoine Pitrou10a99b02011-10-11 15:45:56 +0200171endpats = {"'": Single, '"': Double,
172 "'''": Single3, '"""': Double3,
173 "r'''": Single3, 'r"""': Double3,
174 "b'''": Single3, 'b"""': Double3,
Antoine Pitrou10a99b02011-10-11 15:45:56 +0200175 "R'''": Single3, 'R"""': Double3,
176 "B'''": Single3, 'B"""': Double3,
Armin Ronacherc0eaeca2012-03-04 13:07:57 +0000177 "br'''": Single3, 'br"""': Double3,
Antoine Pitrou10a99b02011-10-11 15:45:56 +0200178 "bR'''": Single3, 'bR"""': Double3,
179 "Br'''": Single3, 'Br"""': Double3,
180 "BR'''": Single3, 'BR"""': Double3,
Armin Ronacherc0eaeca2012-03-04 13:07:57 +0000181 "rb'''": Single3, 'rb"""': Double3,
182 "Rb'''": Single3, 'Rb"""': Double3,
183 "rB'''": Single3, 'rB"""': Double3,
184 "RB'''": Single3, 'RB"""': Double3,
Armin Ronacher6ecf77b2012-03-04 12:04:06 +0000185 "u'''": Single3, 'u"""': Double3,
Armin Ronacher6ecf77b2012-03-04 12:04:06 +0000186 "R'''": Single3, 'R"""': Double3,
187 "U'''": Single3, 'U"""': Double3,
Armin Ronacher6ecf77b2012-03-04 12:04:06 +0000188 'r': None, 'R': None, 'b': None, 'B': None,
189 'u': None, 'U': None}
Guido van Rossum4d8e8591992-01-01 19:34:47 +0000190
Guido van Rossum9d6897a2002-08-24 06:54:19 +0000191triple_quoted = {}
192for t in ("'''", '"""',
193 "r'''", 'r"""', "R'''", 'R"""',
Guido van Rossum4fe72f92007-11-12 17:40:10 +0000194 "b'''", 'b"""', "B'''", 'B"""',
195 "br'''", 'br"""', "Br'''", 'Br"""',
Armin Ronacher6ecf77b2012-03-04 12:04:06 +0000196 "bR'''", 'bR"""', "BR'''", 'BR"""',
Armin Ronacherc0eaeca2012-03-04 13:07:57 +0000197 "rb'''", 'rb"""', "rB'''", 'rB"""',
198 "Rb'''", 'Rb"""', "RB'''", 'RB"""',
Armin Ronacher6ecf77b2012-03-04 12:04:06 +0000199 "u'''", 'u"""', "U'''", 'U"""',
Christian Heimes0b3847d2012-06-20 11:17:58 +0200200 ):
Guido van Rossum9d6897a2002-08-24 06:54:19 +0000201 triple_quoted[t] = t
202single_quoted = {}
203for t in ("'", '"',
204 "r'", 'r"', "R'", 'R"',
Guido van Rossum4fe72f92007-11-12 17:40:10 +0000205 "b'", 'b"', "B'", 'B"',
206 "br'", 'br"', "Br'", 'Br"',
Armin Ronacher6ecf77b2012-03-04 12:04:06 +0000207 "bR'", 'bR"', "BR'", 'BR"' ,
Armin Ronacherc0eaeca2012-03-04 13:07:57 +0000208 "rb'", 'rb"', "rB'", 'rB"',
209 "Rb'", 'Rb"', "RB'", 'RB"' ,
Armin Ronacher6ecf77b2012-03-04 12:04:06 +0000210 "u'", 'u"', "U'", 'U"',
Christian Heimes0b3847d2012-06-20 11:17:58 +0200211 ):
Guido van Rossum9d6897a2002-08-24 06:54:19 +0000212 single_quoted[t] = t
213
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000214tabsize = 8
Fred Drake9b8d8012000-08-17 04:45:13 +0000215
Ka-Ping Yee28c62bb2001-03-23 05:22:49 +0000216class TokenError(Exception): pass
217
218class StopTokenizing(Exception): pass
Fred Drake9b8d8012000-08-17 04:45:13 +0000219
Tim Peters5ca576e2001-06-18 22:08:13 +0000220
Thomas Wouters89f507f2006-12-13 04:49:30 +0000221class Untokenizer:
222
223 def __init__(self):
224 self.tokens = []
225 self.prev_row = 1
226 self.prev_col = 0
Trent Nelson428de652008-03-18 22:41:35 +0000227 self.encoding = None
Thomas Wouters89f507f2006-12-13 04:49:30 +0000228
229 def add_whitespace(self, start):
230 row, col = start
231 assert row <= self.prev_row
232 col_offset = col - self.prev_col
233 if col_offset:
234 self.tokens.append(" " * col_offset)
235
236 def untokenize(self, iterable):
237 for t in iterable:
238 if len(t) == 2:
239 self.compat(t, iterable)
240 break
241 tok_type, token, start, end, line = t
Trent Nelson428de652008-03-18 22:41:35 +0000242 if tok_type == ENCODING:
243 self.encoding = token
244 continue
Thomas Wouters89f507f2006-12-13 04:49:30 +0000245 self.add_whitespace(start)
246 self.tokens.append(token)
247 self.prev_row, self.prev_col = end
248 if tok_type in (NEWLINE, NL):
249 self.prev_row += 1
250 self.prev_col = 0
251 return "".join(self.tokens)
252
253 def compat(self, token, iterable):
254 startline = False
255 indents = []
256 toks_append = self.tokens.append
257 toknum, tokval = token
Trent Nelson428de652008-03-18 22:41:35 +0000258
Thomas Wouters89f507f2006-12-13 04:49:30 +0000259 if toknum in (NAME, NUMBER):
260 tokval += ' '
261 if toknum in (NEWLINE, NL):
262 startline = True
Christian Heimesba4af492008-03-28 00:55:15 +0000263 prevstring = False
Thomas Wouters89f507f2006-12-13 04:49:30 +0000264 for tok in iterable:
265 toknum, tokval = tok[:2]
Trent Nelson428de652008-03-18 22:41:35 +0000266 if toknum == ENCODING:
267 self.encoding = tokval
268 continue
Thomas Wouters89f507f2006-12-13 04:49:30 +0000269
270 if toknum in (NAME, NUMBER):
271 tokval += ' '
272
Christian Heimesba4af492008-03-28 00:55:15 +0000273 # Insert a space between two consecutive strings
274 if toknum == STRING:
275 if prevstring:
276 tokval = ' ' + tokval
277 prevstring = True
278 else:
279 prevstring = False
280
Thomas Wouters89f507f2006-12-13 04:49:30 +0000281 if toknum == INDENT:
282 indents.append(tokval)
283 continue
284 elif toknum == DEDENT:
285 indents.pop()
286 continue
287 elif toknum in (NEWLINE, NL):
288 startline = True
289 elif startline and indents:
290 toks_append(indents[-1])
291 startline = False
292 toks_append(tokval)
Raymond Hettinger68c04532005-06-10 11:05:19 +0000293
Trent Nelson428de652008-03-18 22:41:35 +0000294
Raymond Hettinger68c04532005-06-10 11:05:19 +0000295def untokenize(iterable):
296 """Transform tokens back into Python source code.
Trent Nelson428de652008-03-18 22:41:35 +0000297 It returns a bytes object, encoded using the ENCODING
298 token, which is the first token sequence output by tokenize.
Raymond Hettinger68c04532005-06-10 11:05:19 +0000299
300 Each element returned by the iterable must be a token sequence
Thomas Wouters89f507f2006-12-13 04:49:30 +0000301 with at least two elements, a token number and token value. If
302 only two tokens are passed, the resulting output is poor.
Raymond Hettinger68c04532005-06-10 11:05:19 +0000303
Thomas Wouters89f507f2006-12-13 04:49:30 +0000304 Round-trip invariant for full input:
305 Untokenized source will match input source exactly
306
307 Round-trip invariant for limited intput:
Trent Nelson428de652008-03-18 22:41:35 +0000308 # Output bytes will tokenize the back to the input
309 t1 = [tok[:2] for tok in tokenize(f.readline)]
Raymond Hettinger68c04532005-06-10 11:05:19 +0000310 newcode = untokenize(t1)
Trent Nelson428de652008-03-18 22:41:35 +0000311 readline = BytesIO(newcode).readline
312 t2 = [tok[:2] for tok in tokenize(readline)]
Raymond Hettinger68c04532005-06-10 11:05:19 +0000313 assert t1 == t2
314 """
Thomas Wouters89f507f2006-12-13 04:49:30 +0000315 ut = Untokenizer()
Trent Nelson428de652008-03-18 22:41:35 +0000316 out = ut.untokenize(iterable)
317 if ut.encoding is not None:
318 out = out.encode(ut.encoding)
319 return out
Raymond Hettinger68c04532005-06-10 11:05:19 +0000320
Trent Nelson428de652008-03-18 22:41:35 +0000321
Benjamin Petersond3afada2009-10-09 21:43:09 +0000322def _get_normal_name(orig_enc):
323 """Imitates get_normal_name in tokenizer.c."""
324 # Only care about the first 12 characters.
325 enc = orig_enc[:12].lower().replace("_", "-")
326 if enc == "utf-8" or enc.startswith("utf-8-"):
327 return "utf-8"
328 if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \
329 enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")):
330 return "iso-8859-1"
331 return orig_enc
332
Trent Nelson428de652008-03-18 22:41:35 +0000333def detect_encoding(readline):
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000334 """
Trent Nelson428de652008-03-18 22:41:35 +0000335 The detect_encoding() function is used to detect the encoding that should
Florent Xicluna43e4ea12010-09-03 19:54:02 +0000336 be used to decode a Python source file. It requires one argment, readline,
Trent Nelson428de652008-03-18 22:41:35 +0000337 in the same way as the tokenize() generator.
338
339 It will call readline a maximum of twice, and return the encoding used
Florent Xicluna43e4ea12010-09-03 19:54:02 +0000340 (as a string) and a list of any lines (left as bytes) it has read in.
Trent Nelson428de652008-03-18 22:41:35 +0000341
342 It detects the encoding from the presence of a utf-8 bom or an encoding
Florent Xicluna43e4ea12010-09-03 19:54:02 +0000343 cookie as specified in pep-0263. If both a bom and a cookie are present,
344 but disagree, a SyntaxError will be raised. If the encoding cookie is an
345 invalid charset, raise a SyntaxError. Note that if a utf-8 bom is found,
Benjamin Peterson689a5582010-03-18 22:29:52 +0000346 'utf-8-sig' is returned.
Trent Nelson428de652008-03-18 22:41:35 +0000347
348 If no encoding is specified, then the default of 'utf-8' will be returned.
349 """
Brett Cannonc33f3f22012-04-20 13:23:54 -0400350 try:
351 filename = readline.__self__.name
352 except AttributeError:
353 filename = None
Trent Nelson428de652008-03-18 22:41:35 +0000354 bom_found = False
355 encoding = None
Benjamin Peterson689a5582010-03-18 22:29:52 +0000356 default = 'utf-8'
Trent Nelson428de652008-03-18 22:41:35 +0000357 def read_or_stop():
358 try:
359 return readline()
360 except StopIteration:
361 return b''
362
363 def find_cookie(line):
364 try:
Martin v. Löwis63674f42012-04-20 14:36:47 +0200365 # Decode as UTF-8. Either the line is an encoding declaration,
366 # in which case it should be pure ASCII, or it must be UTF-8
367 # per default encoding.
368 line_string = line.decode('utf-8')
Trent Nelson428de652008-03-18 22:41:35 +0000369 except UnicodeDecodeError:
Brett Cannonc33f3f22012-04-20 13:23:54 -0400370 msg = "invalid or missing encoding declaration"
371 if filename is not None:
372 msg = '{} for {!r}'.format(msg, filename)
373 raise SyntaxError(msg)
Benjamin Peterson433f32c2008-12-12 01:25:05 +0000374
375 matches = cookie_re.findall(line_string)
376 if not matches:
377 return None
Benjamin Petersond3afada2009-10-09 21:43:09 +0000378 encoding = _get_normal_name(matches[0])
Benjamin Peterson433f32c2008-12-12 01:25:05 +0000379 try:
380 codec = lookup(encoding)
381 except LookupError:
382 # This behaviour mimics the Python interpreter
Brett Cannonc33f3f22012-04-20 13:23:54 -0400383 if filename is None:
384 msg = "unknown encoding: " + encoding
385 else:
386 msg = "unknown encoding for {!r}: {}".format(filename,
387 encoding)
388 raise SyntaxError(msg)
Benjamin Peterson433f32c2008-12-12 01:25:05 +0000389
Benjamin Peterson1613ed82010-03-18 22:34:15 +0000390 if bom_found:
Florent Xicluna11f0b412012-07-07 12:13:35 +0200391 if encoding != 'utf-8':
Benjamin Peterson1613ed82010-03-18 22:34:15 +0000392 # This behaviour mimics the Python interpreter
Brett Cannonc33f3f22012-04-20 13:23:54 -0400393 if filename is None:
394 msg = 'encoding problem: utf-8'
395 else:
396 msg = 'encoding problem for {!r}: utf-8'.format(filename)
397 raise SyntaxError(msg)
Benjamin Peterson1613ed82010-03-18 22:34:15 +0000398 encoding += '-sig'
Benjamin Peterson433f32c2008-12-12 01:25:05 +0000399 return encoding
Trent Nelson428de652008-03-18 22:41:35 +0000400
401 first = read_or_stop()
Benjamin Peterson433f32c2008-12-12 01:25:05 +0000402 if first.startswith(BOM_UTF8):
Trent Nelson428de652008-03-18 22:41:35 +0000403 bom_found = True
404 first = first[3:]
Benjamin Peterson689a5582010-03-18 22:29:52 +0000405 default = 'utf-8-sig'
Trent Nelson428de652008-03-18 22:41:35 +0000406 if not first:
Benjamin Peterson689a5582010-03-18 22:29:52 +0000407 return default, []
Trent Nelson428de652008-03-18 22:41:35 +0000408
409 encoding = find_cookie(first)
410 if encoding:
411 return encoding, [first]
412
413 second = read_or_stop()
414 if not second:
Benjamin Peterson689a5582010-03-18 22:29:52 +0000415 return default, [first]
Trent Nelson428de652008-03-18 22:41:35 +0000416
417 encoding = find_cookie(second)
418 if encoding:
419 return encoding, [first, second]
420
Benjamin Peterson689a5582010-03-18 22:29:52 +0000421 return default, [first, second]
Trent Nelson428de652008-03-18 22:41:35 +0000422
423
Victor Stinner58c07522010-11-09 01:08:59 +0000424def open(filename):
425 """Open a file in read only mode using the encoding detected by
426 detect_encoding().
427 """
Brett Cannonf3042782011-02-22 03:25:12 +0000428 buffer = builtins.open(filename, 'rb')
Victor Stinner58c07522010-11-09 01:08:59 +0000429 encoding, lines = detect_encoding(buffer.readline)
430 buffer.seek(0)
431 text = TextIOWrapper(buffer, encoding, line_buffering=True)
432 text.mode = 'r'
433 return text
434
435
Trent Nelson428de652008-03-18 22:41:35 +0000436def tokenize(readline):
437 """
438 The tokenize() generator requires one argment, readline, which
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000439 must be a callable object which provides the same interface as the
Florent Xicluna43e4ea12010-09-03 19:54:02 +0000440 readline() method of built-in file objects. Each call to the function
Trent Nelson428de652008-03-18 22:41:35 +0000441 should return one line of input as bytes. Alternately, readline
Raymond Hettinger68c04532005-06-10 11:05:19 +0000442 can be a callable function terminating with StopIteration:
Trent Nelson428de652008-03-18 22:41:35 +0000443 readline = open(myfile, 'rb').__next__ # Example of alternate readline
Tim Peters8ac14952002-05-23 15:15:30 +0000444
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000445 The generator produces 5-tuples with these members: the token type; the
446 token string; a 2-tuple (srow, scol) of ints specifying the row and
447 column where the token begins in the source; a 2-tuple (erow, ecol) of
448 ints specifying the row and column where the token ends in the source;
Florent Xicluna43e4ea12010-09-03 19:54:02 +0000449 and the line on which the token was found. The line passed is the
Tim Peters8ac14952002-05-23 15:15:30 +0000450 logical line; continuation lines are included.
Trent Nelson428de652008-03-18 22:41:35 +0000451
452 The first token sequence will always be an ENCODING token
453 which tells you which encoding was used to decode the bytes stream.
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000454 """
Benjamin Peterson21db77e2009-11-14 16:27:26 +0000455 # This import is here to avoid problems when the itertools module is not
456 # built yet and tokenize is imported.
Benjamin Peterson81dd8b92009-11-14 18:09:17 +0000457 from itertools import chain, repeat
Trent Nelson428de652008-03-18 22:41:35 +0000458 encoding, consumed = detect_encoding(readline)
Benjamin Peterson81dd8b92009-11-14 18:09:17 +0000459 rl_gen = iter(readline, b"")
460 empty = repeat(b"")
461 return _tokenize(chain(consumed, rl_gen, empty).__next__, encoding)
Trent Nelson428de652008-03-18 22:41:35 +0000462
463
464def _tokenize(readline, encoding):
Guido van Rossum1aec3231997-04-08 14:24:39 +0000465 lnum = parenlev = continued = 0
Benjamin Peterson33856de2010-08-30 14:41:20 +0000466 numchars = '0123456789'
Guido van Rossumde655271997-04-09 17:15:54 +0000467 contstr, needcont = '', 0
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000468 contline = None
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000469 indents = [0]
Guido van Rossum1aec3231997-04-08 14:24:39 +0000470
Trent Nelson428de652008-03-18 22:41:35 +0000471 if encoding is not None:
Benjamin Peterson689a5582010-03-18 22:29:52 +0000472 if encoding == "utf-8-sig":
473 # BOM will already have been stripped.
474 encoding = "utf-8"
Raymond Hettingera48db392009-04-29 00:34:27 +0000475 yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '')
Benjamin Peterson0fe14382008-06-05 23:07:42 +0000476 while True: # loop over lines in stream
Raymond Hettinger68c04532005-06-10 11:05:19 +0000477 try:
478 line = readline()
479 except StopIteration:
Trent Nelson428de652008-03-18 22:41:35 +0000480 line = b''
481
482 if encoding is not None:
483 line = line.decode(encoding)
Benjamin Petersona0dfa822009-11-13 02:25:08 +0000484 lnum += 1
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000485 pos, max = 0, len(line)
486
487 if contstr: # continued string
Guido van Rossumde655271997-04-09 17:15:54 +0000488 if not line:
Collin Winterce36ad82007-08-30 01:19:48 +0000489 raise TokenError("EOF in multi-line string", strstart)
Guido van Rossum3b631771997-10-27 20:44:15 +0000490 endmatch = endprog.match(line)
491 if endmatch:
492 pos = end = endmatch.end(0)
Raymond Hettingera48db392009-04-29 00:34:27 +0000493 yield TokenInfo(STRING, contstr + line[:end],
Thomas Wouters89f507f2006-12-13 04:49:30 +0000494 strstart, (lnum, end), contline + line)
Guido van Rossumde655271997-04-09 17:15:54 +0000495 contstr, needcont = '', 0
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000496 contline = None
Guido van Rossumde655271997-04-09 17:15:54 +0000497 elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
Raymond Hettingera48db392009-04-29 00:34:27 +0000498 yield TokenInfo(ERRORTOKEN, contstr + line,
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000499 strstart, (lnum, len(line)), contline)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000500 contstr = ''
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000501 contline = None
Guido van Rossumde655271997-04-09 17:15:54 +0000502 continue
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000503 else:
504 contstr = contstr + line
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000505 contline = contline + line
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000506 continue
507
Guido van Rossum1aec3231997-04-08 14:24:39 +0000508 elif parenlev == 0 and not continued: # new statement
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000509 if not line: break
510 column = 0
Guido van Rossum1aec3231997-04-08 14:24:39 +0000511 while pos < max: # measure leading whitespace
Benjamin Petersona0dfa822009-11-13 02:25:08 +0000512 if line[pos] == ' ':
513 column += 1
514 elif line[pos] == '\t':
515 column = (column//tabsize + 1)*tabsize
516 elif line[pos] == '\f':
517 column = 0
518 else:
519 break
520 pos += 1
521 if pos == max:
522 break
Guido van Rossum1aec3231997-04-08 14:24:39 +0000523
524 if line[pos] in '#\r\n': # skip comments or blank lines
Thomas Wouters89f507f2006-12-13 04:49:30 +0000525 if line[pos] == '#':
526 comment_token = line[pos:].rstrip('\r\n')
527 nl_pos = pos + len(comment_token)
Raymond Hettingera48db392009-04-29 00:34:27 +0000528 yield TokenInfo(COMMENT, comment_token,
Thomas Wouters89f507f2006-12-13 04:49:30 +0000529 (lnum, pos), (lnum, pos + len(comment_token)), line)
Raymond Hettingera48db392009-04-29 00:34:27 +0000530 yield TokenInfo(NL, line[nl_pos:],
Thomas Wouters89f507f2006-12-13 04:49:30 +0000531 (lnum, nl_pos), (lnum, len(line)), line)
532 else:
Raymond Hettingera48db392009-04-29 00:34:27 +0000533 yield TokenInfo((NL, COMMENT)[line[pos] == '#'], line[pos:],
Guido van Rossum1aec3231997-04-08 14:24:39 +0000534 (lnum, pos), (lnum, len(line)), line)
535 continue
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000536
537 if column > indents[-1]: # count indents or dedents
538 indents.append(column)
Raymond Hettingera48db392009-04-29 00:34:27 +0000539 yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000540 while column < indents[-1]:
Raymond Hettingerda99d1c2005-06-21 07:43:58 +0000541 if column not in indents:
542 raise IndentationError(
Thomas Wouters00ee7ba2006-08-21 19:07:27 +0000543 "unindent does not match any outer indentation level",
544 ("<tokenize>", lnum, pos, line))
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000545 indents = indents[:-1]
Raymond Hettingera48db392009-04-29 00:34:27 +0000546 yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000547
548 else: # continued statement
Guido van Rossumde655271997-04-09 17:15:54 +0000549 if not line:
Collin Winterce36ad82007-08-30 01:19:48 +0000550 raise TokenError("EOF in multi-line statement", (lnum, 0))
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000551 continued = 0
552
553 while pos < max:
Antoine Pitrou10a99b02011-10-11 15:45:56 +0200554 pseudomatch = _compile(PseudoToken).match(line, pos)
Guido van Rossum3b631771997-10-27 20:44:15 +0000555 if pseudomatch: # scan for tokens
556 start, end = pseudomatch.span(1)
Guido van Rossumde655271997-04-09 17:15:54 +0000557 spos, epos, pos = (lnum, start), (lnum, end), end
Ezio Melotti2cc3b4b2012-11-03 17:38:43 +0200558 if start == end:
559 continue
Guido van Rossum1aec3231997-04-08 14:24:39 +0000560 token, initial = line[start:end], line[start]
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000561
Georg Brandldde00282007-03-18 19:01:53 +0000562 if (initial in numchars or # ordinary number
563 (initial == '.' and token != '.' and token != '...')):
Raymond Hettingera48db392009-04-29 00:34:27 +0000564 yield TokenInfo(NUMBER, token, spos, epos, line)
Guido van Rossum1aec3231997-04-08 14:24:39 +0000565 elif initial in '\r\n':
Raymond Hettingera48db392009-04-29 00:34:27 +0000566 yield TokenInfo(NL if parenlev > 0 else NEWLINE,
Thomas Wouters89f507f2006-12-13 04:49:30 +0000567 token, spos, epos, line)
Guido van Rossum1aec3231997-04-08 14:24:39 +0000568 elif initial == '#':
Thomas Wouters89f507f2006-12-13 04:49:30 +0000569 assert not token.endswith("\n")
Raymond Hettingera48db392009-04-29 00:34:27 +0000570 yield TokenInfo(COMMENT, token, spos, epos, line)
Guido van Rossum9d6897a2002-08-24 06:54:19 +0000571 elif token in triple_quoted:
Antoine Pitrou10a99b02011-10-11 15:45:56 +0200572 endprog = _compile(endpats[token])
Guido van Rossum3b631771997-10-27 20:44:15 +0000573 endmatch = endprog.match(line, pos)
574 if endmatch: # all on one line
575 pos = endmatch.end(0)
Guido van Rossum1aec3231997-04-08 14:24:39 +0000576 token = line[start:pos]
Raymond Hettingera48db392009-04-29 00:34:27 +0000577 yield TokenInfo(STRING, token, spos, (lnum, pos), line)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000578 else:
Guido van Rossum1aec3231997-04-08 14:24:39 +0000579 strstart = (lnum, start) # multiple lines
580 contstr = line[start:]
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000581 contline = line
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000582 break
Guido van Rossum9d6897a2002-08-24 06:54:19 +0000583 elif initial in single_quoted or \
584 token[:2] in single_quoted or \
585 token[:3] in single_quoted:
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000586 if token[-1] == '\n': # continued string
Guido van Rossum1aec3231997-04-08 14:24:39 +0000587 strstart = (lnum, start)
Antoine Pitrou10a99b02011-10-11 15:45:56 +0200588 endprog = _compile(endpats[initial] or
589 endpats[token[1]] or
590 endpats[token[2]])
Guido van Rossumde655271997-04-09 17:15:54 +0000591 contstr, needcont = line[start:], 1
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000592 contline = line
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000593 break
594 else: # ordinary string
Raymond Hettingera48db392009-04-29 00:34:27 +0000595 yield TokenInfo(STRING, token, spos, epos, line)
Benjamin Peterson33856de2010-08-30 14:41:20 +0000596 elif initial.isidentifier(): # ordinary name
Raymond Hettingera48db392009-04-29 00:34:27 +0000597 yield TokenInfo(NAME, token, spos, epos, line)
Guido van Rossum3b631771997-10-27 20:44:15 +0000598 elif initial == '\\': # continued stmt
599 continued = 1
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000600 else:
Benjamin Petersona0dfa822009-11-13 02:25:08 +0000601 if initial in '([{':
602 parenlev += 1
603 elif initial in ')]}':
604 parenlev -= 1
Raymond Hettingera48db392009-04-29 00:34:27 +0000605 yield TokenInfo(OP, token, spos, epos, line)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000606 else:
Raymond Hettingera48db392009-04-29 00:34:27 +0000607 yield TokenInfo(ERRORTOKEN, line[pos],
Guido van Rossumde655271997-04-09 17:15:54 +0000608 (lnum, pos), (lnum, pos+1), line)
Benjamin Petersona0dfa822009-11-13 02:25:08 +0000609 pos += 1
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000610
611 for indent in indents[1:]: # pop remaining indent levels
Raymond Hettingera48db392009-04-29 00:34:27 +0000612 yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '')
613 yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '')
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000614
Trent Nelson428de652008-03-18 22:41:35 +0000615
616# An undocumented, backwards compatible, API for all the places in the standard
617# library that expect to be able to use tokenize with strings
618def generate_tokens(readline):
619 return _tokenize(readline, None)
Raymond Hettinger6c60d092010-09-09 04:32:39 +0000620
Meador Inge14c0f032011-10-07 08:53:38 -0500621def main():
622 import argparse
623
624 # Helper error handling routines
625 def perror(message):
626 print(message, file=sys.stderr)
627
628 def error(message, filename=None, location=None):
629 if location:
630 args = (filename,) + location + (message,)
631 perror("%s:%d:%d: error: %s" % args)
632 elif filename:
633 perror("%s: error: %s" % (filename, message))
634 else:
635 perror("error: %s" % message)
636 sys.exit(1)
637
638 # Parse the arguments and options
639 parser = argparse.ArgumentParser(prog='python -m tokenize')
640 parser.add_argument(dest='filename', nargs='?',
641 metavar='filename.py',
642 help='the file to tokenize; defaults to stdin')
Meador Inge00c7f852012-01-19 00:44:45 -0600643 parser.add_argument('-e', '--exact', dest='exact', action='store_true',
644 help='display token names using the exact type')
Meador Inge14c0f032011-10-07 08:53:38 -0500645 args = parser.parse_args()
646
647 try:
648 # Tokenize the input
649 if args.filename:
650 filename = args.filename
651 with builtins.open(filename, 'rb') as f:
652 tokens = list(tokenize(f.readline))
653 else:
654 filename = "<stdin>"
655 tokens = _tokenize(sys.stdin.readline, None)
656
657 # Output the tokenization
658 for token in tokens:
Meador Inge00c7f852012-01-19 00:44:45 -0600659 token_type = token.type
660 if args.exact:
661 token_type = token.exact_type
Meador Inge14c0f032011-10-07 08:53:38 -0500662 token_range = "%d,%d-%d,%d:" % (token.start + token.end)
663 print("%-20s%-15s%-15r" %
Meador Inge00c7f852012-01-19 00:44:45 -0600664 (token_range, tok_name[token_type], token.string))
Meador Inge14c0f032011-10-07 08:53:38 -0500665 except IndentationError as err:
666 line, column = err.args[1][1:3]
667 error(err.args[0], filename, (line, column))
668 except TokenError as err:
669 line, column = err.args[1]
670 error(err.args[0], filename, (line, column))
671 except SyntaxError as err:
672 error(err, filename)
Andrew Svetlovf7a17b42012-12-25 16:47:37 +0200673 except OSError as err:
Meador Inge14c0f032011-10-07 08:53:38 -0500674 error(err)
675 except KeyboardInterrupt:
676 print("interrupted\n")
677 except Exception as err:
678 perror("unexpected error: %s" % err)
679 raise
680
Raymond Hettinger6c60d092010-09-09 04:32:39 +0000681if __name__ == "__main__":
Meador Inge14c0f032011-10-07 08:53:38 -0500682 main()