blob: 51da4847f27d7f2fb3ff031d748d9f36cc0e728a [file] [log] [blame]
Guido van Rossumb51eaa11997-03-07 00:21:55 +00001"""Tokenization help for Python programs.
Guido van Rossum4d8e8591992-01-01 19:34:47 +00002
Florent Xicluna43e4ea12010-09-03 19:54:02 +00003tokenize(readline) is a generator that breaks a stream of bytes into
4Python tokens. It decodes the bytes according to PEP-0263 for
5determining source file encoding.
Trent Nelson428de652008-03-18 22:41:35 +00006
Florent Xicluna43e4ea12010-09-03 19:54:02 +00007It accepts a readline-like method which is called repeatedly to get the
8next line of input (or b"" for EOF). It generates 5-tuples with these
9members:
Tim Peters4efb6e92001-06-29 23:51:08 +000010
11 the token type (see token.py)
12 the token (a string)
13 the starting (row, column) indices of the token (a 2-tuple of ints)
14 the ending (row, column) indices of the token (a 2-tuple of ints)
15 the original line (string)
16
17It is designed to match the working of the Python tokenizer exactly, except
18that it produces COMMENT tokens for comments and gives type OP for all
Florent Xicluna43e4ea12010-09-03 19:54:02 +000019operators. Additionally, all token lists start with an ENCODING token
20which tells you which encoding was used to decode the bytes stream.
21"""
Guido van Rossumb51eaa11997-03-07 00:21:55 +000022
Ka-Ping Yee244c5932001-03-01 13:56:40 +000023__author__ = 'Ka-Ping Yee <ping@lfw.org>'
Trent Nelson428de652008-03-18 22:41:35 +000024__credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
25 'Skip Montanaro, Raymond Hettinger, Trent Nelson, '
26 'Michael Foord')
Benjamin Peterson433f32c2008-12-12 01:25:05 +000027from codecs import lookup, BOM_UTF8
Raymond Hettinger3fb79c72010-09-09 07:15:18 +000028import collections
Victor Stinner58c07522010-11-09 01:08:59 +000029from io import TextIOWrapper
Terry Jan Reedy5b8d2c32014-02-17 23:12:16 -050030from itertools import chain
31import re
32import sys
33from token import *
34
Serhiy Storchakadafea852013-09-16 23:51:56 +030035cookie_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)', re.ASCII)
Serhiy Storchaka768c16c2014-01-09 18:36:09 +020036blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII)
Guido van Rossum4d8e8591992-01-01 19:34:47 +000037
Skip Montanaro40fc1602001-03-01 04:27:19 +000038import token
Alexander Belopolskyb9d10d02010-11-11 14:07:41 +000039__all__ = token.__all__ + ["COMMENT", "tokenize", "detect_encoding",
40 "NL", "untokenize", "ENCODING", "TokenInfo"]
Skip Montanaro40fc1602001-03-01 04:27:19 +000041del token
42
Guido van Rossum1aec3231997-04-08 14:24:39 +000043COMMENT = N_TOKENS
44tok_name[COMMENT] = 'COMMENT'
Guido van Rossuma90c78b1998-04-03 16:05:38 +000045NL = N_TOKENS + 1
46tok_name[NL] = 'NL'
Trent Nelson428de652008-03-18 22:41:35 +000047ENCODING = N_TOKENS + 2
48tok_name[ENCODING] = 'ENCODING'
49N_TOKENS += 3
Meador Inge00c7f852012-01-19 00:44:45 -060050EXACT_TOKEN_TYPES = {
51 '(': LPAR,
52 ')': RPAR,
53 '[': LSQB,
54 ']': RSQB,
55 ':': COLON,
56 ',': COMMA,
57 ';': SEMI,
58 '+': PLUS,
59 '-': MINUS,
60 '*': STAR,
61 '/': SLASH,
62 '|': VBAR,
63 '&': AMPER,
64 '<': LESS,
65 '>': GREATER,
66 '=': EQUAL,
67 '.': DOT,
68 '%': PERCENT,
69 '{': LBRACE,
70 '}': RBRACE,
71 '==': EQEQUAL,
72 '!=': NOTEQUAL,
73 '<=': LESSEQUAL,
74 '>=': GREATEREQUAL,
75 '~': TILDE,
76 '^': CIRCUMFLEX,
77 '<<': LEFTSHIFT,
78 '>>': RIGHTSHIFT,
79 '**': DOUBLESTAR,
80 '+=': PLUSEQUAL,
81 '-=': MINEQUAL,
82 '*=': STAREQUAL,
83 '/=': SLASHEQUAL,
84 '%=': PERCENTEQUAL,
85 '&=': AMPEREQUAL,
86 '|=': VBAREQUAL,
87 '^=': CIRCUMFLEXEQUAL,
88 '<<=': LEFTSHIFTEQUAL,
89 '>>=': RIGHTSHIFTEQUAL,
90 '**=': DOUBLESTAREQUAL,
91 '//': DOUBLESLASH,
92 '//=': DOUBLESLASHEQUAL,
Benjamin Petersond51374e2014-04-09 23:55:56 -040093 '@': AT,
94 '@=': ATEQUAL,
Meador Inge00c7f852012-01-19 00:44:45 -060095}
Guido van Rossum1aec3231997-04-08 14:24:39 +000096
Raymond Hettinger3fb79c72010-09-09 07:15:18 +000097class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')):
Raymond Hettingeraa17a7f2009-04-29 14:21:25 +000098 def __repr__(self):
Raymond Hettingera0e79402010-09-09 08:29:05 +000099 annotated_type = '%d (%s)' % (self.type, tok_name[self.type])
100 return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' %
101 self._replace(type=annotated_type))
Raymond Hettingeraa17a7f2009-04-29 14:21:25 +0000102
Meador Inge00c7f852012-01-19 00:44:45 -0600103 @property
104 def exact_type(self):
105 if self.type == OP and self.string in EXACT_TOKEN_TYPES:
106 return EXACT_TOKEN_TYPES[self.string]
107 else:
108 return self.type
109
Eric S. Raymondb08b2d32001-02-09 11:10:16 +0000110def group(*choices): return '(' + '|'.join(choices) + ')'
Guido van Rossum68468eb2003-02-27 20:14:51 +0000111def any(*choices): return group(*choices) + '*'
112def maybe(*choices): return group(*choices) + '?'
Guido van Rossum4d8e8591992-01-01 19:34:47 +0000113
Antoine Pitroufd036452008-08-19 17:56:33 +0000114# Note: we use unicode matching for names ("\w") but ascii matching for
115# number literals.
Guido van Rossum3b631771997-10-27 20:44:15 +0000116Whitespace = r'[ \f\t]*'
117Comment = r'#[^\r\n]*'
118Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
Benjamin Peterson33856de2010-08-30 14:41:20 +0000119Name = r'\w+'
Guido van Rossum4d8e8591992-01-01 19:34:47 +0000120
Antoine Pitroufd036452008-08-19 17:56:33 +0000121Hexnumber = r'0[xX][0-9a-fA-F]+'
Georg Brandlfceab5a2008-01-19 20:08:23 +0000122Binnumber = r'0[bB][01]+'
123Octnumber = r'0[oO][0-7]+'
Antoine Pitroufd036452008-08-19 17:56:33 +0000124Decnumber = r'(?:0+|[1-9][0-9]*)'
Guido van Rossumcd16bf62007-06-13 18:07:49 +0000125Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
Antoine Pitroufd036452008-08-19 17:56:33 +0000126Exponent = r'[eE][-+]?[0-9]+'
127Pointfloat = group(r'[0-9]+\.[0-9]*', r'\.[0-9]+') + maybe(Exponent)
128Expfloat = r'[0-9]+' + Exponent
Guido van Rossum1aec3231997-04-08 14:24:39 +0000129Floatnumber = group(Pointfloat, Expfloat)
Antoine Pitroufd036452008-08-19 17:56:33 +0000130Imagnumber = group(r'[0-9]+[jJ]', Floatnumber + r'[jJ]')
Guido van Rossum1aec3231997-04-08 14:24:39 +0000131Number = group(Imagnumber, Floatnumber, Intnumber)
Guido van Rossum4d8e8591992-01-01 19:34:47 +0000132
Christian Heimes0b3847d2012-06-20 11:17:58 +0200133StringPrefix = r'(?:[bB][rR]?|[rR][bB]?|[uU])?'
Armin Ronacherc0eaeca2012-03-04 13:07:57 +0000134
Tim Petersde495832000-10-07 05:09:39 +0000135# Tail end of ' string.
136Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
137# Tail end of " string.
138Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
139# Tail end of ''' string.
140Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
141# Tail end of """ string.
142Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
Armin Ronacherc0eaeca2012-03-04 13:07:57 +0000143Triple = group(StringPrefix + "'''", StringPrefix + '"""')
Tim Petersde495832000-10-07 05:09:39 +0000144# Single-line ' or " string.
Armin Ronacherc0eaeca2012-03-04 13:07:57 +0000145String = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
146 StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
Guido van Rossum4d8e8591992-01-01 19:34:47 +0000147
Tim Petersde495832000-10-07 05:09:39 +0000148# Because of leftmost-then-longest match semantics, be sure to put the
149# longest operators first (e.g., if = came before ==, == would get
150# recognized as two instances of =).
Guido van Rossumb053cd82006-08-24 03:53:23 +0000151Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=",
Neal Norwitzc1505362006-12-28 06:47:50 +0000152 r"//=?", r"->",
Benjamin Petersond51374e2014-04-09 23:55:56 -0400153 r"[+\-*/%&@|^=<>]=?",
Tim Petersde495832000-10-07 05:09:39 +0000154 r"~")
Thomas Wouterse1519a12000-08-24 21:44:52 +0000155
Guido van Rossum4d8e8591992-01-01 19:34:47 +0000156Bracket = '[][(){}]'
Georg Brandldde00282007-03-18 19:01:53 +0000157Special = group(r'\r?\n', r'\.\.\.', r'[:;.,@]')
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000158Funny = group(Operator, Bracket, Special)
Guido van Rossum4d8e8591992-01-01 19:34:47 +0000159
Guido van Rossum3b631771997-10-27 20:44:15 +0000160PlainToken = group(Number, Funny, String, Name)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000161Token = Ignore + PlainToken
Guido van Rossum4d8e8591992-01-01 19:34:47 +0000162
Tim Petersde495832000-10-07 05:09:39 +0000163# First (or only) line of ' or " string.
Armin Ronacherc0eaeca2012-03-04 13:07:57 +0000164ContStr = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
Ka-Ping Yee1ff08b12001-01-15 22:04:30 +0000165 group("'", r'\\\r?\n'),
Armin Ronacherc0eaeca2012-03-04 13:07:57 +0000166 StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
Ka-Ping Yee1ff08b12001-01-15 22:04:30 +0000167 group('"', r'\\\r?\n'))
Ezio Melotti2cc3b4b2012-11-03 17:38:43 +0200168PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple)
Guido van Rossum3b631771997-10-27 20:44:15 +0000169PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
Guido van Rossum1aec3231997-04-08 14:24:39 +0000170
Benjamin Peterson33856de2010-08-30 14:41:20 +0000171def _compile(expr):
172 return re.compile(expr, re.UNICODE)
173
Antoine Pitrou10a99b02011-10-11 15:45:56 +0200174endpats = {"'": Single, '"': Double,
175 "'''": Single3, '"""': Double3,
176 "r'''": Single3, 'r"""': Double3,
177 "b'''": Single3, 'b"""': Double3,
Antoine Pitrou10a99b02011-10-11 15:45:56 +0200178 "R'''": Single3, 'R"""': Double3,
179 "B'''": Single3, 'B"""': Double3,
Armin Ronacherc0eaeca2012-03-04 13:07:57 +0000180 "br'''": Single3, 'br"""': Double3,
Antoine Pitrou10a99b02011-10-11 15:45:56 +0200181 "bR'''": Single3, 'bR"""': Double3,
182 "Br'''": Single3, 'Br"""': Double3,
183 "BR'''": Single3, 'BR"""': Double3,
Armin Ronacherc0eaeca2012-03-04 13:07:57 +0000184 "rb'''": Single3, 'rb"""': Double3,
185 "Rb'''": Single3, 'Rb"""': Double3,
186 "rB'''": Single3, 'rB"""': Double3,
187 "RB'''": Single3, 'RB"""': Double3,
Armin Ronacher6ecf77b2012-03-04 12:04:06 +0000188 "u'''": Single3, 'u"""': Double3,
Armin Ronacher6ecf77b2012-03-04 12:04:06 +0000189 "R'''": Single3, 'R"""': Double3,
190 "U'''": Single3, 'U"""': Double3,
Armin Ronacher6ecf77b2012-03-04 12:04:06 +0000191 'r': None, 'R': None, 'b': None, 'B': None,
192 'u': None, 'U': None}
Guido van Rossum4d8e8591992-01-01 19:34:47 +0000193
Guido van Rossum9d6897a2002-08-24 06:54:19 +0000194triple_quoted = {}
195for t in ("'''", '"""',
196 "r'''", 'r"""', "R'''", 'R"""',
Guido van Rossum4fe72f92007-11-12 17:40:10 +0000197 "b'''", 'b"""', "B'''", 'B"""',
198 "br'''", 'br"""', "Br'''", 'Br"""',
Armin Ronacher6ecf77b2012-03-04 12:04:06 +0000199 "bR'''", 'bR"""', "BR'''", 'BR"""',
Armin Ronacherc0eaeca2012-03-04 13:07:57 +0000200 "rb'''", 'rb"""', "rB'''", 'rB"""',
201 "Rb'''", 'Rb"""', "RB'''", 'RB"""',
Armin Ronacher6ecf77b2012-03-04 12:04:06 +0000202 "u'''", 'u"""', "U'''", 'U"""',
Christian Heimes0b3847d2012-06-20 11:17:58 +0200203 ):
Guido van Rossum9d6897a2002-08-24 06:54:19 +0000204 triple_quoted[t] = t
205single_quoted = {}
206for t in ("'", '"',
207 "r'", 'r"', "R'", 'R"',
Guido van Rossum4fe72f92007-11-12 17:40:10 +0000208 "b'", 'b"', "B'", 'B"',
209 "br'", 'br"', "Br'", 'Br"',
Armin Ronacher6ecf77b2012-03-04 12:04:06 +0000210 "bR'", 'bR"', "BR'", 'BR"' ,
Armin Ronacherc0eaeca2012-03-04 13:07:57 +0000211 "rb'", 'rb"', "rB'", 'rB"',
212 "Rb'", 'Rb"', "RB'", 'RB"' ,
Armin Ronacher6ecf77b2012-03-04 12:04:06 +0000213 "u'", 'u"', "U'", 'U"',
Christian Heimes0b3847d2012-06-20 11:17:58 +0200214 ):
Guido van Rossum9d6897a2002-08-24 06:54:19 +0000215 single_quoted[t] = t
216
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000217tabsize = 8
Fred Drake9b8d8012000-08-17 04:45:13 +0000218
Ka-Ping Yee28c62bb2001-03-23 05:22:49 +0000219class TokenError(Exception): pass
220
221class StopTokenizing(Exception): pass
Fred Drake9b8d8012000-08-17 04:45:13 +0000222
Tim Peters5ca576e2001-06-18 22:08:13 +0000223
Thomas Wouters89f507f2006-12-13 04:49:30 +0000224class Untokenizer:
225
226 def __init__(self):
227 self.tokens = []
228 self.prev_row = 1
229 self.prev_col = 0
Trent Nelson428de652008-03-18 22:41:35 +0000230 self.encoding = None
Thomas Wouters89f507f2006-12-13 04:49:30 +0000231
232 def add_whitespace(self, start):
233 row, col = start
Terry Jan Reedy5e6db312014-02-17 16:45:48 -0500234 if row < self.prev_row or row == self.prev_row and col < self.prev_col:
235 raise ValueError("start ({},{}) precedes previous end ({},{})"
236 .format(row, col, self.prev_row, self.prev_col))
Terry Jan Reedy9dc3a362014-02-23 23:33:08 -0500237 row_offset = row - self.prev_row
Terry Jan Reedyf106f8f2014-02-23 23:39:57 -0500238 if row_offset:
Terry Jan Reedy9dc3a362014-02-23 23:33:08 -0500239 self.tokens.append("\\\n" * row_offset)
240 self.prev_col = 0
Thomas Wouters89f507f2006-12-13 04:49:30 +0000241 col_offset = col - self.prev_col
242 if col_offset:
243 self.tokens.append(" " * col_offset)
244
245 def untokenize(self, iterable):
Terry Jan Reedy5b8d2c32014-02-17 23:12:16 -0500246 it = iter(iterable)
247 for t in it:
Thomas Wouters89f507f2006-12-13 04:49:30 +0000248 if len(t) == 2:
Terry Jan Reedy5b8d2c32014-02-17 23:12:16 -0500249 self.compat(t, it)
Thomas Wouters89f507f2006-12-13 04:49:30 +0000250 break
251 tok_type, token, start, end, line = t
Trent Nelson428de652008-03-18 22:41:35 +0000252 if tok_type == ENCODING:
253 self.encoding = token
254 continue
Terry Jan Reedy9dc3a362014-02-23 23:33:08 -0500255 if tok_type == ENDMARKER:
256 break
Thomas Wouters89f507f2006-12-13 04:49:30 +0000257 self.add_whitespace(start)
258 self.tokens.append(token)
259 self.prev_row, self.prev_col = end
260 if tok_type in (NEWLINE, NL):
261 self.prev_row += 1
262 self.prev_col = 0
263 return "".join(self.tokens)
264
265 def compat(self, token, iterable):
Thomas Wouters89f507f2006-12-13 04:49:30 +0000266 indents = []
267 toks_append = self.tokens.append
Terry Jan Reedy5b8d2c32014-02-17 23:12:16 -0500268 startline = token[0] in (NEWLINE, NL)
Christian Heimesba4af492008-03-28 00:55:15 +0000269 prevstring = False
Terry Jan Reedy5b8d2c32014-02-17 23:12:16 -0500270
271 for tok in chain([token], iterable):
Thomas Wouters89f507f2006-12-13 04:49:30 +0000272 toknum, tokval = tok[:2]
Trent Nelson428de652008-03-18 22:41:35 +0000273 if toknum == ENCODING:
274 self.encoding = tokval
275 continue
Thomas Wouters89f507f2006-12-13 04:49:30 +0000276
277 if toknum in (NAME, NUMBER):
278 tokval += ' '
279
Christian Heimesba4af492008-03-28 00:55:15 +0000280 # Insert a space between two consecutive strings
281 if toknum == STRING:
282 if prevstring:
283 tokval = ' ' + tokval
284 prevstring = True
285 else:
286 prevstring = False
287
Thomas Wouters89f507f2006-12-13 04:49:30 +0000288 if toknum == INDENT:
289 indents.append(tokval)
290 continue
291 elif toknum == DEDENT:
292 indents.pop()
293 continue
294 elif toknum in (NEWLINE, NL):
295 startline = True
296 elif startline and indents:
297 toks_append(indents[-1])
298 startline = False
299 toks_append(tokval)
Raymond Hettinger68c04532005-06-10 11:05:19 +0000300
Trent Nelson428de652008-03-18 22:41:35 +0000301
Raymond Hettinger68c04532005-06-10 11:05:19 +0000302def untokenize(iterable):
303 """Transform tokens back into Python source code.
Trent Nelson428de652008-03-18 22:41:35 +0000304 It returns a bytes object, encoded using the ENCODING
305 token, which is the first token sequence output by tokenize.
Raymond Hettinger68c04532005-06-10 11:05:19 +0000306
307 Each element returned by the iterable must be a token sequence
Thomas Wouters89f507f2006-12-13 04:49:30 +0000308 with at least two elements, a token number and token value. If
309 only two tokens are passed, the resulting output is poor.
Raymond Hettinger68c04532005-06-10 11:05:19 +0000310
Thomas Wouters89f507f2006-12-13 04:49:30 +0000311 Round-trip invariant for full input:
312 Untokenized source will match input source exactly
313
314 Round-trip invariant for limited intput:
Trent Nelson428de652008-03-18 22:41:35 +0000315 # Output bytes will tokenize the back to the input
316 t1 = [tok[:2] for tok in tokenize(f.readline)]
Raymond Hettinger68c04532005-06-10 11:05:19 +0000317 newcode = untokenize(t1)
Trent Nelson428de652008-03-18 22:41:35 +0000318 readline = BytesIO(newcode).readline
319 t2 = [tok[:2] for tok in tokenize(readline)]
Raymond Hettinger68c04532005-06-10 11:05:19 +0000320 assert t1 == t2
321 """
Thomas Wouters89f507f2006-12-13 04:49:30 +0000322 ut = Untokenizer()
Trent Nelson428de652008-03-18 22:41:35 +0000323 out = ut.untokenize(iterable)
324 if ut.encoding is not None:
325 out = out.encode(ut.encoding)
326 return out
Raymond Hettinger68c04532005-06-10 11:05:19 +0000327
Trent Nelson428de652008-03-18 22:41:35 +0000328
Benjamin Petersond3afada2009-10-09 21:43:09 +0000329def _get_normal_name(orig_enc):
330 """Imitates get_normal_name in tokenizer.c."""
331 # Only care about the first 12 characters.
332 enc = orig_enc[:12].lower().replace("_", "-")
333 if enc == "utf-8" or enc.startswith("utf-8-"):
334 return "utf-8"
335 if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \
336 enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")):
337 return "iso-8859-1"
338 return orig_enc
339
Trent Nelson428de652008-03-18 22:41:35 +0000340def detect_encoding(readline):
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000341 """
Trent Nelson428de652008-03-18 22:41:35 +0000342 The detect_encoding() function is used to detect the encoding that should
Ezio Melotti4bcc7962013-11-25 05:14:51 +0200343 be used to decode a Python source file. It requires one argument, readline,
Trent Nelson428de652008-03-18 22:41:35 +0000344 in the same way as the tokenize() generator.
345
346 It will call readline a maximum of twice, and return the encoding used
Florent Xicluna43e4ea12010-09-03 19:54:02 +0000347 (as a string) and a list of any lines (left as bytes) it has read in.
Trent Nelson428de652008-03-18 22:41:35 +0000348
349 It detects the encoding from the presence of a utf-8 bom or an encoding
Florent Xicluna43e4ea12010-09-03 19:54:02 +0000350 cookie as specified in pep-0263. If both a bom and a cookie are present,
351 but disagree, a SyntaxError will be raised. If the encoding cookie is an
352 invalid charset, raise a SyntaxError. Note that if a utf-8 bom is found,
Benjamin Peterson689a5582010-03-18 22:29:52 +0000353 'utf-8-sig' is returned.
Trent Nelson428de652008-03-18 22:41:35 +0000354
355 If no encoding is specified, then the default of 'utf-8' will be returned.
356 """
Brett Cannonc33f3f22012-04-20 13:23:54 -0400357 try:
358 filename = readline.__self__.name
359 except AttributeError:
360 filename = None
Trent Nelson428de652008-03-18 22:41:35 +0000361 bom_found = False
362 encoding = None
Benjamin Peterson689a5582010-03-18 22:29:52 +0000363 default = 'utf-8'
Trent Nelson428de652008-03-18 22:41:35 +0000364 def read_or_stop():
365 try:
366 return readline()
367 except StopIteration:
368 return b''
369
370 def find_cookie(line):
371 try:
Martin v. Löwis63674f42012-04-20 14:36:47 +0200372 # Decode as UTF-8. Either the line is an encoding declaration,
373 # in which case it should be pure ASCII, or it must be UTF-8
374 # per default encoding.
375 line_string = line.decode('utf-8')
Trent Nelson428de652008-03-18 22:41:35 +0000376 except UnicodeDecodeError:
Brett Cannonc33f3f22012-04-20 13:23:54 -0400377 msg = "invalid or missing encoding declaration"
378 if filename is not None:
379 msg = '{} for {!r}'.format(msg, filename)
380 raise SyntaxError(msg)
Benjamin Peterson433f32c2008-12-12 01:25:05 +0000381
Serhiy Storchakadafea852013-09-16 23:51:56 +0300382 match = cookie_re.match(line_string)
383 if not match:
Benjamin Peterson433f32c2008-12-12 01:25:05 +0000384 return None
Serhiy Storchakadafea852013-09-16 23:51:56 +0300385 encoding = _get_normal_name(match.group(1))
Benjamin Peterson433f32c2008-12-12 01:25:05 +0000386 try:
387 codec = lookup(encoding)
388 except LookupError:
389 # This behaviour mimics the Python interpreter
Brett Cannonc33f3f22012-04-20 13:23:54 -0400390 if filename is None:
391 msg = "unknown encoding: " + encoding
392 else:
393 msg = "unknown encoding for {!r}: {}".format(filename,
394 encoding)
395 raise SyntaxError(msg)
Benjamin Peterson433f32c2008-12-12 01:25:05 +0000396
Benjamin Peterson1613ed82010-03-18 22:34:15 +0000397 if bom_found:
Florent Xicluna11f0b412012-07-07 12:13:35 +0200398 if encoding != 'utf-8':
Benjamin Peterson1613ed82010-03-18 22:34:15 +0000399 # This behaviour mimics the Python interpreter
Brett Cannonc33f3f22012-04-20 13:23:54 -0400400 if filename is None:
401 msg = 'encoding problem: utf-8'
402 else:
403 msg = 'encoding problem for {!r}: utf-8'.format(filename)
404 raise SyntaxError(msg)
Benjamin Peterson1613ed82010-03-18 22:34:15 +0000405 encoding += '-sig'
Benjamin Peterson433f32c2008-12-12 01:25:05 +0000406 return encoding
Trent Nelson428de652008-03-18 22:41:35 +0000407
408 first = read_or_stop()
Benjamin Peterson433f32c2008-12-12 01:25:05 +0000409 if first.startswith(BOM_UTF8):
Trent Nelson428de652008-03-18 22:41:35 +0000410 bom_found = True
411 first = first[3:]
Benjamin Peterson689a5582010-03-18 22:29:52 +0000412 default = 'utf-8-sig'
Trent Nelson428de652008-03-18 22:41:35 +0000413 if not first:
Benjamin Peterson689a5582010-03-18 22:29:52 +0000414 return default, []
Trent Nelson428de652008-03-18 22:41:35 +0000415
416 encoding = find_cookie(first)
417 if encoding:
418 return encoding, [first]
Serhiy Storchaka768c16c2014-01-09 18:36:09 +0200419 if not blank_re.match(first):
420 return default, [first]
Trent Nelson428de652008-03-18 22:41:35 +0000421
422 second = read_or_stop()
423 if not second:
Benjamin Peterson689a5582010-03-18 22:29:52 +0000424 return default, [first]
Trent Nelson428de652008-03-18 22:41:35 +0000425
426 encoding = find_cookie(second)
427 if encoding:
428 return encoding, [first, second]
429
Benjamin Peterson689a5582010-03-18 22:29:52 +0000430 return default, [first, second]
Trent Nelson428de652008-03-18 22:41:35 +0000431
432
Victor Stinner96917502014-12-05 10:17:10 +0100433_builtin_open = open
434
Victor Stinner58c07522010-11-09 01:08:59 +0000435def open(filename):
436 """Open a file in read only mode using the encoding detected by
437 detect_encoding().
438 """
Victor Stinner96917502014-12-05 10:17:10 +0100439 buffer = _builtin_open(filename, 'rb')
Victor Stinner58c07522010-11-09 01:08:59 +0000440 encoding, lines = detect_encoding(buffer.readline)
441 buffer.seek(0)
442 text = TextIOWrapper(buffer, encoding, line_buffering=True)
443 text.mode = 'r'
444 return text
445
446
Trent Nelson428de652008-03-18 22:41:35 +0000447def tokenize(readline):
448 """
449 The tokenize() generator requires one argment, readline, which
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000450 must be a callable object which provides the same interface as the
Florent Xicluna43e4ea12010-09-03 19:54:02 +0000451 readline() method of built-in file objects. Each call to the function
Trent Nelson428de652008-03-18 22:41:35 +0000452 should return one line of input as bytes. Alternately, readline
Raymond Hettinger68c04532005-06-10 11:05:19 +0000453 can be a callable function terminating with StopIteration:
Trent Nelson428de652008-03-18 22:41:35 +0000454 readline = open(myfile, 'rb').__next__ # Example of alternate readline
Tim Peters8ac14952002-05-23 15:15:30 +0000455
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000456 The generator produces 5-tuples with these members: the token type; the
457 token string; a 2-tuple (srow, scol) of ints specifying the row and
458 column where the token begins in the source; a 2-tuple (erow, ecol) of
459 ints specifying the row and column where the token ends in the source;
Florent Xicluna43e4ea12010-09-03 19:54:02 +0000460 and the line on which the token was found. The line passed is the
Tim Peters8ac14952002-05-23 15:15:30 +0000461 logical line; continuation lines are included.
Trent Nelson428de652008-03-18 22:41:35 +0000462
463 The first token sequence will always be an ENCODING token
464 which tells you which encoding was used to decode the bytes stream.
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000465 """
Benjamin Peterson21db77e2009-11-14 16:27:26 +0000466 # This import is here to avoid problems when the itertools module is not
467 # built yet and tokenize is imported.
Benjamin Peterson81dd8b92009-11-14 18:09:17 +0000468 from itertools import chain, repeat
Trent Nelson428de652008-03-18 22:41:35 +0000469 encoding, consumed = detect_encoding(readline)
Benjamin Peterson81dd8b92009-11-14 18:09:17 +0000470 rl_gen = iter(readline, b"")
471 empty = repeat(b"")
472 return _tokenize(chain(consumed, rl_gen, empty).__next__, encoding)
Trent Nelson428de652008-03-18 22:41:35 +0000473
474
475def _tokenize(readline, encoding):
Guido van Rossum1aec3231997-04-08 14:24:39 +0000476 lnum = parenlev = continued = 0
Benjamin Peterson33856de2010-08-30 14:41:20 +0000477 numchars = '0123456789'
Guido van Rossumde655271997-04-09 17:15:54 +0000478 contstr, needcont = '', 0
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000479 contline = None
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000480 indents = [0]
Guido van Rossum1aec3231997-04-08 14:24:39 +0000481
Trent Nelson428de652008-03-18 22:41:35 +0000482 if encoding is not None:
Benjamin Peterson689a5582010-03-18 22:29:52 +0000483 if encoding == "utf-8-sig":
484 # BOM will already have been stripped.
485 encoding = "utf-8"
Raymond Hettingera48db392009-04-29 00:34:27 +0000486 yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '')
Benjamin Peterson0fe14382008-06-05 23:07:42 +0000487 while True: # loop over lines in stream
Raymond Hettinger68c04532005-06-10 11:05:19 +0000488 try:
489 line = readline()
490 except StopIteration:
Trent Nelson428de652008-03-18 22:41:35 +0000491 line = b''
492
493 if encoding is not None:
494 line = line.decode(encoding)
Benjamin Petersona0dfa822009-11-13 02:25:08 +0000495 lnum += 1
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000496 pos, max = 0, len(line)
497
498 if contstr: # continued string
Guido van Rossumde655271997-04-09 17:15:54 +0000499 if not line:
Collin Winterce36ad82007-08-30 01:19:48 +0000500 raise TokenError("EOF in multi-line string", strstart)
Guido van Rossum3b631771997-10-27 20:44:15 +0000501 endmatch = endprog.match(line)
502 if endmatch:
503 pos = end = endmatch.end(0)
Raymond Hettingera48db392009-04-29 00:34:27 +0000504 yield TokenInfo(STRING, contstr + line[:end],
Thomas Wouters89f507f2006-12-13 04:49:30 +0000505 strstart, (lnum, end), contline + line)
Guido van Rossumde655271997-04-09 17:15:54 +0000506 contstr, needcont = '', 0
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000507 contline = None
Guido van Rossumde655271997-04-09 17:15:54 +0000508 elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
Raymond Hettingera48db392009-04-29 00:34:27 +0000509 yield TokenInfo(ERRORTOKEN, contstr + line,
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000510 strstart, (lnum, len(line)), contline)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000511 contstr = ''
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000512 contline = None
Guido van Rossumde655271997-04-09 17:15:54 +0000513 continue
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000514 else:
515 contstr = contstr + line
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000516 contline = contline + line
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000517 continue
518
Guido van Rossum1aec3231997-04-08 14:24:39 +0000519 elif parenlev == 0 and not continued: # new statement
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000520 if not line: break
521 column = 0
Guido van Rossum1aec3231997-04-08 14:24:39 +0000522 while pos < max: # measure leading whitespace
Benjamin Petersona0dfa822009-11-13 02:25:08 +0000523 if line[pos] == ' ':
524 column += 1
525 elif line[pos] == '\t':
526 column = (column//tabsize + 1)*tabsize
527 elif line[pos] == '\f':
528 column = 0
529 else:
530 break
531 pos += 1
532 if pos == max:
533 break
Guido van Rossum1aec3231997-04-08 14:24:39 +0000534
535 if line[pos] in '#\r\n': # skip comments or blank lines
Thomas Wouters89f507f2006-12-13 04:49:30 +0000536 if line[pos] == '#':
537 comment_token = line[pos:].rstrip('\r\n')
538 nl_pos = pos + len(comment_token)
Raymond Hettingera48db392009-04-29 00:34:27 +0000539 yield TokenInfo(COMMENT, comment_token,
Thomas Wouters89f507f2006-12-13 04:49:30 +0000540 (lnum, pos), (lnum, pos + len(comment_token)), line)
Raymond Hettingera48db392009-04-29 00:34:27 +0000541 yield TokenInfo(NL, line[nl_pos:],
Thomas Wouters89f507f2006-12-13 04:49:30 +0000542 (lnum, nl_pos), (lnum, len(line)), line)
543 else:
Raymond Hettingera48db392009-04-29 00:34:27 +0000544 yield TokenInfo((NL, COMMENT)[line[pos] == '#'], line[pos:],
Guido van Rossum1aec3231997-04-08 14:24:39 +0000545 (lnum, pos), (lnum, len(line)), line)
546 continue
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000547
548 if column > indents[-1]: # count indents or dedents
549 indents.append(column)
Raymond Hettingera48db392009-04-29 00:34:27 +0000550 yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000551 while column < indents[-1]:
Raymond Hettingerda99d1c2005-06-21 07:43:58 +0000552 if column not in indents:
553 raise IndentationError(
Thomas Wouters00ee7ba2006-08-21 19:07:27 +0000554 "unindent does not match any outer indentation level",
555 ("<tokenize>", lnum, pos, line))
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000556 indents = indents[:-1]
Raymond Hettingera48db392009-04-29 00:34:27 +0000557 yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000558
559 else: # continued statement
Guido van Rossumde655271997-04-09 17:15:54 +0000560 if not line:
Collin Winterce36ad82007-08-30 01:19:48 +0000561 raise TokenError("EOF in multi-line statement", (lnum, 0))
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000562 continued = 0
563
564 while pos < max:
Antoine Pitrou10a99b02011-10-11 15:45:56 +0200565 pseudomatch = _compile(PseudoToken).match(line, pos)
Guido van Rossum3b631771997-10-27 20:44:15 +0000566 if pseudomatch: # scan for tokens
567 start, end = pseudomatch.span(1)
Guido van Rossumde655271997-04-09 17:15:54 +0000568 spos, epos, pos = (lnum, start), (lnum, end), end
Ezio Melotti2cc3b4b2012-11-03 17:38:43 +0200569 if start == end:
570 continue
Guido van Rossum1aec3231997-04-08 14:24:39 +0000571 token, initial = line[start:end], line[start]
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000572
Georg Brandldde00282007-03-18 19:01:53 +0000573 if (initial in numchars or # ordinary number
574 (initial == '.' and token != '.' and token != '...')):
Raymond Hettingera48db392009-04-29 00:34:27 +0000575 yield TokenInfo(NUMBER, token, spos, epos, line)
Guido van Rossum1aec3231997-04-08 14:24:39 +0000576 elif initial in '\r\n':
Raymond Hettingera48db392009-04-29 00:34:27 +0000577 yield TokenInfo(NL if parenlev > 0 else NEWLINE,
Thomas Wouters89f507f2006-12-13 04:49:30 +0000578 token, spos, epos, line)
Guido van Rossum1aec3231997-04-08 14:24:39 +0000579 elif initial == '#':
Thomas Wouters89f507f2006-12-13 04:49:30 +0000580 assert not token.endswith("\n")
Raymond Hettingera48db392009-04-29 00:34:27 +0000581 yield TokenInfo(COMMENT, token, spos, epos, line)
Guido van Rossum9d6897a2002-08-24 06:54:19 +0000582 elif token in triple_quoted:
Antoine Pitrou10a99b02011-10-11 15:45:56 +0200583 endprog = _compile(endpats[token])
Guido van Rossum3b631771997-10-27 20:44:15 +0000584 endmatch = endprog.match(line, pos)
585 if endmatch: # all on one line
586 pos = endmatch.end(0)
Guido van Rossum1aec3231997-04-08 14:24:39 +0000587 token = line[start:pos]
Raymond Hettingera48db392009-04-29 00:34:27 +0000588 yield TokenInfo(STRING, token, spos, (lnum, pos), line)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000589 else:
Guido van Rossum1aec3231997-04-08 14:24:39 +0000590 strstart = (lnum, start) # multiple lines
591 contstr = line[start:]
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000592 contline = line
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000593 break
Guido van Rossum9d6897a2002-08-24 06:54:19 +0000594 elif initial in single_quoted or \
595 token[:2] in single_quoted or \
596 token[:3] in single_quoted:
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000597 if token[-1] == '\n': # continued string
Guido van Rossum1aec3231997-04-08 14:24:39 +0000598 strstart = (lnum, start)
Antoine Pitrou10a99b02011-10-11 15:45:56 +0200599 endprog = _compile(endpats[initial] or
600 endpats[token[1]] or
601 endpats[token[2]])
Guido van Rossumde655271997-04-09 17:15:54 +0000602 contstr, needcont = line[start:], 1
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000603 contline = line
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000604 break
605 else: # ordinary string
Raymond Hettingera48db392009-04-29 00:34:27 +0000606 yield TokenInfo(STRING, token, spos, epos, line)
Benjamin Peterson33856de2010-08-30 14:41:20 +0000607 elif initial.isidentifier(): # ordinary name
Raymond Hettingera48db392009-04-29 00:34:27 +0000608 yield TokenInfo(NAME, token, spos, epos, line)
Guido van Rossum3b631771997-10-27 20:44:15 +0000609 elif initial == '\\': # continued stmt
610 continued = 1
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000611 else:
Benjamin Petersona0dfa822009-11-13 02:25:08 +0000612 if initial in '([{':
613 parenlev += 1
614 elif initial in ')]}':
615 parenlev -= 1
Raymond Hettingera48db392009-04-29 00:34:27 +0000616 yield TokenInfo(OP, token, spos, epos, line)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000617 else:
Raymond Hettingera48db392009-04-29 00:34:27 +0000618 yield TokenInfo(ERRORTOKEN, line[pos],
Guido van Rossumde655271997-04-09 17:15:54 +0000619 (lnum, pos), (lnum, pos+1), line)
Benjamin Petersona0dfa822009-11-13 02:25:08 +0000620 pos += 1
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000621
622 for indent in indents[1:]: # pop remaining indent levels
Raymond Hettingera48db392009-04-29 00:34:27 +0000623 yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '')
624 yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '')
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000625
Trent Nelson428de652008-03-18 22:41:35 +0000626
627# An undocumented, backwards compatible, API for all the places in the standard
628# library that expect to be able to use tokenize with strings
629def generate_tokens(readline):
630 return _tokenize(readline, None)
Raymond Hettinger6c60d092010-09-09 04:32:39 +0000631
Meador Inge14c0f032011-10-07 08:53:38 -0500632def main():
633 import argparse
634
635 # Helper error handling routines
636 def perror(message):
637 print(message, file=sys.stderr)
638
639 def error(message, filename=None, location=None):
640 if location:
641 args = (filename,) + location + (message,)
642 perror("%s:%d:%d: error: %s" % args)
643 elif filename:
644 perror("%s: error: %s" % (filename, message))
645 else:
646 perror("error: %s" % message)
647 sys.exit(1)
648
649 # Parse the arguments and options
650 parser = argparse.ArgumentParser(prog='python -m tokenize')
651 parser.add_argument(dest='filename', nargs='?',
652 metavar='filename.py',
653 help='the file to tokenize; defaults to stdin')
Meador Inge00c7f852012-01-19 00:44:45 -0600654 parser.add_argument('-e', '--exact', dest='exact', action='store_true',
655 help='display token names using the exact type')
Meador Inge14c0f032011-10-07 08:53:38 -0500656 args = parser.parse_args()
657
658 try:
659 # Tokenize the input
660 if args.filename:
661 filename = args.filename
Victor Stinner96917502014-12-05 10:17:10 +0100662 with _builtin_open(filename, 'rb') as f:
Meador Inge14c0f032011-10-07 08:53:38 -0500663 tokens = list(tokenize(f.readline))
664 else:
665 filename = "<stdin>"
666 tokens = _tokenize(sys.stdin.readline, None)
667
668 # Output the tokenization
669 for token in tokens:
Meador Inge00c7f852012-01-19 00:44:45 -0600670 token_type = token.type
671 if args.exact:
672 token_type = token.exact_type
Meador Inge14c0f032011-10-07 08:53:38 -0500673 token_range = "%d,%d-%d,%d:" % (token.start + token.end)
674 print("%-20s%-15s%-15r" %
Meador Inge00c7f852012-01-19 00:44:45 -0600675 (token_range, tok_name[token_type], token.string))
Meador Inge14c0f032011-10-07 08:53:38 -0500676 except IndentationError as err:
677 line, column = err.args[1][1:3]
678 error(err.args[0], filename, (line, column))
679 except TokenError as err:
680 line, column = err.args[1]
681 error(err.args[0], filename, (line, column))
682 except SyntaxError as err:
683 error(err, filename)
Andrew Svetlovf7a17b42012-12-25 16:47:37 +0200684 except OSError as err:
Meador Inge14c0f032011-10-07 08:53:38 -0500685 error(err)
686 except KeyboardInterrupt:
687 print("interrupted\n")
688 except Exception as err:
689 perror("unexpected error: %s" % err)
690 raise
691
Raymond Hettinger6c60d092010-09-09 04:32:39 +0000692if __name__ == "__main__":
Meador Inge14c0f032011-10-07 08:53:38 -0500693 main()