blob: 7e6fa1219acfd78d3bacd27a874414c132645867 [file] [log] [blame]
Guido van Rossumb51eaa11997-03-07 00:21:55 +00001"""Tokenization help for Python programs.
Guido van Rossum4d8e8591992-01-01 19:34:47 +00002
Tim Peters4efb6e92001-06-29 23:51:08 +00003generate_tokens(readline) is a generator that breaks a stream of
Guido van Rossum1aec3231997-04-08 14:24:39 +00004text into Python tokens. It accepts a readline-like method which is called
Tim Peters4efb6e92001-06-29 23:51:08 +00005repeatedly to get the next line of input (or "" for EOF). It generates
65-tuples with these members:
7
8 the token type (see token.py)
9 the token (a string)
10 the starting (row, column) indices of the token (a 2-tuple of ints)
11 the ending (row, column) indices of the token (a 2-tuple of ints)
12 the original line (string)
13
14It is designed to match the working of the Python tokenizer exactly, except
15that it produces COMMENT tokens for comments and gives type OP for all
16operators
17
18Older entry points
19 tokenize_loop(readline, tokeneater)
20 tokenize(readline, tokeneater=printtoken)
21are the same, except instead of generating tokens, tokeneater is a callback
22function to which the 5 fields described above are passed as 5 arguments,
23each time a new token is found."""
Guido van Rossumb51eaa11997-03-07 00:21:55 +000024
Ka-Ping Yee244c5932001-03-01 13:56:40 +000025__author__ = 'Ka-Ping Yee <ping@lfw.org>'
Ka-Ping Yee4f64c132001-03-01 17:11:17 +000026__credits__ = \
27 'GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro'
Guido van Rossumb51eaa11997-03-07 00:21:55 +000028
Guido van Rossum3b631771997-10-27 20:44:15 +000029import string, re
Guido van Rossumfc6f5331997-03-07 00:21:12 +000030from token import *
Guido van Rossum4d8e8591992-01-01 19:34:47 +000031
Skip Montanaro40fc1602001-03-01 04:27:19 +000032import token
Raymond Hettinger78a7aee2002-11-05 06:06:02 +000033__all__ = [x for x in dir(token) if x[0] != '_'] + ["COMMENT", "tokenize",
34 "generate_tokens", "NL"]
Neal Norwitze98d16e2002-03-26 16:20:26 +000035del x
Skip Montanaro40fc1602001-03-01 04:27:19 +000036del token
37
Guido van Rossum1aec3231997-04-08 14:24:39 +000038COMMENT = N_TOKENS
39tok_name[COMMENT] = 'COMMENT'
Guido van Rossuma90c78b1998-04-03 16:05:38 +000040NL = N_TOKENS + 1
41tok_name[NL] = 'NL'
Skip Montanaro40fc1602001-03-01 04:27:19 +000042N_TOKENS += 2
Guido van Rossum1aec3231997-04-08 14:24:39 +000043
Eric S. Raymondb08b2d32001-02-09 11:10:16 +000044def group(*choices): return '(' + '|'.join(choices) + ')'
Guido van Rossum68468eb2003-02-27 20:14:51 +000045def any(*choices): return group(*choices) + '*'
46def maybe(*choices): return group(*choices) + '?'
Guido van Rossum4d8e8591992-01-01 19:34:47 +000047
Guido van Rossum3b631771997-10-27 20:44:15 +000048Whitespace = r'[ \f\t]*'
49Comment = r'#[^\r\n]*'
50Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
51Name = r'[a-zA-Z_]\w*'
Guido van Rossum4d8e8591992-01-01 19:34:47 +000052
Guido van Rossum3b631771997-10-27 20:44:15 +000053Hexnumber = r'0[xX][\da-fA-F]*[lL]?'
54Octnumber = r'0[0-7]*[lL]?'
55Decnumber = r'[1-9]\d*[lL]?'
Guido van Rossum1aec3231997-04-08 14:24:39 +000056Intnumber = group(Hexnumber, Octnumber, Decnumber)
Guido van Rossum3b631771997-10-27 20:44:15 +000057Exponent = r'[eE][-+]?\d+'
58Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent)
Tim Petersd507dab2001-08-30 20:51:59 +000059Expfloat = r'\d+' + Exponent
Guido van Rossum1aec3231997-04-08 14:24:39 +000060Floatnumber = group(Pointfloat, Expfloat)
Tim Petersd507dab2001-08-30 20:51:59 +000061Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]')
Guido van Rossum1aec3231997-04-08 14:24:39 +000062Number = group(Imagnumber, Floatnumber, Intnumber)
Guido van Rossum4d8e8591992-01-01 19:34:47 +000063
Tim Petersde495832000-10-07 05:09:39 +000064# Tail end of ' string.
65Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
66# Tail end of " string.
67Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
68# Tail end of ''' string.
69Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
70# Tail end of """ string.
71Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
Ka-Ping Yee1ff08b12001-01-15 22:04:30 +000072Triple = group("[uU]?[rR]?'''", '[uU]?[rR]?"""')
Tim Petersde495832000-10-07 05:09:39 +000073# Single-line ' or " string.
Ka-Ping Yee1ff08b12001-01-15 22:04:30 +000074String = group(r"[uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
75 r'[uU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
Guido van Rossum4d8e8591992-01-01 19:34:47 +000076
Tim Petersde495832000-10-07 05:09:39 +000077# Because of leftmost-then-longest match semantics, be sure to put the
78# longest operators first (e.g., if = came before ==, == would get
79# recognized as two instances of =).
80Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=",
Guido van Rossum96204f52001-08-08 05:04:07 +000081 r"//=?",
Tim Petersde495832000-10-07 05:09:39 +000082 r"[+\-*/%&|^=<>]=?",
83 r"~")
Thomas Wouterse1519a12000-08-24 21:44:52 +000084
Guido van Rossum4d8e8591992-01-01 19:34:47 +000085Bracket = '[][(){}]'
Guido van Rossum3b631771997-10-27 20:44:15 +000086Special = group(r'\r?\n', r'[:;.,`]')
Guido van Rossumfc6f5331997-03-07 00:21:12 +000087Funny = group(Operator, Bracket, Special)
Guido van Rossum4d8e8591992-01-01 19:34:47 +000088
Guido van Rossum3b631771997-10-27 20:44:15 +000089PlainToken = group(Number, Funny, String, Name)
Guido van Rossumfc6f5331997-03-07 00:21:12 +000090Token = Ignore + PlainToken
Guido van Rossum4d8e8591992-01-01 19:34:47 +000091
Tim Petersde495832000-10-07 05:09:39 +000092# First (or only) line of ' or " string.
Ka-Ping Yee1ff08b12001-01-15 22:04:30 +000093ContStr = group(r"[uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
94 group("'", r'\\\r?\n'),
95 r'[uU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
96 group('"', r'\\\r?\n'))
Guido van Rossum3b631771997-10-27 20:44:15 +000097PseudoExtras = group(r'\\\r?\n', Comment, Triple)
98PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
Guido van Rossum1aec3231997-04-08 14:24:39 +000099
Guido van Rossum3b631771997-10-27 20:44:15 +0000100tokenprog, pseudoprog, single3prog, double3prog = map(
101 re.compile, (Token, PseudoToken, Single3, Double3))
Guido van Rossumfefc9221997-10-27 21:17:24 +0000102endprogs = {"'": re.compile(Single), '"': re.compile(Double),
Guido van Rossum3b631771997-10-27 20:44:15 +0000103 "'''": single3prog, '"""': double3prog,
Guido van Rossumfefc9221997-10-27 21:17:24 +0000104 "r'''": single3prog, 'r"""': double3prog,
Ka-Ping Yee1ff08b12001-01-15 22:04:30 +0000105 "u'''": single3prog, 'u"""': double3prog,
106 "ur'''": single3prog, 'ur"""': double3prog,
107 "R'''": single3prog, 'R"""': double3prog,
108 "U'''": single3prog, 'U"""': double3prog,
109 "uR'''": single3prog, 'uR"""': double3prog,
110 "Ur'''": single3prog, 'Ur"""': double3prog,
111 "UR'''": single3prog, 'UR"""': double3prog,
112 'r': None, 'R': None, 'u': None, 'U': None}
Guido van Rossum4d8e8591992-01-01 19:34:47 +0000113
Guido van Rossum9d6897a2002-08-24 06:54:19 +0000114triple_quoted = {}
115for t in ("'''", '"""',
116 "r'''", 'r"""', "R'''", 'R"""',
117 "u'''", 'u"""', "U'''", 'U"""',
118 "ur'''", 'ur"""', "Ur'''", 'Ur"""',
119 "uR'''", 'uR"""', "UR'''", 'UR"""'):
120 triple_quoted[t] = t
121single_quoted = {}
122for t in ("'", '"',
123 "r'", 'r"', "R'", 'R"',
124 "u'", 'u"', "U'", 'U"',
125 "ur'", 'ur"', "Ur'", 'Ur"',
126 "uR'", 'uR"', "UR'", 'UR"' ):
127 single_quoted[t] = t
128
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000129tabsize = 8
Fred Drake9b8d8012000-08-17 04:45:13 +0000130
Ka-Ping Yee28c62bb2001-03-23 05:22:49 +0000131class TokenError(Exception): pass
132
133class StopTokenizing(Exception): pass
Fred Drake9b8d8012000-08-17 04:45:13 +0000134
Guido van Rossum1aec3231997-04-08 14:24:39 +0000135def printtoken(type, token, (srow, scol), (erow, ecol), line): # for testing
136 print "%d,%d-%d,%d:\t%s\t%s" % \
137 (srow, scol, erow, ecol, tok_name[type], repr(token))
Guido van Rossum4d8e8591992-01-01 19:34:47 +0000138
Guido van Rossum1aec3231997-04-08 14:24:39 +0000139def tokenize(readline, tokeneater=printtoken):
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000140 """
141 The tokenize() function accepts two parameters: one representing the
142 input stream, and one providing an output mechanism for tokenize().
Tim Peters8ac14952002-05-23 15:15:30 +0000143
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000144 The first parameter, readline, must be a callable object which provides
145 the same interface as the readline() method of built-in file objects.
Tim Peters8ac14952002-05-23 15:15:30 +0000146 Each call to the function should return one line of input as a string.
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000147
148 The second parameter, tokeneater, must also be a callable object. It is
149 called once for each token, with five arguments, corresponding to the
Tim Peters8ac14952002-05-23 15:15:30 +0000150 tuples generated by generate_tokens().
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000151 """
Ka-Ping Yee28c62bb2001-03-23 05:22:49 +0000152 try:
153 tokenize_loop(readline, tokeneater)
154 except StopTokenizing:
155 pass
156
Tim Peters4efb6e92001-06-29 23:51:08 +0000157# backwards compatible interface
Ka-Ping Yee28c62bb2001-03-23 05:22:49 +0000158def tokenize_loop(readline, tokeneater):
Tim Peters5ca576e2001-06-18 22:08:13 +0000159 for token_info in generate_tokens(readline):
Guido van Rossum68468eb2003-02-27 20:14:51 +0000160 tokeneater(*token_info)
Tim Peters5ca576e2001-06-18 22:08:13 +0000161
162def generate_tokens(readline):
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000163 """
164 The generate_tokens() generator requires one argment, readline, which
165 must be a callable object which provides the same interface as the
166 readline() method of built-in file objects. Each call to the function
167 should return one line of input as a string.
Tim Peters8ac14952002-05-23 15:15:30 +0000168
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000169 The generator produces 5-tuples with these members: the token type; the
170 token string; a 2-tuple (srow, scol) of ints specifying the row and
171 column where the token begins in the source; a 2-tuple (erow, ecol) of
172 ints specifying the row and column where the token ends in the source;
173 and the line on which the token was found. The line passed is the
Tim Peters8ac14952002-05-23 15:15:30 +0000174 logical line; continuation lines are included.
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000175 """
Guido van Rossum1aec3231997-04-08 14:24:39 +0000176 lnum = parenlev = continued = 0
Fred Drake79e75e12001-07-20 19:05:50 +0000177 namechars, numchars = string.ascii_letters + '_', '0123456789'
Guido van Rossumde655271997-04-09 17:15:54 +0000178 contstr, needcont = '', 0
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000179 contline = None
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000180 indents = [0]
Guido van Rossum1aec3231997-04-08 14:24:39 +0000181
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000182 while 1: # loop over lines in stream
183 line = readline()
Guido van Rossum1aec3231997-04-08 14:24:39 +0000184 lnum = lnum + 1
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000185 pos, max = 0, len(line)
186
187 if contstr: # continued string
Guido van Rossumde655271997-04-09 17:15:54 +0000188 if not line:
189 raise TokenError, ("EOF in multi-line string", strstart)
Guido van Rossum3b631771997-10-27 20:44:15 +0000190 endmatch = endprog.match(line)
191 if endmatch:
192 pos = end = endmatch.end(0)
Tim Peters5ca576e2001-06-18 22:08:13 +0000193 yield (STRING, contstr + line[:end],
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000194 strstart, (lnum, end), contline + line)
Guido van Rossumde655271997-04-09 17:15:54 +0000195 contstr, needcont = '', 0
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000196 contline = None
Guido van Rossumde655271997-04-09 17:15:54 +0000197 elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
Tim Peters5ca576e2001-06-18 22:08:13 +0000198 yield (ERRORTOKEN, contstr + line,
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000199 strstart, (lnum, len(line)), contline)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000200 contstr = ''
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000201 contline = None
Guido van Rossumde655271997-04-09 17:15:54 +0000202 continue
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000203 else:
204 contstr = contstr + line
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000205 contline = contline + line
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000206 continue
207
Guido van Rossum1aec3231997-04-08 14:24:39 +0000208 elif parenlev == 0 and not continued: # new statement
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000209 if not line: break
210 column = 0
Guido van Rossum1aec3231997-04-08 14:24:39 +0000211 while pos < max: # measure leading whitespace
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000212 if line[pos] == ' ': column = column + 1
Guido van Rossum1aec3231997-04-08 14:24:39 +0000213 elif line[pos] == '\t': column = (column/tabsize + 1)*tabsize
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000214 elif line[pos] == '\f': column = 0
215 else: break
216 pos = pos + 1
Guido van Rossumde655271997-04-09 17:15:54 +0000217 if pos == max: break
Guido van Rossum1aec3231997-04-08 14:24:39 +0000218
219 if line[pos] in '#\r\n': # skip comments or blank lines
Tim Peters5ca576e2001-06-18 22:08:13 +0000220 yield ((NL, COMMENT)[line[pos] == '#'], line[pos:],
Guido van Rossum1aec3231997-04-08 14:24:39 +0000221 (lnum, pos), (lnum, len(line)), line)
222 continue
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000223
224 if column > indents[-1]: # count indents or dedents
225 indents.append(column)
Tim Peters5ca576e2001-06-18 22:08:13 +0000226 yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000227 while column < indents[-1]:
228 indents = indents[:-1]
Tim Peters5ca576e2001-06-18 22:08:13 +0000229 yield (DEDENT, '', (lnum, pos), (lnum, pos), line)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000230
231 else: # continued statement
Guido van Rossumde655271997-04-09 17:15:54 +0000232 if not line:
233 raise TokenError, ("EOF in multi-line statement", (lnum, 0))
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000234 continued = 0
235
236 while pos < max:
Guido van Rossum3b631771997-10-27 20:44:15 +0000237 pseudomatch = pseudoprog.match(line, pos)
238 if pseudomatch: # scan for tokens
239 start, end = pseudomatch.span(1)
Guido van Rossumde655271997-04-09 17:15:54 +0000240 spos, epos, pos = (lnum, start), (lnum, end), end
Guido van Rossum1aec3231997-04-08 14:24:39 +0000241 token, initial = line[start:end], line[start]
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000242
Ka-Ping Yee28c62bb2001-03-23 05:22:49 +0000243 if initial in numchars or \
244 (initial == '.' and token != '.'): # ordinary number
Tim Peters5ca576e2001-06-18 22:08:13 +0000245 yield (NUMBER, token, spos, epos, line)
Guido van Rossum1aec3231997-04-08 14:24:39 +0000246 elif initial in '\r\n':
Tim Peters5ca576e2001-06-18 22:08:13 +0000247 yield (parenlev > 0 and NL or NEWLINE,
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000248 token, spos, epos, line)
Guido van Rossum1aec3231997-04-08 14:24:39 +0000249 elif initial == '#':
Tim Peters5ca576e2001-06-18 22:08:13 +0000250 yield (COMMENT, token, spos, epos, line)
Guido van Rossum9d6897a2002-08-24 06:54:19 +0000251 elif token in triple_quoted:
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000252 endprog = endprogs[token]
Guido van Rossum3b631771997-10-27 20:44:15 +0000253 endmatch = endprog.match(line, pos)
254 if endmatch: # all on one line
255 pos = endmatch.end(0)
Guido van Rossum1aec3231997-04-08 14:24:39 +0000256 token = line[start:pos]
Tim Peters5ca576e2001-06-18 22:08:13 +0000257 yield (STRING, token, spos, (lnum, pos), line)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000258 else:
Guido van Rossum1aec3231997-04-08 14:24:39 +0000259 strstart = (lnum, start) # multiple lines
260 contstr = line[start:]
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000261 contline = line
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000262 break
Guido van Rossum9d6897a2002-08-24 06:54:19 +0000263 elif initial in single_quoted or \
264 token[:2] in single_quoted or \
265 token[:3] in single_quoted:
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000266 if token[-1] == '\n': # continued string
Guido van Rossum1aec3231997-04-08 14:24:39 +0000267 strstart = (lnum, start)
Ka-Ping Yee1ff08b12001-01-15 22:04:30 +0000268 endprog = (endprogs[initial] or endprogs[token[1]] or
269 endprogs[token[2]])
Guido van Rossumde655271997-04-09 17:15:54 +0000270 contstr, needcont = line[start:], 1
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000271 contline = line
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000272 break
273 else: # ordinary string
Tim Peters5ca576e2001-06-18 22:08:13 +0000274 yield (STRING, token, spos, epos, line)
Guido van Rossum3b631771997-10-27 20:44:15 +0000275 elif initial in namechars: # ordinary name
Tim Peters5ca576e2001-06-18 22:08:13 +0000276 yield (NAME, token, spos, epos, line)
Guido van Rossum3b631771997-10-27 20:44:15 +0000277 elif initial == '\\': # continued stmt
278 continued = 1
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000279 else:
Guido van Rossum1aec3231997-04-08 14:24:39 +0000280 if initial in '([{': parenlev = parenlev + 1
281 elif initial in ')]}': parenlev = parenlev - 1
Tim Peters5ca576e2001-06-18 22:08:13 +0000282 yield (OP, token, spos, epos, line)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000283 else:
Tim Peters5ca576e2001-06-18 22:08:13 +0000284 yield (ERRORTOKEN, line[pos],
Guido van Rossumde655271997-04-09 17:15:54 +0000285 (lnum, pos), (lnum, pos+1), line)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000286 pos = pos + 1
287
288 for indent in indents[1:]: # pop remaining indent levels
Tim Peters5ca576e2001-06-18 22:08:13 +0000289 yield (DEDENT, '', (lnum, 0), (lnum, 0), '')
290 yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '')
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000291
292if __name__ == '__main__': # testing
293 import sys
Guido van Rossumde655271997-04-09 17:15:54 +0000294 if len(sys.argv) > 1: tokenize(open(sys.argv[1]).readline)
Guido van Rossum2b1566b1997-06-03 22:05:15 +0000295 else: tokenize(sys.stdin.readline)