blob: 0db3867283cd8406ca041e2af632a6af38ac4639 [file] [log] [blame]
Guido van Rossumb51eaa11997-03-07 00:21:55 +00001"""Tokenization help for Python programs.
Guido van Rossum4d8e8591992-01-01 19:34:47 +00002
Tim Peters4efb6e92001-06-29 23:51:08 +00003generate_tokens(readline) is a generator that breaks a stream of
Guido van Rossum1aec3231997-04-08 14:24:39 +00004text into Python tokens. It accepts a readline-like method which is called
Tim Peters4efb6e92001-06-29 23:51:08 +00005repeatedly to get the next line of input (or "" for EOF). It generates
65-tuples with these members:
7
8 the token type (see token.py)
9 the token (a string)
10 the starting (row, column) indices of the token (a 2-tuple of ints)
11 the ending (row, column) indices of the token (a 2-tuple of ints)
12 the original line (string)
13
14It is designed to match the working of the Python tokenizer exactly, except
15that it produces COMMENT tokens for comments and gives type OP for all
16operators
17
18Older entry points
19 tokenize_loop(readline, tokeneater)
20 tokenize(readline, tokeneater=printtoken)
21are the same, except instead of generating tokens, tokeneater is a callback
22function to which the 5 fields described above are passed as 5 arguments,
23each time a new token is found."""
Guido van Rossumb51eaa11997-03-07 00:21:55 +000024
Ka-Ping Yee244c5932001-03-01 13:56:40 +000025__author__ = 'Ka-Ping Yee <ping@lfw.org>'
Ka-Ping Yee4f64c132001-03-01 17:11:17 +000026__credits__ = \
27 'GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro'
Guido van Rossumb51eaa11997-03-07 00:21:55 +000028
Guido van Rossum3b631771997-10-27 20:44:15 +000029import string, re
Guido van Rossumfc6f5331997-03-07 00:21:12 +000030from token import *
Guido van Rossum4d8e8591992-01-01 19:34:47 +000031
Skip Montanaro40fc1602001-03-01 04:27:19 +000032import token
Raymond Hettinger78a7aee2002-11-05 06:06:02 +000033__all__ = [x for x in dir(token) if x[0] != '_'] + ["COMMENT", "tokenize",
Raymond Hettinger68c04532005-06-10 11:05:19 +000034 "generate_tokens", "NL", "untokenize"]
Neal Norwitze98d16e2002-03-26 16:20:26 +000035del x
Skip Montanaro40fc1602001-03-01 04:27:19 +000036del token
37
Guido van Rossum1aec3231997-04-08 14:24:39 +000038COMMENT = N_TOKENS
39tok_name[COMMENT] = 'COMMENT'
Guido van Rossuma90c78b1998-04-03 16:05:38 +000040NL = N_TOKENS + 1
41tok_name[NL] = 'NL'
Skip Montanaro40fc1602001-03-01 04:27:19 +000042N_TOKENS += 2
Guido van Rossum1aec3231997-04-08 14:24:39 +000043
Eric S. Raymondb08b2d32001-02-09 11:10:16 +000044def group(*choices): return '(' + '|'.join(choices) + ')'
Guido van Rossum68468eb2003-02-27 20:14:51 +000045def any(*choices): return group(*choices) + '*'
46def maybe(*choices): return group(*choices) + '?'
Guido van Rossum4d8e8591992-01-01 19:34:47 +000047
Guido van Rossum3b631771997-10-27 20:44:15 +000048Whitespace = r'[ \f\t]*'
49Comment = r'#[^\r\n]*'
50Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
51Name = r'[a-zA-Z_]\w*'
Guido van Rossum4d8e8591992-01-01 19:34:47 +000052
Guido van Rossum3b631771997-10-27 20:44:15 +000053Hexnumber = r'0[xX][\da-fA-F]*[lL]?'
54Octnumber = r'0[0-7]*[lL]?'
55Decnumber = r'[1-9]\d*[lL]?'
Guido van Rossum1aec3231997-04-08 14:24:39 +000056Intnumber = group(Hexnumber, Octnumber, Decnumber)
Guido van Rossum3b631771997-10-27 20:44:15 +000057Exponent = r'[eE][-+]?\d+'
58Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent)
Tim Petersd507dab2001-08-30 20:51:59 +000059Expfloat = r'\d+' + Exponent
Guido van Rossum1aec3231997-04-08 14:24:39 +000060Floatnumber = group(Pointfloat, Expfloat)
Tim Petersd507dab2001-08-30 20:51:59 +000061Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]')
Guido van Rossum1aec3231997-04-08 14:24:39 +000062Number = group(Imagnumber, Floatnumber, Intnumber)
Guido van Rossum4d8e8591992-01-01 19:34:47 +000063
Tim Petersde495832000-10-07 05:09:39 +000064# Tail end of ' string.
65Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
66# Tail end of " string.
67Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
68# Tail end of ''' string.
69Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
70# Tail end of """ string.
71Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
Ka-Ping Yee1ff08b12001-01-15 22:04:30 +000072Triple = group("[uU]?[rR]?'''", '[uU]?[rR]?"""')
Tim Petersde495832000-10-07 05:09:39 +000073# Single-line ' or " string.
Ka-Ping Yee1ff08b12001-01-15 22:04:30 +000074String = group(r"[uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
75 r'[uU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
Guido van Rossum4d8e8591992-01-01 19:34:47 +000076
Tim Petersde495832000-10-07 05:09:39 +000077# Because of leftmost-then-longest match semantics, be sure to put the
78# longest operators first (e.g., if = came before ==, == would get
79# recognized as two instances of =).
80Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=",
Guido van Rossum96204f52001-08-08 05:04:07 +000081 r"//=?",
Tim Petersde495832000-10-07 05:09:39 +000082 r"[+\-*/%&|^=<>]=?",
83 r"~")
Thomas Wouterse1519a12000-08-24 21:44:52 +000084
Guido van Rossum4d8e8591992-01-01 19:34:47 +000085Bracket = '[][(){}]'
Anthony Baxterc2a5a632004-08-02 06:10:11 +000086Special = group(r'\r?\n', r'[:;.,`@]')
Guido van Rossumfc6f5331997-03-07 00:21:12 +000087Funny = group(Operator, Bracket, Special)
Guido van Rossum4d8e8591992-01-01 19:34:47 +000088
Guido van Rossum3b631771997-10-27 20:44:15 +000089PlainToken = group(Number, Funny, String, Name)
Guido van Rossumfc6f5331997-03-07 00:21:12 +000090Token = Ignore + PlainToken
Guido van Rossum4d8e8591992-01-01 19:34:47 +000091
Tim Petersde495832000-10-07 05:09:39 +000092# First (or only) line of ' or " string.
Ka-Ping Yee1ff08b12001-01-15 22:04:30 +000093ContStr = group(r"[uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
94 group("'", r'\\\r?\n'),
95 r'[uU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
96 group('"', r'\\\r?\n'))
Guido van Rossum3b631771997-10-27 20:44:15 +000097PseudoExtras = group(r'\\\r?\n', Comment, Triple)
98PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
Guido van Rossum1aec3231997-04-08 14:24:39 +000099
Guido van Rossum3b631771997-10-27 20:44:15 +0000100tokenprog, pseudoprog, single3prog, double3prog = map(
101 re.compile, (Token, PseudoToken, Single3, Double3))
Guido van Rossumfefc9221997-10-27 21:17:24 +0000102endprogs = {"'": re.compile(Single), '"': re.compile(Double),
Guido van Rossum3b631771997-10-27 20:44:15 +0000103 "'''": single3prog, '"""': double3prog,
Guido van Rossumfefc9221997-10-27 21:17:24 +0000104 "r'''": single3prog, 'r"""': double3prog,
Ka-Ping Yee1ff08b12001-01-15 22:04:30 +0000105 "u'''": single3prog, 'u"""': double3prog,
106 "ur'''": single3prog, 'ur"""': double3prog,
107 "R'''": single3prog, 'R"""': double3prog,
108 "U'''": single3prog, 'U"""': double3prog,
109 "uR'''": single3prog, 'uR"""': double3prog,
110 "Ur'''": single3prog, 'Ur"""': double3prog,
111 "UR'''": single3prog, 'UR"""': double3prog,
112 'r': None, 'R': None, 'u': None, 'U': None}
Guido van Rossum4d8e8591992-01-01 19:34:47 +0000113
Guido van Rossum9d6897a2002-08-24 06:54:19 +0000114triple_quoted = {}
115for t in ("'''", '"""',
116 "r'''", 'r"""', "R'''", 'R"""',
117 "u'''", 'u"""', "U'''", 'U"""',
118 "ur'''", 'ur"""', "Ur'''", 'Ur"""',
119 "uR'''", 'uR"""', "UR'''", 'UR"""'):
120 triple_quoted[t] = t
121single_quoted = {}
122for t in ("'", '"',
123 "r'", 'r"', "R'", 'R"',
124 "u'", 'u"', "U'", 'U"',
125 "ur'", 'ur"', "Ur'", 'Ur"',
126 "uR'", 'uR"', "UR'", 'UR"' ):
127 single_quoted[t] = t
128
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000129tabsize = 8
Fred Drake9b8d8012000-08-17 04:45:13 +0000130
Ka-Ping Yee28c62bb2001-03-23 05:22:49 +0000131class TokenError(Exception): pass
132
133class StopTokenizing(Exception): pass
Fred Drake9b8d8012000-08-17 04:45:13 +0000134
Guido van Rossum1aec3231997-04-08 14:24:39 +0000135def printtoken(type, token, (srow, scol), (erow, ecol), line): # for testing
136 print "%d,%d-%d,%d:\t%s\t%s" % \
137 (srow, scol, erow, ecol, tok_name[type], repr(token))
Guido van Rossum4d8e8591992-01-01 19:34:47 +0000138
Guido van Rossum1aec3231997-04-08 14:24:39 +0000139def tokenize(readline, tokeneater=printtoken):
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000140 """
141 The tokenize() function accepts two parameters: one representing the
142 input stream, and one providing an output mechanism for tokenize().
Tim Peters8ac14952002-05-23 15:15:30 +0000143
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000144 The first parameter, readline, must be a callable object which provides
145 the same interface as the readline() method of built-in file objects.
Tim Peters8ac14952002-05-23 15:15:30 +0000146 Each call to the function should return one line of input as a string.
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000147
148 The second parameter, tokeneater, must also be a callable object. It is
149 called once for each token, with five arguments, corresponding to the
Tim Peters8ac14952002-05-23 15:15:30 +0000150 tuples generated by generate_tokens().
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000151 """
Ka-Ping Yee28c62bb2001-03-23 05:22:49 +0000152 try:
153 tokenize_loop(readline, tokeneater)
154 except StopTokenizing:
155 pass
156
Tim Peters4efb6e92001-06-29 23:51:08 +0000157# backwards compatible interface
Ka-Ping Yee28c62bb2001-03-23 05:22:49 +0000158def tokenize_loop(readline, tokeneater):
Tim Peters5ca576e2001-06-18 22:08:13 +0000159 for token_info in generate_tokens(readline):
Guido van Rossum68468eb2003-02-27 20:14:51 +0000160 tokeneater(*token_info)
Tim Peters5ca576e2001-06-18 22:08:13 +0000161
Raymond Hettinger68c04532005-06-10 11:05:19 +0000162
163def untokenize(iterable):
164 """Transform tokens back into Python source code.
165
166 Each element returned by the iterable must be a token sequence
167 with at least two elements, a token number and token value.
168
169 Round-trip invariant:
170 # Output text will tokenize the back to the input
171 t1 = [tok[:2] for tok in generate_tokens(f.readline)]
172 newcode = untokenize(t1)
173 readline = iter(newcode.splitlines(1)).next
Amaury Forgeot d'Arc198e3532008-03-27 23:41:59 +0000174 t2 = [tok[:2] for tok in generate_tokens(readline)]
Raymond Hettinger68c04532005-06-10 11:05:19 +0000175 assert t1 == t2
176 """
177
178 startline = False
Amaury Forgeot d'Arc198e3532008-03-27 23:41:59 +0000179 prevstring = False
Raymond Hettinger68c04532005-06-10 11:05:19 +0000180 indents = []
181 toks = []
182 toks_append = toks.append
183 for tok in iterable:
184 toknum, tokval = tok[:2]
185
Guido van Rossumc259cc92006-03-30 21:43:35 +0000186 if toknum in (NAME, NUMBER):
Raymond Hettinger68c04532005-06-10 11:05:19 +0000187 tokval += ' '
188
Amaury Forgeot d'Arc198e3532008-03-27 23:41:59 +0000189 # Insert a space between two consecutive strings
190 if toknum == STRING:
191 if prevstring:
192 tokval = ' ' + tokval
193 prevstring = True
194 else:
195 prevstring = False
196
Raymond Hettinger68c04532005-06-10 11:05:19 +0000197 if toknum == INDENT:
198 indents.append(tokval)
199 continue
200 elif toknum == DEDENT:
201 indents.pop()
202 continue
203 elif toknum in (NEWLINE, COMMENT, NL):
204 startline = True
205 elif startline and indents:
206 toks_append(indents[-1])
207 startline = False
208 toks_append(tokval)
209 return ''.join(toks)
210
211
Tim Peters5ca576e2001-06-18 22:08:13 +0000212def generate_tokens(readline):
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000213 """
214 The generate_tokens() generator requires one argment, readline, which
215 must be a callable object which provides the same interface as the
216 readline() method of built-in file objects. Each call to the function
Raymond Hettinger68c04532005-06-10 11:05:19 +0000217 should return one line of input as a string. Alternately, readline
218 can be a callable function terminating with StopIteration:
219 readline = open(myfile).next # Example of alternate readline
Tim Peters8ac14952002-05-23 15:15:30 +0000220
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000221 The generator produces 5-tuples with these members: the token type; the
222 token string; a 2-tuple (srow, scol) of ints specifying the row and
223 column where the token begins in the source; a 2-tuple (erow, ecol) of
224 ints specifying the row and column where the token ends in the source;
225 and the line on which the token was found. The line passed is the
Tim Peters8ac14952002-05-23 15:15:30 +0000226 logical line; continuation lines are included.
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000227 """
Guido van Rossum1aec3231997-04-08 14:24:39 +0000228 lnum = parenlev = continued = 0
Fred Drake79e75e12001-07-20 19:05:50 +0000229 namechars, numchars = string.ascii_letters + '_', '0123456789'
Guido van Rossumde655271997-04-09 17:15:54 +0000230 contstr, needcont = '', 0
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000231 contline = None
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000232 indents = [0]
Guido van Rossum1aec3231997-04-08 14:24:39 +0000233
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000234 while 1: # loop over lines in stream
Raymond Hettinger68c04532005-06-10 11:05:19 +0000235 try:
236 line = readline()
237 except StopIteration:
238 line = ''
Guido van Rossum1aec3231997-04-08 14:24:39 +0000239 lnum = lnum + 1
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000240 pos, max = 0, len(line)
241
242 if contstr: # continued string
Guido van Rossumde655271997-04-09 17:15:54 +0000243 if not line:
244 raise TokenError, ("EOF in multi-line string", strstart)
Guido van Rossum3b631771997-10-27 20:44:15 +0000245 endmatch = endprog.match(line)
246 if endmatch:
247 pos = end = endmatch.end(0)
Tim Peters5ca576e2001-06-18 22:08:13 +0000248 yield (STRING, contstr + line[:end],
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000249 strstart, (lnum, end), contline + line)
Guido van Rossumde655271997-04-09 17:15:54 +0000250 contstr, needcont = '', 0
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000251 contline = None
Guido van Rossumde655271997-04-09 17:15:54 +0000252 elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
Tim Peters5ca576e2001-06-18 22:08:13 +0000253 yield (ERRORTOKEN, contstr + line,
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000254 strstart, (lnum, len(line)), contline)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000255 contstr = ''
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000256 contline = None
Guido van Rossumde655271997-04-09 17:15:54 +0000257 continue
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000258 else:
259 contstr = contstr + line
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000260 contline = contline + line
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000261 continue
262
Guido van Rossum1aec3231997-04-08 14:24:39 +0000263 elif parenlev == 0 and not continued: # new statement
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000264 if not line: break
265 column = 0
Guido van Rossum1aec3231997-04-08 14:24:39 +0000266 while pos < max: # measure leading whitespace
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000267 if line[pos] == ' ': column = column + 1
Guido van Rossum1aec3231997-04-08 14:24:39 +0000268 elif line[pos] == '\t': column = (column/tabsize + 1)*tabsize
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000269 elif line[pos] == '\f': column = 0
270 else: break
271 pos = pos + 1
Guido van Rossumde655271997-04-09 17:15:54 +0000272 if pos == max: break
Guido van Rossum1aec3231997-04-08 14:24:39 +0000273
274 if line[pos] in '#\r\n': # skip comments or blank lines
Tim Peters5ca576e2001-06-18 22:08:13 +0000275 yield ((NL, COMMENT)[line[pos] == '#'], line[pos:],
Guido van Rossum1aec3231997-04-08 14:24:39 +0000276 (lnum, pos), (lnum, len(line)), line)
277 continue
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000278
279 if column > indents[-1]: # count indents or dedents
280 indents.append(column)
Tim Peters5ca576e2001-06-18 22:08:13 +0000281 yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000282 while column < indents[-1]:
Raymond Hettingerda99d1c2005-06-21 07:43:58 +0000283 if column not in indents:
284 raise IndentationError(
Georg Brandl2463f8f2006-08-14 21:34:08 +0000285 "unindent does not match any outer indentation level",
286 ("<tokenize>", lnum, pos, line))
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000287 indents = indents[:-1]
Tim Peters5ca576e2001-06-18 22:08:13 +0000288 yield (DEDENT, '', (lnum, pos), (lnum, pos), line)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000289
290 else: # continued statement
Guido van Rossumde655271997-04-09 17:15:54 +0000291 if not line:
292 raise TokenError, ("EOF in multi-line statement", (lnum, 0))
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000293 continued = 0
294
295 while pos < max:
Guido van Rossum3b631771997-10-27 20:44:15 +0000296 pseudomatch = pseudoprog.match(line, pos)
297 if pseudomatch: # scan for tokens
298 start, end = pseudomatch.span(1)
Guido van Rossumde655271997-04-09 17:15:54 +0000299 spos, epos, pos = (lnum, start), (lnum, end), end
Guido van Rossum1aec3231997-04-08 14:24:39 +0000300 token, initial = line[start:end], line[start]
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000301
Ka-Ping Yee28c62bb2001-03-23 05:22:49 +0000302 if initial in numchars or \
303 (initial == '.' and token != '.'): # ordinary number
Tim Peters5ca576e2001-06-18 22:08:13 +0000304 yield (NUMBER, token, spos, epos, line)
Guido van Rossum1aec3231997-04-08 14:24:39 +0000305 elif initial in '\r\n':
Tim Peters5ca576e2001-06-18 22:08:13 +0000306 yield (parenlev > 0 and NL or NEWLINE,
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000307 token, spos, epos, line)
Guido van Rossum1aec3231997-04-08 14:24:39 +0000308 elif initial == '#':
Tim Peters5ca576e2001-06-18 22:08:13 +0000309 yield (COMMENT, token, spos, epos, line)
Guido van Rossum9d6897a2002-08-24 06:54:19 +0000310 elif token in triple_quoted:
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000311 endprog = endprogs[token]
Guido van Rossum3b631771997-10-27 20:44:15 +0000312 endmatch = endprog.match(line, pos)
313 if endmatch: # all on one line
314 pos = endmatch.end(0)
Guido van Rossum1aec3231997-04-08 14:24:39 +0000315 token = line[start:pos]
Tim Peters5ca576e2001-06-18 22:08:13 +0000316 yield (STRING, token, spos, (lnum, pos), line)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000317 else:
Guido van Rossum1aec3231997-04-08 14:24:39 +0000318 strstart = (lnum, start) # multiple lines
319 contstr = line[start:]
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000320 contline = line
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000321 break
Guido van Rossum9d6897a2002-08-24 06:54:19 +0000322 elif initial in single_quoted or \
323 token[:2] in single_quoted or \
324 token[:3] in single_quoted:
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000325 if token[-1] == '\n': # continued string
Guido van Rossum1aec3231997-04-08 14:24:39 +0000326 strstart = (lnum, start)
Ka-Ping Yee1ff08b12001-01-15 22:04:30 +0000327 endprog = (endprogs[initial] or endprogs[token[1]] or
328 endprogs[token[2]])
Guido van Rossumde655271997-04-09 17:15:54 +0000329 contstr, needcont = line[start:], 1
Guido van Rossuma90c78b1998-04-03 16:05:38 +0000330 contline = line
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000331 break
332 else: # ordinary string
Tim Peters5ca576e2001-06-18 22:08:13 +0000333 yield (STRING, token, spos, epos, line)
Guido van Rossum3b631771997-10-27 20:44:15 +0000334 elif initial in namechars: # ordinary name
Tim Peters5ca576e2001-06-18 22:08:13 +0000335 yield (NAME, token, spos, epos, line)
Guido van Rossum3b631771997-10-27 20:44:15 +0000336 elif initial == '\\': # continued stmt
337 continued = 1
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000338 else:
Guido van Rossum1aec3231997-04-08 14:24:39 +0000339 if initial in '([{': parenlev = parenlev + 1
340 elif initial in ')]}': parenlev = parenlev - 1
Tim Peters5ca576e2001-06-18 22:08:13 +0000341 yield (OP, token, spos, epos, line)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000342 else:
Tim Peters5ca576e2001-06-18 22:08:13 +0000343 yield (ERRORTOKEN, line[pos],
Guido van Rossumde655271997-04-09 17:15:54 +0000344 (lnum, pos), (lnum, pos+1), line)
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000345 pos = pos + 1
346
347 for indent in indents[1:]: # pop remaining indent levels
Tim Peters5ca576e2001-06-18 22:08:13 +0000348 yield (DEDENT, '', (lnum, 0), (lnum, 0), '')
349 yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '')
Guido van Rossumfc6f5331997-03-07 00:21:12 +0000350
351if __name__ == '__main__': # testing
352 import sys
Guido van Rossumde655271997-04-09 17:15:54 +0000353 if len(sys.argv) > 1: tokenize(open(sys.argv[1]).readline)
Guido van Rossum2b1566b1997-06-03 22:05:15 +0000354 else: tokenize(sys.stdin.readline)