Guido van Rossum | b51eaa1 | 1997-03-07 00:21:55 +0000 | [diff] [blame] | 1 | """Tokenization help for Python programs. |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 2 | |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 3 | This module exports a function called 'tokenize()' that breaks a stream of |
| 4 | text into Python tokens. It accepts a readline-like method which is called |
| 5 | repeatedly to get the next line of input (or "" for EOF) and a "token-eater" |
| 6 | function which is called once for each token found. The latter function is |
| 7 | passed the token type, a string containing the token, the starting and |
| 8 | ending (row, column) coordinates of the token, and the original line. It is |
| 9 | designed to match the working of the Python tokenizer exactly, except that |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 10 | it produces COMMENT tokens for comments and gives type OP for all operators.""" |
Guido van Rossum | b51eaa1 | 1997-03-07 00:21:55 +0000 | [diff] [blame] | 11 | |
Ka-Ping Yee | 244c593 | 2001-03-01 13:56:40 +0000 | [diff] [blame^] | 12 | __author__ = 'Ka-Ping Yee <ping@lfw.org>' |
| 13 | __credits__ = 'first version, 26 October 1997; patched, GvR 3/30/98' |
Guido van Rossum | b51eaa1 | 1997-03-07 00:21:55 +0000 | [diff] [blame] | 14 | |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 15 | import string, re |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 16 | from token import * |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 17 | |
Skip Montanaro | 40fc160 | 2001-03-01 04:27:19 +0000 | [diff] [blame] | 18 | import token |
| 19 | __all__ = [x for x in dir(token) if x[0] != '_'] + ["COMMENT", "tokenize", "NL"] |
| 20 | del token |
| 21 | |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 22 | COMMENT = N_TOKENS |
| 23 | tok_name[COMMENT] = 'COMMENT' |
Guido van Rossum | a90c78b | 1998-04-03 16:05:38 +0000 | [diff] [blame] | 24 | NL = N_TOKENS + 1 |
| 25 | tok_name[NL] = 'NL' |
Skip Montanaro | 40fc160 | 2001-03-01 04:27:19 +0000 | [diff] [blame] | 26 | N_TOKENS += 2 |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 27 | |
| 28 | # Changes from 1.3: |
| 29 | # Ignore now accepts \f as whitespace. Operator now includes '**'. |
| 30 | # Ignore and Special now accept \n or \r\n at the end of a line. |
| 31 | # Imagnumber is new. Expfloat is corrected to reject '0e4'. |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 32 | # Note: to quote a backslash in a regex, it must be doubled in a r'aw' string. |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 33 | |
Eric S. Raymond | b08b2d3 | 2001-02-09 11:10:16 +0000 | [diff] [blame] | 34 | def group(*choices): return '(' + '|'.join(choices) + ')' |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 35 | def any(*choices): return apply(group, choices) + '*' |
| 36 | def maybe(*choices): return apply(group, choices) + '?' |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 37 | |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 38 | Whitespace = r'[ \f\t]*' |
| 39 | Comment = r'#[^\r\n]*' |
| 40 | Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) |
| 41 | Name = r'[a-zA-Z_]\w*' |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 42 | |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 43 | Hexnumber = r'0[xX][\da-fA-F]*[lL]?' |
| 44 | Octnumber = r'0[0-7]*[lL]?' |
| 45 | Decnumber = r'[1-9]\d*[lL]?' |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 46 | Intnumber = group(Hexnumber, Octnumber, Decnumber) |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 47 | Exponent = r'[eE][-+]?\d+' |
| 48 | Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent) |
| 49 | Expfloat = r'[1-9]\d*' + Exponent |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 50 | Floatnumber = group(Pointfloat, Expfloat) |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 51 | Imagnumber = group(r'0[jJ]', r'[1-9]\d*[jJ]', Floatnumber + r'[jJ]') |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 52 | Number = group(Imagnumber, Floatnumber, Intnumber) |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 53 | |
Tim Peters | de49583 | 2000-10-07 05:09:39 +0000 | [diff] [blame] | 54 | # Tail end of ' string. |
| 55 | Single = r"[^'\\]*(?:\\.[^'\\]*)*'" |
| 56 | # Tail end of " string. |
| 57 | Double = r'[^"\\]*(?:\\.[^"\\]*)*"' |
| 58 | # Tail end of ''' string. |
| 59 | Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''" |
| 60 | # Tail end of """ string. |
| 61 | Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""' |
Ka-Ping Yee | 1ff08b1 | 2001-01-15 22:04:30 +0000 | [diff] [blame] | 62 | Triple = group("[uU]?[rR]?'''", '[uU]?[rR]?"""') |
Tim Peters | de49583 | 2000-10-07 05:09:39 +0000 | [diff] [blame] | 63 | # Single-line ' or " string. |
Ka-Ping Yee | 1ff08b1 | 2001-01-15 22:04:30 +0000 | [diff] [blame] | 64 | String = group(r"[uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'", |
| 65 | r'[uU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"') |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 66 | |
Tim Peters | de49583 | 2000-10-07 05:09:39 +0000 | [diff] [blame] | 67 | # Because of leftmost-then-longest match semantics, be sure to put the |
| 68 | # longest operators first (e.g., if = came before ==, == would get |
| 69 | # recognized as two instances of =). |
| 70 | Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=", |
| 71 | r"[+\-*/%&|^=<>]=?", |
| 72 | r"~") |
Thomas Wouters | e1519a1 | 2000-08-24 21:44:52 +0000 | [diff] [blame] | 73 | |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 74 | Bracket = '[][(){}]' |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 75 | Special = group(r'\r?\n', r'[:;.,`]') |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 76 | Funny = group(Operator, Bracket, Special) |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 77 | |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 78 | PlainToken = group(Number, Funny, String, Name) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 79 | Token = Ignore + PlainToken |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 80 | |
Tim Peters | de49583 | 2000-10-07 05:09:39 +0000 | [diff] [blame] | 81 | # First (or only) line of ' or " string. |
Ka-Ping Yee | 1ff08b1 | 2001-01-15 22:04:30 +0000 | [diff] [blame] | 82 | ContStr = group(r"[uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" + |
| 83 | group("'", r'\\\r?\n'), |
| 84 | r'[uU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' + |
| 85 | group('"', r'\\\r?\n')) |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 86 | PseudoExtras = group(r'\\\r?\n', Comment, Triple) |
| 87 | PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 88 | |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 89 | tokenprog, pseudoprog, single3prog, double3prog = map( |
| 90 | re.compile, (Token, PseudoToken, Single3, Double3)) |
Guido van Rossum | fefc922 | 1997-10-27 21:17:24 +0000 | [diff] [blame] | 91 | endprogs = {"'": re.compile(Single), '"': re.compile(Double), |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 92 | "'''": single3prog, '"""': double3prog, |
Guido van Rossum | fefc922 | 1997-10-27 21:17:24 +0000 | [diff] [blame] | 93 | "r'''": single3prog, 'r"""': double3prog, |
Ka-Ping Yee | 1ff08b1 | 2001-01-15 22:04:30 +0000 | [diff] [blame] | 94 | "u'''": single3prog, 'u"""': double3prog, |
| 95 | "ur'''": single3prog, 'ur"""': double3prog, |
| 96 | "R'''": single3prog, 'R"""': double3prog, |
| 97 | "U'''": single3prog, 'U"""': double3prog, |
| 98 | "uR'''": single3prog, 'uR"""': double3prog, |
| 99 | "Ur'''": single3prog, 'Ur"""': double3prog, |
| 100 | "UR'''": single3prog, 'UR"""': double3prog, |
| 101 | 'r': None, 'R': None, 'u': None, 'U': None} |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 102 | |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 103 | tabsize = 8 |
Fred Drake | 9b8d801 | 2000-08-17 04:45:13 +0000 | [diff] [blame] | 104 | |
| 105 | class TokenError(Exception): |
| 106 | pass |
| 107 | |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 108 | def printtoken(type, token, (srow, scol), (erow, ecol), line): # for testing |
| 109 | print "%d,%d-%d,%d:\t%s\t%s" % \ |
| 110 | (srow, scol, erow, ecol, tok_name[type], repr(token)) |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 111 | |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 112 | def tokenize(readline, tokeneater=printtoken): |
| 113 | lnum = parenlev = continued = 0 |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 114 | namechars, numchars = string.letters + '_', string.digits |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 115 | contstr, needcont = '', 0 |
Guido van Rossum | a90c78b | 1998-04-03 16:05:38 +0000 | [diff] [blame] | 116 | contline = None |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 117 | indents = [0] |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 118 | |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 119 | while 1: # loop over lines in stream |
| 120 | line = readline() |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 121 | lnum = lnum + 1 |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 122 | pos, max = 0, len(line) |
| 123 | |
| 124 | if contstr: # continued string |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 125 | if not line: |
| 126 | raise TokenError, ("EOF in multi-line string", strstart) |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 127 | endmatch = endprog.match(line) |
| 128 | if endmatch: |
| 129 | pos = end = endmatch.end(0) |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 130 | tokeneater(STRING, contstr + line[:end], |
Guido van Rossum | a90c78b | 1998-04-03 16:05:38 +0000 | [diff] [blame] | 131 | strstart, (lnum, end), contline + line) |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 132 | contstr, needcont = '', 0 |
Guido van Rossum | a90c78b | 1998-04-03 16:05:38 +0000 | [diff] [blame] | 133 | contline = None |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 134 | elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n': |
| 135 | tokeneater(ERRORTOKEN, contstr + line, |
Guido van Rossum | a90c78b | 1998-04-03 16:05:38 +0000 | [diff] [blame] | 136 | strstart, (lnum, len(line)), contline) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 137 | contstr = '' |
Guido van Rossum | a90c78b | 1998-04-03 16:05:38 +0000 | [diff] [blame] | 138 | contline = None |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 139 | continue |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 140 | else: |
| 141 | contstr = contstr + line |
Guido van Rossum | a90c78b | 1998-04-03 16:05:38 +0000 | [diff] [blame] | 142 | contline = contline + line |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 143 | continue |
| 144 | |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 145 | elif parenlev == 0 and not continued: # new statement |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 146 | if not line: break |
| 147 | column = 0 |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 148 | while pos < max: # measure leading whitespace |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 149 | if line[pos] == ' ': column = column + 1 |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 150 | elif line[pos] == '\t': column = (column/tabsize + 1)*tabsize |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 151 | elif line[pos] == '\f': column = 0 |
| 152 | else: break |
| 153 | pos = pos + 1 |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 154 | if pos == max: break |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 155 | |
| 156 | if line[pos] in '#\r\n': # skip comments or blank lines |
Guido van Rossum | a90c78b | 1998-04-03 16:05:38 +0000 | [diff] [blame] | 157 | tokeneater((NL, COMMENT)[line[pos] == '#'], line[pos:], |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 158 | (lnum, pos), (lnum, len(line)), line) |
| 159 | continue |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 160 | |
| 161 | if column > indents[-1]: # count indents or dedents |
| 162 | indents.append(column) |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 163 | tokeneater(INDENT, line[:pos], (lnum, 0), (lnum, pos), line) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 164 | while column < indents[-1]: |
| 165 | indents = indents[:-1] |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 166 | tokeneater(DEDENT, '', (lnum, pos), (lnum, pos), line) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 167 | |
| 168 | else: # continued statement |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 169 | if not line: |
| 170 | raise TokenError, ("EOF in multi-line statement", (lnum, 0)) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 171 | continued = 0 |
| 172 | |
| 173 | while pos < max: |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 174 | pseudomatch = pseudoprog.match(line, pos) |
| 175 | if pseudomatch: # scan for tokens |
| 176 | start, end = pseudomatch.span(1) |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 177 | spos, epos, pos = (lnum, start), (lnum, end), end |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 178 | token, initial = line[start:end], line[start] |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 179 | |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 180 | if initial in numchars \ |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 181 | or (initial == '.' and token != '.'): # ordinary number |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 182 | tokeneater(NUMBER, token, spos, epos, line) |
| 183 | elif initial in '\r\n': |
Guido van Rossum | a90c78b | 1998-04-03 16:05:38 +0000 | [diff] [blame] | 184 | tokeneater(parenlev > 0 and NL or NEWLINE, |
| 185 | token, spos, epos, line) |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 186 | elif initial == '#': |
| 187 | tokeneater(COMMENT, token, spos, epos, line) |
Guido van Rossum | fefc922 | 1997-10-27 21:17:24 +0000 | [diff] [blame] | 188 | elif token in ("'''", '"""', # triple-quoted |
Ka-Ping Yee | 1ff08b1 | 2001-01-15 22:04:30 +0000 | [diff] [blame] | 189 | "r'''", 'r"""', "R'''", 'R"""', |
| 190 | "u'''", 'u"""', "U'''", 'U"""', |
| 191 | "ur'''", 'ur"""', "Ur'''", 'Ur"""', |
| 192 | "uR'''", 'uR"""', "UR'''", 'UR"""'): |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 193 | endprog = endprogs[token] |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 194 | endmatch = endprog.match(line, pos) |
| 195 | if endmatch: # all on one line |
| 196 | pos = endmatch.end(0) |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 197 | token = line[start:pos] |
| 198 | tokeneater(STRING, token, spos, (lnum, pos), line) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 199 | else: |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 200 | strstart = (lnum, start) # multiple lines |
| 201 | contstr = line[start:] |
Guido van Rossum | a90c78b | 1998-04-03 16:05:38 +0000 | [diff] [blame] | 202 | contline = line |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 203 | break |
Guido van Rossum | fefc922 | 1997-10-27 21:17:24 +0000 | [diff] [blame] | 204 | elif initial in ("'", '"') or \ |
Ka-Ping Yee | 1ff08b1 | 2001-01-15 22:04:30 +0000 | [diff] [blame] | 205 | token[:2] in ("r'", 'r"', "R'", 'R"', |
| 206 | "u'", 'u"', "U'", 'U"') or \ |
| 207 | token[:3] in ("ur'", 'ur"', "Ur'", 'Ur"', |
| 208 | "uR'", 'uR"', "UR'", 'UR"' ): |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 209 | if token[-1] == '\n': # continued string |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 210 | strstart = (lnum, start) |
Ka-Ping Yee | 1ff08b1 | 2001-01-15 22:04:30 +0000 | [diff] [blame] | 211 | endprog = (endprogs[initial] or endprogs[token[1]] or |
| 212 | endprogs[token[2]]) |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 213 | contstr, needcont = line[start:], 1 |
Guido van Rossum | a90c78b | 1998-04-03 16:05:38 +0000 | [diff] [blame] | 214 | contline = line |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 215 | break |
| 216 | else: # ordinary string |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 217 | tokeneater(STRING, token, spos, epos, line) |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 218 | elif initial in namechars: # ordinary name |
| 219 | tokeneater(NAME, token, spos, epos, line) |
| 220 | elif initial == '\\': # continued stmt |
| 221 | continued = 1 |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 222 | else: |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 223 | if initial in '([{': parenlev = parenlev + 1 |
| 224 | elif initial in ')]}': parenlev = parenlev - 1 |
| 225 | tokeneater(OP, token, spos, epos, line) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 226 | else: |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 227 | tokeneater(ERRORTOKEN, line[pos], |
| 228 | (lnum, pos), (lnum, pos+1), line) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 229 | pos = pos + 1 |
| 230 | |
| 231 | for indent in indents[1:]: # pop remaining indent levels |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 232 | tokeneater(DEDENT, '', (lnum, 0), (lnum, 0), '') |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 233 | tokeneater(ENDMARKER, '', (lnum, 0), (lnum, 0), '') |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 234 | |
| 235 | if __name__ == '__main__': # testing |
| 236 | import sys |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 237 | if len(sys.argv) > 1: tokenize(open(sys.argv[1]).readline) |
Guido van Rossum | 2b1566b | 1997-06-03 22:05:15 +0000 | [diff] [blame] | 238 | else: tokenize(sys.stdin.readline) |