Guido van Rossum | b51eaa1 | 1997-03-07 00:21:55 +0000 | [diff] [blame] | 1 | """Tokenization help for Python programs. |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 2 | |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 3 | This module exports a function called 'tokenize()' that breaks a stream of |
| 4 | text into Python tokens. It accepts a readline-like method which is called |
| 5 | repeatedly to get the next line of input (or "" for EOF) and a "token-eater" |
| 6 | function which is called once for each token found. The latter function is |
| 7 | passed the token type, a string containing the token, the starting and |
| 8 | ending (row, column) coordinates of the token, and the original line. It is |
| 9 | designed to match the working of the Python tokenizer exactly, except that |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 10 | it produces COMMENT tokens for comments and gives type OP for all operators.""" |
Guido van Rossum | b51eaa1 | 1997-03-07 00:21:55 +0000 | [diff] [blame] | 11 | |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 12 | __version__ = "Ka-Ping Yee, 26 October 1997" |
Guido van Rossum | b51eaa1 | 1997-03-07 00:21:55 +0000 | [diff] [blame] | 13 | |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 14 | import string, re |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 15 | from token import * |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 16 | |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 17 | COMMENT = N_TOKENS |
| 18 | tok_name[COMMENT] = 'COMMENT' |
| 19 | |
| 20 | # Changes from 1.3: |
| 21 | # Ignore now accepts \f as whitespace. Operator now includes '**'. |
| 22 | # Ignore and Special now accept \n or \r\n at the end of a line. |
| 23 | # Imagnumber is new. Expfloat is corrected to reject '0e4'. |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 24 | # Note: to quote a backslash in a regex, it must be doubled in a r'aw' string. |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 25 | |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 26 | def group(*choices): return '(' + string.join(choices, '|') + ')' |
| 27 | def any(*choices): return apply(group, choices) + '*' |
| 28 | def maybe(*choices): return apply(group, choices) + '?' |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 29 | |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 30 | Whitespace = r'[ \f\t]*' |
| 31 | Comment = r'#[^\r\n]*' |
| 32 | Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) |
| 33 | Name = r'[a-zA-Z_]\w*' |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 34 | |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 35 | Hexnumber = r'0[xX][\da-fA-F]*[lL]?' |
| 36 | Octnumber = r'0[0-7]*[lL]?' |
| 37 | Decnumber = r'[1-9]\d*[lL]?' |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 38 | Intnumber = group(Hexnumber, Octnumber, Decnumber) |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 39 | Exponent = r'[eE][-+]?\d+' |
| 40 | Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent) |
| 41 | Expfloat = r'[1-9]\d*' + Exponent |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 42 | Floatnumber = group(Pointfloat, Expfloat) |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 43 | Imagnumber = group(r'0[jJ]', r'[1-9]\d*[jJ]', Floatnumber + r'[jJ]') |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 44 | Number = group(Imagnumber, Floatnumber, Intnumber) |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 45 | |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 46 | Single = any(r"[^'\\]", r'\\.') + "'" |
| 47 | Double = any(r'[^"\\]', r'\\.') + '"' |
| 48 | Single3 = any(r"[^'\\]",r'\\.',r"'[^'\\]",r"'\\.",r"''[^'\\]",r"''\\.") + "'''" |
| 49 | Double3 = any(r'[^"\\]',r'\\.',r'"[^"\\]',r'"\\.',r'""[^"\\]',r'""\\.') + '"""' |
Guido van Rossum | fefc922 | 1997-10-27 21:17:24 +0000 | [diff] [blame] | 50 | Triple = group("[rR]?'''", '[rR]?"""') |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 51 | String = group("[rR]?'" + any(r"[^\n'\\]", r'\\.') + "'", |
| 52 | '[rR]?"' + any(r'[^\n"\\]', r'\\.') + '"') |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 53 | |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 54 | Operator = group('\+', '\-', '\*\*', '\*', '\^', '~', '/', '%', '&', '\|', |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 55 | '<<', '>>', '==', '<=', '<>', '!=', '>=', '=', '<', '>') |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 56 | Bracket = '[][(){}]' |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 57 | Special = group(r'\r?\n', r'[:;.,`]') |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 58 | Funny = group(Operator, Bracket, Special) |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 59 | |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 60 | PlainToken = group(Number, Funny, String, Name) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 61 | Token = Ignore + PlainToken |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 62 | |
Guido van Rossum | fefc922 | 1997-10-27 21:17:24 +0000 | [diff] [blame] | 63 | ContStr = group("[rR]?'" + any(r'\\.', r"[^\n'\\]") + group("'", r'\\\r?\n'), |
| 64 | '[rR]?"' + any(r'\\.', r'[^\n"\\]') + group('"', r'\\\r?\n')) |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 65 | PseudoExtras = group(r'\\\r?\n', Comment, Triple) |
| 66 | PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 67 | |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 68 | tokenprog, pseudoprog, single3prog, double3prog = map( |
| 69 | re.compile, (Token, PseudoToken, Single3, Double3)) |
Guido van Rossum | fefc922 | 1997-10-27 21:17:24 +0000 | [diff] [blame] | 70 | endprogs = {"'": re.compile(Single), '"': re.compile(Double), |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 71 | "'''": single3prog, '"""': double3prog, |
Guido van Rossum | fefc922 | 1997-10-27 21:17:24 +0000 | [diff] [blame] | 72 | "r'''": single3prog, 'r"""': double3prog, |
| 73 | "R'''": single3prog, 'R"""': double3prog, 'r': None, 'R': None} |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 74 | |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 75 | tabsize = 8 |
| 76 | TokenError = 'TokenError' |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 77 | def printtoken(type, token, (srow, scol), (erow, ecol), line): # for testing |
| 78 | print "%d,%d-%d,%d:\t%s\t%s" % \ |
| 79 | (srow, scol, erow, ecol, tok_name[type], repr(token)) |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 80 | |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 81 | def tokenize(readline, tokeneater=printtoken): |
| 82 | lnum = parenlev = continued = 0 |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 83 | namechars, numchars = string.letters + '_', string.digits |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 84 | contstr, needcont = '', 0 |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 85 | indents = [0] |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 86 | |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 87 | while 1: # loop over lines in stream |
| 88 | line = readline() |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 89 | lnum = lnum + 1 |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 90 | pos, max = 0, len(line) |
| 91 | |
| 92 | if contstr: # continued string |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 93 | if not line: |
| 94 | raise TokenError, ("EOF in multi-line string", strstart) |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 95 | endmatch = endprog.match(line) |
| 96 | if endmatch: |
| 97 | pos = end = endmatch.end(0) |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 98 | tokeneater(STRING, contstr + line[:end], |
| 99 | strstart, (lnum, end), line) |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 100 | contstr, needcont = '', 0 |
| 101 | elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n': |
| 102 | tokeneater(ERRORTOKEN, contstr + line, |
| 103 | strstart, (lnum, len(line)), line) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 104 | contstr = '' |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 105 | continue |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 106 | else: |
| 107 | contstr = contstr + line |
| 108 | continue |
| 109 | |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 110 | elif parenlev == 0 and not continued: # new statement |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 111 | if not line: break |
| 112 | column = 0 |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 113 | while pos < max: # measure leading whitespace |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 114 | if line[pos] == ' ': column = column + 1 |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 115 | elif line[pos] == '\t': column = (column/tabsize + 1)*tabsize |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 116 | elif line[pos] == '\f': column = 0 |
| 117 | else: break |
| 118 | pos = pos + 1 |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 119 | if pos == max: break |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 120 | |
| 121 | if line[pos] in '#\r\n': # skip comments or blank lines |
| 122 | tokeneater((NEWLINE, COMMENT)[line[pos] == '#'], line[pos:], |
| 123 | (lnum, pos), (lnum, len(line)), line) |
| 124 | continue |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 125 | |
| 126 | if column > indents[-1]: # count indents or dedents |
| 127 | indents.append(column) |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 128 | tokeneater(INDENT, line[:pos], (lnum, 0), (lnum, pos), line) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 129 | while column < indents[-1]: |
| 130 | indents = indents[:-1] |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 131 | tokeneater(DEDENT, '', (lnum, pos), (lnum, pos), line) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 132 | |
| 133 | else: # continued statement |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 134 | if not line: |
| 135 | raise TokenError, ("EOF in multi-line statement", (lnum, 0)) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 136 | continued = 0 |
| 137 | |
| 138 | while pos < max: |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 139 | pseudomatch = pseudoprog.match(line, pos) |
| 140 | if pseudomatch: # scan for tokens |
| 141 | start, end = pseudomatch.span(1) |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 142 | spos, epos, pos = (lnum, start), (lnum, end), end |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 143 | token, initial = line[start:end], line[start] |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 144 | |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 145 | if initial in numchars \ |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 146 | or (initial == '.' and token != '.'): # ordinary number |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 147 | tokeneater(NUMBER, token, spos, epos, line) |
| 148 | elif initial in '\r\n': |
| 149 | tokeneater(NEWLINE, token, spos, epos, line) |
| 150 | elif initial == '#': |
| 151 | tokeneater(COMMENT, token, spos, epos, line) |
Guido van Rossum | fefc922 | 1997-10-27 21:17:24 +0000 | [diff] [blame] | 152 | elif token in ("'''", '"""', # triple-quoted |
| 153 | "r'''", 'r"""', "R'''", 'R"""'): |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 154 | endprog = endprogs[token] |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 155 | endmatch = endprog.match(line, pos) |
| 156 | if endmatch: # all on one line |
| 157 | pos = endmatch.end(0) |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 158 | token = line[start:pos] |
| 159 | tokeneater(STRING, token, spos, (lnum, pos), line) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 160 | else: |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 161 | strstart = (lnum, start) # multiple lines |
| 162 | contstr = line[start:] |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 163 | break |
Guido van Rossum | fefc922 | 1997-10-27 21:17:24 +0000 | [diff] [blame] | 164 | elif initial in ("'", '"') or \ |
| 165 | token[:2] in ("r'", 'r"', "R'", 'R"'): |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 166 | if token[-1] == '\n': # continued string |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 167 | strstart = (lnum, start) |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 168 | endprog = endprogs[initial] or endprogs[token[1]] |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 169 | contstr, needcont = line[start:], 1 |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 170 | break |
| 171 | else: # ordinary string |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 172 | tokeneater(STRING, token, spos, epos, line) |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 173 | elif initial in namechars: # ordinary name |
| 174 | tokeneater(NAME, token, spos, epos, line) |
| 175 | elif initial == '\\': # continued stmt |
| 176 | continued = 1 |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 177 | else: |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 178 | if initial in '([{': parenlev = parenlev + 1 |
| 179 | elif initial in ')]}': parenlev = parenlev - 1 |
| 180 | tokeneater(OP, token, spos, epos, line) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 181 | else: |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 182 | tokeneater(ERRORTOKEN, line[pos], |
| 183 | (lnum, pos), (lnum, pos+1), line) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 184 | pos = pos + 1 |
| 185 | |
| 186 | for indent in indents[1:]: # pop remaining indent levels |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 187 | tokeneater(DEDENT, '', (lnum, 0), (lnum, 0), '') |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 188 | tokeneater(ENDMARKER, '', (lnum, 0), (lnum, 0), '') |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 189 | |
| 190 | if __name__ == '__main__': # testing |
| 191 | import sys |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 192 | if len(sys.argv) > 1: tokenize(open(sys.argv[1]).readline) |
Guido van Rossum | 2b1566b | 1997-06-03 22:05:15 +0000 | [diff] [blame] | 193 | else: tokenize(sys.stdin.readline) |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 194 | |