Guido van Rossum | b51eaa1 | 1997-03-07 00:21:55 +0000 | [diff] [blame] | 1 | """Tokenization help for Python programs. |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 2 | |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 3 | This module exports a function called 'tokenize()' that breaks a stream of |
| 4 | text into Python tokens. It accepts a readline-like method which is called |
| 5 | repeatedly to get the next line of input (or "" for EOF) and a "token-eater" |
| 6 | function which is called once for each token found. The latter function is |
| 7 | passed the token type, a string containing the token, the starting and |
| 8 | ending (row, column) coordinates of the token, and the original line. It is |
| 9 | designed to match the working of the Python tokenizer exactly, except that |
| 10 | it produces COMMENT tokens for comments and gives type OP for all operators. |
Guido van Rossum | b51eaa1 | 1997-03-07 00:21:55 +0000 | [diff] [blame] | 11 | |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 12 | For compatibility with the older 'tokenize' module, this also compiles a |
| 13 | regular expression into 'tokenprog' that matches Python tokens in individual |
| 14 | lines of text, leaving the token in 'tokenprog.group(3)', but does not |
| 15 | handle indentation, continuations, or multi-line strings.""" |
Guido van Rossum | b51eaa1 | 1997-03-07 00:21:55 +0000 | [diff] [blame] | 16 | |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 17 | __version__ = "Ka-Ping Yee, 29 March 1997" |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 18 | |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 19 | import string, regex |
| 20 | from token import * |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 21 | |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 22 | COMMENT = N_TOKENS |
| 23 | tok_name[COMMENT] = 'COMMENT' |
| 24 | |
| 25 | # Changes from 1.3: |
| 26 | # Ignore now accepts \f as whitespace. Operator now includes '**'. |
| 27 | # Ignore and Special now accept \n or \r\n at the end of a line. |
| 28 | # Imagnumber is new. Expfloat is corrected to reject '0e4'. |
| 29 | # Note: to get a quoted backslash in a regex, it must be enclosed in brackets. |
| 30 | |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 31 | def group(*choices): return '\(' + string.join(choices, '\|') + '\)' |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 32 | |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 33 | Whitespace = '[ \f\t]*' |
| 34 | Comment = '\(#[^\r\n]*\)' |
| 35 | Ignore = Whitespace + group('[\]\r?\n' + Whitespace)+'*' + Comment+'?' |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 36 | Name = '[a-zA-Z_][a-zA-Z0-9_]*' |
| 37 | |
| 38 | Hexnumber = '0[xX][0-9a-fA-F]*[lL]?' |
| 39 | Octnumber = '0[0-7]*[lL]?' |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 40 | Decnumber = '[1-9][0-9]*[lL]?' |
| 41 | Intnumber = group(Hexnumber, Octnumber, Decnumber) |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 42 | Exponent = '[eE][-+]?[0-9]+' |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 43 | Pointfloat = group('[0-9]+\.[0-9]*', '\.[0-9]+') + group(Exponent) + '?' |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 44 | Expfloat = '[1-9][0-9]*' + Exponent |
| 45 | Floatnumber = group(Pointfloat, Expfloat) |
| 46 | Imagnumber = group('0[jJ]', '[1-9][0-9]*[jJ]', Floatnumber + '[jJ]') |
| 47 | Number = group(Imagnumber, Floatnumber, Intnumber) |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 48 | |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 49 | Single = group("[^'\]", "[\].") + "*'" |
| 50 | Double = group('[^"\]', '[\].') + '*"' |
| 51 | Single3 = group("[^'\]","[\].","'[^'\]","'[\].","''[^'\]","''[\].") + "*'''" |
| 52 | Double3 = group('[^"\]','[\].','"[^"\]','"[\].','""[^"\]','""[\].') + '*"""' |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 53 | Triple = group("'''", '"""') |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 54 | String = group("'" + group("[^\n'\]", "[\].") + "*'", |
| 55 | '"' + group('[^\n"\]', '[\].') + '*"') |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 56 | |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 57 | Operator = group('\+', '\-', '\*\*', '\*', '\^', '~', '/', '%', '&', '|', |
| 58 | '<<', '>>', '==', '<=', '<>', '!=', '>=', '=', '<', '>') |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 59 | Bracket = '[][(){}]' |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 60 | Special = group('\r?\n', '[:;.,`]') |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 61 | Funny = group(Operator, Bracket, Special) |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 62 | |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 63 | PlainToken = group(Name, Number, String, Funny) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 64 | Token = Ignore + PlainToken |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 65 | |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 66 | ContStr = group("'" + group('[\].', "[^\n'\]")+'*' + group("'", '[\]\r?\n'), |
| 67 | '"' + group('[\].', '[^\n"\]')+'*' + group('"', '[\]\r?\n')) |
| 68 | PseudoExtras = group('[\]\r?\n', Comment, Triple) |
| 69 | PseudoToken = Whitespace + group(PseudoExtras, Name, Number, ContStr, Funny) |
| 70 | |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 71 | try: |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 72 | saved_syntax = regex.set_syntax(0) # use default syntax |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 73 | tokenprog = regex.compile(Token) |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 74 | pseudoprog = regex.compile(PseudoToken) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 75 | endprogs = { '\'': regex.compile(Single), '"': regex.compile(Double), |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 76 | '\'\'\'': regex.compile(Single3), '"""': regex.compile(Double3) } |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 77 | finally: |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 78 | regex.set_syntax(saved_syntax) # restore original syntax |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 79 | |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 80 | tabsize = 8 |
| 81 | TokenError = 'TokenError' |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 82 | def printtoken(type, token, (srow, scol), (erow, ecol), line): # for testing |
| 83 | print "%d,%d-%d,%d:\t%s\t%s" % \ |
| 84 | (srow, scol, erow, ecol, tok_name[type], repr(token)) |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 85 | |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 86 | def tokenize(readline, tokeneater=printtoken): |
| 87 | lnum = parenlev = continued = 0 |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 88 | namechars, numchars = string.letters + '_', string.digits |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 89 | contstr, needcont = '', 0 |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 90 | indents = [0] |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 91 | |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 92 | while 1: # loop over lines in stream |
| 93 | line = readline() |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 94 | lnum = lnum + 1 |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 95 | pos, max = 0, len(line) |
| 96 | |
| 97 | if contstr: # continued string |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 98 | if not line: |
| 99 | raise TokenError, ("EOF in multi-line string", strstart) |
| 100 | if endprog.match(line) >= 0: |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 101 | pos = end = endprog.regs[0][1] |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 102 | tokeneater(STRING, contstr + line[:end], |
| 103 | strstart, (lnum, end), line) |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 104 | contstr, needcont = '', 0 |
| 105 | elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n': |
| 106 | tokeneater(ERRORTOKEN, contstr + line, |
| 107 | strstart, (lnum, len(line)), line) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 108 | contstr = '' |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 109 | continue |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 110 | else: |
| 111 | contstr = contstr + line |
| 112 | continue |
| 113 | |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 114 | elif parenlev == 0 and not continued: # new statement |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 115 | if not line: break |
| 116 | column = 0 |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 117 | while pos < max: # measure leading whitespace |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 118 | if line[pos] == ' ': column = column + 1 |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 119 | elif line[pos] == '\t': column = (column/tabsize + 1)*tabsize |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 120 | elif line[pos] == '\f': column = 0 |
| 121 | else: break |
| 122 | pos = pos + 1 |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 123 | if pos == max: break |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 124 | |
| 125 | if line[pos] in '#\r\n': # skip comments or blank lines |
| 126 | tokeneater((NEWLINE, COMMENT)[line[pos] == '#'], line[pos:], |
| 127 | (lnum, pos), (lnum, len(line)), line) |
| 128 | continue |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 129 | |
| 130 | if column > indents[-1]: # count indents or dedents |
| 131 | indents.append(column) |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 132 | tokeneater(INDENT, line[:pos], (lnum, 0), (lnum, pos), line) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 133 | while column < indents[-1]: |
| 134 | indents = indents[:-1] |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 135 | tokeneater(DEDENT, '', (lnum, pos), (lnum, pos), line) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 136 | |
| 137 | else: # continued statement |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 138 | if not line: |
| 139 | raise TokenError, ("EOF in multi-line statement", (lnum, 0)) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 140 | continued = 0 |
| 141 | |
| 142 | while pos < max: |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 143 | if pseudoprog.match(line, pos) > 0: # scan for tokens |
| 144 | start, end = pseudoprog.regs[1] |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 145 | spos, epos, pos = (lnum, start), (lnum, end), end |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 146 | token, initial = line[start:end], line[start] |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 147 | |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 148 | if initial in namechars: # ordinary name |
| 149 | tokeneater(NAME, token, spos, epos, line) |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 150 | elif initial in numchars \ |
| 151 | or (initial == '.' and token != '.'): # ordinary number |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 152 | tokeneater(NUMBER, token, spos, epos, line) |
| 153 | elif initial in '\r\n': |
| 154 | tokeneater(NEWLINE, token, spos, epos, line) |
| 155 | elif initial == '#': |
| 156 | tokeneater(COMMENT, token, spos, epos, line) |
| 157 | elif initial == '\\': # continued stmt |
| 158 | continued = 1 |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 159 | elif token in ('\'\'\'', '"""'): # triple-quoted |
| 160 | endprog = endprogs[token] |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 161 | if endprog.match(line, pos) >= 0: # all on one line |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 162 | pos = endprog.regs[0][1] |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 163 | token = line[start:pos] |
| 164 | tokeneater(STRING, token, spos, (lnum, pos), line) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 165 | else: |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 166 | strstart = (lnum, start) # multiple lines |
| 167 | contstr = line[start:] |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 168 | break |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 169 | elif initial in '\'"': |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 170 | if token[-1] == '\n': # continued string |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 171 | strstart = (lnum, start) |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 172 | endprog = endprogs[initial] |
| 173 | contstr, needcont = line[start:], 1 |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 174 | break |
| 175 | else: # ordinary string |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 176 | tokeneater(STRING, token, spos, epos, line) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 177 | else: |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 178 | if initial in '([{': parenlev = parenlev + 1 |
| 179 | elif initial in ')]}': parenlev = parenlev - 1 |
| 180 | tokeneater(OP, token, spos, epos, line) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 181 | else: |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 182 | tokeneater(ERRORTOKEN, line[pos], |
| 183 | (lnum, pos), (lnum, pos+1), line) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 184 | pos = pos + 1 |
| 185 | |
| 186 | for indent in indents[1:]: # pop remaining indent levels |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 187 | tokeneater(DEDENT, '', (lnum, 0), (lnum, 0), '') |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 188 | tokeneater(ENDMARKER, '', (lnum, 0), (lnum, 0), '') |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 189 | |
| 190 | if __name__ == '__main__': # testing |
| 191 | import sys |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 192 | if len(sys.argv) > 1: tokenize(open(sys.argv[1]).readline) |
| 193 | else: tokenize(syss.tdin.readline) |