Guido van Rossum | b51eaa1 | 1997-03-07 00:21:55 +0000 | [diff] [blame] | 1 | """Tokenization help for Python programs. |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 2 | |
Tim Peters | 4efb6e9 | 2001-06-29 23:51:08 +0000 | [diff] [blame] | 3 | generate_tokens(readline) is a generator that breaks a stream of |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 4 | text into Python tokens. It accepts a readline-like method which is called |
Tim Peters | 4efb6e9 | 2001-06-29 23:51:08 +0000 | [diff] [blame] | 5 | repeatedly to get the next line of input (or "" for EOF). It generates |
| 6 | 5-tuples with these members: |
| 7 | |
| 8 | the token type (see token.py) |
| 9 | the token (a string) |
| 10 | the starting (row, column) indices of the token (a 2-tuple of ints) |
| 11 | the ending (row, column) indices of the token (a 2-tuple of ints) |
| 12 | the original line (string) |
| 13 | |
| 14 | It is designed to match the working of the Python tokenizer exactly, except |
| 15 | that it produces COMMENT tokens for comments and gives type OP for all |
| 16 | operators |
| 17 | |
| 18 | Older entry points |
| 19 | tokenize_loop(readline, tokeneater) |
| 20 | tokenize(readline, tokeneater=printtoken) |
| 21 | are the same, except instead of generating tokens, tokeneater is a callback |
| 22 | function to which the 5 fields described above are passed as 5 arguments, |
| 23 | each time a new token is found.""" |
Guido van Rossum | b51eaa1 | 1997-03-07 00:21:55 +0000 | [diff] [blame] | 24 | |
Guido van Rossum | b09f7ed | 2001-07-15 21:08:29 +0000 | [diff] [blame] | 25 | from __future__ import generators |
| 26 | |
Ka-Ping Yee | 244c593 | 2001-03-01 13:56:40 +0000 | [diff] [blame] | 27 | __author__ = 'Ka-Ping Yee <ping@lfw.org>' |
Ka-Ping Yee | 4f64c13 | 2001-03-01 17:11:17 +0000 | [diff] [blame] | 28 | __credits__ = \ |
| 29 | 'GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro' |
Guido van Rossum | b51eaa1 | 1997-03-07 00:21:55 +0000 | [diff] [blame] | 30 | |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 31 | import string, re |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 32 | from token import * |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 33 | |
Skip Montanaro | 40fc160 | 2001-03-01 04:27:19 +0000 | [diff] [blame] | 34 | import token |
| 35 | __all__ = [x for x in dir(token) if x[0] != '_'] + ["COMMENT", "tokenize", "NL"] |
Neal Norwitz | e98d16e | 2002-03-26 16:20:26 +0000 | [diff] [blame] | 36 | del x |
Skip Montanaro | 40fc160 | 2001-03-01 04:27:19 +0000 | [diff] [blame] | 37 | del token |
| 38 | |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 39 | COMMENT = N_TOKENS |
| 40 | tok_name[COMMENT] = 'COMMENT' |
Guido van Rossum | a90c78b | 1998-04-03 16:05:38 +0000 | [diff] [blame] | 41 | NL = N_TOKENS + 1 |
| 42 | tok_name[NL] = 'NL' |
Skip Montanaro | 40fc160 | 2001-03-01 04:27:19 +0000 | [diff] [blame] | 43 | N_TOKENS += 2 |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 44 | |
Eric S. Raymond | b08b2d3 | 2001-02-09 11:10:16 +0000 | [diff] [blame] | 45 | def group(*choices): return '(' + '|'.join(choices) + ')' |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 46 | def any(*choices): return apply(group, choices) + '*' |
| 47 | def maybe(*choices): return apply(group, choices) + '?' |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 48 | |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 49 | Whitespace = r'[ \f\t]*' |
| 50 | Comment = r'#[^\r\n]*' |
| 51 | Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) |
| 52 | Name = r'[a-zA-Z_]\w*' |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 53 | |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 54 | Hexnumber = r'0[xX][\da-fA-F]*[lL]?' |
| 55 | Octnumber = r'0[0-7]*[lL]?' |
| 56 | Decnumber = r'[1-9]\d*[lL]?' |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 57 | Intnumber = group(Hexnumber, Octnumber, Decnumber) |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 58 | Exponent = r'[eE][-+]?\d+' |
| 59 | Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent) |
Tim Peters | d507dab | 2001-08-30 20:51:59 +0000 | [diff] [blame] | 60 | Expfloat = r'\d+' + Exponent |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 61 | Floatnumber = group(Pointfloat, Expfloat) |
Tim Peters | d507dab | 2001-08-30 20:51:59 +0000 | [diff] [blame] | 62 | Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]') |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 63 | Number = group(Imagnumber, Floatnumber, Intnumber) |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 64 | |
Tim Peters | de49583 | 2000-10-07 05:09:39 +0000 | [diff] [blame] | 65 | # Tail end of ' string. |
| 66 | Single = r"[^'\\]*(?:\\.[^'\\]*)*'" |
| 67 | # Tail end of " string. |
| 68 | Double = r'[^"\\]*(?:\\.[^"\\]*)*"' |
| 69 | # Tail end of ''' string. |
| 70 | Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''" |
| 71 | # Tail end of """ string. |
| 72 | Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""' |
Ka-Ping Yee | 1ff08b1 | 2001-01-15 22:04:30 +0000 | [diff] [blame] | 73 | Triple = group("[uU]?[rR]?'''", '[uU]?[rR]?"""') |
Tim Peters | de49583 | 2000-10-07 05:09:39 +0000 | [diff] [blame] | 74 | # Single-line ' or " string. |
Ka-Ping Yee | 1ff08b1 | 2001-01-15 22:04:30 +0000 | [diff] [blame] | 75 | String = group(r"[uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'", |
| 76 | r'[uU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"') |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 77 | |
Tim Peters | de49583 | 2000-10-07 05:09:39 +0000 | [diff] [blame] | 78 | # Because of leftmost-then-longest match semantics, be sure to put the |
| 79 | # longest operators first (e.g., if = came before ==, == would get |
| 80 | # recognized as two instances of =). |
| 81 | Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=", |
Guido van Rossum | 96204f5 | 2001-08-08 05:04:07 +0000 | [diff] [blame] | 82 | r"//=?", |
Tim Peters | de49583 | 2000-10-07 05:09:39 +0000 | [diff] [blame] | 83 | r"[+\-*/%&|^=<>]=?", |
| 84 | r"~") |
Thomas Wouters | e1519a1 | 2000-08-24 21:44:52 +0000 | [diff] [blame] | 85 | |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 86 | Bracket = '[][(){}]' |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 87 | Special = group(r'\r?\n', r'[:;.,`]') |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 88 | Funny = group(Operator, Bracket, Special) |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 89 | |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 90 | PlainToken = group(Number, Funny, String, Name) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 91 | Token = Ignore + PlainToken |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 92 | |
Tim Peters | de49583 | 2000-10-07 05:09:39 +0000 | [diff] [blame] | 93 | # First (or only) line of ' or " string. |
Ka-Ping Yee | 1ff08b1 | 2001-01-15 22:04:30 +0000 | [diff] [blame] | 94 | ContStr = group(r"[uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" + |
| 95 | group("'", r'\\\r?\n'), |
| 96 | r'[uU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' + |
| 97 | group('"', r'\\\r?\n')) |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 98 | PseudoExtras = group(r'\\\r?\n', Comment, Triple) |
| 99 | PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 100 | |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 101 | tokenprog, pseudoprog, single3prog, double3prog = map( |
| 102 | re.compile, (Token, PseudoToken, Single3, Double3)) |
Guido van Rossum | fefc922 | 1997-10-27 21:17:24 +0000 | [diff] [blame] | 103 | endprogs = {"'": re.compile(Single), '"': re.compile(Double), |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 104 | "'''": single3prog, '"""': double3prog, |
Guido van Rossum | fefc922 | 1997-10-27 21:17:24 +0000 | [diff] [blame] | 105 | "r'''": single3prog, 'r"""': double3prog, |
Ka-Ping Yee | 1ff08b1 | 2001-01-15 22:04:30 +0000 | [diff] [blame] | 106 | "u'''": single3prog, 'u"""': double3prog, |
| 107 | "ur'''": single3prog, 'ur"""': double3prog, |
| 108 | "R'''": single3prog, 'R"""': double3prog, |
| 109 | "U'''": single3prog, 'U"""': double3prog, |
| 110 | "uR'''": single3prog, 'uR"""': double3prog, |
| 111 | "Ur'''": single3prog, 'Ur"""': double3prog, |
| 112 | "UR'''": single3prog, 'UR"""': double3prog, |
| 113 | 'r': None, 'R': None, 'u': None, 'U': None} |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 114 | |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 115 | tabsize = 8 |
Fred Drake | 9b8d801 | 2000-08-17 04:45:13 +0000 | [diff] [blame] | 116 | |
Ka-Ping Yee | 28c62bb | 2001-03-23 05:22:49 +0000 | [diff] [blame] | 117 | class TokenError(Exception): pass |
| 118 | |
| 119 | class StopTokenizing(Exception): pass |
Fred Drake | 9b8d801 | 2000-08-17 04:45:13 +0000 | [diff] [blame] | 120 | |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 121 | def printtoken(type, token, (srow, scol), (erow, ecol), line): # for testing |
| 122 | print "%d,%d-%d,%d:\t%s\t%s" % \ |
| 123 | (srow, scol, erow, ecol, tok_name[type], repr(token)) |
Guido van Rossum | 4d8e859 | 1992-01-01 19:34:47 +0000 | [diff] [blame] | 124 | |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 125 | def tokenize(readline, tokeneater=printtoken): |
Ka-Ping Yee | 28c62bb | 2001-03-23 05:22:49 +0000 | [diff] [blame] | 126 | try: |
| 127 | tokenize_loop(readline, tokeneater) |
| 128 | except StopTokenizing: |
| 129 | pass |
| 130 | |
Tim Peters | 4efb6e9 | 2001-06-29 23:51:08 +0000 | [diff] [blame] | 131 | # backwards compatible interface |
Ka-Ping Yee | 28c62bb | 2001-03-23 05:22:49 +0000 | [diff] [blame] | 132 | def tokenize_loop(readline, tokeneater): |
Tim Peters | 5ca576e | 2001-06-18 22:08:13 +0000 | [diff] [blame] | 133 | for token_info in generate_tokens(readline): |
| 134 | apply(tokeneater, token_info) |
| 135 | |
| 136 | def generate_tokens(readline): |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 137 | lnum = parenlev = continued = 0 |
Fred Drake | 79e75e1 | 2001-07-20 19:05:50 +0000 | [diff] [blame] | 138 | namechars, numchars = string.ascii_letters + '_', '0123456789' |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 139 | contstr, needcont = '', 0 |
Guido van Rossum | a90c78b | 1998-04-03 16:05:38 +0000 | [diff] [blame] | 140 | contline = None |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 141 | indents = [0] |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 142 | |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 143 | while 1: # loop over lines in stream |
| 144 | line = readline() |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 145 | lnum = lnum + 1 |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 146 | pos, max = 0, len(line) |
| 147 | |
| 148 | if contstr: # continued string |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 149 | if not line: |
| 150 | raise TokenError, ("EOF in multi-line string", strstart) |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 151 | endmatch = endprog.match(line) |
| 152 | if endmatch: |
| 153 | pos = end = endmatch.end(0) |
Tim Peters | 5ca576e | 2001-06-18 22:08:13 +0000 | [diff] [blame] | 154 | yield (STRING, contstr + line[:end], |
Guido van Rossum | a90c78b | 1998-04-03 16:05:38 +0000 | [diff] [blame] | 155 | strstart, (lnum, end), contline + line) |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 156 | contstr, needcont = '', 0 |
Guido van Rossum | a90c78b | 1998-04-03 16:05:38 +0000 | [diff] [blame] | 157 | contline = None |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 158 | elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n': |
Tim Peters | 5ca576e | 2001-06-18 22:08:13 +0000 | [diff] [blame] | 159 | yield (ERRORTOKEN, contstr + line, |
Guido van Rossum | a90c78b | 1998-04-03 16:05:38 +0000 | [diff] [blame] | 160 | strstart, (lnum, len(line)), contline) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 161 | contstr = '' |
Guido van Rossum | a90c78b | 1998-04-03 16:05:38 +0000 | [diff] [blame] | 162 | contline = None |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 163 | continue |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 164 | else: |
| 165 | contstr = contstr + line |
Guido van Rossum | a90c78b | 1998-04-03 16:05:38 +0000 | [diff] [blame] | 166 | contline = contline + line |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 167 | continue |
| 168 | |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 169 | elif parenlev == 0 and not continued: # new statement |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 170 | if not line: break |
| 171 | column = 0 |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 172 | while pos < max: # measure leading whitespace |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 173 | if line[pos] == ' ': column = column + 1 |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 174 | elif line[pos] == '\t': column = (column/tabsize + 1)*tabsize |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 175 | elif line[pos] == '\f': column = 0 |
| 176 | else: break |
| 177 | pos = pos + 1 |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 178 | if pos == max: break |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 179 | |
| 180 | if line[pos] in '#\r\n': # skip comments or blank lines |
Tim Peters | 5ca576e | 2001-06-18 22:08:13 +0000 | [diff] [blame] | 181 | yield ((NL, COMMENT)[line[pos] == '#'], line[pos:], |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 182 | (lnum, pos), (lnum, len(line)), line) |
| 183 | continue |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 184 | |
| 185 | if column > indents[-1]: # count indents or dedents |
| 186 | indents.append(column) |
Tim Peters | 5ca576e | 2001-06-18 22:08:13 +0000 | [diff] [blame] | 187 | yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 188 | while column < indents[-1]: |
| 189 | indents = indents[:-1] |
Tim Peters | 5ca576e | 2001-06-18 22:08:13 +0000 | [diff] [blame] | 190 | yield (DEDENT, '', (lnum, pos), (lnum, pos), line) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 191 | |
| 192 | else: # continued statement |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 193 | if not line: |
| 194 | raise TokenError, ("EOF in multi-line statement", (lnum, 0)) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 195 | continued = 0 |
| 196 | |
| 197 | while pos < max: |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 198 | pseudomatch = pseudoprog.match(line, pos) |
| 199 | if pseudomatch: # scan for tokens |
| 200 | start, end = pseudomatch.span(1) |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 201 | spos, epos, pos = (lnum, start), (lnum, end), end |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 202 | token, initial = line[start:end], line[start] |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 203 | |
Ka-Ping Yee | 28c62bb | 2001-03-23 05:22:49 +0000 | [diff] [blame] | 204 | if initial in numchars or \ |
| 205 | (initial == '.' and token != '.'): # ordinary number |
Tim Peters | 5ca576e | 2001-06-18 22:08:13 +0000 | [diff] [blame] | 206 | yield (NUMBER, token, spos, epos, line) |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 207 | elif initial in '\r\n': |
Tim Peters | 5ca576e | 2001-06-18 22:08:13 +0000 | [diff] [blame] | 208 | yield (parenlev > 0 and NL or NEWLINE, |
Guido van Rossum | a90c78b | 1998-04-03 16:05:38 +0000 | [diff] [blame] | 209 | token, spos, epos, line) |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 210 | elif initial == '#': |
Tim Peters | 5ca576e | 2001-06-18 22:08:13 +0000 | [diff] [blame] | 211 | yield (COMMENT, token, spos, epos, line) |
Guido van Rossum | fefc922 | 1997-10-27 21:17:24 +0000 | [diff] [blame] | 212 | elif token in ("'''", '"""', # triple-quoted |
Ka-Ping Yee | 1ff08b1 | 2001-01-15 22:04:30 +0000 | [diff] [blame] | 213 | "r'''", 'r"""', "R'''", 'R"""', |
| 214 | "u'''", 'u"""', "U'''", 'U"""', |
| 215 | "ur'''", 'ur"""', "Ur'''", 'Ur"""', |
| 216 | "uR'''", 'uR"""', "UR'''", 'UR"""'): |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 217 | endprog = endprogs[token] |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 218 | endmatch = endprog.match(line, pos) |
| 219 | if endmatch: # all on one line |
| 220 | pos = endmatch.end(0) |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 221 | token = line[start:pos] |
Tim Peters | 5ca576e | 2001-06-18 22:08:13 +0000 | [diff] [blame] | 222 | yield (STRING, token, spos, (lnum, pos), line) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 223 | else: |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 224 | strstart = (lnum, start) # multiple lines |
| 225 | contstr = line[start:] |
Guido van Rossum | a90c78b | 1998-04-03 16:05:38 +0000 | [diff] [blame] | 226 | contline = line |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 227 | break |
Guido van Rossum | fefc922 | 1997-10-27 21:17:24 +0000 | [diff] [blame] | 228 | elif initial in ("'", '"') or \ |
Ka-Ping Yee | 1ff08b1 | 2001-01-15 22:04:30 +0000 | [diff] [blame] | 229 | token[:2] in ("r'", 'r"', "R'", 'R"', |
| 230 | "u'", 'u"', "U'", 'U"') or \ |
| 231 | token[:3] in ("ur'", 'ur"', "Ur'", 'Ur"', |
| 232 | "uR'", 'uR"', "UR'", 'UR"' ): |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 233 | if token[-1] == '\n': # continued string |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 234 | strstart = (lnum, start) |
Ka-Ping Yee | 1ff08b1 | 2001-01-15 22:04:30 +0000 | [diff] [blame] | 235 | endprog = (endprogs[initial] or endprogs[token[1]] or |
| 236 | endprogs[token[2]]) |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 237 | contstr, needcont = line[start:], 1 |
Guido van Rossum | a90c78b | 1998-04-03 16:05:38 +0000 | [diff] [blame] | 238 | contline = line |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 239 | break |
| 240 | else: # ordinary string |
Tim Peters | 5ca576e | 2001-06-18 22:08:13 +0000 | [diff] [blame] | 241 | yield (STRING, token, spos, epos, line) |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 242 | elif initial in namechars: # ordinary name |
Tim Peters | 5ca576e | 2001-06-18 22:08:13 +0000 | [diff] [blame] | 243 | yield (NAME, token, spos, epos, line) |
Guido van Rossum | 3b63177 | 1997-10-27 20:44:15 +0000 | [diff] [blame] | 244 | elif initial == '\\': # continued stmt |
| 245 | continued = 1 |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 246 | else: |
Guido van Rossum | 1aec323 | 1997-04-08 14:24:39 +0000 | [diff] [blame] | 247 | if initial in '([{': parenlev = parenlev + 1 |
| 248 | elif initial in ')]}': parenlev = parenlev - 1 |
Tim Peters | 5ca576e | 2001-06-18 22:08:13 +0000 | [diff] [blame] | 249 | yield (OP, token, spos, epos, line) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 250 | else: |
Tim Peters | 5ca576e | 2001-06-18 22:08:13 +0000 | [diff] [blame] | 251 | yield (ERRORTOKEN, line[pos], |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 252 | (lnum, pos), (lnum, pos+1), line) |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 253 | pos = pos + 1 |
| 254 | |
| 255 | for indent in indents[1:]: # pop remaining indent levels |
Tim Peters | 5ca576e | 2001-06-18 22:08:13 +0000 | [diff] [blame] | 256 | yield (DEDENT, '', (lnum, 0), (lnum, 0), '') |
| 257 | yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '') |
Guido van Rossum | fc6f533 | 1997-03-07 00:21:12 +0000 | [diff] [blame] | 258 | |
| 259 | if __name__ == '__main__': # testing |
| 260 | import sys |
Guido van Rossum | de65527 | 1997-04-09 17:15:54 +0000 | [diff] [blame] | 261 | if len(sys.argv) > 1: tokenize(open(sys.argv[1]).readline) |
Guido van Rossum | 2b1566b | 1997-06-03 22:05:15 +0000 | [diff] [blame] | 262 | else: tokenize(sys.stdin.readline) |