Ka-Ping's muich improved version of March 26, 1997:
# Ignore now accepts \f as whitespace. Operator now includes '**'.
# Ignore and Special now accept \n or \r\n at the end of a line.
# Imagnumber is new. Expfloat is corrected to reject '0e4'.
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 06ed746..7fe6fc1 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -1,158 +1,182 @@
"""Tokenization help for Python programs.
-This module compiles a regular expression that recognizes Python
-tokens in individual lines of text. The regular expression handles
-everything except indentation, continuations, and triple-quoted
-strings. The function 'tokenize.tokenize()' takes care of these
-things for streams of text. It accepts a readline-like function which
-is called repeatedly to come up with the next input line (or "" for
-EOF), and a "token-eater" function which is called for each token
-found, passing its type, a string containing the token, the line
-number, the line, and the starting and ending positions of the token
-within the line. It is designed to match the working of the Python
-tokenizer exactly.
+This module exports a function called 'tokenize()' that breaks a stream of
+text into Python tokens. It accepts a readline-like method which is called
+repeatedly to get the next line of input (or "" for EOF) and a "token-eater"
+function which is called once for each token found. The latter function is
+passed the token type, a string containing the token, the starting and
+ending (row, column) coordinates of the token, and the original line. It is
+designed to match the working of the Python tokenizer exactly, except that
+it produces COMMENT tokens for comments and gives type OP for all operators.
-"""
+For compatibility with the older 'tokenize' module, this also compiles a
+regular expression into 'tokenprog' that matches Python tokens in individual
+lines of text, leaving the token in 'tokenprog.group(3)', but does not
+handle indentation, continuations, or multi-line strings."""
-__version__ = "Ka-Ping Yee, 4 March 1997, updated by GvR, 10 March 1997"
+__version__ = "Ka-Ping Yee, 26 March 1997"
import string, regex
from token import *
+COMMENT = N_TOKENS
+tok_name[COMMENT] = 'COMMENT'
+
+# Changes from 1.3:
+# Ignore now accepts \f as whitespace. Operator now includes '**'.
+# Ignore and Special now accept \n or \r\n at the end of a line.
+# Imagnumber is new. Expfloat is corrected to reject '0e4'.
+# Note: to get a quoted backslash in a regex, it must be enclosed in brackets.
+
def group(*choices): return '\(' + string.join(choices, '\|') + '\)'
-Ignore = '[ \f\t]*\([\]\r?\n[ \t]*\)*\(#.*\)?'
+Whitespace = '[ \f\t]*'
+Comment = '\(#[^\r\n]*\)'
+Ignore = Whitespace + group('[\]\r?\n' + Whitespace)+'*' + Comment+'?'
Name = '[a-zA-Z_][a-zA-Z0-9_]*'
-ImagZero = '0[jJ]' # This is not caught by any of the following
Hexnumber = '0[xX][0-9a-fA-F]*[lL]?'
Octnumber = '0[0-7]*[lL]?'
-Decnumber = '[1-9][0-9]*[lLjJ]?'
-Intnumber = group(ImagZero, Hexnumber, Octnumber, Decnumber)
+Decnumber = '[1-9][0-9]*[lL]?'
+Intnumber = group(Hexnumber, Octnumber, Decnumber)
Exponent = '[eE][-+]?[0-9]+'
Pointfloat = group('[0-9]+\.[0-9]*', '\.[0-9]+') + group(Exponent) + '?'
-Expfloat = '[0-9]+' + Exponent
-Floatnumber = group(Pointfloat, Expfloat) + "[jJ]?"
-Number = group(Floatnumber, Intnumber)
+Expfloat = '[1-9][0-9]*' + Exponent
+Floatnumber = group(Pointfloat, Expfloat)
+Imagnumber = group('0[jJ]', '[1-9][0-9]*[jJ]', Floatnumber + '[jJ]')
+Number = group(Imagnumber, Floatnumber, Intnumber)
-Single = group('^\'', '[^\]\'')
+Single = group("^'", "[^\]'")
Double = group('^"', '[^\]"')
-Tsingle = group('^\'\'\'', '[^\]\'\'\'')
-Tdouble = group('^"""', '[^\]"""')
-Triple = group('\'\'\'', '"""')
-String = group('\'' + group('[\].', '[^\'\]')+ '*' + group('\'', '[\]\n'),
- '"' + group('[\].', '[^"\]') + '*' + group('"', '[\]\n'))
+Single3 = group("^'''", "[^\]'''")
+Double3 = group('^"""', '[^\]"""')
+Triple = group("'''", '"""')
+String = group("'" + group('[\].', "[^\n'\]") + "*'",
+ '"' + group('[\].', '[^\n"\]') + '*"')
Operator = group('\+', '\-', '\*\*', '\*', '\^', '~', '/', '%', '&', '|',
'<<', '>>', '==', '<=', '<>', '!=', '>=', '=', '<', '>')
Bracket = '[][(){}]'
-Special = group('[\]?\r?\n', '[:;.,`\f]')
+Special = group('\r?\n', '[:;.,`]')
Funny = group(Operator, Bracket, Special)
-PlainToken = group(Name, Number, Triple, String, Funny)
+PlainToken = group(Name, Number, String, Funny)
Token = Ignore + PlainToken
+ContStr = group("'" + group('[\].', "[^\n'\]")+'*' + group("'", '[\]\r?\n'),
+ '"' + group('[\].', '[^\n"\]')+'*' + group('"', '[\]\r?\n'))
+PseudoExtras = group('[\]\r?\n', Comment, Triple)
+PseudoToken = Whitespace + group(PseudoExtras, Name, Number, ContStr, Funny)
+
try:
- save_syntax = regex.set_syntax(0) # use default syntax
+ saved_syntax = regex.set_syntax(0) # use default syntax
tokenprog = regex.compile(Token)
+ pseudoprog = regex.compile(PseudoToken)
endprogs = { '\'': regex.compile(Single), '"': regex.compile(Double),
- '\'\'\'': regex.compile(Tsingle), '"""': regex.compile(Tdouble) }
+ '\'\'\'': regex.compile(Single3), '"""': regex.compile(Double3) }
finally:
- regex.set_syntax(save_syntax) # restore original syntax
+ regex.set_syntax(saved_syntax) # restore original syntax
tabsize = 8
TokenError = 'TokenError'
-def printtoken(type, string, linenum, line, start, end): # for testing
- print `linenum` + ':', tok_name[type], repr(string)
+def printtoken(type, token, (srow, scol), (erow, ecol), line): # for testing
+ print "%d,%d-%d,%d:\t%s\t%s" % \
+ (srow, scol, erow, ecol, tok_name[type], repr(token))
-def tokenize(readline, tokeneater = printtoken):
- linenum = parenlev = continued = 0
+def tokenize(readline, tokeneater=printtoken):
+ lnum = parenlev = continued = 0
namechars, numchars = string.letters + '_', string.digits
contstr = ''
indents = [0]
+
while 1: # loop over lines in stream
line = readline()
- linenum = linenum + 1
- if line[-2:] == '\r\n': line = line[:-2] + '\n'
+ lnum = lnum + 1
pos, max = 0, len(line)
if contstr: # continued string
if not line: raise TokenError, "EOF within multi-line string"
- if contstr[-2:] == '\\\n': contstr = contstr[:-2] + '\n'
if endprog.search(line) >= 0:
pos = end = endprog.regs[0][1]
- tokeneater(STRING, contstr + line[:end], linenum, line, 0, 0)
+ tokeneater(STRING, contstr + line[:end],
+ strstart, (lnum, end), line)
contstr = ''
else:
contstr = contstr + line
continue
- elif parenlev == 0 and not continued: # this is a new statement
+ elif parenlev == 0 and not continued: # new statement
if not line: break
column = 0
- while 1: # measure leading whitespace
+ while pos < max: # measure leading whitespace
if line[pos] == ' ': column = column + 1
- elif line[pos] == '\t': column = (column/tabsize + 1) * tabsize
+ elif line[pos] == '\t': column = (column/tabsize + 1)*tabsize
elif line[pos] == '\f': column = 0
else: break
pos = pos + 1
- if line[pos] in '#\n': continue # skip comments or blank lines
+ if pos == max: break
+
+ if line[pos] in '#\r\n': # skip comments or blank lines
+ tokeneater((NEWLINE, COMMENT)[line[pos] == '#'], line[pos:],
+ (lnum, pos), (lnum, len(line)), line)
+ continue
if column > indents[-1]: # count indents or dedents
indents.append(column)
- tokeneater(INDENT, '\t', linenum, line, 0, 0)
+ tokeneater(INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
while column < indents[-1]:
indents = indents[:-1]
- tokeneater(DEDENT, '\t', linenum, line, 0, 0)
+ tokeneater(DEDENT, line[:pos], (lnum, 0), (lnum, pos), line)
else: # continued statement
if not line: raise TokenError, "EOF within multi-line statement"
continued = 0
while pos < max:
- if tokenprog.match(line, pos) > 0: # scan for tokens
- start, end = tokenprog.regs[3]
- token = line[start:end]
+ if pseudoprog.match(line, pos) > 0: # scan for tokens
+ start, end = pseudoprog.regs[1]
+ spos, epos = (lnum, start), (lnum, end)
+ token, initial = line[start:end], line[start]
pos = end
- if token[0] in namechars: # ordinary name
- tokeneater(NAME, token, linenum, line, start, end)
- elif token[0] in numchars: # ordinary number
- tokeneater(NUMBER, token, linenum, line, start, end)
-
+ if initial in namechars: # ordinary name
+ tokeneater(NAME, token, spos, epos, line)
+ elif initial in numchars: # ordinary number
+ tokeneater(NUMBER, token, spos, epos, line)
+ elif initial in '\r\n':
+ tokeneater(NEWLINE, token, spos, epos, line)
+ elif initial == '#':
+ tokeneater(COMMENT, token, spos, epos, line)
+ elif initial == '\\': # continued stmt
+ continued = 1
elif token in ('\'\'\'', '"""'): # triple-quoted
endprog = endprogs[token]
if endprog.search(line, pos) >= 0: # all on one line
pos = endprog.regs[0][1]
- token = line[start:pos]
- tokeneater(STRING, token, linenum, line, start, pos)
+ token = line[start:pos]
+ tokeneater(STRING, token, spos, (lnum, pos), line)
else:
- contstr = line[start:] # multiple lines
+ strstart = (lnum, start) # multiple lines
+ contstr = line[start:]
break
- elif token[0] in '\'"':
+ elif initial in '\'"':
if token[-1] == '\n': # continued string
- endprog, contstr = endprogs[token[0]], line[start:]
+ strstart = (lnum, start)
+ endprog, contstr = endprogs[initial], line[start:]
break
else: # ordinary string
- tokeneater(STRING, token, linenum, line, start, end)
-
- elif token[0] == '\n':
- tokeneater(NEWLINE, token, linenum, line, start, end)
- elif token[0] == '\\': # continued stmt
- continued = 1
-
+ tokeneater(STRING, token, spos, epos, line)
else:
- if token[0] in '([{': parenlev = parenlev + 1
- if token[0] in ')]}': parenlev = parenlev - 1
- tokeneater(OP, token, linenum, line, start, end)
+ if initial in '([{': parenlev = parenlev + 1
+ elif initial in ')]}': parenlev = parenlev - 1
+ tokeneater(OP, token, spos, epos, line)
else:
- tokeneater(ERRORTOKEN, line[pos], linenum, line, pos, pos + 1)
+ tokeneater(ERRORTOKEN, line[pos], spos, (lnum, pos+1), line)
pos = pos + 1
for indent in indents[1:]: # pop remaining indent levels
- tokeneater(DEDENT, '\t', linenum, line, 0, 0)
+ tokeneater(DEDENT, '', (lnum, 0), (lnum, 0), '')
if __name__ == '__main__': # testing
import sys
- file = open(sys.argv[-1])
- tokenize(file.readline)
+ tokenize(open(sys.argv[-1]).readline)