#16152: fix tokenize to ignore whitespace at the end of the code when no newline is found. Patch by Ned Batchelder.
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 59081d3..29c9e29 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -108,7 +108,7 @@
group("'", r'\\\r?\n'),
r'[bB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
group('"', r'\\\r?\n'))
-PseudoExtras = group(r'\\\r?\n', Comment, Triple)
+PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple)
PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
def _compile(expr):
@@ -473,6 +473,8 @@
if pseudomatch: # scan for tokens
start, end = pseudomatch.span(1)
spos, epos, pos = (lnum, start), (lnum, end), end
+ if start == end:
+ continue
token, initial = line[start:end], line[start]
if (initial in numchars or # ordinary number