#16152: merge with 3.2.
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 213e9b4..b4a58f0 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -1109,6 +1109,10 @@
token.NAME, token.AMPER, token.NUMBER,
token.RPAR)
+ def test_pathological_trailing_whitespace(self):
+ # See http://bugs.python.org/issue16152
+ self.assertExactTypeEqual('@ ', token.AT)
+
__test__ = {"doctests" : doctests, 'decistmt': decistmt}
def test_main():
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index d669412..cbf91ef 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -162,7 +162,7 @@
group("'", r'\\\r?\n'),
StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
group('"', r'\\\r?\n'))
-PseudoExtras = group(r'\\\r?\n', Comment, Triple)
+PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple)
PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
def _compile(expr):
@@ -555,6 +555,8 @@
if pseudomatch: # scan for tokens
start, end = pseudomatch.span(1)
spos, epos, pos = (lnum, start), (lnum, end), end
+ if start == end:
+ continue
token, initial = line[start:end], line[start]
if (initial in numchars or # ordinary number
diff --git a/Misc/ACKS b/Misc/ACKS
index d80ead0..ee1ce67 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -77,6 +77,7 @@
Don Bashford
Pior Bastida
Nick Bastin
+Ned Batchelder
Jeff Bauer
Michael R Bax
Anthony Baxter
diff --git a/Misc/NEWS b/Misc/NEWS
index 618f34b..eaf6b3a 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -61,6 +61,9 @@
Library
-------
+- Issue #16152: fix tokenize to ignore whitespace at the end of the code when
+ no newline is found. Patch by Ned Batchelder.
+
- Issue #1207589: Add Cut/Copy/Paste items to IDLE right click Context Menu
Patch by Todd Rovito.