#16152: fix tokenize to ignore whitespace at the end of the code when no newline is found. Patch by Ned Batchelder.
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index a51e781..489f68f 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -550,6 +550,10 @@
NAME 'pass' (3, 9) (3, 13)
DEDENT '' (4, 0) (4, 0)
DEDENT '' (4, 0) (4, 0)
+
+Pathological whitespace (http://bugs.python.org/issue16152)
+ >>> dump_tokens("@ ")
+ OP '@' (1, 0) (1, 1)
"""
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 1cba6e5..ca7b074 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -95,7 +95,7 @@
group("'", r'\\\r?\n'),
r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
group('"', r'\\\r?\n'))
-PseudoExtras = group(r'\\\r?\n', Comment, Triple)
+PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple)
PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
tokenprog, pseudoprog, single3prog, double3prog = map(
@@ -362,6 +362,8 @@
if pseudomatch: # scan for tokens
start, end = pseudomatch.span(1)
spos, epos, pos = (lnum, start), (lnum, end), end
+ if start == end:
+ continue
token, initial = line[start:end], line[start]
if initial in numchars or \
diff --git a/Misc/ACKS b/Misc/ACKS
index 129a116..01bc45e 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -64,6 +64,7 @@
Ulf Bartelt
Don Bashford
Nick Bastin
+Ned Batchelder
Jeff Bauer
Mike Bayer
Michael R Bax
diff --git a/Misc/NEWS b/Misc/NEWS
index da21101..fedcd35 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -130,6 +130,9 @@
Library
-------
+- Issue #16152: fix tokenize to ignore whitespace at the end of the code when
+ no newline is found. Patch by Ned Batchelder.
+
- Issue #1207589: Add Cut/Copy/Paste items to IDLE right click Context Menu
Patch by Todd Rovito.