commit 2cc3b4ba9ffa658784da03f14a0a068e2c61d1b3
author Ezio Melotti <ezio.melotti@gmail.com> Sat Nov 03 17:38:43 2012 +0200
committer Ezio Melotti <ezio.melotti@gmail.com> Sat Nov 03 17:38:43 2012 +0200
tree 66423060d67ba1f6ad182450edb895cc29ca28f1
parent 4552e3f95c3382a4665cb8adab343521f8898331

#16152: fix tokenize to ignore whitespace at the end of the code when no newline is found.  Patch by Ned Batchelder.

Lib/test/test_tokenize.py

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index b6a9ca1..f9652ce 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -552,6 +552,11 @@
     DEDENT     ''            (4, 0) (4, 0)
     DEDENT     ''            (4, 0) (4, 0)
 
+Pathological whitespace (http://bugs.python.org/issue16152)
+    >>> dump_tokens("@          ")
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    OP         '@'           (1, 0) (1, 1)
+
 Non-ascii identifiers
 
     >>> dump_tokens("Örter = 'places'\\ngrün = 'green'")