Issue #8478: Untokenizer.compat now processes first token from iterator input.
Patch based on lines from Georg Brandl, Eric Snow, and Gareth Rees.
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index a4963f8..bbac59b 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -627,9 +627,17 @@
'start (1,3) precedes previous end (2,2)')
self.assertRaises(ValueError, u.add_whitespace, (2,1))
-__test__ = {"doctests" : doctests, 'decistmt': decistmt}
+ def test_iter_compat(self):
+ u = Untokenizer()
+ token = (NAME, 'Hello')
+ u.compat(token, iter([]))
+ self.assertEqual(u.tokens, ["Hello "])
+ u = Untokenizer()
+ self.assertEqual(u.untokenize(iter([token])), 'Hello ')
+__test__ = {"doctests" : doctests, 'decistmt': decistmt}
+
def test_main():
from test import test_tokenize
test_support.run_doctest(test_tokenize, True)