Issue #8478: Untokenizer.compat now processes first token from iterator input.
Patch based on lines from Georg Brandl, Eric Snow, and Gareth Rees.
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index c156450..7356a88 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -25,12 +25,14 @@
'Skip Montanaro, Raymond Hettinger, Trent Nelson, '
'Michael Foord')
import builtins
-import re
-import sys
-from token import *
from codecs import lookup, BOM_UTF8
import collections
from io import TextIOWrapper
+from itertools import chain
+import re
+import sys
+from token import *
+
cookie_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)', re.ASCII)
blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII)
@@ -237,9 +239,10 @@
self.tokens.append(" " * col_offset)
def untokenize(self, iterable):
- for t in iterable:
+ it = iter(iterable)
+ for t in it:
if len(t) == 2:
- self.compat(t, iterable)
+ self.compat(t, it)
break
tok_type, token, start, end, line = t
if tok_type == ENCODING:
@@ -254,17 +257,12 @@
return "".join(self.tokens)
def compat(self, token, iterable):
- startline = False
indents = []
toks_append = self.tokens.append
- toknum, tokval = token
-
- if toknum in (NAME, NUMBER):
- tokval += ' '
- if toknum in (NEWLINE, NL):
- startline = True
+ startline = token[0] in (NEWLINE, NL)
prevstring = False
- for tok in iterable:
+
+ for tok in chain([token], iterable):
toknum, tokval = tok[:2]
if toknum == ENCODING:
self.encoding = tokval