Issue2495: tokenize.untokenize did not insert space between two consecutive string literals:
"" "" becomes """", which is invalid code.
Backport of r61979.
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index a9be4cf..0db3867 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -171,11 +171,12 @@
t1 = [tok[:2] for tok in generate_tokens(f.readline)]
newcode = untokenize(t1)
readline = iter(newcode.splitlines(1)).next
- t2 = [tok[:2] for tokin generate_tokens(readline)]
+ t2 = [tok[:2] for tok in generate_tokens(readline)]
assert t1 == t2
"""
startline = False
+ prevstring = False
indents = []
toks = []
toks_append = toks.append
@@ -185,6 +186,14 @@
if toknum in (NAME, NUMBER):
tokval += ' '
+ # Insert a space between two consecutive strings
+ if toknum == STRING:
+ if prevstring:
+ tokval = ' ' + tokval
+ prevstring = True
+ else:
+ prevstring = False
+
if toknum == INDENT:
indents.append(tokval)
continue