Environment.lex returns unicode tokens now, even if the input data was a bytestring.
--HG--
branch : trunk
diff --git a/jinja2/lexer.py b/jinja2/lexer.py
index 7f0b33f..92ff12e 100644
--- a/jinja2/lexer.py
+++ b/jinja2/lexer.py
@@ -371,7 +371,6 @@
converted into types and postprocessed. For example comments are removed,
integers and floats converted, strings unescaped etc.
"""
- source = unicode(source)
def generate():
for lineno, token, value in self.tokeniter(source, name, filename):
if token in ('comment_begin', 'comment', 'comment_end',
@@ -425,7 +424,7 @@
wants. The parser uses the `tokenize` function with returns a
`TokenStream` and postprocessed tokens.
"""
- source = '\n'.join(source.splitlines())
+ source = u'\n'.join(unicode(source).splitlines())
pos = 0
lineno = 1
stack = ['root']