[svn] merged newparser into trunk

--HG--
branch : trunk
diff --git a/tests/runtime/bigtable.py b/tests/runtime/bigtable.py
index b1576df..ce853d9 100644
--- a/tests/runtime/bigtable.py
+++ b/tests/runtime/bigtable.py
@@ -7,12 +7,16 @@
 # Author: Jonas Borgström <jonas@edgewall.com>
 # Author: Armin Ronacher <armin.ronacher@active-4.com>
 
-import cgi
+import os
 import sys
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
+
+import cgi
 import timeit
 import jdebug
 from StringIO import StringIO
 
+
 try:
     from genshi.builder import tag
     from genshi.template import MarkupTemplate
diff --git a/tests/runtime/exception.py b/tests/runtime/exception.py
index 0af7955..9dffaeb 100644
--- a/tests/runtime/exception.py
+++ b/tests/runtime/exception.py
@@ -1,5 +1,8 @@
-import jdebug
+import os
 import sys
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
+
+import jdebug
 from jinja import Environment, DictLoader
 from jinja.exceptions import TemplateNotFound
 from wsgiref.simple_server import make_server
@@ -35,6 +38,7 @@
       <li><a href="code_runtime_error">runtime error in code</a></li>
       <li><a href="syntax_from_string">a syntax error from string</a></li>
       <li><a href="runtime_from_string">runtime error from a string</a></li>
+      <li><a href="multiple_templates">multiple templates</a></li>
     </ul>
   </body>
 </html>
@@ -53,17 +57,31 @@
     '/nested_syntax_error': u'''
 {% include 'syntax_broken' %}
     ''',
-
     '/code_runtime_error': u'''We have a runtime error here:
     {{ broken() }}''',
+    '/multiple_templates': '''\
+{{ fire_multiple_broken() }}
+''',
 
     'runtime_broken': '''\
 This is an included template
 {% set a = 1 / 0 %}''',
     'syntax_broken': '''\
 This is an included template
-{% raw %}just some foo'''
+{% raw %}just some foo''',
+    'multiple_broken': '''\
+Just some context:
+{% include 'macro_broken' %}
+{{ broken() }}
+''',
+    'macro_broken': '''\
+{% macro broken %}
+    {{ 1 / 0 }}
+{% endmacro %}
+'''
 }))
+e.globals['fire_multiple_broken'] = lambda: \
+    e.get_template('multiple_broken').render()
 
 FAILING_STRING_TEMPLATE = '{{ 1 / 0 }}'
 BROKEN_STRING_TEMPLATE = '{% if foo %}...{% endfor %}'
@@ -91,5 +109,5 @@
 
 if __name__ == '__main__':
     from werkzeug.debug import DebuggedApplication
-    app = DebuggedApplication(test)
+    app = DebuggedApplication(test, True)
     make_server("localhost", 7000, app).serve_forever()
diff --git a/tests/test_lexer.py b/tests/test_lexer.py
index 6ac49d4..cc85b19 100644
--- a/tests/test_lexer.py
+++ b/tests/test_lexer.py
@@ -14,6 +14,7 @@
   <li>{item}</li>
 <!--- endfor -->
 </ul>'''
+BYTEFALLBACK = u'''{{ 'foo'|pprint }}|{{ 'bär'|pprint }}'''
 
 
 def test_balancing():
@@ -29,3 +30,25 @@
     tmpl = env.from_string(COMMENTS)
     assert tmpl.render(seq=range(3)) == ("<ul>\n  <li>0</li>\n  "
                                          "<li>1</li>\n  <li>2</li>\n</ul>")
+
+
+def test_string_escapes(env):
+    for char in u'\0', u'\u2668', u'\xe4', u'\t', u'\r', u'\n':
+        tmpl = env.from_string('{{ %s }}' % repr(char)[1:])
+        assert tmpl.render() == char
+    assert env.from_string('{{ "\N{HOT SPRINGS}" }}').render() == u'\u2668'
+
+
+def test_bytefallback(env):
+    tmpl = env.from_string(BYTEFALLBACK)
+    assert tmpl.render() == u"'foo'|u'b\\xe4r'"
+
+
+def test_operators(env):
+    from jinja.lexer import operators
+    for test, expect in operators.iteritems():
+        if test in '([{}])':
+            continue
+        stream = env.lexer.tokenize('{{ %s }}' % test)
+        stream.next()
+        assert stream.current.type == expect
diff --git a/tests/test_loaders.py b/tests/test_loaders.py
index 2335b9c..a1d21ad 100644
--- a/tests/test_loaders.py
+++ b/tests/test_loaders.py
@@ -46,14 +46,19 @@
 
 def test_package_loader():
     env = Environment(loader=package_loader)
-    tmpl = env.get_template('test.html')
-    assert tmpl.render().strip() == 'BAR'
-    try:
-        env.get_template('missing.html')
-    except TemplateNotFound:
-        pass
-    else:
-        raise AssertionError('expected template exception')
+    for x in xrange(2):
+        tmpl = env.get_template('test.html')
+        assert tmpl.render().strip() == 'BAR'
+        try:
+            env.get_template('missing.html')
+        except TemplateNotFound:
+            pass
+        else:
+            raise AssertionError('expected template exception')
+
+        # second run in native mode (no pkg_resources)
+        package_loader.force_native = True
+        del package_loader._load_func
 
 
 def test_filesystem_loader():
diff --git a/tests/test_macros.py b/tests/test_macros.py
index cf84f95..7e6d54a 100644
--- a/tests/test_macros.py
+++ b/tests/test_macros.py
@@ -55,6 +55,8 @@
 {{ test() }}\
 '''
 
+INCLUDETEMPLATE = '''{% macro test(foo) %}[{{ foo }}]{% endmacro %}'''
+
 
 def test_simple(env):
     tmpl = env.from_string(SIMPLE)
@@ -105,3 +107,8 @@
 def test_caller_undefined(env):
     tmpl = env.from_string(CALLERUNDEFINED)
     assert tmpl.render() == 'True'
+
+
+def test_include(env):
+    tmpl = env.from_string('{% include "include" %}{{ test("foo") }}')
+    assert tmpl.render() == '[foo]'
diff --git a/tests/test_security.py b/tests/test_security.py
index 95d3371..78a1e0a 100644
--- a/tests/test_security.py
+++ b/tests/test_security.py
@@ -43,9 +43,9 @@
 >>> env.from_string("{% for item.attribute in seq %}...{% endfor %}")
 Traceback (most recent call last):
     ...
-TemplateSyntaxError: can't assign to expression. (line 1)
+TemplateSyntaxError: cannot assign to expression (line 1)
 >>> env.from_string("{% for foo, bar.baz in seq %}...{% endfor %}")
 Traceback (most recent call last):
     ...
-TemplateSyntaxError: can't assign to expression. (line 1)
+TemplateSyntaxError: cannot assign to expression (line 1)
 '''
diff --git a/tests/test_syntax.py b/tests/test_syntax.py
index e6a8714..abcd058 100644
--- a/tests/test_syntax.py
+++ b/tests/test_syntax.py
@@ -6,20 +6,43 @@
     :copyright: 2007 by Armin Ronacher.
     :license: BSD, see LICENSE for more details.
 """
+from jinja import Environment, DictLoader
+from jinja.exceptions import TemplateSyntaxError
+
 
 CALL = '''{{ foo('a', c='d', e='f', *['b'], **{'g': 'h'}) }}'''
 SLICING = '''{{ [1, 2, 3][:] }}|{{ [1, 2, 3][::-1] }}'''
 ATTR = '''{{ foo.bar }}|{{ foo['bar'] }}'''
 SUBSCRIPT = '''{{ foo[0] }}|{{ foo[-1] }}'''
 KEYATTR = '''{{ {'items': 'foo'}.items }}|{{ {}.items() }}'''
-TUPLE = '''{{ () }}'''
+TUPLE = '''{{ () }}|{{ (1,) }}|{{ (1, 2) }}'''
 MATH = '''{{ (1 + 1 * 2) - 3 / 2 }}|{{ 2**3 }}'''
 DIV = '''{{ 3 // 2 }}|{{ 3 / 2 }}|{{ 3 % 2 }}'''
 UNARY = '''{{ +3 }}|{{ -3 }}'''
+CONCAT = '''{{ [1, 2] ~ 'foo' }}'''
 COMPARE = '''{{ 1 > 0 }}|{{ 1 >= 1 }}|{{ 2 < 3 }}|{{ 2 == 2 }}|{{ 1 <= 1 }}'''
-LITERALS = '''{{ [] }}|{{ {} }}|{{ '' }}'''
+INOP = '''{{ 1 in [1, 2, 3] }}|{{ 1 not in [1, 2, 3] }}'''
+LITERALS = '''{{ [] }}|{{ {} }}|{{ () }}|{{ '' }}|{{ @() }}'''
 BOOL = '''{{ true and false }}|{{ false or true }}|{{ not false }}'''
 GROUPING = '''{{ (true and false) or (false and true) and not false }}'''
+CONDEXPR = '''{{ 0 if true else 1 }}'''
+DJANGOATTR = '''{{ [1, 2, 3].0 }}'''
+FILTERPRIORITY = '''{{ "foo"|upper + "bar"|upper }}'''
+REGEX = '''{{ @/\S+/.findall('foo bar baz') }}'''
+TUPLETEMPLATES = [
+    '{{ () }}',
+    '{{ (1, 2) }}',
+    '{{ (1, 2,) }}',
+    '{{ 1, }}',
+    '{{ 1, 2 }}',
+    '{% for foo, bar in seq %}...{% endfor %}',
+    '{% for x in foo, bar %}...{% endfor %}',
+    '{% for x in foo, %}...{% endfor %}',
+    '{% for x in foo, recursive %}...{% endfor %}',
+    '{% for x in foo, bar recursive %}...{% endfor %}',
+    '{% for x, in foo, recursive %}...{% endfor %}'
+]
+TRAILINGCOMMA = '''{{ (1, 2,) }}|{{ [1, 2,] }}|{{ {1: 2,} }}|{{ @(1, 2,) }}'''
 
 
 def test_call():
@@ -52,7 +75,7 @@
 
 def test_tuple(env):
     tmpl = env.from_string(TUPLE)
-    assert tmpl.render() == '[]'
+    assert tmpl.render() == '()|(1,)|(1, 2)'
 
 
 def test_math(env):
@@ -70,14 +93,24 @@
     assert tmpl.render() == '3|-3'
 
 
+def test_concat(env):
+    tmpl = env.from_string(CONCAT)
+    assert tmpl.render() == '[1, 2]foo'
+
+
 def test_compare(env):
     tmpl = env.from_string(COMPARE)
     assert tmpl.render() == 'True|True|True|True|True'
 
 
+def test_inop(env):
+    tmpl = env.from_string(INOP)
+    assert tmpl.render() == 'True|False'
+
+
 def test_literals(env):
     tmpl = env.from_string(LITERALS)
-    assert tmpl.render() == '[]|{}|'
+    assert tmpl.render() == '[]|{}|()||set([])'
 
 
 def test_bool(env):
@@ -88,3 +121,78 @@
 def test_grouping(env):
     tmpl = env.from_string(GROUPING)
     assert tmpl.render() == 'False'
+
+
+def test_django_attr(env):
+    tmpl = env.from_string(DJANGOATTR)
+    assert tmpl.render() == '1'
+
+
+def test_conditional_expression(env):
+    tmpl = env.from_string(CONDEXPR)
+    assert tmpl.render() == '0'
+
+
+def test_filter_priority(env):
+    tmpl = env.from_string(FILTERPRIORITY)
+    assert tmpl.render() == 'FOOBAR'
+
+
+def test_function_calls(env):
+    tests = [
+        (True, '*foo, bar'),
+        (True, '*foo, *bar'),
+        (True, '*foo, bar=42'),
+        (True, '**foo, *bar'),
+        (True, '**foo, bar'),
+        (False, 'foo, bar'),
+        (False, 'foo, bar=42'),
+        (False, 'foo, bar=23, *args'),
+        (False, 'a, b=c, *d, **e'),
+        (False, '*foo, **bar')
+    ]
+    for should_fail, sig in tests:
+        if should_fail:
+            try:
+                print env.from_string('{{ foo(%s) }}' % sig)
+            except TemplateSyntaxError:
+                continue
+            assert False, 'expected syntax error'
+        else:
+            env.from_string('foo(%s)' % sig)
+
+
+def test_regex(env):
+    tmpl = env.from_string(REGEX)
+    assert tmpl.render() == "['foo', 'bar', 'baz']"
+
+
+def test_tuple_expr(env):
+    for tmpl in TUPLETEMPLATES:
+        assert env.from_string(tmpl)
+
+
+def test_trailing_comma(env):
+    tmpl = env.from_string(TRAILINGCOMMA)
+    assert tmpl.render() == '(1, 2)|[1, 2]|{1: 2}|set([1, 2])'
+
+
+def test_extends_position():
+    env = Environment(loader=DictLoader({
+        'empty': '[{% block empty %}{% endblock %}]'
+    }))
+    tests = [
+        ('{% extends "empty" %}', '[!]'),
+        ('  {% extends "empty" %}', '[!]'),
+        ('  !\n', '  !\n!'),
+        ('{# foo #}  {% extends "empty" %}', '[!]'),
+        ('{% set foo = "blub" %}{% extends "empty" %}', None)
+    ]
+
+    for tmpl, expected_output in tests:
+        try:
+            tmpl = env.from_string(tmpl + '{% block empty %}!{% endblock %}')
+        except TemplateSyntaxError:
+            assert expected_output is None, 'got syntax error'
+        else:
+            assert expected_output == tmpl.render()
diff --git a/tests/test_tests.py b/tests/test_tests.py
index c28cd90..9d9a33f 100644
--- a/tests/test_tests.py
+++ b/tests/test_tests.py
@@ -11,7 +11,8 @@
 EVEN = '''{{ 1 is even }}|{{ 2 is even }}'''
 LOWER = '''{{ "foo" is lower }}|{{ "FOO" is lower }}'''
 MATCHING = '''{{ "42" is matching('^\\d+$') }}|\
-{{ "foo" is matching('^\\d+$') }}'''
+{{ "foo" is matching('^\\d+$') }}|\
+{{ "foo bar" is matching @/^foo\\s+BAR$/i }}'''
 NUMERIC = '''{{ "43" is numeric }}|{{ "foo" is numeric }}|\
 {{ 42 is numeric }}'''
 ODD = '''{{ 1 is odd }}|{{ 2 is odd }}'''
@@ -20,6 +21,7 @@
 {{ 42 is sequence }}'''
 UPPER = '''{{ "FOO" is upper }}|{{ "foo" is upper }}'''
 SAMEAS = '''{{ foo is sameas(false) }}|{{ 0 is sameas(false) }}'''
+NOPARENFORARG1 = '''{{ foo is sameas none }}'''
 
 
 def test_defined(env):
@@ -39,7 +41,7 @@
 
 def test_matching(env):
     tmpl = env.from_string(MATCHING)
-    assert tmpl.render() == 'True|False'
+    assert tmpl.render() == 'True|False|True'
 
 
 def test_numeric(env):
@@ -65,3 +67,8 @@
 def test_sameas(env):
     tmpl = env.from_string(SAMEAS)
     assert tmpl.render(foo=False) == 'True|False'
+
+
+def test_no_paren_for_arg1(env):
+    tmpl = env.from_string(NOPARENFORARG1)
+    assert tmpl.render(foo=None) == 'True'
diff --git a/tests/test_various.py b/tests/test_various.py
index cea56b8..9fb483f 100644
--- a/tests/test_various.py
+++ b/tests/test_various.py
@@ -29,11 +29,16 @@
 {{ while }}
 {{ pass }}
 {{ finally }}'''
-LIGHTKW = '''{{ call }}'''
 UNPACKING = '''{% for a, b, c in [[1, 2, 3]] %}{{ a }}|{{ b }}|{{ c }}{% endfor %}'''
 RAW = '''{% raw %}{{ FOO }} and {% BAR %}{% endraw %}'''
 CONST = '''{{ true }}|{{ false }}|{{ none }}|{{ undefined }}|\
 {{ none is defined }}|{{ undefined is defined }}'''
+LOCALSET = '''{% set foo = 0 %}\
+{% for item in [1, 2] %}{% set foo = 1 %}{% endfor %}\
+{{ foo }}'''
+NONLOCALSET = '''{% set foo = 0 %}\
+{% for item in [1, 2] %}{% set foo = 1! %}{% endfor %}\
+{{ foo }}'''
 CONSTASS1 = '''{% set true = 42 %}'''
 CONSTASS2 = '''{% for undefined in seq %}{% endfor %}'''
 
@@ -42,10 +47,6 @@
     env.from_string(KEYWORDS)
 
 
-def test_lightkw(env):
-    env.from_string(LIGHTKW)
-
-
 def test_unpacking(env):
     tmpl = env.from_string(UNPACKING)
     assert tmpl.render() == '1|2|3'
@@ -100,3 +101,13 @@
             pass
         else:
             raise AssertionError('expected syntax error')
+
+
+def test_localset(env):
+    tmpl = env.from_string(LOCALSET)
+    assert tmpl.render() == '0'
+
+
+def test_nonlocalset(env):
+    tmpl = env.from_string(NONLOCALSET)
+    assert tmpl.render() == '1'