blob: 2f1310b4a310f8cb8dde85489f4d46d7a41b4695 [file] [log] [blame]
Fred Drake79ca79d2000-08-21 22:30:53 +00001import parser
Benjamin Petersonc0747cf2008-11-03 20:31:38 +00002import os
Fred Drake58422e52001-06-04 03:56:24 +00003import unittest
Christian Heimesb186d002008-03-18 15:15:01 +00004import sys
Benjamin Petersonee8712c2008-05-20 21:35:26 +00005from test import support
Fred Drake79ca79d2000-08-21 22:30:53 +00006
7#
8# First, we test that we can generate trees from valid source fragments,
9# and that these valid trees are indeed allowed by the tree-loading side
10# of the parser module.
11#
12
Fred Drake58422e52001-06-04 03:56:24 +000013class RoundtripLegalSyntaxTestCase(unittest.TestCase):
Guido van Rossum32c2ae72002-08-22 19:45:32 +000014
Fred Drake58422e52001-06-04 03:56:24 +000015 def roundtrip(self, f, s):
16 st1 = f(s)
17 t = st1.totuple()
18 try:
Fred Drake6e4f2c02001-07-17 19:33:25 +000019 st2 = parser.sequence2st(t)
Guido van Rossumb940e112007-01-10 16:19:56 +000020 except parser.ParserError as why:
Anthony Baxterc2a5a632004-08-02 06:10:11 +000021 self.fail("could not roundtrip %r: %s" % (s, why))
Fred Drake79ca79d2000-08-21 22:30:53 +000022
Fred Drake58422e52001-06-04 03:56:24 +000023 self.assertEquals(t, st2.totuple(),
24 "could not re-generate syntax tree")
Fred Drake28f739a2000-08-25 22:42:40 +000025
Fred Drake58422e52001-06-04 03:56:24 +000026 def check_expr(self, s):
27 self.roundtrip(parser.expr, s)
Fred Drake28f739a2000-08-25 22:42:40 +000028
Benjamin Petersonf216c942008-10-31 02:28:05 +000029 def test_flags_passed(self):
30 # The unicode literals flags has to be passed from the paser to AST
31 # generation.
32 suite = parser.suite("from __future__ import unicode_literals; x = ''")
33 code = suite.compile()
34 scope = {}
35 exec(code, {}, scope)
36 self.assertTrue(isinstance(scope["x"], str))
37
Fred Drake58422e52001-06-04 03:56:24 +000038 def check_suite(self, s):
39 self.roundtrip(parser.suite, s)
Fred Drake28f739a2000-08-25 22:42:40 +000040
Fred Drakecf580c72001-07-17 03:01:29 +000041 def test_yield_statement(self):
Tim Peters496563a2002-04-01 00:28:59 +000042 self.check_suite("def f(): yield 1")
Phillip J. Eby0d6615f2005-08-02 00:46:46 +000043 self.check_suite("def f(): yield")
44 self.check_suite("def f(): x += yield")
45 self.check_suite("def f(): x = yield 1")
46 self.check_suite("def f(): x = y = yield 1")
47 self.check_suite("def f(): x = yield")
48 self.check_suite("def f(): x = y = yield")
49 self.check_suite("def f(): 1 + (yield)*2")
50 self.check_suite("def f(): (yield 1)*2")
Tim Peters496563a2002-04-01 00:28:59 +000051 self.check_suite("def f(): return; yield 1")
52 self.check_suite("def f(): yield 1; return")
53 self.check_suite("def f():\n"
Fred Drakecf580c72001-07-17 03:01:29 +000054 " for x in range(30):\n"
55 " yield x\n")
Phillip J. Eby0d6615f2005-08-02 00:46:46 +000056 self.check_suite("def f():\n"
57 " if (yield):\n"
58 " yield x\n")
Fred Drakecf580c72001-07-17 03:01:29 +000059
Fred Drake58422e52001-06-04 03:56:24 +000060 def test_expressions(self):
61 self.check_expr("foo(1)")
62 self.check_expr("[1, 2, 3]")
63 self.check_expr("[x**3 for x in range(20)]")
64 self.check_expr("[x**3 for x in range(20) if x % 3]")
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000065 self.check_expr("[x**3 for x in range(20) if x % 2 if x % 3]")
66 self.check_expr("list(x**3 for x in range(20))")
67 self.check_expr("list(x**3 for x in range(20) if x % 3)")
68 self.check_expr("list(x**3 for x in range(20) if x % 2 if x % 3)")
Fred Drake58422e52001-06-04 03:56:24 +000069 self.check_expr("foo(*args)")
70 self.check_expr("foo(*args, **kw)")
71 self.check_expr("foo(**kw)")
72 self.check_expr("foo(key=value)")
73 self.check_expr("foo(key=value, *args)")
74 self.check_expr("foo(key=value, *args, **kw)")
75 self.check_expr("foo(key=value, **kw)")
76 self.check_expr("foo(a, b, c, *args)")
77 self.check_expr("foo(a, b, c, *args, **kw)")
78 self.check_expr("foo(a, b, c, **kw)")
Benjamin Peterson3938a902008-08-20 02:33:00 +000079 self.check_expr("foo(a, *args, keyword=23)")
Fred Drake58422e52001-06-04 03:56:24 +000080 self.check_expr("foo + bar")
Michael W. Hudson5e83b7a2003-01-29 14:20:23 +000081 self.check_expr("foo - bar")
82 self.check_expr("foo * bar")
83 self.check_expr("foo / bar")
84 self.check_expr("foo // bar")
Fred Drake58422e52001-06-04 03:56:24 +000085 self.check_expr("lambda: 0")
86 self.check_expr("lambda x: 0")
87 self.check_expr("lambda *y: 0")
88 self.check_expr("lambda *y, **z: 0")
89 self.check_expr("lambda **z: 0")
90 self.check_expr("lambda x, y: 0")
91 self.check_expr("lambda foo=bar: 0")
92 self.check_expr("lambda foo=bar, spaz=nifty+spit: 0")
93 self.check_expr("lambda foo=bar, **z: 0")
94 self.check_expr("lambda foo=bar, blaz=blat+2, **z: 0")
95 self.check_expr("lambda foo=bar, blaz=blat+2, *y, **z: 0")
96 self.check_expr("lambda x, *y, **z: 0")
Raymond Hettinger354433a2004-05-19 08:20:33 +000097 self.check_expr("(x for x in range(10))")
98 self.check_expr("foo(x for x in range(10))")
Fred Drake79ca79d2000-08-21 22:30:53 +000099
Fred Drake58422e52001-06-04 03:56:24 +0000100 def test_simple_expression(self):
101 # expr_stmt
102 self.check_suite("a")
Fred Drake79ca79d2000-08-21 22:30:53 +0000103
Fred Drake58422e52001-06-04 03:56:24 +0000104 def test_simple_assignments(self):
105 self.check_suite("a = b")
106 self.check_suite("a = b = c = d = e")
Fred Drake28f739a2000-08-25 22:42:40 +0000107
Fred Drake58422e52001-06-04 03:56:24 +0000108 def test_simple_augmented_assignments(self):
109 self.check_suite("a += b")
110 self.check_suite("a -= b")
111 self.check_suite("a *= b")
112 self.check_suite("a /= b")
Michael W. Hudson5e83b7a2003-01-29 14:20:23 +0000113 self.check_suite("a //= b")
Fred Drake58422e52001-06-04 03:56:24 +0000114 self.check_suite("a %= b")
115 self.check_suite("a &= b")
116 self.check_suite("a |= b")
117 self.check_suite("a ^= b")
118 self.check_suite("a <<= b")
119 self.check_suite("a >>= b")
120 self.check_suite("a **= b")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000121
Fred Drake58422e52001-06-04 03:56:24 +0000122 def test_function_defs(self):
123 self.check_suite("def f(): pass")
124 self.check_suite("def f(*args): pass")
125 self.check_suite("def f(*args, **kw): pass")
126 self.check_suite("def f(**kw): pass")
127 self.check_suite("def f(foo=bar): pass")
128 self.check_suite("def f(foo=bar, *args): pass")
129 self.check_suite("def f(foo=bar, *args, **kw): pass")
130 self.check_suite("def f(foo=bar, **kw): pass")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000131
Fred Drake58422e52001-06-04 03:56:24 +0000132 self.check_suite("def f(a, b): pass")
133 self.check_suite("def f(a, b, *args): pass")
134 self.check_suite("def f(a, b, *args, **kw): pass")
135 self.check_suite("def f(a, b, **kw): pass")
136 self.check_suite("def f(a, b, foo=bar): pass")
137 self.check_suite("def f(a, b, foo=bar, *args): pass")
138 self.check_suite("def f(a, b, foo=bar, *args, **kw): pass")
139 self.check_suite("def f(a, b, foo=bar, **kw): pass")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000140
Anthony Baxterc2a5a632004-08-02 06:10:11 +0000141 self.check_suite("@staticmethod\n"
142 "def f(): pass")
143 self.check_suite("@staticmethod\n"
144 "@funcattrs(x, y)\n"
145 "def f(): pass")
146 self.check_suite("@funcattrs()\n"
147 "def f(): pass")
148
Brett Cannonf4189912005-04-09 02:30:16 +0000149 def test_class_defs(self):
150 self.check_suite("class foo():pass")
Guido van Rossumfc158e22007-11-15 19:17:28 +0000151 self.check_suite("class foo(object):pass")
Tim Peterse8906822005-04-20 17:45:13 +0000152
Fred Drake58422e52001-06-04 03:56:24 +0000153 def test_import_from_statement(self):
154 self.check_suite("from sys.path import *")
155 self.check_suite("from sys.path import dirname")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000156 self.check_suite("from sys.path import (dirname)")
157 self.check_suite("from sys.path import (dirname,)")
Fred Drake58422e52001-06-04 03:56:24 +0000158 self.check_suite("from sys.path import dirname as my_dirname")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000159 self.check_suite("from sys.path import (dirname as my_dirname)")
160 self.check_suite("from sys.path import (dirname as my_dirname,)")
Fred Drake58422e52001-06-04 03:56:24 +0000161 self.check_suite("from sys.path import dirname, basename")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000162 self.check_suite("from sys.path import (dirname, basename)")
163 self.check_suite("from sys.path import (dirname, basename,)")
Fred Drake58422e52001-06-04 03:56:24 +0000164 self.check_suite(
165 "from sys.path import dirname as my_dirname, basename")
166 self.check_suite(
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000167 "from sys.path import (dirname as my_dirname, basename)")
168 self.check_suite(
169 "from sys.path import (dirname as my_dirname, basename,)")
170 self.check_suite(
Fred Drake58422e52001-06-04 03:56:24 +0000171 "from sys.path import dirname, basename as my_basename")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000172 self.check_suite(
173 "from sys.path import (dirname, basename as my_basename)")
174 self.check_suite(
175 "from sys.path import (dirname, basename as my_basename,)")
Benjamin Petersonc0747cf2008-11-03 20:31:38 +0000176 self.check_suite("from .bogus import x")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000177
Fred Drake58422e52001-06-04 03:56:24 +0000178 def test_basic_import_statement(self):
179 self.check_suite("import sys")
180 self.check_suite("import sys as system")
181 self.check_suite("import sys, math")
182 self.check_suite("import sys as system, math")
183 self.check_suite("import sys, math as my_math")
Fred Drake79ca79d2000-08-21 22:30:53 +0000184
Neal Norwitz9caf9c02003-02-10 01:54:06 +0000185 def test_pep263(self):
186 self.check_suite("# -*- coding: iso-8859-1 -*-\n"
187 "pass\n")
188
189 def test_assert(self):
190 self.check_suite("assert alo < ahi and blo < bhi\n")
191
Benjamin Peterson4469d0c2008-11-30 22:46:23 +0000192 def test_with(self):
193 self.check_suite("with open('x'): pass\n")
194 self.check_suite("with open('x') as f: pass\n")
195
Georg Brandleee31162008-12-07 15:15:22 +0000196 def test_try_stmt(self):
197 self.check_suite("try: pass\nexcept: pass\n")
198 self.check_suite("try: pass\nfinally: pass\n")
199 self.check_suite("try: pass\nexcept A: pass\nfinally: pass\n")
200 self.check_suite("try: pass\nexcept A: pass\nexcept: pass\n"
201 "finally: pass\n")
202 self.check_suite("try: pass\nexcept: pass\nelse: pass\n")
203 self.check_suite("try: pass\nexcept: pass\nelse: pass\n"
204 "finally: pass\n")
205
Thomas Wouters89f507f2006-12-13 04:49:30 +0000206 def test_position(self):
207 # An absolutely minimal test of position information. Better
208 # tests would be a big project.
209 code = "def f(x):\n return x + 1\n"
210 st1 = parser.suite(code)
211 st2 = st1.totuple(line_info=1, col_info=1)
212
213 def walk(tree):
214 node_type = tree[0]
215 next = tree[1]
216 if isinstance(next, tuple):
217 for elt in tree[1:]:
218 for x in walk(elt):
219 yield x
220 else:
221 yield tree
222
223 terminals = list(walk(st2))
224 self.assertEqual([
225 (1, 'def', 1, 0),
226 (1, 'f', 1, 4),
227 (7, '(', 1, 5),
228 (1, 'x', 1, 6),
229 (8, ')', 1, 7),
230 (11, ':', 1, 8),
231 (4, '', 1, 9),
232 (5, '', 2, -1),
233 (1, 'return', 2, 4),
234 (1, 'x', 2, 11),
235 (14, '+', 2, 13),
236 (2, '1', 2, 15),
237 (4, '', 2, 16),
238 (6, '', 2, -1),
239 (4, '', 2, -1),
240 (0, '', 2, -1)],
241 terminals)
242
243
Fred Drake79ca79d2000-08-21 22:30:53 +0000244#
245# Second, we take *invalid* trees and make sure we get ParserError
246# rejections for them.
247#
248
Fred Drake58422e52001-06-04 03:56:24 +0000249class IllegalSyntaxTestCase(unittest.TestCase):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000250
Fred Drake58422e52001-06-04 03:56:24 +0000251 def check_bad_tree(self, tree, label):
252 try:
Fred Drake6e4f2c02001-07-17 19:33:25 +0000253 parser.sequence2st(tree)
Fred Drake58422e52001-06-04 03:56:24 +0000254 except parser.ParserError:
255 pass
256 else:
257 self.fail("did not detect invalid tree for %r" % label)
Fred Drake79ca79d2000-08-21 22:30:53 +0000258
Fred Drake58422e52001-06-04 03:56:24 +0000259 def test_junk(self):
260 # not even remotely valid:
261 self.check_bad_tree((1, 2, 3), "<junk>")
262
Fred Drakecf580c72001-07-17 03:01:29 +0000263 def test_illegal_yield_1(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000264 # Illegal yield statement: def f(): return 1; yield 1
Fred Drakecf580c72001-07-17 03:01:29 +0000265 tree = \
266 (257,
267 (264,
268 (285,
269 (259,
270 (1, 'def'),
271 (1, 'f'),
272 (260, (7, '('), (8, ')')),
273 (11, ':'),
274 (291,
275 (4, ''),
276 (5, ''),
277 (264,
278 (265,
279 (266,
280 (272,
281 (275,
282 (1, 'return'),
283 (313,
284 (292,
285 (293,
286 (294,
287 (295,
288 (297,
289 (298,
290 (299,
291 (300,
292 (301,
293 (302, (303, (304, (305, (2, '1')))))))))))))))))),
294 (264,
295 (265,
296 (266,
297 (272,
298 (276,
299 (1, 'yield'),
300 (313,
301 (292,
302 (293,
303 (294,
304 (295,
305 (297,
306 (298,
307 (299,
308 (300,
309 (301,
310 (302,
311 (303, (304, (305, (2, '1')))))))))))))))))),
312 (4, ''))),
313 (6, ''))))),
314 (4, ''),
315 (0, ''))))
316 self.check_bad_tree(tree, "def f():\n return 1\n yield 1")
317
318 def test_illegal_yield_2(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000319 # Illegal return in generator: def f(): return 1; yield 1
Fred Drakecf580c72001-07-17 03:01:29 +0000320 tree = \
321 (257,
322 (264,
323 (265,
324 (266,
325 (278,
326 (1, 'from'),
327 (281, (1, '__future__')),
328 (1, 'import'),
329 (279, (1, 'generators')))),
330 (4, ''))),
331 (264,
332 (285,
333 (259,
334 (1, 'def'),
335 (1, 'f'),
336 (260, (7, '('), (8, ')')),
337 (11, ':'),
338 (291,
339 (4, ''),
340 (5, ''),
341 (264,
342 (265,
343 (266,
344 (272,
345 (275,
346 (1, 'return'),
347 (313,
348 (292,
349 (293,
350 (294,
351 (295,
352 (297,
353 (298,
354 (299,
355 (300,
356 (301,
357 (302, (303, (304, (305, (2, '1')))))))))))))))))),
358 (264,
359 (265,
360 (266,
361 (272,
362 (276,
363 (1, 'yield'),
364 (313,
365 (292,
366 (293,
367 (294,
368 (295,
369 (297,
370 (298,
371 (299,
372 (300,
373 (301,
374 (302,
375 (303, (304, (305, (2, '1')))))))))))))))))),
376 (4, ''))),
377 (6, ''))))),
378 (4, ''),
379 (0, ''))))
380 self.check_bad_tree(tree, "def f():\n return 1\n yield 1")
381
Fred Drake58422e52001-06-04 03:56:24 +0000382 def test_a_comma_comma_c(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000383 # Illegal input: a,,c
Fred Drake58422e52001-06-04 03:56:24 +0000384 tree = \
385 (258,
386 (311,
387 (290,
388 (291,
389 (292,
390 (293,
391 (295,
392 (296,
393 (297,
394 (298, (299, (300, (301, (302, (303, (1, 'a')))))))))))))),
395 (12, ','),
396 (12, ','),
397 (290,
398 (291,
399 (292,
400 (293,
401 (295,
402 (296,
403 (297,
404 (298, (299, (300, (301, (302, (303, (1, 'c'))))))))))))))),
405 (4, ''),
406 (0, ''))
407 self.check_bad_tree(tree, "a,,c")
408
409 def test_illegal_operator(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000410 # Illegal input: a $= b
Fred Drake58422e52001-06-04 03:56:24 +0000411 tree = \
412 (257,
413 (264,
414 (265,
415 (266,
416 (267,
417 (312,
418 (291,
419 (292,
420 (293,
421 (294,
422 (296,
423 (297,
424 (298,
425 (299,
426 (300, (301, (302, (303, (304, (1, 'a'))))))))))))))),
427 (268, (37, '$=')),
428 (312,
429 (291,
430 (292,
431 (293,
432 (294,
433 (296,
434 (297,
435 (298,
436 (299,
437 (300, (301, (302, (303, (304, (1, 'b'))))))))))))))))),
438 (4, ''))),
439 (0, ''))
440 self.check_bad_tree(tree, "a $= b")
Fred Drake79ca79d2000-08-21 22:30:53 +0000441
Neal Norwitz9caf9c02003-02-10 01:54:06 +0000442 def test_malformed_global(self):
443 #doesn't have global keyword in ast
444 tree = (257,
445 (264,
446 (265,
447 (266,
448 (282, (1, 'foo'))), (4, ''))),
449 (4, ''),
Tim Petersf2715e02003-02-19 02:35:07 +0000450 (0, ''))
Neal Norwitz9caf9c02003-02-10 01:54:06 +0000451 self.check_bad_tree(tree, "malformed global ast")
Fred Drake79ca79d2000-08-21 22:30:53 +0000452
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000453
454class CompileTestCase(unittest.TestCase):
455
456 # These tests are very minimal. :-(
457
458 def test_compile_expr(self):
459 st = parser.expr('2 + 3')
460 code = parser.compilest(st)
461 self.assertEquals(eval(code), 5)
462
463 def test_compile_suite(self):
464 st = parser.suite('x = 2; y = x + 3')
465 code = parser.compilest(st)
466 globs = {}
Georg Brandl7cae87c2006-09-06 06:51:57 +0000467 exec(code, globs)
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000468 self.assertEquals(globs['y'], 5)
469
470 def test_compile_error(self):
471 st = parser.suite('1 = 3 + 4')
472 self.assertRaises(SyntaxError, parser.compilest, st)
473
Guido van Rossumb5a755e2007-07-18 18:15:48 +0000474 def test_compile_badunicode(self):
Guido van Rossum7eb6ca52007-07-18 21:00:22 +0000475 st = parser.suite('a = "\\U12345678"')
Guido van Rossumb5a755e2007-07-18 18:15:48 +0000476 self.assertRaises(SyntaxError, parser.compilest, st)
Guido van Rossum7eb6ca52007-07-18 21:00:22 +0000477 st = parser.suite('a = "\\u1"')
Guido van Rossumb5a755e2007-07-18 18:15:48 +0000478 self.assertRaises(SyntaxError, parser.compilest, st)
479
Christian Heimes90c3d9b2008-02-23 13:18:03 +0000480class ParserStackLimitTestCase(unittest.TestCase):
481 """try to push the parser to/over it's limits.
482 see http://bugs.python.org/issue1881 for a discussion
483 """
484 def _nested_expression(self, level):
485 return "["*level+"]"*level
486
487 def test_deeply_nested_list(self):
488 # XXX used to be 99 levels in 2.x
489 e = self._nested_expression(93)
490 st = parser.expr(e)
491 st.compile()
492
493 def test_trigger_memory_error(self):
494 e = self._nested_expression(100)
Christian Heimesb186d002008-03-18 15:15:01 +0000495 print("Expecting 's_push: parser stack overflow' in next line",
496 file=sys.stderr)
Christian Heimes90c3d9b2008-02-23 13:18:03 +0000497 self.assertRaises(MemoryError, parser.expr, e)
498
Fred Drake2e2be372001-09-20 21:33:42 +0000499def test_main():
Benjamin Petersonee8712c2008-05-20 21:35:26 +0000500 support.run_unittest(
Walter Dörwald21d3a322003-05-01 17:45:56 +0000501 RoundtripLegalSyntaxTestCase,
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000502 IllegalSyntaxTestCase,
503 CompileTestCase,
Christian Heimes90c3d9b2008-02-23 13:18:03 +0000504 ParserStackLimitTestCase,
Walter Dörwald21d3a322003-05-01 17:45:56 +0000505 )
Fred Drake2e2be372001-09-20 21:33:42 +0000506
507
508if __name__ == "__main__":
509 test_main()