blob: ba77bb806f38b79cd7228ee8490963520d5fdff6 [file] [log] [blame]
Fred Drake79ca79d2000-08-21 22:30:53 +00001import parser
Fred Drake58422e52001-06-04 03:56:24 +00002import unittest
Martin v. Löwis66e26632008-03-18 13:16:05 +00003import sys
Barry Warsaw04f357c2002-07-23 19:04:11 +00004from test import test_support
Fred Drake79ca79d2000-08-21 22:30:53 +00005
6#
7# First, we test that we can generate trees from valid source fragments,
8# and that these valid trees are indeed allowed by the tree-loading side
9# of the parser module.
10#
11
Fred Drake58422e52001-06-04 03:56:24 +000012class RoundtripLegalSyntaxTestCase(unittest.TestCase):
Guido van Rossum32c2ae72002-08-22 19:45:32 +000013
Fred Drake58422e52001-06-04 03:56:24 +000014 def roundtrip(self, f, s):
15 st1 = f(s)
16 t = st1.totuple()
17 try:
Fred Drake6e4f2c02001-07-17 19:33:25 +000018 st2 = parser.sequence2st(t)
Anthony Baxterc2a5a632004-08-02 06:10:11 +000019 except parser.ParserError, why:
20 self.fail("could not roundtrip %r: %s" % (s, why))
Fred Drake79ca79d2000-08-21 22:30:53 +000021
Fred Drake58422e52001-06-04 03:56:24 +000022 self.assertEquals(t, st2.totuple(),
23 "could not re-generate syntax tree")
Fred Drake28f739a2000-08-25 22:42:40 +000024
Fred Drake58422e52001-06-04 03:56:24 +000025 def check_expr(self, s):
26 self.roundtrip(parser.expr, s)
Fred Drake28f739a2000-08-25 22:42:40 +000027
Fred Drake58422e52001-06-04 03:56:24 +000028 def check_suite(self, s):
29 self.roundtrip(parser.suite, s)
Fred Drake28f739a2000-08-25 22:42:40 +000030
Fred Drakecf580c72001-07-17 03:01:29 +000031 def test_yield_statement(self):
Tim Peters496563a2002-04-01 00:28:59 +000032 self.check_suite("def f(): yield 1")
Phillip J. Eby0d6615f2005-08-02 00:46:46 +000033 self.check_suite("def f(): yield")
34 self.check_suite("def f(): x += yield")
35 self.check_suite("def f(): x = yield 1")
36 self.check_suite("def f(): x = y = yield 1")
37 self.check_suite("def f(): x = yield")
38 self.check_suite("def f(): x = y = yield")
39 self.check_suite("def f(): 1 + (yield)*2")
40 self.check_suite("def f(): (yield 1)*2")
Tim Peters496563a2002-04-01 00:28:59 +000041 self.check_suite("def f(): return; yield 1")
42 self.check_suite("def f(): yield 1; return")
43 self.check_suite("def f():\n"
Fred Drakecf580c72001-07-17 03:01:29 +000044 " for x in range(30):\n"
45 " yield x\n")
Phillip J. Eby0d6615f2005-08-02 00:46:46 +000046 self.check_suite("def f():\n"
47 " if (yield):\n"
48 " yield x\n")
Fred Drakecf580c72001-07-17 03:01:29 +000049
Fred Drake58422e52001-06-04 03:56:24 +000050 def test_expressions(self):
51 self.check_expr("foo(1)")
52 self.check_expr("[1, 2, 3]")
53 self.check_expr("[x**3 for x in range(20)]")
54 self.check_expr("[x**3 for x in range(20) if x % 3]")
Neal Norwitzd3a91622006-04-12 05:27:46 +000055 self.check_expr("[x**3 for x in range(20) if x % 2 if x % 3]")
56 self.check_expr("list(x**3 for x in range(20))")
57 self.check_expr("list(x**3 for x in range(20) if x % 3)")
58 self.check_expr("list(x**3 for x in range(20) if x % 2 if x % 3)")
Fred Drake58422e52001-06-04 03:56:24 +000059 self.check_expr("foo(*args)")
60 self.check_expr("foo(*args, **kw)")
61 self.check_expr("foo(**kw)")
62 self.check_expr("foo(key=value)")
63 self.check_expr("foo(key=value, *args)")
64 self.check_expr("foo(key=value, *args, **kw)")
65 self.check_expr("foo(key=value, **kw)")
66 self.check_expr("foo(a, b, c, *args)")
67 self.check_expr("foo(a, b, c, *args, **kw)")
68 self.check_expr("foo(a, b, c, **kw)")
Benjamin Petersonbd6a05f2008-08-19 22:06:11 +000069 self.check_expr("foo(a, *args, keyword=23)")
Fred Drake58422e52001-06-04 03:56:24 +000070 self.check_expr("foo + bar")
Michael W. Hudson5e83b7a2003-01-29 14:20:23 +000071 self.check_expr("foo - bar")
72 self.check_expr("foo * bar")
73 self.check_expr("foo / bar")
74 self.check_expr("foo // bar")
Fred Drake58422e52001-06-04 03:56:24 +000075 self.check_expr("lambda: 0")
76 self.check_expr("lambda x: 0")
77 self.check_expr("lambda *y: 0")
78 self.check_expr("lambda *y, **z: 0")
79 self.check_expr("lambda **z: 0")
80 self.check_expr("lambda x, y: 0")
81 self.check_expr("lambda foo=bar: 0")
82 self.check_expr("lambda foo=bar, spaz=nifty+spit: 0")
83 self.check_expr("lambda foo=bar, **z: 0")
84 self.check_expr("lambda foo=bar, blaz=blat+2, **z: 0")
85 self.check_expr("lambda foo=bar, blaz=blat+2, *y, **z: 0")
86 self.check_expr("lambda x, *y, **z: 0")
Raymond Hettinger354433a2004-05-19 08:20:33 +000087 self.check_expr("(x for x in range(10))")
88 self.check_expr("foo(x for x in range(10))")
Fred Drake79ca79d2000-08-21 22:30:53 +000089
Fred Drake58422e52001-06-04 03:56:24 +000090 def test_print(self):
91 self.check_suite("print")
92 self.check_suite("print 1")
93 self.check_suite("print 1,")
94 self.check_suite("print >>fp")
95 self.check_suite("print >>fp, 1")
96 self.check_suite("print >>fp, 1,")
Fred Drake79ca79d2000-08-21 22:30:53 +000097
Fred Drake58422e52001-06-04 03:56:24 +000098 def test_simple_expression(self):
99 # expr_stmt
100 self.check_suite("a")
Fred Drake79ca79d2000-08-21 22:30:53 +0000101
Fred Drake58422e52001-06-04 03:56:24 +0000102 def test_simple_assignments(self):
103 self.check_suite("a = b")
104 self.check_suite("a = b = c = d = e")
Fred Drake28f739a2000-08-25 22:42:40 +0000105
Fred Drake58422e52001-06-04 03:56:24 +0000106 def test_simple_augmented_assignments(self):
107 self.check_suite("a += b")
108 self.check_suite("a -= b")
109 self.check_suite("a *= b")
110 self.check_suite("a /= b")
Michael W. Hudson5e83b7a2003-01-29 14:20:23 +0000111 self.check_suite("a //= b")
Fred Drake58422e52001-06-04 03:56:24 +0000112 self.check_suite("a %= b")
113 self.check_suite("a &= b")
114 self.check_suite("a |= b")
115 self.check_suite("a ^= b")
116 self.check_suite("a <<= b")
117 self.check_suite("a >>= b")
118 self.check_suite("a **= b")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000119
Fred Drake58422e52001-06-04 03:56:24 +0000120 def test_function_defs(self):
121 self.check_suite("def f(): pass")
122 self.check_suite("def f(*args): pass")
123 self.check_suite("def f(*args, **kw): pass")
124 self.check_suite("def f(**kw): pass")
125 self.check_suite("def f(foo=bar): pass")
126 self.check_suite("def f(foo=bar, *args): pass")
127 self.check_suite("def f(foo=bar, *args, **kw): pass")
128 self.check_suite("def f(foo=bar, **kw): pass")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000129
Fred Drake58422e52001-06-04 03:56:24 +0000130 self.check_suite("def f(a, b): pass")
131 self.check_suite("def f(a, b, *args): pass")
132 self.check_suite("def f(a, b, *args, **kw): pass")
133 self.check_suite("def f(a, b, **kw): pass")
134 self.check_suite("def f(a, b, foo=bar): pass")
135 self.check_suite("def f(a, b, foo=bar, *args): pass")
136 self.check_suite("def f(a, b, foo=bar, *args, **kw): pass")
137 self.check_suite("def f(a, b, foo=bar, **kw): pass")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000138
Anthony Baxterc2a5a632004-08-02 06:10:11 +0000139 self.check_suite("@staticmethod\n"
140 "def f(): pass")
141 self.check_suite("@staticmethod\n"
142 "@funcattrs(x, y)\n"
143 "def f(): pass")
144 self.check_suite("@funcattrs()\n"
145 "def f(): pass")
146
Brett Cannonf4189912005-04-09 02:30:16 +0000147 def test_class_defs(self):
148 self.check_suite("class foo():pass")
Tim Peterse8906822005-04-20 17:45:13 +0000149
Fred Drake58422e52001-06-04 03:56:24 +0000150 def test_import_from_statement(self):
151 self.check_suite("from sys.path import *")
152 self.check_suite("from sys.path import dirname")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000153 self.check_suite("from sys.path import (dirname)")
154 self.check_suite("from sys.path import (dirname,)")
Fred Drake58422e52001-06-04 03:56:24 +0000155 self.check_suite("from sys.path import dirname as my_dirname")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000156 self.check_suite("from sys.path import (dirname as my_dirname)")
157 self.check_suite("from sys.path import (dirname as my_dirname,)")
Fred Drake58422e52001-06-04 03:56:24 +0000158 self.check_suite("from sys.path import dirname, basename")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000159 self.check_suite("from sys.path import (dirname, basename)")
160 self.check_suite("from sys.path import (dirname, basename,)")
Fred Drake58422e52001-06-04 03:56:24 +0000161 self.check_suite(
162 "from sys.path import dirname as my_dirname, basename")
163 self.check_suite(
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000164 "from sys.path import (dirname as my_dirname, basename)")
165 self.check_suite(
166 "from sys.path import (dirname as my_dirname, basename,)")
167 self.check_suite(
Fred Drake58422e52001-06-04 03:56:24 +0000168 "from sys.path import dirname, basename as my_basename")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000169 self.check_suite(
170 "from sys.path import (dirname, basename as my_basename)")
171 self.check_suite(
172 "from sys.path import (dirname, basename as my_basename,)")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000173
Fred Drake58422e52001-06-04 03:56:24 +0000174 def test_basic_import_statement(self):
175 self.check_suite("import sys")
176 self.check_suite("import sys as system")
177 self.check_suite("import sys, math")
178 self.check_suite("import sys as system, math")
179 self.check_suite("import sys, math as my_math")
Fred Drake79ca79d2000-08-21 22:30:53 +0000180
Neal Norwitz9caf9c02003-02-10 01:54:06 +0000181 def test_pep263(self):
182 self.check_suite("# -*- coding: iso-8859-1 -*-\n"
183 "pass\n")
184
185 def test_assert(self):
186 self.check_suite("assert alo < ahi and blo < bhi\n")
187
Jeremy Hylton60e96f62006-08-22 20:46:00 +0000188 def test_position(self):
189 # An absolutely minimal test of position information. Better
190 # tests would be a big project.
191 code = "def f(x):\n return x + 1\n"
192 st1 = parser.suite(code)
193 st2 = st1.totuple(line_info=1, col_info=1)
194
195 def walk(tree):
196 node_type = tree[0]
197 next = tree[1]
198 if isinstance(next, tuple):
199 for elt in tree[1:]:
200 for x in walk(elt):
201 yield x
202 else:
203 yield tree
Tim Peters147f9ae2006-08-25 22:05:39 +0000204
Jeremy Hylton60e96f62006-08-22 20:46:00 +0000205 terminals = list(walk(st2))
206 self.assertEqual([
207 (1, 'def', 1, 0),
208 (1, 'f', 1, 4),
209 (7, '(', 1, 5),
210 (1, 'x', 1, 6),
211 (8, ')', 1, 7),
212 (11, ':', 1, 8),
213 (4, '', 1, 9),
214 (5, '', 2, -1),
215 (1, 'return', 2, 4),
216 (1, 'x', 2, 11),
217 (14, '+', 2, 13),
218 (2, '1', 2, 15),
219 (4, '', 2, 16),
220 (6, '', 2, -1),
221 (4, '', 2, -1),
222 (0, '', 2, -1)],
223 terminals)
224
225
Fred Drake79ca79d2000-08-21 22:30:53 +0000226#
227# Second, we take *invalid* trees and make sure we get ParserError
228# rejections for them.
229#
230
Fred Drake58422e52001-06-04 03:56:24 +0000231class IllegalSyntaxTestCase(unittest.TestCase):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000232
Fred Drake58422e52001-06-04 03:56:24 +0000233 def check_bad_tree(self, tree, label):
234 try:
Fred Drake6e4f2c02001-07-17 19:33:25 +0000235 parser.sequence2st(tree)
Fred Drake58422e52001-06-04 03:56:24 +0000236 except parser.ParserError:
237 pass
238 else:
239 self.fail("did not detect invalid tree for %r" % label)
Fred Drake79ca79d2000-08-21 22:30:53 +0000240
Fred Drake58422e52001-06-04 03:56:24 +0000241 def test_junk(self):
242 # not even remotely valid:
243 self.check_bad_tree((1, 2, 3), "<junk>")
244
Fred Drakecf580c72001-07-17 03:01:29 +0000245 def test_illegal_yield_1(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000246 # Illegal yield statement: def f(): return 1; yield 1
Fred Drakecf580c72001-07-17 03:01:29 +0000247 tree = \
248 (257,
249 (264,
250 (285,
251 (259,
252 (1, 'def'),
253 (1, 'f'),
254 (260, (7, '('), (8, ')')),
255 (11, ':'),
256 (291,
257 (4, ''),
258 (5, ''),
259 (264,
260 (265,
261 (266,
262 (272,
263 (275,
264 (1, 'return'),
265 (313,
266 (292,
267 (293,
268 (294,
269 (295,
270 (297,
271 (298,
272 (299,
273 (300,
274 (301,
275 (302, (303, (304, (305, (2, '1')))))))))))))))))),
276 (264,
277 (265,
278 (266,
279 (272,
280 (276,
281 (1, 'yield'),
282 (313,
283 (292,
284 (293,
285 (294,
286 (295,
287 (297,
288 (298,
289 (299,
290 (300,
291 (301,
292 (302,
293 (303, (304, (305, (2, '1')))))))))))))))))),
294 (4, ''))),
295 (6, ''))))),
296 (4, ''),
297 (0, ''))))
298 self.check_bad_tree(tree, "def f():\n return 1\n yield 1")
299
300 def test_illegal_yield_2(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000301 # Illegal return in generator: def f(): return 1; yield 1
Fred Drakecf580c72001-07-17 03:01:29 +0000302 tree = \
303 (257,
304 (264,
305 (265,
306 (266,
307 (278,
308 (1, 'from'),
309 (281, (1, '__future__')),
310 (1, 'import'),
311 (279, (1, 'generators')))),
312 (4, ''))),
313 (264,
314 (285,
315 (259,
316 (1, 'def'),
317 (1, 'f'),
318 (260, (7, '('), (8, ')')),
319 (11, ':'),
320 (291,
321 (4, ''),
322 (5, ''),
323 (264,
324 (265,
325 (266,
326 (272,
327 (275,
328 (1, 'return'),
329 (313,
330 (292,
331 (293,
332 (294,
333 (295,
334 (297,
335 (298,
336 (299,
337 (300,
338 (301,
339 (302, (303, (304, (305, (2, '1')))))))))))))))))),
340 (264,
341 (265,
342 (266,
343 (272,
344 (276,
345 (1, 'yield'),
346 (313,
347 (292,
348 (293,
349 (294,
350 (295,
351 (297,
352 (298,
353 (299,
354 (300,
355 (301,
356 (302,
357 (303, (304, (305, (2, '1')))))))))))))))))),
358 (4, ''))),
359 (6, ''))))),
360 (4, ''),
361 (0, ''))))
362 self.check_bad_tree(tree, "def f():\n return 1\n yield 1")
363
Fred Drake58422e52001-06-04 03:56:24 +0000364 def test_print_chevron_comma(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000365 # Illegal input: print >>fp,
Fred Drake58422e52001-06-04 03:56:24 +0000366 tree = \
367 (257,
368 (264,
369 (265,
370 (266,
371 (268,
372 (1, 'print'),
373 (35, '>>'),
374 (290,
375 (291,
376 (292,
377 (293,
378 (295,
379 (296,
380 (297,
381 (298, (299, (300, (301, (302, (303, (1, 'fp')))))))))))))),
382 (12, ','))),
383 (4, ''))),
384 (0, ''))
385 self.check_bad_tree(tree, "print >>fp,")
386
387 def test_a_comma_comma_c(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000388 # Illegal input: a,,c
Fred Drake58422e52001-06-04 03:56:24 +0000389 tree = \
390 (258,
391 (311,
392 (290,
393 (291,
394 (292,
395 (293,
396 (295,
397 (296,
398 (297,
399 (298, (299, (300, (301, (302, (303, (1, 'a')))))))))))))),
400 (12, ','),
401 (12, ','),
402 (290,
403 (291,
404 (292,
405 (293,
406 (295,
407 (296,
408 (297,
409 (298, (299, (300, (301, (302, (303, (1, 'c'))))))))))))))),
410 (4, ''),
411 (0, ''))
412 self.check_bad_tree(tree, "a,,c")
413
414 def test_illegal_operator(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000415 # Illegal input: a $= b
Fred Drake58422e52001-06-04 03:56:24 +0000416 tree = \
417 (257,
418 (264,
419 (265,
420 (266,
421 (267,
422 (312,
423 (291,
424 (292,
425 (293,
426 (294,
427 (296,
428 (297,
429 (298,
430 (299,
431 (300, (301, (302, (303, (304, (1, 'a'))))))))))))))),
432 (268, (37, '$=')),
433 (312,
434 (291,
435 (292,
436 (293,
437 (294,
438 (296,
439 (297,
440 (298,
441 (299,
442 (300, (301, (302, (303, (304, (1, 'b'))))))))))))))))),
443 (4, ''))),
444 (0, ''))
445 self.check_bad_tree(tree, "a $= b")
Fred Drake79ca79d2000-08-21 22:30:53 +0000446
Neal Norwitz9caf9c02003-02-10 01:54:06 +0000447 def test_malformed_global(self):
448 #doesn't have global keyword in ast
449 tree = (257,
450 (264,
451 (265,
452 (266,
453 (282, (1, 'foo'))), (4, ''))),
454 (4, ''),
Tim Petersf2715e02003-02-19 02:35:07 +0000455 (0, ''))
Neal Norwitz9caf9c02003-02-10 01:54:06 +0000456 self.check_bad_tree(tree, "malformed global ast")
Fred Drake79ca79d2000-08-21 22:30:53 +0000457
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000458
459class CompileTestCase(unittest.TestCase):
460
461 # These tests are very minimal. :-(
462
463 def test_compile_expr(self):
464 st = parser.expr('2 + 3')
465 code = parser.compilest(st)
466 self.assertEquals(eval(code), 5)
467
468 def test_compile_suite(self):
469 st = parser.suite('x = 2; y = x + 3')
470 code = parser.compilest(st)
471 globs = {}
472 exec code in globs
473 self.assertEquals(globs['y'], 5)
474
475 def test_compile_error(self):
476 st = parser.suite('1 = 3 + 4')
477 self.assertRaises(SyntaxError, parser.compilest, st)
478
Guido van Rossumb6ac23c2007-07-18 17:19:14 +0000479 def test_compile_badunicode(self):
480 st = parser.suite('a = u"\U12345678"')
481 self.assertRaises(SyntaxError, parser.compilest, st)
482 st = parser.suite('a = u"\u1"')
483 self.assertRaises(SyntaxError, parser.compilest, st)
484
Facundo Batistafc2d0102008-02-23 12:01:13 +0000485class ParserStackLimitTestCase(unittest.TestCase):
486 """try to push the parser to/over it's limits.
487 see http://bugs.python.org/issue1881 for a discussion
488 """
489 def _nested_expression(self, level):
490 return "["*level+"]"*level
491
492 def test_deeply_nested_list(self):
493 e = self._nested_expression(99)
494 st = parser.expr(e)
495 st.compile()
496
497 def test_trigger_memory_error(self):
498 e = self._nested_expression(100)
Martin v. Löwis66e26632008-03-18 13:16:05 +0000499 print >>sys.stderr, "Expecting 's_push: parser stack overflow' in next line"
Facundo Batistafc2d0102008-02-23 12:01:13 +0000500 self.assertRaises(MemoryError, parser.expr, e)
501
Fred Drake2e2be372001-09-20 21:33:42 +0000502def test_main():
Walter Dörwald21d3a322003-05-01 17:45:56 +0000503 test_support.run_unittest(
504 RoundtripLegalSyntaxTestCase,
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000505 IllegalSyntaxTestCase,
506 CompileTestCase,
Facundo Batistafc2d0102008-02-23 12:01:13 +0000507 ParserStackLimitTestCase,
Walter Dörwald21d3a322003-05-01 17:45:56 +0000508 )
Fred Drake2e2be372001-09-20 21:33:42 +0000509
510
511if __name__ == "__main__":
512 test_main()