blob: 9ecca51a36060e7ba729ee3130190e6e9c8b37c4 [file] [log] [blame]
Fred Drake79ca79d2000-08-21 22:30:53 +00001import parser
Fred Drake58422e52001-06-04 03:56:24 +00002import unittest
Martin v. Löwis66e26632008-03-18 13:16:05 +00003import sys
Barry Warsaw04f357c2002-07-23 19:04:11 +00004from test import test_support
Fred Drake79ca79d2000-08-21 22:30:53 +00005
6#
7# First, we test that we can generate trees from valid source fragments,
8# and that these valid trees are indeed allowed by the tree-loading side
9# of the parser module.
10#
11
Fred Drake58422e52001-06-04 03:56:24 +000012class RoundtripLegalSyntaxTestCase(unittest.TestCase):
Guido van Rossum32c2ae72002-08-22 19:45:32 +000013
Fred Drake58422e52001-06-04 03:56:24 +000014 def roundtrip(self, f, s):
15 st1 = f(s)
16 t = st1.totuple()
17 try:
Fred Drake6e4f2c02001-07-17 19:33:25 +000018 st2 = parser.sequence2st(t)
Anthony Baxterc2a5a632004-08-02 06:10:11 +000019 except parser.ParserError, why:
20 self.fail("could not roundtrip %r: %s" % (s, why))
Fred Drake79ca79d2000-08-21 22:30:53 +000021
Fred Drake58422e52001-06-04 03:56:24 +000022 self.assertEquals(t, st2.totuple(),
23 "could not re-generate syntax tree")
Fred Drake28f739a2000-08-25 22:42:40 +000024
Fred Drake58422e52001-06-04 03:56:24 +000025 def check_expr(self, s):
26 self.roundtrip(parser.expr, s)
Fred Drake28f739a2000-08-25 22:42:40 +000027
Benjamin Peterson084ce7a2008-10-31 02:26:20 +000028 def test_flags_passed(self):
29 # The unicode literals flags has to be passed from the paser to AST
30 # generation.
31 suite = parser.suite("from __future__ import unicode_literals; x = ''")
32 code = suite.compile()
33 scope = {}
34 exec code in scope
35 self.assertTrue(isinstance(scope["x"], unicode))
36
Fred Drake58422e52001-06-04 03:56:24 +000037 def check_suite(self, s):
38 self.roundtrip(parser.suite, s)
Fred Drake28f739a2000-08-25 22:42:40 +000039
Fred Drakecf580c72001-07-17 03:01:29 +000040 def test_yield_statement(self):
Tim Peters496563a2002-04-01 00:28:59 +000041 self.check_suite("def f(): yield 1")
Phillip J. Eby0d6615f2005-08-02 00:46:46 +000042 self.check_suite("def f(): yield")
43 self.check_suite("def f(): x += yield")
44 self.check_suite("def f(): x = yield 1")
45 self.check_suite("def f(): x = y = yield 1")
46 self.check_suite("def f(): x = yield")
47 self.check_suite("def f(): x = y = yield")
48 self.check_suite("def f(): 1 + (yield)*2")
49 self.check_suite("def f(): (yield 1)*2")
Tim Peters496563a2002-04-01 00:28:59 +000050 self.check_suite("def f(): return; yield 1")
51 self.check_suite("def f(): yield 1; return")
52 self.check_suite("def f():\n"
Fred Drakecf580c72001-07-17 03:01:29 +000053 " for x in range(30):\n"
54 " yield x\n")
Phillip J. Eby0d6615f2005-08-02 00:46:46 +000055 self.check_suite("def f():\n"
56 " if (yield):\n"
57 " yield x\n")
Fred Drakecf580c72001-07-17 03:01:29 +000058
Fred Drake58422e52001-06-04 03:56:24 +000059 def test_expressions(self):
60 self.check_expr("foo(1)")
61 self.check_expr("[1, 2, 3]")
62 self.check_expr("[x**3 for x in range(20)]")
63 self.check_expr("[x**3 for x in range(20) if x % 3]")
Neal Norwitzd3a91622006-04-12 05:27:46 +000064 self.check_expr("[x**3 for x in range(20) if x % 2 if x % 3]")
65 self.check_expr("list(x**3 for x in range(20))")
66 self.check_expr("list(x**3 for x in range(20) if x % 3)")
67 self.check_expr("list(x**3 for x in range(20) if x % 2 if x % 3)")
Fred Drake58422e52001-06-04 03:56:24 +000068 self.check_expr("foo(*args)")
69 self.check_expr("foo(*args, **kw)")
70 self.check_expr("foo(**kw)")
71 self.check_expr("foo(key=value)")
72 self.check_expr("foo(key=value, *args)")
73 self.check_expr("foo(key=value, *args, **kw)")
74 self.check_expr("foo(key=value, **kw)")
75 self.check_expr("foo(a, b, c, *args)")
76 self.check_expr("foo(a, b, c, *args, **kw)")
77 self.check_expr("foo(a, b, c, **kw)")
Benjamin Petersonbd6a05f2008-08-19 22:06:11 +000078 self.check_expr("foo(a, *args, keyword=23)")
Fred Drake58422e52001-06-04 03:56:24 +000079 self.check_expr("foo + bar")
Michael W. Hudson5e83b7a2003-01-29 14:20:23 +000080 self.check_expr("foo - bar")
81 self.check_expr("foo * bar")
82 self.check_expr("foo / bar")
83 self.check_expr("foo // bar")
Fred Drake58422e52001-06-04 03:56:24 +000084 self.check_expr("lambda: 0")
85 self.check_expr("lambda x: 0")
86 self.check_expr("lambda *y: 0")
87 self.check_expr("lambda *y, **z: 0")
88 self.check_expr("lambda **z: 0")
89 self.check_expr("lambda x, y: 0")
90 self.check_expr("lambda foo=bar: 0")
91 self.check_expr("lambda foo=bar, spaz=nifty+spit: 0")
92 self.check_expr("lambda foo=bar, **z: 0")
93 self.check_expr("lambda foo=bar, blaz=blat+2, **z: 0")
94 self.check_expr("lambda foo=bar, blaz=blat+2, *y, **z: 0")
95 self.check_expr("lambda x, *y, **z: 0")
Raymond Hettinger354433a2004-05-19 08:20:33 +000096 self.check_expr("(x for x in range(10))")
97 self.check_expr("foo(x for x in range(10))")
Fred Drake79ca79d2000-08-21 22:30:53 +000098
Fred Drake58422e52001-06-04 03:56:24 +000099 def test_print(self):
100 self.check_suite("print")
101 self.check_suite("print 1")
102 self.check_suite("print 1,")
103 self.check_suite("print >>fp")
104 self.check_suite("print >>fp, 1")
105 self.check_suite("print >>fp, 1,")
Fred Drake79ca79d2000-08-21 22:30:53 +0000106
Fred Drake58422e52001-06-04 03:56:24 +0000107 def test_simple_expression(self):
108 # expr_stmt
109 self.check_suite("a")
Fred Drake79ca79d2000-08-21 22:30:53 +0000110
Fred Drake58422e52001-06-04 03:56:24 +0000111 def test_simple_assignments(self):
112 self.check_suite("a = b")
113 self.check_suite("a = b = c = d = e")
Fred Drake28f739a2000-08-25 22:42:40 +0000114
Fred Drake58422e52001-06-04 03:56:24 +0000115 def test_simple_augmented_assignments(self):
116 self.check_suite("a += b")
117 self.check_suite("a -= b")
118 self.check_suite("a *= b")
119 self.check_suite("a /= b")
Michael W. Hudson5e83b7a2003-01-29 14:20:23 +0000120 self.check_suite("a //= b")
Fred Drake58422e52001-06-04 03:56:24 +0000121 self.check_suite("a %= b")
122 self.check_suite("a &= b")
123 self.check_suite("a |= b")
124 self.check_suite("a ^= b")
125 self.check_suite("a <<= b")
126 self.check_suite("a >>= b")
127 self.check_suite("a **= b")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000128
Fred Drake58422e52001-06-04 03:56:24 +0000129 def test_function_defs(self):
130 self.check_suite("def f(): pass")
131 self.check_suite("def f(*args): pass")
132 self.check_suite("def f(*args, **kw): pass")
133 self.check_suite("def f(**kw): pass")
134 self.check_suite("def f(foo=bar): pass")
135 self.check_suite("def f(foo=bar, *args): pass")
136 self.check_suite("def f(foo=bar, *args, **kw): pass")
137 self.check_suite("def f(foo=bar, **kw): pass")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000138
Fred Drake58422e52001-06-04 03:56:24 +0000139 self.check_suite("def f(a, b): pass")
140 self.check_suite("def f(a, b, *args): pass")
141 self.check_suite("def f(a, b, *args, **kw): pass")
142 self.check_suite("def f(a, b, **kw): pass")
143 self.check_suite("def f(a, b, foo=bar): pass")
144 self.check_suite("def f(a, b, foo=bar, *args): pass")
145 self.check_suite("def f(a, b, foo=bar, *args, **kw): pass")
146 self.check_suite("def f(a, b, foo=bar, **kw): pass")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000147
Anthony Baxterc2a5a632004-08-02 06:10:11 +0000148 self.check_suite("@staticmethod\n"
149 "def f(): pass")
150 self.check_suite("@staticmethod\n"
151 "@funcattrs(x, y)\n"
152 "def f(): pass")
153 self.check_suite("@funcattrs()\n"
154 "def f(): pass")
155
Brett Cannonf4189912005-04-09 02:30:16 +0000156 def test_class_defs(self):
157 self.check_suite("class foo():pass")
Tim Peterse8906822005-04-20 17:45:13 +0000158
Fred Drake58422e52001-06-04 03:56:24 +0000159 def test_import_from_statement(self):
160 self.check_suite("from sys.path import *")
161 self.check_suite("from sys.path import dirname")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000162 self.check_suite("from sys.path import (dirname)")
163 self.check_suite("from sys.path import (dirname,)")
Fred Drake58422e52001-06-04 03:56:24 +0000164 self.check_suite("from sys.path import dirname as my_dirname")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000165 self.check_suite("from sys.path import (dirname as my_dirname)")
166 self.check_suite("from sys.path import (dirname as my_dirname,)")
Fred Drake58422e52001-06-04 03:56:24 +0000167 self.check_suite("from sys.path import dirname, basename")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000168 self.check_suite("from sys.path import (dirname, basename)")
169 self.check_suite("from sys.path import (dirname, basename,)")
Fred Drake58422e52001-06-04 03:56:24 +0000170 self.check_suite(
171 "from sys.path import dirname as my_dirname, basename")
172 self.check_suite(
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000173 "from sys.path import (dirname as my_dirname, basename)")
174 self.check_suite(
175 "from sys.path import (dirname as my_dirname, basename,)")
176 self.check_suite(
Fred Drake58422e52001-06-04 03:56:24 +0000177 "from sys.path import dirname, basename as my_basename")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000178 self.check_suite(
179 "from sys.path import (dirname, basename as my_basename)")
180 self.check_suite(
181 "from sys.path import (dirname, basename as my_basename,)")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000182
Fred Drake58422e52001-06-04 03:56:24 +0000183 def test_basic_import_statement(self):
184 self.check_suite("import sys")
185 self.check_suite("import sys as system")
186 self.check_suite("import sys, math")
187 self.check_suite("import sys as system, math")
188 self.check_suite("import sys, math as my_math")
Fred Drake79ca79d2000-08-21 22:30:53 +0000189
Neal Norwitz9caf9c02003-02-10 01:54:06 +0000190 def test_pep263(self):
191 self.check_suite("# -*- coding: iso-8859-1 -*-\n"
192 "pass\n")
193
194 def test_assert(self):
195 self.check_suite("assert alo < ahi and blo < bhi\n")
196
Jeremy Hylton60e96f62006-08-22 20:46:00 +0000197 def test_position(self):
198 # An absolutely minimal test of position information. Better
199 # tests would be a big project.
200 code = "def f(x):\n return x + 1\n"
201 st1 = parser.suite(code)
202 st2 = st1.totuple(line_info=1, col_info=1)
203
204 def walk(tree):
205 node_type = tree[0]
206 next = tree[1]
207 if isinstance(next, tuple):
208 for elt in tree[1:]:
209 for x in walk(elt):
210 yield x
211 else:
212 yield tree
Tim Peters147f9ae2006-08-25 22:05:39 +0000213
Jeremy Hylton60e96f62006-08-22 20:46:00 +0000214 terminals = list(walk(st2))
215 self.assertEqual([
216 (1, 'def', 1, 0),
217 (1, 'f', 1, 4),
218 (7, '(', 1, 5),
219 (1, 'x', 1, 6),
220 (8, ')', 1, 7),
221 (11, ':', 1, 8),
222 (4, '', 1, 9),
223 (5, '', 2, -1),
224 (1, 'return', 2, 4),
225 (1, 'x', 2, 11),
226 (14, '+', 2, 13),
227 (2, '1', 2, 15),
228 (4, '', 2, 16),
229 (6, '', 2, -1),
230 (4, '', 2, -1),
231 (0, '', 2, -1)],
232 terminals)
233
234
Fred Drake79ca79d2000-08-21 22:30:53 +0000235#
236# Second, we take *invalid* trees and make sure we get ParserError
237# rejections for them.
238#
239
Fred Drake58422e52001-06-04 03:56:24 +0000240class IllegalSyntaxTestCase(unittest.TestCase):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000241
Fred Drake58422e52001-06-04 03:56:24 +0000242 def check_bad_tree(self, tree, label):
243 try:
Fred Drake6e4f2c02001-07-17 19:33:25 +0000244 parser.sequence2st(tree)
Fred Drake58422e52001-06-04 03:56:24 +0000245 except parser.ParserError:
246 pass
247 else:
248 self.fail("did not detect invalid tree for %r" % label)
Fred Drake79ca79d2000-08-21 22:30:53 +0000249
Fred Drake58422e52001-06-04 03:56:24 +0000250 def test_junk(self):
251 # not even remotely valid:
252 self.check_bad_tree((1, 2, 3), "<junk>")
253
Fred Drakecf580c72001-07-17 03:01:29 +0000254 def test_illegal_yield_1(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000255 # Illegal yield statement: def f(): return 1; yield 1
Fred Drakecf580c72001-07-17 03:01:29 +0000256 tree = \
257 (257,
258 (264,
259 (285,
260 (259,
261 (1, 'def'),
262 (1, 'f'),
263 (260, (7, '('), (8, ')')),
264 (11, ':'),
265 (291,
266 (4, ''),
267 (5, ''),
268 (264,
269 (265,
270 (266,
271 (272,
272 (275,
273 (1, 'return'),
274 (313,
275 (292,
276 (293,
277 (294,
278 (295,
279 (297,
280 (298,
281 (299,
282 (300,
283 (301,
284 (302, (303, (304, (305, (2, '1')))))))))))))))))),
285 (264,
286 (265,
287 (266,
288 (272,
289 (276,
290 (1, 'yield'),
291 (313,
292 (292,
293 (293,
294 (294,
295 (295,
296 (297,
297 (298,
298 (299,
299 (300,
300 (301,
301 (302,
302 (303, (304, (305, (2, '1')))))))))))))))))),
303 (4, ''))),
304 (6, ''))))),
305 (4, ''),
306 (0, ''))))
307 self.check_bad_tree(tree, "def f():\n return 1\n yield 1")
308
309 def test_illegal_yield_2(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000310 # Illegal return in generator: def f(): return 1; yield 1
Fred Drakecf580c72001-07-17 03:01:29 +0000311 tree = \
312 (257,
313 (264,
314 (265,
315 (266,
316 (278,
317 (1, 'from'),
318 (281, (1, '__future__')),
319 (1, 'import'),
320 (279, (1, 'generators')))),
321 (4, ''))),
322 (264,
323 (285,
324 (259,
325 (1, 'def'),
326 (1, 'f'),
327 (260, (7, '('), (8, ')')),
328 (11, ':'),
329 (291,
330 (4, ''),
331 (5, ''),
332 (264,
333 (265,
334 (266,
335 (272,
336 (275,
337 (1, 'return'),
338 (313,
339 (292,
340 (293,
341 (294,
342 (295,
343 (297,
344 (298,
345 (299,
346 (300,
347 (301,
348 (302, (303, (304, (305, (2, '1')))))))))))))))))),
349 (264,
350 (265,
351 (266,
352 (272,
353 (276,
354 (1, 'yield'),
355 (313,
356 (292,
357 (293,
358 (294,
359 (295,
360 (297,
361 (298,
362 (299,
363 (300,
364 (301,
365 (302,
366 (303, (304, (305, (2, '1')))))))))))))))))),
367 (4, ''))),
368 (6, ''))))),
369 (4, ''),
370 (0, ''))))
371 self.check_bad_tree(tree, "def f():\n return 1\n yield 1")
372
Fred Drake58422e52001-06-04 03:56:24 +0000373 def test_print_chevron_comma(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000374 # Illegal input: print >>fp,
Fred Drake58422e52001-06-04 03:56:24 +0000375 tree = \
376 (257,
377 (264,
378 (265,
379 (266,
380 (268,
381 (1, 'print'),
382 (35, '>>'),
383 (290,
384 (291,
385 (292,
386 (293,
387 (295,
388 (296,
389 (297,
390 (298, (299, (300, (301, (302, (303, (1, 'fp')))))))))))))),
391 (12, ','))),
392 (4, ''))),
393 (0, ''))
394 self.check_bad_tree(tree, "print >>fp,")
395
396 def test_a_comma_comma_c(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000397 # Illegal input: a,,c
Fred Drake58422e52001-06-04 03:56:24 +0000398 tree = \
399 (258,
400 (311,
401 (290,
402 (291,
403 (292,
404 (293,
405 (295,
406 (296,
407 (297,
408 (298, (299, (300, (301, (302, (303, (1, 'a')))))))))))))),
409 (12, ','),
410 (12, ','),
411 (290,
412 (291,
413 (292,
414 (293,
415 (295,
416 (296,
417 (297,
418 (298, (299, (300, (301, (302, (303, (1, 'c'))))))))))))))),
419 (4, ''),
420 (0, ''))
421 self.check_bad_tree(tree, "a,,c")
422
423 def test_illegal_operator(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000424 # Illegal input: a $= b
Fred Drake58422e52001-06-04 03:56:24 +0000425 tree = \
426 (257,
427 (264,
428 (265,
429 (266,
430 (267,
431 (312,
432 (291,
433 (292,
434 (293,
435 (294,
436 (296,
437 (297,
438 (298,
439 (299,
440 (300, (301, (302, (303, (304, (1, 'a'))))))))))))))),
441 (268, (37, '$=')),
442 (312,
443 (291,
444 (292,
445 (293,
446 (294,
447 (296,
448 (297,
449 (298,
450 (299,
451 (300, (301, (302, (303, (304, (1, 'b'))))))))))))))))),
452 (4, ''))),
453 (0, ''))
454 self.check_bad_tree(tree, "a $= b")
Fred Drake79ca79d2000-08-21 22:30:53 +0000455
Neal Norwitz9caf9c02003-02-10 01:54:06 +0000456 def test_malformed_global(self):
457 #doesn't have global keyword in ast
458 tree = (257,
459 (264,
460 (265,
461 (266,
462 (282, (1, 'foo'))), (4, ''))),
463 (4, ''),
Tim Petersf2715e02003-02-19 02:35:07 +0000464 (0, ''))
Neal Norwitz9caf9c02003-02-10 01:54:06 +0000465 self.check_bad_tree(tree, "malformed global ast")
Fred Drake79ca79d2000-08-21 22:30:53 +0000466
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000467
468class CompileTestCase(unittest.TestCase):
469
470 # These tests are very minimal. :-(
471
472 def test_compile_expr(self):
473 st = parser.expr('2 + 3')
474 code = parser.compilest(st)
475 self.assertEquals(eval(code), 5)
476
477 def test_compile_suite(self):
478 st = parser.suite('x = 2; y = x + 3')
479 code = parser.compilest(st)
480 globs = {}
481 exec code in globs
482 self.assertEquals(globs['y'], 5)
483
484 def test_compile_error(self):
485 st = parser.suite('1 = 3 + 4')
486 self.assertRaises(SyntaxError, parser.compilest, st)
487
Guido van Rossumb6ac23c2007-07-18 17:19:14 +0000488 def test_compile_badunicode(self):
489 st = parser.suite('a = u"\U12345678"')
490 self.assertRaises(SyntaxError, parser.compilest, st)
491 st = parser.suite('a = u"\u1"')
492 self.assertRaises(SyntaxError, parser.compilest, st)
493
Facundo Batistafc2d0102008-02-23 12:01:13 +0000494class ParserStackLimitTestCase(unittest.TestCase):
495 """try to push the parser to/over it's limits.
496 see http://bugs.python.org/issue1881 for a discussion
497 """
498 def _nested_expression(self, level):
499 return "["*level+"]"*level
500
501 def test_deeply_nested_list(self):
502 e = self._nested_expression(99)
503 st = parser.expr(e)
504 st.compile()
505
506 def test_trigger_memory_error(self):
507 e = self._nested_expression(100)
Martin v. Löwis66e26632008-03-18 13:16:05 +0000508 print >>sys.stderr, "Expecting 's_push: parser stack overflow' in next line"
Facundo Batistafc2d0102008-02-23 12:01:13 +0000509 self.assertRaises(MemoryError, parser.expr, e)
510
Fred Drake2e2be372001-09-20 21:33:42 +0000511def test_main():
Walter Dörwald21d3a322003-05-01 17:45:56 +0000512 test_support.run_unittest(
513 RoundtripLegalSyntaxTestCase,
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000514 IllegalSyntaxTestCase,
515 CompileTestCase,
Facundo Batistafc2d0102008-02-23 12:01:13 +0000516 ParserStackLimitTestCase,
Walter Dörwald21d3a322003-05-01 17:45:56 +0000517 )
Fred Drake2e2be372001-09-20 21:33:42 +0000518
519
520if __name__ == "__main__":
521 test_main()