blob: 7d059c20f20cdbdafd1f8734ef7eb3448fac7908 [file] [log] [blame]
Fred Drake79ca79d2000-08-21 22:30:53 +00001import parser
Benjamin Peterson6624a9f2008-11-03 15:14:51 +00002import os
Fred Drake58422e52001-06-04 03:56:24 +00003import unittest
Martin v. Löwis66e26632008-03-18 13:16:05 +00004import sys
Barry Warsaw04f357c2002-07-23 19:04:11 +00005from test import test_support
Fred Drake79ca79d2000-08-21 22:30:53 +00006
7#
8# First, we test that we can generate trees from valid source fragments,
9# and that these valid trees are indeed allowed by the tree-loading side
10# of the parser module.
11#
12
Fred Drake58422e52001-06-04 03:56:24 +000013class RoundtripLegalSyntaxTestCase(unittest.TestCase):
Guido van Rossum32c2ae72002-08-22 19:45:32 +000014
Fred Drake58422e52001-06-04 03:56:24 +000015 def roundtrip(self, f, s):
16 st1 = f(s)
17 t = st1.totuple()
18 try:
Fred Drake6e4f2c02001-07-17 19:33:25 +000019 st2 = parser.sequence2st(t)
Anthony Baxterc2a5a632004-08-02 06:10:11 +000020 except parser.ParserError, why:
21 self.fail("could not roundtrip %r: %s" % (s, why))
Fred Drake79ca79d2000-08-21 22:30:53 +000022
Fred Drake58422e52001-06-04 03:56:24 +000023 self.assertEquals(t, st2.totuple(),
24 "could not re-generate syntax tree")
Fred Drake28f739a2000-08-25 22:42:40 +000025
Fred Drake58422e52001-06-04 03:56:24 +000026 def check_expr(self, s):
27 self.roundtrip(parser.expr, s)
Fred Drake28f739a2000-08-25 22:42:40 +000028
Benjamin Petersondcee09d2008-10-31 02:16:05 +000029 def test_flags_passed(self):
30 # The unicode literals flags has to be passed from the paser to AST
31 # generation.
32 suite = parser.suite("from __future__ import unicode_literals; x = ''")
33 code = suite.compile()
34 scope = {}
35 exec code in scope
36 self.assertTrue(isinstance(scope["x"], unicode))
37
Fred Drake58422e52001-06-04 03:56:24 +000038 def check_suite(self, s):
39 self.roundtrip(parser.suite, s)
Fred Drake28f739a2000-08-25 22:42:40 +000040
Fred Drakecf580c72001-07-17 03:01:29 +000041 def test_yield_statement(self):
Tim Peters496563a2002-04-01 00:28:59 +000042 self.check_suite("def f(): yield 1")
Phillip J. Eby0d6615f2005-08-02 00:46:46 +000043 self.check_suite("def f(): yield")
44 self.check_suite("def f(): x += yield")
45 self.check_suite("def f(): x = yield 1")
46 self.check_suite("def f(): x = y = yield 1")
47 self.check_suite("def f(): x = yield")
48 self.check_suite("def f(): x = y = yield")
49 self.check_suite("def f(): 1 + (yield)*2")
50 self.check_suite("def f(): (yield 1)*2")
Tim Peters496563a2002-04-01 00:28:59 +000051 self.check_suite("def f(): return; yield 1")
52 self.check_suite("def f(): yield 1; return")
53 self.check_suite("def f():\n"
Fred Drakecf580c72001-07-17 03:01:29 +000054 " for x in range(30):\n"
55 " yield x\n")
Phillip J. Eby0d6615f2005-08-02 00:46:46 +000056 self.check_suite("def f():\n"
57 " if (yield):\n"
58 " yield x\n")
Fred Drakecf580c72001-07-17 03:01:29 +000059
Fred Drake58422e52001-06-04 03:56:24 +000060 def test_expressions(self):
61 self.check_expr("foo(1)")
62 self.check_expr("[1, 2, 3]")
63 self.check_expr("[x**3 for x in range(20)]")
64 self.check_expr("[x**3 for x in range(20) if x % 3]")
Neal Norwitzd3a91622006-04-12 05:27:46 +000065 self.check_expr("[x**3 for x in range(20) if x % 2 if x % 3]")
66 self.check_expr("list(x**3 for x in range(20))")
67 self.check_expr("list(x**3 for x in range(20) if x % 3)")
68 self.check_expr("list(x**3 for x in range(20) if x % 2 if x % 3)")
Fred Drake58422e52001-06-04 03:56:24 +000069 self.check_expr("foo(*args)")
70 self.check_expr("foo(*args, **kw)")
71 self.check_expr("foo(**kw)")
72 self.check_expr("foo(key=value)")
73 self.check_expr("foo(key=value, *args)")
74 self.check_expr("foo(key=value, *args, **kw)")
75 self.check_expr("foo(key=value, **kw)")
76 self.check_expr("foo(a, b, c, *args)")
77 self.check_expr("foo(a, b, c, *args, **kw)")
78 self.check_expr("foo(a, b, c, **kw)")
Benjamin Petersonbd6a05f2008-08-19 22:06:11 +000079 self.check_expr("foo(a, *args, keyword=23)")
Fred Drake58422e52001-06-04 03:56:24 +000080 self.check_expr("foo + bar")
Michael W. Hudson5e83b7a2003-01-29 14:20:23 +000081 self.check_expr("foo - bar")
82 self.check_expr("foo * bar")
83 self.check_expr("foo / bar")
84 self.check_expr("foo // bar")
Fred Drake58422e52001-06-04 03:56:24 +000085 self.check_expr("lambda: 0")
86 self.check_expr("lambda x: 0")
87 self.check_expr("lambda *y: 0")
88 self.check_expr("lambda *y, **z: 0")
89 self.check_expr("lambda **z: 0")
90 self.check_expr("lambda x, y: 0")
91 self.check_expr("lambda foo=bar: 0")
92 self.check_expr("lambda foo=bar, spaz=nifty+spit: 0")
93 self.check_expr("lambda foo=bar, **z: 0")
94 self.check_expr("lambda foo=bar, blaz=blat+2, **z: 0")
95 self.check_expr("lambda foo=bar, blaz=blat+2, *y, **z: 0")
96 self.check_expr("lambda x, *y, **z: 0")
Raymond Hettinger354433a2004-05-19 08:20:33 +000097 self.check_expr("(x for x in range(10))")
98 self.check_expr("foo(x for x in range(10))")
Fred Drake79ca79d2000-08-21 22:30:53 +000099
Fred Drake58422e52001-06-04 03:56:24 +0000100 def test_print(self):
101 self.check_suite("print")
102 self.check_suite("print 1")
103 self.check_suite("print 1,")
104 self.check_suite("print >>fp")
105 self.check_suite("print >>fp, 1")
106 self.check_suite("print >>fp, 1,")
Fred Drake79ca79d2000-08-21 22:30:53 +0000107
Fred Drake58422e52001-06-04 03:56:24 +0000108 def test_simple_expression(self):
109 # expr_stmt
110 self.check_suite("a")
Fred Drake79ca79d2000-08-21 22:30:53 +0000111
Fred Drake58422e52001-06-04 03:56:24 +0000112 def test_simple_assignments(self):
113 self.check_suite("a = b")
114 self.check_suite("a = b = c = d = e")
Fred Drake28f739a2000-08-25 22:42:40 +0000115
Fred Drake58422e52001-06-04 03:56:24 +0000116 def test_simple_augmented_assignments(self):
117 self.check_suite("a += b")
118 self.check_suite("a -= b")
119 self.check_suite("a *= b")
120 self.check_suite("a /= b")
Michael W. Hudson5e83b7a2003-01-29 14:20:23 +0000121 self.check_suite("a //= b")
Fred Drake58422e52001-06-04 03:56:24 +0000122 self.check_suite("a %= b")
123 self.check_suite("a &= b")
124 self.check_suite("a |= b")
125 self.check_suite("a ^= b")
126 self.check_suite("a <<= b")
127 self.check_suite("a >>= b")
128 self.check_suite("a **= b")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000129
Fred Drake58422e52001-06-04 03:56:24 +0000130 def test_function_defs(self):
131 self.check_suite("def f(): pass")
132 self.check_suite("def f(*args): pass")
133 self.check_suite("def f(*args, **kw): pass")
134 self.check_suite("def f(**kw): pass")
135 self.check_suite("def f(foo=bar): pass")
136 self.check_suite("def f(foo=bar, *args): pass")
137 self.check_suite("def f(foo=bar, *args, **kw): pass")
138 self.check_suite("def f(foo=bar, **kw): pass")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000139
Fred Drake58422e52001-06-04 03:56:24 +0000140 self.check_suite("def f(a, b): pass")
141 self.check_suite("def f(a, b, *args): pass")
142 self.check_suite("def f(a, b, *args, **kw): pass")
143 self.check_suite("def f(a, b, **kw): pass")
144 self.check_suite("def f(a, b, foo=bar): pass")
145 self.check_suite("def f(a, b, foo=bar, *args): pass")
146 self.check_suite("def f(a, b, foo=bar, *args, **kw): pass")
147 self.check_suite("def f(a, b, foo=bar, **kw): pass")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000148
Anthony Baxterc2a5a632004-08-02 06:10:11 +0000149 self.check_suite("@staticmethod\n"
150 "def f(): pass")
151 self.check_suite("@staticmethod\n"
152 "@funcattrs(x, y)\n"
153 "def f(): pass")
154 self.check_suite("@funcattrs()\n"
155 "def f(): pass")
156
Brett Cannonf4189912005-04-09 02:30:16 +0000157 def test_class_defs(self):
158 self.check_suite("class foo():pass")
Tim Peterse8906822005-04-20 17:45:13 +0000159
Fred Drake58422e52001-06-04 03:56:24 +0000160 def test_import_from_statement(self):
161 self.check_suite("from sys.path import *")
162 self.check_suite("from sys.path import dirname")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000163 self.check_suite("from sys.path import (dirname)")
164 self.check_suite("from sys.path import (dirname,)")
Fred Drake58422e52001-06-04 03:56:24 +0000165 self.check_suite("from sys.path import dirname as my_dirname")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000166 self.check_suite("from sys.path import (dirname as my_dirname)")
167 self.check_suite("from sys.path import (dirname as my_dirname,)")
Fred Drake58422e52001-06-04 03:56:24 +0000168 self.check_suite("from sys.path import dirname, basename")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000169 self.check_suite("from sys.path import (dirname, basename)")
170 self.check_suite("from sys.path import (dirname, basename,)")
Fred Drake58422e52001-06-04 03:56:24 +0000171 self.check_suite(
172 "from sys.path import dirname as my_dirname, basename")
173 self.check_suite(
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000174 "from sys.path import (dirname as my_dirname, basename)")
175 self.check_suite(
176 "from sys.path import (dirname as my_dirname, basename,)")
177 self.check_suite(
Fred Drake58422e52001-06-04 03:56:24 +0000178 "from sys.path import dirname, basename as my_basename")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000179 self.check_suite(
180 "from sys.path import (dirname, basename as my_basename)")
181 self.check_suite(
182 "from sys.path import (dirname, basename as my_basename,)")
Benjamin Peterson6624a9f2008-11-03 15:14:51 +0000183 self.check_suite("from .bogus import x")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000184
Fred Drake58422e52001-06-04 03:56:24 +0000185 def test_basic_import_statement(self):
186 self.check_suite("import sys")
187 self.check_suite("import sys as system")
188 self.check_suite("import sys, math")
189 self.check_suite("import sys as system, math")
190 self.check_suite("import sys, math as my_math")
Fred Drake79ca79d2000-08-21 22:30:53 +0000191
Neal Norwitz9caf9c02003-02-10 01:54:06 +0000192 def test_pep263(self):
193 self.check_suite("# -*- coding: iso-8859-1 -*-\n"
194 "pass\n")
195
196 def test_assert(self):
197 self.check_suite("assert alo < ahi and blo < bhi\n")
198
Benjamin Peterson9dfe6a82008-11-24 04:09:03 +0000199 def test_with(self):
200 self.check_suite("with open('x'): pass\n")
201 self.check_suite("with open('x') as f: pass\n")
Georg Brandl944f6842009-05-25 21:02:56 +0000202 self.check_suite("with open('x') as f, open('y') as g: pass\n")
Benjamin Peterson9dfe6a82008-11-24 04:09:03 +0000203
Georg Brandlfe879e82008-12-05 12:09:41 +0000204 def test_try_stmt(self):
205 self.check_suite("try: pass\nexcept: pass\n")
206 self.check_suite("try: pass\nfinally: pass\n")
207 self.check_suite("try: pass\nexcept A: pass\nfinally: pass\n")
208 self.check_suite("try: pass\nexcept A: pass\nexcept: pass\n"
209 "finally: pass\n")
210 self.check_suite("try: pass\nexcept: pass\nelse: pass\n")
211 self.check_suite("try: pass\nexcept: pass\nelse: pass\n"
212 "finally: pass\n")
213
Jeremy Hylton60e96f62006-08-22 20:46:00 +0000214 def test_position(self):
215 # An absolutely minimal test of position information. Better
216 # tests would be a big project.
217 code = "def f(x):\n return x + 1\n"
218 st1 = parser.suite(code)
219 st2 = st1.totuple(line_info=1, col_info=1)
220
221 def walk(tree):
222 node_type = tree[0]
223 next = tree[1]
224 if isinstance(next, tuple):
225 for elt in tree[1:]:
226 for x in walk(elt):
227 yield x
228 else:
229 yield tree
Tim Peters147f9ae2006-08-25 22:05:39 +0000230
Jeremy Hylton60e96f62006-08-22 20:46:00 +0000231 terminals = list(walk(st2))
232 self.assertEqual([
233 (1, 'def', 1, 0),
234 (1, 'f', 1, 4),
235 (7, '(', 1, 5),
236 (1, 'x', 1, 6),
237 (8, ')', 1, 7),
238 (11, ':', 1, 8),
239 (4, '', 1, 9),
240 (5, '', 2, -1),
241 (1, 'return', 2, 4),
242 (1, 'x', 2, 11),
243 (14, '+', 2, 13),
244 (2, '1', 2, 15),
245 (4, '', 2, 16),
246 (6, '', 2, -1),
247 (4, '', 2, -1),
248 (0, '', 2, -1)],
249 terminals)
250
251
Fred Drake79ca79d2000-08-21 22:30:53 +0000252#
253# Second, we take *invalid* trees and make sure we get ParserError
254# rejections for them.
255#
256
Fred Drake58422e52001-06-04 03:56:24 +0000257class IllegalSyntaxTestCase(unittest.TestCase):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000258
Fred Drake58422e52001-06-04 03:56:24 +0000259 def check_bad_tree(self, tree, label):
260 try:
Fred Drake6e4f2c02001-07-17 19:33:25 +0000261 parser.sequence2st(tree)
Fred Drake58422e52001-06-04 03:56:24 +0000262 except parser.ParserError:
263 pass
264 else:
265 self.fail("did not detect invalid tree for %r" % label)
Fred Drake79ca79d2000-08-21 22:30:53 +0000266
Fred Drake58422e52001-06-04 03:56:24 +0000267 def test_junk(self):
268 # not even remotely valid:
269 self.check_bad_tree((1, 2, 3), "<junk>")
270
Fred Drakecf580c72001-07-17 03:01:29 +0000271 def test_illegal_yield_1(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000272 # Illegal yield statement: def f(): return 1; yield 1
Fred Drakecf580c72001-07-17 03:01:29 +0000273 tree = \
274 (257,
275 (264,
276 (285,
277 (259,
278 (1, 'def'),
279 (1, 'f'),
280 (260, (7, '('), (8, ')')),
281 (11, ':'),
282 (291,
283 (4, ''),
284 (5, ''),
285 (264,
286 (265,
287 (266,
288 (272,
289 (275,
290 (1, 'return'),
291 (313,
292 (292,
293 (293,
294 (294,
295 (295,
296 (297,
297 (298,
298 (299,
299 (300,
300 (301,
301 (302, (303, (304, (305, (2, '1')))))))))))))))))),
302 (264,
303 (265,
304 (266,
305 (272,
306 (276,
307 (1, 'yield'),
308 (313,
309 (292,
310 (293,
311 (294,
312 (295,
313 (297,
314 (298,
315 (299,
316 (300,
317 (301,
318 (302,
319 (303, (304, (305, (2, '1')))))))))))))))))),
320 (4, ''))),
321 (6, ''))))),
322 (4, ''),
323 (0, ''))))
324 self.check_bad_tree(tree, "def f():\n return 1\n yield 1")
325
326 def test_illegal_yield_2(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000327 # Illegal return in generator: def f(): return 1; yield 1
Fred Drakecf580c72001-07-17 03:01:29 +0000328 tree = \
329 (257,
330 (264,
331 (265,
332 (266,
333 (278,
334 (1, 'from'),
335 (281, (1, '__future__')),
336 (1, 'import'),
337 (279, (1, 'generators')))),
338 (4, ''))),
339 (264,
340 (285,
341 (259,
342 (1, 'def'),
343 (1, 'f'),
344 (260, (7, '('), (8, ')')),
345 (11, ':'),
346 (291,
347 (4, ''),
348 (5, ''),
349 (264,
350 (265,
351 (266,
352 (272,
353 (275,
354 (1, 'return'),
355 (313,
356 (292,
357 (293,
358 (294,
359 (295,
360 (297,
361 (298,
362 (299,
363 (300,
364 (301,
365 (302, (303, (304, (305, (2, '1')))))))))))))))))),
366 (264,
367 (265,
368 (266,
369 (272,
370 (276,
371 (1, 'yield'),
372 (313,
373 (292,
374 (293,
375 (294,
376 (295,
377 (297,
378 (298,
379 (299,
380 (300,
381 (301,
382 (302,
383 (303, (304, (305, (2, '1')))))))))))))))))),
384 (4, ''))),
385 (6, ''))))),
386 (4, ''),
387 (0, ''))))
388 self.check_bad_tree(tree, "def f():\n return 1\n yield 1")
389
Fred Drake58422e52001-06-04 03:56:24 +0000390 def test_print_chevron_comma(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000391 # Illegal input: print >>fp,
Fred Drake58422e52001-06-04 03:56:24 +0000392 tree = \
393 (257,
394 (264,
395 (265,
396 (266,
397 (268,
398 (1, 'print'),
399 (35, '>>'),
400 (290,
401 (291,
402 (292,
403 (293,
404 (295,
405 (296,
406 (297,
407 (298, (299, (300, (301, (302, (303, (1, 'fp')))))))))))))),
408 (12, ','))),
409 (4, ''))),
410 (0, ''))
411 self.check_bad_tree(tree, "print >>fp,")
412
413 def test_a_comma_comma_c(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000414 # Illegal input: a,,c
Fred Drake58422e52001-06-04 03:56:24 +0000415 tree = \
416 (258,
417 (311,
418 (290,
419 (291,
420 (292,
421 (293,
422 (295,
423 (296,
424 (297,
425 (298, (299, (300, (301, (302, (303, (1, 'a')))))))))))))),
426 (12, ','),
427 (12, ','),
428 (290,
429 (291,
430 (292,
431 (293,
432 (295,
433 (296,
434 (297,
435 (298, (299, (300, (301, (302, (303, (1, 'c'))))))))))))))),
436 (4, ''),
437 (0, ''))
438 self.check_bad_tree(tree, "a,,c")
439
440 def test_illegal_operator(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000441 # Illegal input: a $= b
Fred Drake58422e52001-06-04 03:56:24 +0000442 tree = \
443 (257,
444 (264,
445 (265,
446 (266,
447 (267,
448 (312,
449 (291,
450 (292,
451 (293,
452 (294,
453 (296,
454 (297,
455 (298,
456 (299,
457 (300, (301, (302, (303, (304, (1, 'a'))))))))))))))),
458 (268, (37, '$=')),
459 (312,
460 (291,
461 (292,
462 (293,
463 (294,
464 (296,
465 (297,
466 (298,
467 (299,
468 (300, (301, (302, (303, (304, (1, 'b'))))))))))))))))),
469 (4, ''))),
470 (0, ''))
471 self.check_bad_tree(tree, "a $= b")
Fred Drake79ca79d2000-08-21 22:30:53 +0000472
Neal Norwitz9caf9c02003-02-10 01:54:06 +0000473 def test_malformed_global(self):
474 #doesn't have global keyword in ast
475 tree = (257,
476 (264,
477 (265,
478 (266,
479 (282, (1, 'foo'))), (4, ''))),
480 (4, ''),
Tim Petersf2715e02003-02-19 02:35:07 +0000481 (0, ''))
Neal Norwitz9caf9c02003-02-10 01:54:06 +0000482 self.check_bad_tree(tree, "malformed global ast")
Fred Drake79ca79d2000-08-21 22:30:53 +0000483
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000484
485class CompileTestCase(unittest.TestCase):
486
487 # These tests are very minimal. :-(
488
489 def test_compile_expr(self):
490 st = parser.expr('2 + 3')
491 code = parser.compilest(st)
492 self.assertEquals(eval(code), 5)
493
494 def test_compile_suite(self):
495 st = parser.suite('x = 2; y = x + 3')
496 code = parser.compilest(st)
497 globs = {}
498 exec code in globs
499 self.assertEquals(globs['y'], 5)
500
501 def test_compile_error(self):
502 st = parser.suite('1 = 3 + 4')
503 self.assertRaises(SyntaxError, parser.compilest, st)
504
Guido van Rossumb6ac23c2007-07-18 17:19:14 +0000505 def test_compile_badunicode(self):
506 st = parser.suite('a = u"\U12345678"')
507 self.assertRaises(SyntaxError, parser.compilest, st)
508 st = parser.suite('a = u"\u1"')
509 self.assertRaises(SyntaxError, parser.compilest, st)
510
Facundo Batistafc2d0102008-02-23 12:01:13 +0000511class ParserStackLimitTestCase(unittest.TestCase):
512 """try to push the parser to/over it's limits.
513 see http://bugs.python.org/issue1881 for a discussion
514 """
515 def _nested_expression(self, level):
516 return "["*level+"]"*level
517
518 def test_deeply_nested_list(self):
519 e = self._nested_expression(99)
520 st = parser.expr(e)
521 st.compile()
522
523 def test_trigger_memory_error(self):
524 e = self._nested_expression(100)
Martin v. Löwis66e26632008-03-18 13:16:05 +0000525 print >>sys.stderr, "Expecting 's_push: parser stack overflow' in next line"
Facundo Batistafc2d0102008-02-23 12:01:13 +0000526 self.assertRaises(MemoryError, parser.expr, e)
527
Fred Drake2e2be372001-09-20 21:33:42 +0000528def test_main():
Walter Dörwald21d3a322003-05-01 17:45:56 +0000529 test_support.run_unittest(
530 RoundtripLegalSyntaxTestCase,
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000531 IllegalSyntaxTestCase,
532 CompileTestCase,
Facundo Batistafc2d0102008-02-23 12:01:13 +0000533 ParserStackLimitTestCase,
Walter Dörwald21d3a322003-05-01 17:45:56 +0000534 )
Fred Drake2e2be372001-09-20 21:33:42 +0000535
536
537if __name__ == "__main__":
538 test_main()