blob: cec4c70aa203c3e3f8f27536d57c8911d46eec92 [file] [log] [blame]
Fred Drake79ca79d2000-08-21 22:30:53 +00001import parser
Fred Drake58422e52001-06-04 03:56:24 +00002import unittest
Christian Heimesb186d002008-03-18 15:15:01 +00003import sys
Benjamin Petersonee8712c2008-05-20 21:35:26 +00004from test import support
Fred Drake79ca79d2000-08-21 22:30:53 +00005
6#
7# First, we test that we can generate trees from valid source fragments,
8# and that these valid trees are indeed allowed by the tree-loading side
9# of the parser module.
10#
11
Fred Drake58422e52001-06-04 03:56:24 +000012class RoundtripLegalSyntaxTestCase(unittest.TestCase):
Guido van Rossum32c2ae72002-08-22 19:45:32 +000013
Fred Drake58422e52001-06-04 03:56:24 +000014 def roundtrip(self, f, s):
15 st1 = f(s)
16 t = st1.totuple()
17 try:
Fred Drake6e4f2c02001-07-17 19:33:25 +000018 st2 = parser.sequence2st(t)
Guido van Rossumb940e112007-01-10 16:19:56 +000019 except parser.ParserError as why:
Anthony Baxterc2a5a632004-08-02 06:10:11 +000020 self.fail("could not roundtrip %r: %s" % (s, why))
Fred Drake79ca79d2000-08-21 22:30:53 +000021
Fred Drake58422e52001-06-04 03:56:24 +000022 self.assertEquals(t, st2.totuple(),
23 "could not re-generate syntax tree")
Fred Drake28f739a2000-08-25 22:42:40 +000024
Fred Drake58422e52001-06-04 03:56:24 +000025 def check_expr(self, s):
26 self.roundtrip(parser.expr, s)
Fred Drake28f739a2000-08-25 22:42:40 +000027
Benjamin Petersonf216c942008-10-31 02:28:05 +000028 def test_flags_passed(self):
29 # The unicode literals flags has to be passed from the paser to AST
30 # generation.
31 suite = parser.suite("from __future__ import unicode_literals; x = ''")
32 code = suite.compile()
33 scope = {}
34 exec(code, {}, scope)
35 self.assertTrue(isinstance(scope["x"], str))
36
Fred Drake58422e52001-06-04 03:56:24 +000037 def check_suite(self, s):
38 self.roundtrip(parser.suite, s)
Fred Drake28f739a2000-08-25 22:42:40 +000039
Fred Drakecf580c72001-07-17 03:01:29 +000040 def test_yield_statement(self):
Tim Peters496563a2002-04-01 00:28:59 +000041 self.check_suite("def f(): yield 1")
Phillip J. Eby0d6615f2005-08-02 00:46:46 +000042 self.check_suite("def f(): yield")
43 self.check_suite("def f(): x += yield")
44 self.check_suite("def f(): x = yield 1")
45 self.check_suite("def f(): x = y = yield 1")
46 self.check_suite("def f(): x = yield")
47 self.check_suite("def f(): x = y = yield")
48 self.check_suite("def f(): 1 + (yield)*2")
49 self.check_suite("def f(): (yield 1)*2")
Tim Peters496563a2002-04-01 00:28:59 +000050 self.check_suite("def f(): return; yield 1")
51 self.check_suite("def f(): yield 1; return")
52 self.check_suite("def f():\n"
Fred Drakecf580c72001-07-17 03:01:29 +000053 " for x in range(30):\n"
54 " yield x\n")
Phillip J. Eby0d6615f2005-08-02 00:46:46 +000055 self.check_suite("def f():\n"
56 " if (yield):\n"
57 " yield x\n")
Fred Drakecf580c72001-07-17 03:01:29 +000058
Fred Drake58422e52001-06-04 03:56:24 +000059 def test_expressions(self):
60 self.check_expr("foo(1)")
61 self.check_expr("[1, 2, 3]")
62 self.check_expr("[x**3 for x in range(20)]")
63 self.check_expr("[x**3 for x in range(20) if x % 3]")
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000064 self.check_expr("[x**3 for x in range(20) if x % 2 if x % 3]")
65 self.check_expr("list(x**3 for x in range(20))")
66 self.check_expr("list(x**3 for x in range(20) if x % 3)")
67 self.check_expr("list(x**3 for x in range(20) if x % 2 if x % 3)")
Fred Drake58422e52001-06-04 03:56:24 +000068 self.check_expr("foo(*args)")
69 self.check_expr("foo(*args, **kw)")
70 self.check_expr("foo(**kw)")
71 self.check_expr("foo(key=value)")
72 self.check_expr("foo(key=value, *args)")
73 self.check_expr("foo(key=value, *args, **kw)")
74 self.check_expr("foo(key=value, **kw)")
75 self.check_expr("foo(a, b, c, *args)")
76 self.check_expr("foo(a, b, c, *args, **kw)")
77 self.check_expr("foo(a, b, c, **kw)")
Benjamin Peterson3938a902008-08-20 02:33:00 +000078 self.check_expr("foo(a, *args, keyword=23)")
Fred Drake58422e52001-06-04 03:56:24 +000079 self.check_expr("foo + bar")
Michael W. Hudson5e83b7a2003-01-29 14:20:23 +000080 self.check_expr("foo - bar")
81 self.check_expr("foo * bar")
82 self.check_expr("foo / bar")
83 self.check_expr("foo // bar")
Fred Drake58422e52001-06-04 03:56:24 +000084 self.check_expr("lambda: 0")
85 self.check_expr("lambda x: 0")
86 self.check_expr("lambda *y: 0")
87 self.check_expr("lambda *y, **z: 0")
88 self.check_expr("lambda **z: 0")
89 self.check_expr("lambda x, y: 0")
90 self.check_expr("lambda foo=bar: 0")
91 self.check_expr("lambda foo=bar, spaz=nifty+spit: 0")
92 self.check_expr("lambda foo=bar, **z: 0")
93 self.check_expr("lambda foo=bar, blaz=blat+2, **z: 0")
94 self.check_expr("lambda foo=bar, blaz=blat+2, *y, **z: 0")
95 self.check_expr("lambda x, *y, **z: 0")
Raymond Hettinger354433a2004-05-19 08:20:33 +000096 self.check_expr("(x for x in range(10))")
97 self.check_expr("foo(x for x in range(10))")
Fred Drake79ca79d2000-08-21 22:30:53 +000098
Fred Drake58422e52001-06-04 03:56:24 +000099 def test_simple_expression(self):
100 # expr_stmt
101 self.check_suite("a")
Fred Drake79ca79d2000-08-21 22:30:53 +0000102
Fred Drake58422e52001-06-04 03:56:24 +0000103 def test_simple_assignments(self):
104 self.check_suite("a = b")
105 self.check_suite("a = b = c = d = e")
Fred Drake28f739a2000-08-25 22:42:40 +0000106
Fred Drake58422e52001-06-04 03:56:24 +0000107 def test_simple_augmented_assignments(self):
108 self.check_suite("a += b")
109 self.check_suite("a -= b")
110 self.check_suite("a *= b")
111 self.check_suite("a /= b")
Michael W. Hudson5e83b7a2003-01-29 14:20:23 +0000112 self.check_suite("a //= b")
Fred Drake58422e52001-06-04 03:56:24 +0000113 self.check_suite("a %= b")
114 self.check_suite("a &= b")
115 self.check_suite("a |= b")
116 self.check_suite("a ^= b")
117 self.check_suite("a <<= b")
118 self.check_suite("a >>= b")
119 self.check_suite("a **= b")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000120
Fred Drake58422e52001-06-04 03:56:24 +0000121 def test_function_defs(self):
122 self.check_suite("def f(): pass")
123 self.check_suite("def f(*args): pass")
124 self.check_suite("def f(*args, **kw): pass")
125 self.check_suite("def f(**kw): pass")
126 self.check_suite("def f(foo=bar): pass")
127 self.check_suite("def f(foo=bar, *args): pass")
128 self.check_suite("def f(foo=bar, *args, **kw): pass")
129 self.check_suite("def f(foo=bar, **kw): pass")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000130
Fred Drake58422e52001-06-04 03:56:24 +0000131 self.check_suite("def f(a, b): pass")
132 self.check_suite("def f(a, b, *args): pass")
133 self.check_suite("def f(a, b, *args, **kw): pass")
134 self.check_suite("def f(a, b, **kw): pass")
135 self.check_suite("def f(a, b, foo=bar): pass")
136 self.check_suite("def f(a, b, foo=bar, *args): pass")
137 self.check_suite("def f(a, b, foo=bar, *args, **kw): pass")
138 self.check_suite("def f(a, b, foo=bar, **kw): pass")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000139
Anthony Baxterc2a5a632004-08-02 06:10:11 +0000140 self.check_suite("@staticmethod\n"
141 "def f(): pass")
142 self.check_suite("@staticmethod\n"
143 "@funcattrs(x, y)\n"
144 "def f(): pass")
145 self.check_suite("@funcattrs()\n"
146 "def f(): pass")
147
Brett Cannonf4189912005-04-09 02:30:16 +0000148 def test_class_defs(self):
149 self.check_suite("class foo():pass")
Guido van Rossumfc158e22007-11-15 19:17:28 +0000150 self.check_suite("class foo(object):pass")
Tim Peterse8906822005-04-20 17:45:13 +0000151
Fred Drake58422e52001-06-04 03:56:24 +0000152 def test_import_from_statement(self):
153 self.check_suite("from sys.path import *")
154 self.check_suite("from sys.path import dirname")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000155 self.check_suite("from sys.path import (dirname)")
156 self.check_suite("from sys.path import (dirname,)")
Fred Drake58422e52001-06-04 03:56:24 +0000157 self.check_suite("from sys.path import dirname as my_dirname")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000158 self.check_suite("from sys.path import (dirname as my_dirname)")
159 self.check_suite("from sys.path import (dirname as my_dirname,)")
Fred Drake58422e52001-06-04 03:56:24 +0000160 self.check_suite("from sys.path import dirname, basename")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000161 self.check_suite("from sys.path import (dirname, basename)")
162 self.check_suite("from sys.path import (dirname, basename,)")
Fred Drake58422e52001-06-04 03:56:24 +0000163 self.check_suite(
164 "from sys.path import dirname as my_dirname, basename")
165 self.check_suite(
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000166 "from sys.path import (dirname as my_dirname, basename)")
167 self.check_suite(
168 "from sys.path import (dirname as my_dirname, basename,)")
169 self.check_suite(
Fred Drake58422e52001-06-04 03:56:24 +0000170 "from sys.path import dirname, basename as my_basename")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000171 self.check_suite(
172 "from sys.path import (dirname, basename as my_basename)")
173 self.check_suite(
174 "from sys.path import (dirname, basename as my_basename,)")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000175
Fred Drake58422e52001-06-04 03:56:24 +0000176 def test_basic_import_statement(self):
177 self.check_suite("import sys")
178 self.check_suite("import sys as system")
179 self.check_suite("import sys, math")
180 self.check_suite("import sys as system, math")
181 self.check_suite("import sys, math as my_math")
Fred Drake79ca79d2000-08-21 22:30:53 +0000182
Neal Norwitz9caf9c02003-02-10 01:54:06 +0000183 def test_pep263(self):
184 self.check_suite("# -*- coding: iso-8859-1 -*-\n"
185 "pass\n")
186
187 def test_assert(self):
188 self.check_suite("assert alo < ahi and blo < bhi\n")
189
Thomas Wouters89f507f2006-12-13 04:49:30 +0000190 def test_position(self):
191 # An absolutely minimal test of position information. Better
192 # tests would be a big project.
193 code = "def f(x):\n return x + 1\n"
194 st1 = parser.suite(code)
195 st2 = st1.totuple(line_info=1, col_info=1)
196
197 def walk(tree):
198 node_type = tree[0]
199 next = tree[1]
200 if isinstance(next, tuple):
201 for elt in tree[1:]:
202 for x in walk(elt):
203 yield x
204 else:
205 yield tree
206
207 terminals = list(walk(st2))
208 self.assertEqual([
209 (1, 'def', 1, 0),
210 (1, 'f', 1, 4),
211 (7, '(', 1, 5),
212 (1, 'x', 1, 6),
213 (8, ')', 1, 7),
214 (11, ':', 1, 8),
215 (4, '', 1, 9),
216 (5, '', 2, -1),
217 (1, 'return', 2, 4),
218 (1, 'x', 2, 11),
219 (14, '+', 2, 13),
220 (2, '1', 2, 15),
221 (4, '', 2, 16),
222 (6, '', 2, -1),
223 (4, '', 2, -1),
224 (0, '', 2, -1)],
225 terminals)
226
227
Fred Drake79ca79d2000-08-21 22:30:53 +0000228#
229# Second, we take *invalid* trees and make sure we get ParserError
230# rejections for them.
231#
232
Fred Drake58422e52001-06-04 03:56:24 +0000233class IllegalSyntaxTestCase(unittest.TestCase):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000234
Fred Drake58422e52001-06-04 03:56:24 +0000235 def check_bad_tree(self, tree, label):
236 try:
Fred Drake6e4f2c02001-07-17 19:33:25 +0000237 parser.sequence2st(tree)
Fred Drake58422e52001-06-04 03:56:24 +0000238 except parser.ParserError:
239 pass
240 else:
241 self.fail("did not detect invalid tree for %r" % label)
Fred Drake79ca79d2000-08-21 22:30:53 +0000242
Fred Drake58422e52001-06-04 03:56:24 +0000243 def test_junk(self):
244 # not even remotely valid:
245 self.check_bad_tree((1, 2, 3), "<junk>")
246
Fred Drakecf580c72001-07-17 03:01:29 +0000247 def test_illegal_yield_1(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000248 # Illegal yield statement: def f(): return 1; yield 1
Fred Drakecf580c72001-07-17 03:01:29 +0000249 tree = \
250 (257,
251 (264,
252 (285,
253 (259,
254 (1, 'def'),
255 (1, 'f'),
256 (260, (7, '('), (8, ')')),
257 (11, ':'),
258 (291,
259 (4, ''),
260 (5, ''),
261 (264,
262 (265,
263 (266,
264 (272,
265 (275,
266 (1, 'return'),
267 (313,
268 (292,
269 (293,
270 (294,
271 (295,
272 (297,
273 (298,
274 (299,
275 (300,
276 (301,
277 (302, (303, (304, (305, (2, '1')))))))))))))))))),
278 (264,
279 (265,
280 (266,
281 (272,
282 (276,
283 (1, 'yield'),
284 (313,
285 (292,
286 (293,
287 (294,
288 (295,
289 (297,
290 (298,
291 (299,
292 (300,
293 (301,
294 (302,
295 (303, (304, (305, (2, '1')))))))))))))))))),
296 (4, ''))),
297 (6, ''))))),
298 (4, ''),
299 (0, ''))))
300 self.check_bad_tree(tree, "def f():\n return 1\n yield 1")
301
302 def test_illegal_yield_2(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000303 # Illegal return in generator: def f(): return 1; yield 1
Fred Drakecf580c72001-07-17 03:01:29 +0000304 tree = \
305 (257,
306 (264,
307 (265,
308 (266,
309 (278,
310 (1, 'from'),
311 (281, (1, '__future__')),
312 (1, 'import'),
313 (279, (1, 'generators')))),
314 (4, ''))),
315 (264,
316 (285,
317 (259,
318 (1, 'def'),
319 (1, 'f'),
320 (260, (7, '('), (8, ')')),
321 (11, ':'),
322 (291,
323 (4, ''),
324 (5, ''),
325 (264,
326 (265,
327 (266,
328 (272,
329 (275,
330 (1, 'return'),
331 (313,
332 (292,
333 (293,
334 (294,
335 (295,
336 (297,
337 (298,
338 (299,
339 (300,
340 (301,
341 (302, (303, (304, (305, (2, '1')))))))))))))))))),
342 (264,
343 (265,
344 (266,
345 (272,
346 (276,
347 (1, 'yield'),
348 (313,
349 (292,
350 (293,
351 (294,
352 (295,
353 (297,
354 (298,
355 (299,
356 (300,
357 (301,
358 (302,
359 (303, (304, (305, (2, '1')))))))))))))))))),
360 (4, ''))),
361 (6, ''))))),
362 (4, ''),
363 (0, ''))))
364 self.check_bad_tree(tree, "def f():\n return 1\n yield 1")
365
Fred Drake58422e52001-06-04 03:56:24 +0000366 def test_a_comma_comma_c(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000367 # Illegal input: a,,c
Fred Drake58422e52001-06-04 03:56:24 +0000368 tree = \
369 (258,
370 (311,
371 (290,
372 (291,
373 (292,
374 (293,
375 (295,
376 (296,
377 (297,
378 (298, (299, (300, (301, (302, (303, (1, 'a')))))))))))))),
379 (12, ','),
380 (12, ','),
381 (290,
382 (291,
383 (292,
384 (293,
385 (295,
386 (296,
387 (297,
388 (298, (299, (300, (301, (302, (303, (1, 'c'))))))))))))))),
389 (4, ''),
390 (0, ''))
391 self.check_bad_tree(tree, "a,,c")
392
393 def test_illegal_operator(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000394 # Illegal input: a $= b
Fred Drake58422e52001-06-04 03:56:24 +0000395 tree = \
396 (257,
397 (264,
398 (265,
399 (266,
400 (267,
401 (312,
402 (291,
403 (292,
404 (293,
405 (294,
406 (296,
407 (297,
408 (298,
409 (299,
410 (300, (301, (302, (303, (304, (1, 'a'))))))))))))))),
411 (268, (37, '$=')),
412 (312,
413 (291,
414 (292,
415 (293,
416 (294,
417 (296,
418 (297,
419 (298,
420 (299,
421 (300, (301, (302, (303, (304, (1, 'b'))))))))))))))))),
422 (4, ''))),
423 (0, ''))
424 self.check_bad_tree(tree, "a $= b")
Fred Drake79ca79d2000-08-21 22:30:53 +0000425
Neal Norwitz9caf9c02003-02-10 01:54:06 +0000426 def test_malformed_global(self):
427 #doesn't have global keyword in ast
428 tree = (257,
429 (264,
430 (265,
431 (266,
432 (282, (1, 'foo'))), (4, ''))),
433 (4, ''),
Tim Petersf2715e02003-02-19 02:35:07 +0000434 (0, ''))
Neal Norwitz9caf9c02003-02-10 01:54:06 +0000435 self.check_bad_tree(tree, "malformed global ast")
Fred Drake79ca79d2000-08-21 22:30:53 +0000436
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000437
438class CompileTestCase(unittest.TestCase):
439
440 # These tests are very minimal. :-(
441
442 def test_compile_expr(self):
443 st = parser.expr('2 + 3')
444 code = parser.compilest(st)
445 self.assertEquals(eval(code), 5)
446
447 def test_compile_suite(self):
448 st = parser.suite('x = 2; y = x + 3')
449 code = parser.compilest(st)
450 globs = {}
Georg Brandl7cae87c2006-09-06 06:51:57 +0000451 exec(code, globs)
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000452 self.assertEquals(globs['y'], 5)
453
454 def test_compile_error(self):
455 st = parser.suite('1 = 3 + 4')
456 self.assertRaises(SyntaxError, parser.compilest, st)
457
Guido van Rossumb5a755e2007-07-18 18:15:48 +0000458 def test_compile_badunicode(self):
Guido van Rossum7eb6ca52007-07-18 21:00:22 +0000459 st = parser.suite('a = "\\U12345678"')
Guido van Rossumb5a755e2007-07-18 18:15:48 +0000460 self.assertRaises(SyntaxError, parser.compilest, st)
Guido van Rossum7eb6ca52007-07-18 21:00:22 +0000461 st = parser.suite('a = "\\u1"')
Guido van Rossumb5a755e2007-07-18 18:15:48 +0000462 self.assertRaises(SyntaxError, parser.compilest, st)
463
Christian Heimes90c3d9b2008-02-23 13:18:03 +0000464class ParserStackLimitTestCase(unittest.TestCase):
465 """try to push the parser to/over it's limits.
466 see http://bugs.python.org/issue1881 for a discussion
467 """
468 def _nested_expression(self, level):
469 return "["*level+"]"*level
470
471 def test_deeply_nested_list(self):
472 # XXX used to be 99 levels in 2.x
473 e = self._nested_expression(93)
474 st = parser.expr(e)
475 st.compile()
476
477 def test_trigger_memory_error(self):
478 e = self._nested_expression(100)
Christian Heimesb186d002008-03-18 15:15:01 +0000479 print("Expecting 's_push: parser stack overflow' in next line",
480 file=sys.stderr)
Christian Heimes90c3d9b2008-02-23 13:18:03 +0000481 self.assertRaises(MemoryError, parser.expr, e)
482
Fred Drake2e2be372001-09-20 21:33:42 +0000483def test_main():
Benjamin Petersonee8712c2008-05-20 21:35:26 +0000484 support.run_unittest(
Walter Dörwald21d3a322003-05-01 17:45:56 +0000485 RoundtripLegalSyntaxTestCase,
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000486 IllegalSyntaxTestCase,
487 CompileTestCase,
Christian Heimes90c3d9b2008-02-23 13:18:03 +0000488 ParserStackLimitTestCase,
Walter Dörwald21d3a322003-05-01 17:45:56 +0000489 )
Fred Drake2e2be372001-09-20 21:33:42 +0000490
491
492if __name__ == "__main__":
493 test_main()