Gitiles
Code Review Sign In
gerrit-public.fairphone.software / platform / external / python / cpython3 / 2442015af26321083d4a2d75c096c8b732f049b2 / . / Lib / test / test_parser.py
blob: e4179620e71f4e78733feaddf07161987f785b86 [file] [log] [blame]
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]1import parser
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]2import unittest
Christian Heimesb186d002008-03-18 15:15:01 +0000[diff] [blame]3import sys
Benjamin Petersonee8712c2008-05-20 21:35:26 +0000[diff] [blame]4from test import support
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]5
6#
7# First, we test that we can generate trees from valid source fragments,
8# and that these valid trees are indeed allowed by the tree-loading side
9# of the parser module.
10#
11
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]12class RoundtripLegalSyntaxTestCase(unittest.TestCase):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000[diff] [blame]13
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]14 def roundtrip(self, f, s):
15 st1 = f(s)
16 t = st1.totuple()
17 try:
Fred Drake6e4f2c02001-07-17 19:33:25 +0000[diff] [blame]18 st2 = parser.sequence2st(t)
Guido van Rossumb940e112007-01-10 16:19:56 +0000[diff] [blame]19 except parser.ParserError as why:
Anthony Baxterc2a5a632004-08-02 06:10:11 +0000[diff] [blame]20 self.fail("could not roundtrip %r: %s" % (s, why))
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]21
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]22 self.assertEquals(t, st2.totuple(),
23 "could not re-generate syntax tree")
Fred Drake28f739a2000-08-25 22:42:40 +0000[diff] [blame]24
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]25 def check_expr(self, s):
26 self.roundtrip(parser.expr, s)
Fred Drake28f739a2000-08-25 22:42:40 +0000[diff] [blame]27
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]28 def check_suite(self, s):
29 self.roundtrip(parser.suite, s)
Fred Drake28f739a2000-08-25 22:42:40 +0000[diff] [blame]30
Fred Drakecf580c72001-07-17 03:01:29 +0000[diff] [blame]31 def test_yield_statement(self):
Tim Peters496563a2002-04-01 00:28:59 +0000[diff] [blame]32 self.check_suite("def f(): yield 1")
Phillip J. Eby0d6615f2005-08-02 00:46:46 +0000[diff] [blame]33 self.check_suite("def f(): yield")
34 self.check_suite("def f(): x += yield")
35 self.check_suite("def f(): x = yield 1")
36 self.check_suite("def f(): x = y = yield 1")
37 self.check_suite("def f(): x = yield")
38 self.check_suite("def f(): x = y = yield")
39 self.check_suite("def f(): 1 + (yield)*2")
40 self.check_suite("def f(): (yield 1)*2")
Tim Peters496563a2002-04-01 00:28:59 +0000[diff] [blame]41 self.check_suite("def f(): return; yield 1")
42 self.check_suite("def f(): yield 1; return")
43 self.check_suite("def f():\n"
Fred Drakecf580c72001-07-17 03:01:29 +0000[diff] [blame]44 " for x in range(30):\n"
45 " yield x\n")
Phillip J. Eby0d6615f2005-08-02 00:46:46 +0000[diff] [blame]46 self.check_suite("def f():\n"
47 " if (yield):\n"
48 " yield x\n")
Fred Drakecf580c72001-07-17 03:01:29 +0000[diff] [blame]49
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]50 def test_expressions(self):
51 self.check_expr("foo(1)")
52 self.check_expr("[1, 2, 3]")
53 self.check_expr("[x**3 for x in range(20)]")
54 self.check_expr("[x**3 for x in range(20) if x % 3]")
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000[diff] [blame]55 self.check_expr("[x**3 for x in range(20) if x % 2 if x % 3]")
56 self.check_expr("list(x**3 for x in range(20))")
57 self.check_expr("list(x**3 for x in range(20) if x % 3)")
58 self.check_expr("list(x**3 for x in range(20) if x % 2 if x % 3)")
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]59 self.check_expr("foo(*args)")
60 self.check_expr("foo(*args, **kw)")
61 self.check_expr("foo(**kw)")
62 self.check_expr("foo(key=value)")
63 self.check_expr("foo(key=value, *args)")
64 self.check_expr("foo(key=value, *args, **kw)")
65 self.check_expr("foo(key=value, **kw)")
66 self.check_expr("foo(a, b, c, *args)")
67 self.check_expr("foo(a, b, c, *args, **kw)")
68 self.check_expr("foo(a, b, c, **kw)")
69 self.check_expr("foo + bar")
Michael W. Hudson5e83b7a2003-01-29 14:20:23 +0000[diff] [blame]70 self.check_expr("foo - bar")
71 self.check_expr("foo * bar")
72 self.check_expr("foo / bar")
73 self.check_expr("foo // bar")
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]74 self.check_expr("lambda: 0")
75 self.check_expr("lambda x: 0")
76 self.check_expr("lambda *y: 0")
77 self.check_expr("lambda *y, **z: 0")
78 self.check_expr("lambda **z: 0")
79 self.check_expr("lambda x, y: 0")
80 self.check_expr("lambda foo=bar: 0")
81 self.check_expr("lambda foo=bar, spaz=nifty+spit: 0")
82 self.check_expr("lambda foo=bar, **z: 0")
83 self.check_expr("lambda foo=bar, blaz=blat+2, **z: 0")
84 self.check_expr("lambda foo=bar, blaz=blat+2, *y, **z: 0")
85 self.check_expr("lambda x, *y, **z: 0")
Raymond Hettinger354433a2004-05-19 08:20:33 +0000[diff] [blame]86 self.check_expr("(x for x in range(10))")
87 self.check_expr("foo(x for x in range(10))")
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]88
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]89 def test_simple_expression(self):
90 # expr_stmt
91 self.check_suite("a")
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]92
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]93 def test_simple_assignments(self):
94 self.check_suite("a = b")
95 self.check_suite("a = b = c = d = e")
Fred Drake28f739a2000-08-25 22:42:40 +0000[diff] [blame]96
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]97 def test_simple_augmented_assignments(self):
98 self.check_suite("a += b")
99 self.check_suite("a -= b")
100 self.check_suite("a *= b")
101 self.check_suite("a /= b")
Michael W. Hudson5e83b7a2003-01-29 14:20:23 +0000[diff] [blame]102 self.check_suite("a //= b")
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]103 self.check_suite("a %= b")
104 self.check_suite("a &= b")
105 self.check_suite("a |= b")
106 self.check_suite("a ^= b")
107 self.check_suite("a <<= b")
108 self.check_suite("a >>= b")
109 self.check_suite("a **= b")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000[diff] [blame]110
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]111 def test_function_defs(self):
112 self.check_suite("def f(): pass")
113 self.check_suite("def f(*args): pass")
114 self.check_suite("def f(*args, **kw): pass")
115 self.check_suite("def f(**kw): pass")
116 self.check_suite("def f(foo=bar): pass")
117 self.check_suite("def f(foo=bar, *args): pass")
118 self.check_suite("def f(foo=bar, *args, **kw): pass")
119 self.check_suite("def f(foo=bar, **kw): pass")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000[diff] [blame]120
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]121 self.check_suite("def f(a, b): pass")
122 self.check_suite("def f(a, b, *args): pass")
123 self.check_suite("def f(a, b, *args, **kw): pass")
124 self.check_suite("def f(a, b, **kw): pass")
125 self.check_suite("def f(a, b, foo=bar): pass")
126 self.check_suite("def f(a, b, foo=bar, *args): pass")
127 self.check_suite("def f(a, b, foo=bar, *args, **kw): pass")
128 self.check_suite("def f(a, b, foo=bar, **kw): pass")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000[diff] [blame]129
Anthony Baxterc2a5a632004-08-02 06:10:11 +0000[diff] [blame]130 self.check_suite("@staticmethod\n"
131 "def f(): pass")
132 self.check_suite("@staticmethod\n"
133 "@funcattrs(x, y)\n"
134 "def f(): pass")
135 self.check_suite("@funcattrs()\n"
136 "def f(): pass")
137
Brett Cannonf4189912005-04-09 02:30:16 +0000[diff] [blame]138 def test_class_defs(self):
139 self.check_suite("class foo():pass")
Guido van Rossumfc158e22007-11-15 19:17:28 +0000[diff] [blame]140 self.check_suite("class foo(object):pass")
Tim Peterse8906822005-04-20 17:45:13 +0000[diff] [blame]141
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]142 def test_import_from_statement(self):
143 self.check_suite("from sys.path import *")
144 self.check_suite("from sys.path import dirname")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000[diff] [blame]145 self.check_suite("from sys.path import (dirname)")
146 self.check_suite("from sys.path import (dirname,)")
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]147 self.check_suite("from sys.path import dirname as my_dirname")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000[diff] [blame]148 self.check_suite("from sys.path import (dirname as my_dirname)")
149 self.check_suite("from sys.path import (dirname as my_dirname,)")
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]150 self.check_suite("from sys.path import dirname, basename")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000[diff] [blame]151 self.check_suite("from sys.path import (dirname, basename)")
152 self.check_suite("from sys.path import (dirname, basename,)")
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]153 self.check_suite(
154 "from sys.path import dirname as my_dirname, basename")
155 self.check_suite(
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000[diff] [blame]156 "from sys.path import (dirname as my_dirname, basename)")
157 self.check_suite(
158 "from sys.path import (dirname as my_dirname, basename,)")
159 self.check_suite(
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]160 "from sys.path import dirname, basename as my_basename")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000[diff] [blame]161 self.check_suite(
162 "from sys.path import (dirname, basename as my_basename)")
163 self.check_suite(
164 "from sys.path import (dirname, basename as my_basename,)")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000[diff] [blame]165
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]166 def test_basic_import_statement(self):
167 self.check_suite("import sys")
168 self.check_suite("import sys as system")
169 self.check_suite("import sys, math")
170 self.check_suite("import sys as system, math")
171 self.check_suite("import sys, math as my_math")
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]172
Neal Norwitz9caf9c02003-02-10 01:54:06 +0000[diff] [blame]173 def test_pep263(self):
174 self.check_suite("# -*- coding: iso-8859-1 -*-\n"
175 "pass\n")
176
177 def test_assert(self):
178 self.check_suite("assert alo < ahi and blo < bhi\n")
179
Thomas Wouters89f507f2006-12-13 04:49:30 +0000[diff] [blame]180 def test_position(self):
181 # An absolutely minimal test of position information. Better
182 # tests would be a big project.
183 code = "def f(x):\n return x + 1\n"
184 st1 = parser.suite(code)
185 st2 = st1.totuple(line_info=1, col_info=1)
186
187 def walk(tree):
188 node_type = tree[0]
189 next = tree[1]
190 if isinstance(next, tuple):
191 for elt in tree[1:]:
192 for x in walk(elt):
193 yield x
194 else:
195 yield tree
196
197 terminals = list(walk(st2))
198 self.assertEqual([
199 (1, 'def', 1, 0),
200 (1, 'f', 1, 4),
201 (7, '(', 1, 5),
202 (1, 'x', 1, 6),
203 (8, ')', 1, 7),
204 (11, ':', 1, 8),
205 (4, '', 1, 9),
206 (5, '', 2, -1),
207 (1, 'return', 2, 4),
208 (1, 'x', 2, 11),
209 (14, '+', 2, 13),
210 (2, '1', 2, 15),
211 (4, '', 2, 16),
212 (6, '', 2, -1),
213 (4, '', 2, -1),
214 (0, '', 2, -1)],
215 terminals)
216
217
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]218#
219# Second, we take *invalid* trees and make sure we get ParserError
220# rejections for them.
221#
222
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]223class IllegalSyntaxTestCase(unittest.TestCase):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000[diff] [blame]224
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]225 def check_bad_tree(self, tree, label):
226 try:
Fred Drake6e4f2c02001-07-17 19:33:25 +0000[diff] [blame]227 parser.sequence2st(tree)
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]228 except parser.ParserError:
229 pass
230 else:
231 self.fail("did not detect invalid tree for %r" % label)
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]232
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]233 def test_junk(self):
234 # not even remotely valid:
235 self.check_bad_tree((1, 2, 3), "<junk>")
236
Fred Drakecf580c72001-07-17 03:01:29 +0000[diff] [blame]237 def test_illegal_yield_1(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000[diff] [blame]238 # Illegal yield statement: def f(): return 1; yield 1
Fred Drakecf580c72001-07-17 03:01:29 +0000[diff] [blame]239 tree = \
240 (257,
241 (264,
242 (285,
243 (259,
244 (1, 'def'),
245 (1, 'f'),
246 (260, (7, '('), (8, ')')),
247 (11, ':'),
248 (291,
249 (4, ''),
250 (5, ''),
251 (264,
252 (265,
253 (266,
254 (272,
255 (275,
256 (1, 'return'),
257 (313,
258 (292,
259 (293,
260 (294,
261 (295,
262 (297,
263 (298,
264 (299,
265 (300,
266 (301,
267 (302, (303, (304, (305, (2, '1')))))))))))))))))),
268 (264,
269 (265,
270 (266,
271 (272,
272 (276,
273 (1, 'yield'),
274 (313,
275 (292,
276 (293,
277 (294,
278 (295,
279 (297,
280 (298,
281 (299,
282 (300,
283 (301,
284 (302,
285 (303, (304, (305, (2, '1')))))))))))))))))),
286 (4, ''))),
287 (6, ''))))),
288 (4, ''),
289 (0, ''))))
290 self.check_bad_tree(tree, "def f():\n return 1\n yield 1")
291
292 def test_illegal_yield_2(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000[diff] [blame]293 # Illegal return in generator: def f(): return 1; yield 1
Fred Drakecf580c72001-07-17 03:01:29 +0000[diff] [blame]294 tree = \
295 (257,
296 (264,
297 (265,
298 (266,
299 (278,
300 (1, 'from'),
301 (281, (1, '__future__')),
302 (1, 'import'),
303 (279, (1, 'generators')))),
304 (4, ''))),
305 (264,
306 (285,
307 (259,
308 (1, 'def'),
309 (1, 'f'),
310 (260, (7, '('), (8, ')')),
311 (11, ':'),
312 (291,
313 (4, ''),
314 (5, ''),
315 (264,
316 (265,
317 (266,
318 (272,
319 (275,
320 (1, 'return'),
321 (313,
322 (292,
323 (293,
324 (294,
325 (295,
326 (297,
327 (298,
328 (299,
329 (300,
330 (301,
331 (302, (303, (304, (305, (2, '1')))))))))))))))))),
332 (264,
333 (265,
334 (266,
335 (272,
336 (276,
337 (1, 'yield'),
338 (313,
339 (292,
340 (293,
341 (294,
342 (295,
343 (297,
344 (298,
345 (299,
346 (300,
347 (301,
348 (302,
349 (303, (304, (305, (2, '1')))))))))))))))))),
350 (4, ''))),
351 (6, ''))))),
352 (4, ''),
353 (0, ''))))
354 self.check_bad_tree(tree, "def f():\n return 1\n yield 1")
355
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]356 def test_a_comma_comma_c(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000[diff] [blame]357 # Illegal input: a,,c
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]358 tree = \
359 (258,
360 (311,
361 (290,
362 (291,
363 (292,
364 (293,
365 (295,
366 (296,
367 (297,
368 (298, (299, (300, (301, (302, (303, (1, 'a')))))))))))))),
369 (12, ','),
370 (12, ','),
371 (290,
372 (291,
373 (292,
374 (293,
375 (295,
376 (296,
377 (297,
378 (298, (299, (300, (301, (302, (303, (1, 'c'))))))))))))))),
379 (4, ''),
380 (0, ''))
381 self.check_bad_tree(tree, "a,,c")
382
383 def test_illegal_operator(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000[diff] [blame]384 # Illegal input: a $= b
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]385 tree = \
386 (257,
387 (264,
388 (265,
389 (266,
390 (267,
391 (312,
392 (291,
393 (292,
394 (293,
395 (294,
396 (296,
397 (297,
398 (298,
399 (299,
400 (300, (301, (302, (303, (304, (1, 'a'))))))))))))))),
401 (268, (37, '$=')),
402 (312,
403 (291,
404 (292,
405 (293,
406 (294,
407 (296,
408 (297,
409 (298,
410 (299,
411 (300, (301, (302, (303, (304, (1, 'b'))))))))))))))))),
412 (4, ''))),
413 (0, ''))
414 self.check_bad_tree(tree, "a $= b")
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]415
Neal Norwitz9caf9c02003-02-10 01:54:06 +0000[diff] [blame]416 def test_malformed_global(self):
417 #doesn't have global keyword in ast
418 tree = (257,
419 (264,
420 (265,
421 (266,
422 (282, (1, 'foo'))), (4, ''))),
423 (4, ''),
Tim Petersf2715e02003-02-19 02:35:07 +0000[diff] [blame]424 (0, ''))
Neal Norwitz9caf9c02003-02-10 01:54:06 +0000[diff] [blame]425 self.check_bad_tree(tree, "malformed global ast")
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]426
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000[diff] [blame]427
428class CompileTestCase(unittest.TestCase):
429
430 # These tests are very minimal. :-(
431
432 def test_compile_expr(self):
433 st = parser.expr('2 + 3')
434 code = parser.compilest(st)
435 self.assertEquals(eval(code), 5)
436
437 def test_compile_suite(self):
438 st = parser.suite('x = 2; y = x + 3')
439 code = parser.compilest(st)
440 globs = {}
Georg Brandl7cae87c2006-09-06 06:51:57 +0000[diff] [blame]441 exec(code, globs)
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000[diff] [blame]442 self.assertEquals(globs['y'], 5)
443
444 def test_compile_error(self):
445 st = parser.suite('1 = 3 + 4')
446 self.assertRaises(SyntaxError, parser.compilest, st)
447
Guido van Rossumb5a755e2007-07-18 18:15:48 +0000[diff] [blame]448 def test_compile_badunicode(self):
Guido van Rossum7eb6ca52007-07-18 21:00:22 +0000[diff] [blame]449 st = parser.suite('a = "\\U12345678"')
Guido van Rossumb5a755e2007-07-18 18:15:48 +0000[diff] [blame]450 self.assertRaises(SyntaxError, parser.compilest, st)
Guido van Rossum7eb6ca52007-07-18 21:00:22 +0000[diff] [blame]451 st = parser.suite('a = "\\u1"')
Guido van Rossumb5a755e2007-07-18 18:15:48 +0000[diff] [blame]452 self.assertRaises(SyntaxError, parser.compilest, st)
453
Christian Heimes90c3d9b2008-02-23 13:18:03 +0000[diff] [blame]454class ParserStackLimitTestCase(unittest.TestCase):
455 """try to push the parser to/over it's limits.
456 see http://bugs.python.org/issue1881 for a discussion
457 """
458 def _nested_expression(self, level):
459 return "["*level+"]"*level
460
461 def test_deeply_nested_list(self):
462 # XXX used to be 99 levels in 2.x
463 e = self._nested_expression(93)
464 st = parser.expr(e)
465 st.compile()
466
467 def test_trigger_memory_error(self):
468 e = self._nested_expression(100)
Christian Heimesb186d002008-03-18 15:15:01 +0000[diff] [blame]469 print("Expecting 's_push: parser stack overflow' in next line",
470 file=sys.stderr)
Christian Heimes90c3d9b2008-02-23 13:18:03 +0000[diff] [blame]471 self.assertRaises(MemoryError, parser.expr, e)
472
Fred Drake2e2be372001-09-20 21:33:42 +0000[diff] [blame]473def test_main():
Benjamin Petersonee8712c2008-05-20 21:35:26 +0000[diff] [blame]474 support.run_unittest(
Walter Dörwald21d3a322003-05-01 17:45:56 +0000[diff] [blame]475 RoundtripLegalSyntaxTestCase,
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000[diff] [blame]476 IllegalSyntaxTestCase,
477 CompileTestCase,
Christian Heimes90c3d9b2008-02-23 13:18:03 +0000[diff] [blame]478 ParserStackLimitTestCase,
Walter Dörwald21d3a322003-05-01 17:45:56 +0000[diff] [blame]479 )
Fred Drake2e2be372001-09-20 21:33:42 +0000[diff] [blame]480
481
482if __name__ == "__main__":
483 test_main()