Gitiles
Code Review Sign In
gerrit-public.fairphone.software / platform / external / python / cpython3 / 3938a90a4a8008b8ef6f26ee4b820ce146211001 / . / Lib / test / test_parser.py
blob: c1a22ae0957568139110233bcf591a654790f44a [file] [log] [blame]
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]1import parser
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]2import unittest
Christian Heimesb186d002008-03-18 15:15:01 +0000[diff] [blame]3import sys
Benjamin Petersonee8712c2008-05-20 21:35:26 +0000[diff] [blame]4from test import support
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]5
6#
7# First, we test that we can generate trees from valid source fragments,
8# and that these valid trees are indeed allowed by the tree-loading side
9# of the parser module.
10#
11
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]12class RoundtripLegalSyntaxTestCase(unittest.TestCase):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000[diff] [blame]13
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]14 def roundtrip(self, f, s):
15 st1 = f(s)
16 t = st1.totuple()
17 try:
Fred Drake6e4f2c02001-07-17 19:33:25 +0000[diff] [blame]18 st2 = parser.sequence2st(t)
Guido van Rossumb940e112007-01-10 16:19:56 +0000[diff] [blame]19 except parser.ParserError as why:
Anthony Baxterc2a5a632004-08-02 06:10:11 +0000[diff] [blame]20 self.fail("could not roundtrip %r: %s" % (s, why))
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]21
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]22 self.assertEquals(t, st2.totuple(),
23 "could not re-generate syntax tree")
Fred Drake28f739a2000-08-25 22:42:40 +0000[diff] [blame]24
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]25 def check_expr(self, s):
26 self.roundtrip(parser.expr, s)
Fred Drake28f739a2000-08-25 22:42:40 +0000[diff] [blame]27
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]28 def check_suite(self, s):
29 self.roundtrip(parser.suite, s)
Fred Drake28f739a2000-08-25 22:42:40 +0000[diff] [blame]30
Fred Drakecf580c72001-07-17 03:01:29 +0000[diff] [blame]31 def test_yield_statement(self):
Tim Peters496563a2002-04-01 00:28:59 +0000[diff] [blame]32 self.check_suite("def f(): yield 1")
Phillip J. Eby0d6615f2005-08-02 00:46:46 +0000[diff] [blame]33 self.check_suite("def f(): yield")
34 self.check_suite("def f(): x += yield")
35 self.check_suite("def f(): x = yield 1")
36 self.check_suite("def f(): x = y = yield 1")
37 self.check_suite("def f(): x = yield")
38 self.check_suite("def f(): x = y = yield")
39 self.check_suite("def f(): 1 + (yield)*2")
40 self.check_suite("def f(): (yield 1)*2")
Tim Peters496563a2002-04-01 00:28:59 +0000[diff] [blame]41 self.check_suite("def f(): return; yield 1")
42 self.check_suite("def f(): yield 1; return")
43 self.check_suite("def f():\n"
Fred Drakecf580c72001-07-17 03:01:29 +0000[diff] [blame]44 " for x in range(30):\n"
45 " yield x\n")
Phillip J. Eby0d6615f2005-08-02 00:46:46 +0000[diff] [blame]46 self.check_suite("def f():\n"
47 " if (yield):\n"
48 " yield x\n")
Fred Drakecf580c72001-07-17 03:01:29 +0000[diff] [blame]49
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]50 def test_expressions(self):
51 self.check_expr("foo(1)")
52 self.check_expr("[1, 2, 3]")
53 self.check_expr("[x**3 for x in range(20)]")
54 self.check_expr("[x**3 for x in range(20) if x % 3]")
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000[diff] [blame]55 self.check_expr("[x**3 for x in range(20) if x % 2 if x % 3]")
56 self.check_expr("list(x**3 for x in range(20))")
57 self.check_expr("list(x**3 for x in range(20) if x % 3)")
58 self.check_expr("list(x**3 for x in range(20) if x % 2 if x % 3)")
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]59 self.check_expr("foo(*args)")
60 self.check_expr("foo(*args, **kw)")
61 self.check_expr("foo(**kw)")
62 self.check_expr("foo(key=value)")
63 self.check_expr("foo(key=value, *args)")
64 self.check_expr("foo(key=value, *args, **kw)")
65 self.check_expr("foo(key=value, **kw)")
66 self.check_expr("foo(a, b, c, *args)")
67 self.check_expr("foo(a, b, c, *args, **kw)")
68 self.check_expr("foo(a, b, c, **kw)")
Benjamin Peterson3938a902008-08-20 02:33:00 +0000[diff] [blame^]69 self.check_expr("foo(a, *args, keyword=23)")
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]70 self.check_expr("foo + bar")
Michael W. Hudson5e83b7a2003-01-29 14:20:23 +0000[diff] [blame]71 self.check_expr("foo - bar")
72 self.check_expr("foo * bar")
73 self.check_expr("foo / bar")
74 self.check_expr("foo // bar")
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]75 self.check_expr("lambda: 0")
76 self.check_expr("lambda x: 0")
77 self.check_expr("lambda *y: 0")
78 self.check_expr("lambda *y, **z: 0")
79 self.check_expr("lambda **z: 0")
80 self.check_expr("lambda x, y: 0")
81 self.check_expr("lambda foo=bar: 0")
82 self.check_expr("lambda foo=bar, spaz=nifty+spit: 0")
83 self.check_expr("lambda foo=bar, **z: 0")
84 self.check_expr("lambda foo=bar, blaz=blat+2, **z: 0")
85 self.check_expr("lambda foo=bar, blaz=blat+2, *y, **z: 0")
86 self.check_expr("lambda x, *y, **z: 0")
Raymond Hettinger354433a2004-05-19 08:20:33 +0000[diff] [blame]87 self.check_expr("(x for x in range(10))")
88 self.check_expr("foo(x for x in range(10))")
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]89
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]90 def test_simple_expression(self):
91 # expr_stmt
92 self.check_suite("a")
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]93
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]94 def test_simple_assignments(self):
95 self.check_suite("a = b")
96 self.check_suite("a = b = c = d = e")
Fred Drake28f739a2000-08-25 22:42:40 +0000[diff] [blame]97
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]98 def test_simple_augmented_assignments(self):
99 self.check_suite("a += b")
100 self.check_suite("a -= b")
101 self.check_suite("a *= b")
102 self.check_suite("a /= b")
Michael W. Hudson5e83b7a2003-01-29 14:20:23 +0000[diff] [blame]103 self.check_suite("a //= b")
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]104 self.check_suite("a %= b")
105 self.check_suite("a &= b")
106 self.check_suite("a |= b")
107 self.check_suite("a ^= b")
108 self.check_suite("a <<= b")
109 self.check_suite("a >>= b")
110 self.check_suite("a **= b")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000[diff] [blame]111
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]112 def test_function_defs(self):
113 self.check_suite("def f(): pass")
114 self.check_suite("def f(*args): pass")
115 self.check_suite("def f(*args, **kw): pass")
116 self.check_suite("def f(**kw): pass")
117 self.check_suite("def f(foo=bar): pass")
118 self.check_suite("def f(foo=bar, *args): pass")
119 self.check_suite("def f(foo=bar, *args, **kw): pass")
120 self.check_suite("def f(foo=bar, **kw): pass")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000[diff] [blame]121
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]122 self.check_suite("def f(a, b): pass")
123 self.check_suite("def f(a, b, *args): pass")
124 self.check_suite("def f(a, b, *args, **kw): pass")
125 self.check_suite("def f(a, b, **kw): pass")
126 self.check_suite("def f(a, b, foo=bar): pass")
127 self.check_suite("def f(a, b, foo=bar, *args): pass")
128 self.check_suite("def f(a, b, foo=bar, *args, **kw): pass")
129 self.check_suite("def f(a, b, foo=bar, **kw): pass")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000[diff] [blame]130
Anthony Baxterc2a5a632004-08-02 06:10:11 +0000[diff] [blame]131 self.check_suite("@staticmethod\n"
132 "def f(): pass")
133 self.check_suite("@staticmethod\n"
134 "@funcattrs(x, y)\n"
135 "def f(): pass")
136 self.check_suite("@funcattrs()\n"
137 "def f(): pass")
138
Brett Cannonf4189912005-04-09 02:30:16 +0000[diff] [blame]139 def test_class_defs(self):
140 self.check_suite("class foo():pass")
Guido van Rossumfc158e22007-11-15 19:17:28 +0000[diff] [blame]141 self.check_suite("class foo(object):pass")
Tim Peterse8906822005-04-20 17:45:13 +0000[diff] [blame]142
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]143 def test_import_from_statement(self):
144 self.check_suite("from sys.path import *")
145 self.check_suite("from sys.path import dirname")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000[diff] [blame]146 self.check_suite("from sys.path import (dirname)")
147 self.check_suite("from sys.path import (dirname,)")
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]148 self.check_suite("from sys.path import dirname as my_dirname")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000[diff] [blame]149 self.check_suite("from sys.path import (dirname as my_dirname)")
150 self.check_suite("from sys.path import (dirname as my_dirname,)")
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]151 self.check_suite("from sys.path import dirname, basename")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000[diff] [blame]152 self.check_suite("from sys.path import (dirname, basename)")
153 self.check_suite("from sys.path import (dirname, basename,)")
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]154 self.check_suite(
155 "from sys.path import dirname as my_dirname, basename")
156 self.check_suite(
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000[diff] [blame]157 "from sys.path import (dirname as my_dirname, basename)")
158 self.check_suite(
159 "from sys.path import (dirname as my_dirname, basename,)")
160 self.check_suite(
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]161 "from sys.path import dirname, basename as my_basename")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000[diff] [blame]162 self.check_suite(
163 "from sys.path import (dirname, basename as my_basename)")
164 self.check_suite(
165 "from sys.path import (dirname, basename as my_basename,)")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000[diff] [blame]166
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]167 def test_basic_import_statement(self):
168 self.check_suite("import sys")
169 self.check_suite("import sys as system")
170 self.check_suite("import sys, math")
171 self.check_suite("import sys as system, math")
172 self.check_suite("import sys, math as my_math")
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]173
Neal Norwitz9caf9c02003-02-10 01:54:06 +0000[diff] [blame]174 def test_pep263(self):
175 self.check_suite("# -*- coding: iso-8859-1 -*-\n"
176 "pass\n")
177
178 def test_assert(self):
179 self.check_suite("assert alo < ahi and blo < bhi\n")
180
Thomas Wouters89f507f2006-12-13 04:49:30 +0000[diff] [blame]181 def test_position(self):
182 # An absolutely minimal test of position information. Better
183 # tests would be a big project.
184 code = "def f(x):\n return x + 1\n"
185 st1 = parser.suite(code)
186 st2 = st1.totuple(line_info=1, col_info=1)
187
188 def walk(tree):
189 node_type = tree[0]
190 next = tree[1]
191 if isinstance(next, tuple):
192 for elt in tree[1:]:
193 for x in walk(elt):
194 yield x
195 else:
196 yield tree
197
198 terminals = list(walk(st2))
199 self.assertEqual([
200 (1, 'def', 1, 0),
201 (1, 'f', 1, 4),
202 (7, '(', 1, 5),
203 (1, 'x', 1, 6),
204 (8, ')', 1, 7),
205 (11, ':', 1, 8),
206 (4, '', 1, 9),
207 (5, '', 2, -1),
208 (1, 'return', 2, 4),
209 (1, 'x', 2, 11),
210 (14, '+', 2, 13),
211 (2, '1', 2, 15),
212 (4, '', 2, 16),
213 (6, '', 2, -1),
214 (4, '', 2, -1),
215 (0, '', 2, -1)],
216 terminals)
217
218
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]219#
220# Second, we take *invalid* trees and make sure we get ParserError
221# rejections for them.
222#
223
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]224class IllegalSyntaxTestCase(unittest.TestCase):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000[diff] [blame]225
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]226 def check_bad_tree(self, tree, label):
227 try:
Fred Drake6e4f2c02001-07-17 19:33:25 +0000[diff] [blame]228 parser.sequence2st(tree)
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]229 except parser.ParserError:
230 pass
231 else:
232 self.fail("did not detect invalid tree for %r" % label)
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]233
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]234 def test_junk(self):
235 # not even remotely valid:
236 self.check_bad_tree((1, 2, 3), "<junk>")
237
Fred Drakecf580c72001-07-17 03:01:29 +0000[diff] [blame]238 def test_illegal_yield_1(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000[diff] [blame]239 # Illegal yield statement: def f(): return 1; yield 1
Fred Drakecf580c72001-07-17 03:01:29 +0000[diff] [blame]240 tree = \
241 (257,
242 (264,
243 (285,
244 (259,
245 (1, 'def'),
246 (1, 'f'),
247 (260, (7, '('), (8, ')')),
248 (11, ':'),
249 (291,
250 (4, ''),
251 (5, ''),
252 (264,
253 (265,
254 (266,
255 (272,
256 (275,
257 (1, 'return'),
258 (313,
259 (292,
260 (293,
261 (294,
262 (295,
263 (297,
264 (298,
265 (299,
266 (300,
267 (301,
268 (302, (303, (304, (305, (2, '1')))))))))))))))))),
269 (264,
270 (265,
271 (266,
272 (272,
273 (276,
274 (1, 'yield'),
275 (313,
276 (292,
277 (293,
278 (294,
279 (295,
280 (297,
281 (298,
282 (299,
283 (300,
284 (301,
285 (302,
286 (303, (304, (305, (2, '1')))))))))))))))))),
287 (4, ''))),
288 (6, ''))))),
289 (4, ''),
290 (0, ''))))
291 self.check_bad_tree(tree, "def f():\n return 1\n yield 1")
292
293 def test_illegal_yield_2(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000[diff] [blame]294 # Illegal return in generator: def f(): return 1; yield 1
Fred Drakecf580c72001-07-17 03:01:29 +0000[diff] [blame]295 tree = \
296 (257,
297 (264,
298 (265,
299 (266,
300 (278,
301 (1, 'from'),
302 (281, (1, '__future__')),
303 (1, 'import'),
304 (279, (1, 'generators')))),
305 (4, ''))),
306 (264,
307 (285,
308 (259,
309 (1, 'def'),
310 (1, 'f'),
311 (260, (7, '('), (8, ')')),
312 (11, ':'),
313 (291,
314 (4, ''),
315 (5, ''),
316 (264,
317 (265,
318 (266,
319 (272,
320 (275,
321 (1, 'return'),
322 (313,
323 (292,
324 (293,
325 (294,
326 (295,
327 (297,
328 (298,
329 (299,
330 (300,
331 (301,
332 (302, (303, (304, (305, (2, '1')))))))))))))))))),
333 (264,
334 (265,
335 (266,
336 (272,
337 (276,
338 (1, 'yield'),
339 (313,
340 (292,
341 (293,
342 (294,
343 (295,
344 (297,
345 (298,
346 (299,
347 (300,
348 (301,
349 (302,
350 (303, (304, (305, (2, '1')))))))))))))))))),
351 (4, ''))),
352 (6, ''))))),
353 (4, ''),
354 (0, ''))))
355 self.check_bad_tree(tree, "def f():\n return 1\n yield 1")
356
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]357 def test_a_comma_comma_c(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000[diff] [blame]358 # Illegal input: a,,c
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]359 tree = \
360 (258,
361 (311,
362 (290,
363 (291,
364 (292,
365 (293,
366 (295,
367 (296,
368 (297,
369 (298, (299, (300, (301, (302, (303, (1, 'a')))))))))))))),
370 (12, ','),
371 (12, ','),
372 (290,
373 (291,
374 (292,
375 (293,
376 (295,
377 (296,
378 (297,
379 (298, (299, (300, (301, (302, (303, (1, 'c'))))))))))))))),
380 (4, ''),
381 (0, ''))
382 self.check_bad_tree(tree, "a,,c")
383
384 def test_illegal_operator(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000[diff] [blame]385 # Illegal input: a $= b
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]386 tree = \
387 (257,
388 (264,
389 (265,
390 (266,
391 (267,
392 (312,
393 (291,
394 (292,
395 (293,
396 (294,
397 (296,
398 (297,
399 (298,
400 (299,
401 (300, (301, (302, (303, (304, (1, 'a'))))))))))))))),
402 (268, (37, '$=')),
403 (312,
404 (291,
405 (292,
406 (293,
407 (294,
408 (296,
409 (297,
410 (298,
411 (299,
412 (300, (301, (302, (303, (304, (1, 'b'))))))))))))))))),
413 (4, ''))),
414 (0, ''))
415 self.check_bad_tree(tree, "a $= b")
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]416
Neal Norwitz9caf9c02003-02-10 01:54:06 +0000[diff] [blame]417 def test_malformed_global(self):
418 #doesn't have global keyword in ast
419 tree = (257,
420 (264,
421 (265,
422 (266,
423 (282, (1, 'foo'))), (4, ''))),
424 (4, ''),
Tim Petersf2715e02003-02-19 02:35:07 +0000[diff] [blame]425 (0, ''))
Neal Norwitz9caf9c02003-02-10 01:54:06 +0000[diff] [blame]426 self.check_bad_tree(tree, "malformed global ast")
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]427
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000[diff] [blame]428
429class CompileTestCase(unittest.TestCase):
430
431 # These tests are very minimal. :-(
432
433 def test_compile_expr(self):
434 st = parser.expr('2 + 3')
435 code = parser.compilest(st)
436 self.assertEquals(eval(code), 5)
437
438 def test_compile_suite(self):
439 st = parser.suite('x = 2; y = x + 3')
440 code = parser.compilest(st)
441 globs = {}
Georg Brandl7cae87c2006-09-06 06:51:57 +0000[diff] [blame]442 exec(code, globs)
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000[diff] [blame]443 self.assertEquals(globs['y'], 5)
444
445 def test_compile_error(self):
446 st = parser.suite('1 = 3 + 4')
447 self.assertRaises(SyntaxError, parser.compilest, st)
448
Guido van Rossumb5a755e2007-07-18 18:15:48 +0000[diff] [blame]449 def test_compile_badunicode(self):
Guido van Rossum7eb6ca52007-07-18 21:00:22 +0000[diff] [blame]450 st = parser.suite('a = "\\U12345678"')
Guido van Rossumb5a755e2007-07-18 18:15:48 +0000[diff] [blame]451 self.assertRaises(SyntaxError, parser.compilest, st)
Guido van Rossum7eb6ca52007-07-18 21:00:22 +0000[diff] [blame]452 st = parser.suite('a = "\\u1"')
Guido van Rossumb5a755e2007-07-18 18:15:48 +0000[diff] [blame]453 self.assertRaises(SyntaxError, parser.compilest, st)
454
Christian Heimes90c3d9b2008-02-23 13:18:03 +0000[diff] [blame]455class ParserStackLimitTestCase(unittest.TestCase):
456 """try to push the parser to/over it's limits.
457 see http://bugs.python.org/issue1881 for a discussion
458 """
459 def _nested_expression(self, level):
460 return "["*level+"]"*level
461
462 def test_deeply_nested_list(self):
463 # XXX used to be 99 levels in 2.x
464 e = self._nested_expression(93)
465 st = parser.expr(e)
466 st.compile()
467
468 def test_trigger_memory_error(self):
469 e = self._nested_expression(100)
Christian Heimesb186d002008-03-18 15:15:01 +0000[diff] [blame]470 print("Expecting 's_push: parser stack overflow' in next line",
471 file=sys.stderr)
Christian Heimes90c3d9b2008-02-23 13:18:03 +0000[diff] [blame]472 self.assertRaises(MemoryError, parser.expr, e)
473
Fred Drake2e2be372001-09-20 21:33:42 +0000[diff] [blame]474def test_main():
Benjamin Petersonee8712c2008-05-20 21:35:26 +0000[diff] [blame]475 support.run_unittest(
Walter Dörwald21d3a322003-05-01 17:45:56 +0000[diff] [blame]476 RoundtripLegalSyntaxTestCase,
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000[diff] [blame]477 IllegalSyntaxTestCase,
478 CompileTestCase,
Christian Heimes90c3d9b2008-02-23 13:18:03 +0000[diff] [blame]479 ParserStackLimitTestCase,
Walter Dörwald21d3a322003-05-01 17:45:56 +0000[diff] [blame]480 )
Fred Drake2e2be372001-09-20 21:33:42 +0000[diff] [blame]481
482
483if __name__ == "__main__":
484 test_main()