Gitiles
Code Review Sign In
gerrit-public.fairphone.software / platform / external / python / cpython2 / bd7bda4345e50fb857270f9d2010f0407136630e / . / Lib / test / test_parser.py
blob: 8cb68eee6fc6aa07c302690a01c1c1af9d695012 [file] [log] [blame]
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]1import parser
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]2import unittest
Martin v. Löwis66e26632008-03-18 13:16:05 +0000[diff] [blame]3import sys
Barry Warsaw04f357c2002-07-23 19:04:11 +0000[diff] [blame]4from test import test_support
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]5
6#
7# First, we test that we can generate trees from valid source fragments,
8# and that these valid trees are indeed allowed by the tree-loading side
9# of the parser module.
10#
11
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]12class RoundtripLegalSyntaxTestCase(unittest.TestCase):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000[diff] [blame]13
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]14 def roundtrip(self, f, s):
15 st1 = f(s)
16 t = st1.totuple()
17 try:
Fred Drake6e4f2c02001-07-17 19:33:25 +0000[diff] [blame]18 st2 = parser.sequence2st(t)
Anthony Baxterc2a5a632004-08-02 06:10:11 +0000[diff] [blame]19 except parser.ParserError, why:
20 self.fail("could not roundtrip %r: %s" % (s, why))
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]21
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]22 self.assertEquals(t, st2.totuple(),
23 "could not re-generate syntax tree")
Fred Drake28f739a2000-08-25 22:42:40 +0000[diff] [blame]24
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]25 def check_expr(self, s):
26 self.roundtrip(parser.expr, s)
Fred Drake28f739a2000-08-25 22:42:40 +0000[diff] [blame]27
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]28 def check_suite(self, s):
29 self.roundtrip(parser.suite, s)
Fred Drake28f739a2000-08-25 22:42:40 +0000[diff] [blame]30
Fred Drakecf580c72001-07-17 03:01:29 +0000[diff] [blame]31 def test_yield_statement(self):
Tim Peters496563a2002-04-01 00:28:59 +0000[diff] [blame]32 self.check_suite("def f(): yield 1")
Phillip J. Eby0d6615f2005-08-02 00:46:46 +0000[diff] [blame]33 self.check_suite("def f(): yield")
34 self.check_suite("def f(): x += yield")
35 self.check_suite("def f(): x = yield 1")
36 self.check_suite("def f(): x = y = yield 1")
37 self.check_suite("def f(): x = yield")
38 self.check_suite("def f(): x = y = yield")
39 self.check_suite("def f(): 1 + (yield)*2")
40 self.check_suite("def f(): (yield 1)*2")
Tim Peters496563a2002-04-01 00:28:59 +0000[diff] [blame]41 self.check_suite("def f(): return; yield 1")
42 self.check_suite("def f(): yield 1; return")
43 self.check_suite("def f():\n"
Fred Drakecf580c72001-07-17 03:01:29 +0000[diff] [blame]44 " for x in range(30):\n"
45 " yield x\n")
Phillip J. Eby0d6615f2005-08-02 00:46:46 +0000[diff] [blame]46 self.check_suite("def f():\n"
47 " if (yield):\n"
48 " yield x\n")
Fred Drakecf580c72001-07-17 03:01:29 +0000[diff] [blame]49
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]50 def test_expressions(self):
51 self.check_expr("foo(1)")
52 self.check_expr("[1, 2, 3]")
53 self.check_expr("[x**3 for x in range(20)]")
54 self.check_expr("[x**3 for x in range(20) if x % 3]")
Neal Norwitzd3a91622006-04-12 05:27:46 +0000[diff] [blame]55 self.check_expr("[x**3 for x in range(20) if x % 2 if x % 3]")
56 self.check_expr("list(x**3 for x in range(20))")
57 self.check_expr("list(x**3 for x in range(20) if x % 3)")
58 self.check_expr("list(x**3 for x in range(20) if x % 2 if x % 3)")
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]59 self.check_expr("foo(*args)")
60 self.check_expr("foo(*args, **kw)")
61 self.check_expr("foo(**kw)")
62 self.check_expr("foo(key=value)")
63 self.check_expr("foo(key=value, *args)")
64 self.check_expr("foo(key=value, *args, **kw)")
65 self.check_expr("foo(key=value, **kw)")
66 self.check_expr("foo(a, b, c, *args)")
67 self.check_expr("foo(a, b, c, *args, **kw)")
68 self.check_expr("foo(a, b, c, **kw)")
69 self.check_expr("foo + bar")
Michael W. Hudson5e83b7a2003-01-29 14:20:23 +0000[diff] [blame]70 self.check_expr("foo - bar")
71 self.check_expr("foo * bar")
72 self.check_expr("foo / bar")
73 self.check_expr("foo // bar")
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]74 self.check_expr("lambda: 0")
75 self.check_expr("lambda x: 0")
76 self.check_expr("lambda *y: 0")
77 self.check_expr("lambda *y, **z: 0")
78 self.check_expr("lambda **z: 0")
79 self.check_expr("lambda x, y: 0")
80 self.check_expr("lambda foo=bar: 0")
81 self.check_expr("lambda foo=bar, spaz=nifty+spit: 0")
82 self.check_expr("lambda foo=bar, **z: 0")
83 self.check_expr("lambda foo=bar, blaz=blat+2, **z: 0")
84 self.check_expr("lambda foo=bar, blaz=blat+2, *y, **z: 0")
85 self.check_expr("lambda x, *y, **z: 0")
Raymond Hettinger354433a2004-05-19 08:20:33 +0000[diff] [blame]86 self.check_expr("(x for x in range(10))")
87 self.check_expr("foo(x for x in range(10))")
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]88
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]89 def test_print(self):
90 self.check_suite("print")
91 self.check_suite("print 1")
92 self.check_suite("print 1,")
93 self.check_suite("print >>fp")
94 self.check_suite("print >>fp, 1")
95 self.check_suite("print >>fp, 1,")
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]96
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]97 def test_simple_expression(self):
98 # expr_stmt
99 self.check_suite("a")
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]100
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]101 def test_simple_assignments(self):
102 self.check_suite("a = b")
103 self.check_suite("a = b = c = d = e")
Fred Drake28f739a2000-08-25 22:42:40 +0000[diff] [blame]104
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]105 def test_simple_augmented_assignments(self):
106 self.check_suite("a += b")
107 self.check_suite("a -= b")
108 self.check_suite("a *= b")
109 self.check_suite("a /= b")
Michael W. Hudson5e83b7a2003-01-29 14:20:23 +0000[diff] [blame]110 self.check_suite("a //= b")
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]111 self.check_suite("a %= b")
112 self.check_suite("a &= b")
113 self.check_suite("a |= b")
114 self.check_suite("a ^= b")
115 self.check_suite("a <<= b")
116 self.check_suite("a >>= b")
117 self.check_suite("a **= b")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000[diff] [blame]118
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]119 def test_function_defs(self):
120 self.check_suite("def f(): pass")
121 self.check_suite("def f(*args): pass")
122 self.check_suite("def f(*args, **kw): pass")
123 self.check_suite("def f(**kw): pass")
124 self.check_suite("def f(foo=bar): pass")
125 self.check_suite("def f(foo=bar, *args): pass")
126 self.check_suite("def f(foo=bar, *args, **kw): pass")
127 self.check_suite("def f(foo=bar, **kw): pass")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000[diff] [blame]128
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]129 self.check_suite("def f(a, b): pass")
130 self.check_suite("def f(a, b, *args): pass")
131 self.check_suite("def f(a, b, *args, **kw): pass")
132 self.check_suite("def f(a, b, **kw): pass")
133 self.check_suite("def f(a, b, foo=bar): pass")
134 self.check_suite("def f(a, b, foo=bar, *args): pass")
135 self.check_suite("def f(a, b, foo=bar, *args, **kw): pass")
136 self.check_suite("def f(a, b, foo=bar, **kw): pass")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000[diff] [blame]137
Anthony Baxterc2a5a632004-08-02 06:10:11 +0000[diff] [blame]138 self.check_suite("@staticmethod\n"
139 "def f(): pass")
140 self.check_suite("@staticmethod\n"
141 "@funcattrs(x, y)\n"
142 "def f(): pass")
143 self.check_suite("@funcattrs()\n"
144 "def f(): pass")
145
Brett Cannonf4189912005-04-09 02:30:16 +0000[diff] [blame]146 def test_class_defs(self):
147 self.check_suite("class foo():pass")
Tim Peterse8906822005-04-20 17:45:13 +0000[diff] [blame]148
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]149 def test_import_from_statement(self):
150 self.check_suite("from sys.path import *")
151 self.check_suite("from sys.path import dirname")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000[diff] [blame]152 self.check_suite("from sys.path import (dirname)")
153 self.check_suite("from sys.path import (dirname,)")
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]154 self.check_suite("from sys.path import dirname as my_dirname")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000[diff] [blame]155 self.check_suite("from sys.path import (dirname as my_dirname)")
156 self.check_suite("from sys.path import (dirname as my_dirname,)")
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]157 self.check_suite("from sys.path import dirname, basename")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000[diff] [blame]158 self.check_suite("from sys.path import (dirname, basename)")
159 self.check_suite("from sys.path import (dirname, basename,)")
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]160 self.check_suite(
161 "from sys.path import dirname as my_dirname, basename")
162 self.check_suite(
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000[diff] [blame]163 "from sys.path import (dirname as my_dirname, basename)")
164 self.check_suite(
165 "from sys.path import (dirname as my_dirname, basename,)")
166 self.check_suite(
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]167 "from sys.path import dirname, basename as my_basename")
Anthony Baxter1a4ddae2004-08-31 10:07:13 +0000[diff] [blame]168 self.check_suite(
169 "from sys.path import (dirname, basename as my_basename)")
170 self.check_suite(
171 "from sys.path import (dirname, basename as my_basename,)")
Fred Drakee3fb18c2001-01-07 06:02:19 +0000[diff] [blame]172
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]173 def test_basic_import_statement(self):
174 self.check_suite("import sys")
175 self.check_suite("import sys as system")
176 self.check_suite("import sys, math")
177 self.check_suite("import sys as system, math")
178 self.check_suite("import sys, math as my_math")
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]179
Neal Norwitz9caf9c02003-02-10 01:54:06 +0000[diff] [blame]180 def test_pep263(self):
181 self.check_suite("# -*- coding: iso-8859-1 -*-\n"
182 "pass\n")
183
184 def test_assert(self):
185 self.check_suite("assert alo < ahi and blo < bhi\n")
186
Jeremy Hylton60e96f62006-08-22 20:46:00 +0000[diff] [blame]187 def test_position(self):
188 # An absolutely minimal test of position information. Better
189 # tests would be a big project.
190 code = "def f(x):\n return x + 1\n"
191 st1 = parser.suite(code)
192 st2 = st1.totuple(line_info=1, col_info=1)
193
194 def walk(tree):
195 node_type = tree[0]
196 next = tree[1]
197 if isinstance(next, tuple):
198 for elt in tree[1:]:
199 for x in walk(elt):
200 yield x
201 else:
202 yield tree
Tim Peters147f9ae2006-08-25 22:05:39 +0000[diff] [blame]203
Jeremy Hylton60e96f62006-08-22 20:46:00 +0000[diff] [blame]204 terminals = list(walk(st2))
205 self.assertEqual([
206 (1, 'def', 1, 0),
207 (1, 'f', 1, 4),
208 (7, '(', 1, 5),
209 (1, 'x', 1, 6),
210 (8, ')', 1, 7),
211 (11, ':', 1, 8),
212 (4, '', 1, 9),
213 (5, '', 2, -1),
214 (1, 'return', 2, 4),
215 (1, 'x', 2, 11),
216 (14, '+', 2, 13),
217 (2, '1', 2, 15),
218 (4, '', 2, 16),
219 (6, '', 2, -1),
220 (4, '', 2, -1),
221 (0, '', 2, -1)],
222 terminals)
223
224
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]225#
226# Second, we take *invalid* trees and make sure we get ParserError
227# rejections for them.
228#
229
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]230class IllegalSyntaxTestCase(unittest.TestCase):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000[diff] [blame]231
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]232 def check_bad_tree(self, tree, label):
233 try:
Fred Drake6e4f2c02001-07-17 19:33:25 +0000[diff] [blame]234 parser.sequence2st(tree)
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]235 except parser.ParserError:
236 pass
237 else:
238 self.fail("did not detect invalid tree for %r" % label)
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]239
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]240 def test_junk(self):
241 # not even remotely valid:
242 self.check_bad_tree((1, 2, 3), "<junk>")
243
Fred Drakecf580c72001-07-17 03:01:29 +0000[diff] [blame]244 def test_illegal_yield_1(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000[diff] [blame]245 # Illegal yield statement: def f(): return 1; yield 1
Fred Drakecf580c72001-07-17 03:01:29 +0000[diff] [blame]246 tree = \
247 (257,
248 (264,
249 (285,
250 (259,
251 (1, 'def'),
252 (1, 'f'),
253 (260, (7, '('), (8, ')')),
254 (11, ':'),
255 (291,
256 (4, ''),
257 (5, ''),
258 (264,
259 (265,
260 (266,
261 (272,
262 (275,
263 (1, 'return'),
264 (313,
265 (292,
266 (293,
267 (294,
268 (295,
269 (297,
270 (298,
271 (299,
272 (300,
273 (301,
274 (302, (303, (304, (305, (2, '1')))))))))))))))))),
275 (264,
276 (265,
277 (266,
278 (272,
279 (276,
280 (1, 'yield'),
281 (313,
282 (292,
283 (293,
284 (294,
285 (295,
286 (297,
287 (298,
288 (299,
289 (300,
290 (301,
291 (302,
292 (303, (304, (305, (2, '1')))))))))))))))))),
293 (4, ''))),
294 (6, ''))))),
295 (4, ''),
296 (0, ''))))
297 self.check_bad_tree(tree, "def f():\n return 1\n yield 1")
298
299 def test_illegal_yield_2(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000[diff] [blame]300 # Illegal return in generator: def f(): return 1; yield 1
Fred Drakecf580c72001-07-17 03:01:29 +0000[diff] [blame]301 tree = \
302 (257,
303 (264,
304 (265,
305 (266,
306 (278,
307 (1, 'from'),
308 (281, (1, '__future__')),
309 (1, 'import'),
310 (279, (1, 'generators')))),
311 (4, ''))),
312 (264,
313 (285,
314 (259,
315 (1, 'def'),
316 (1, 'f'),
317 (260, (7, '('), (8, ')')),
318 (11, ':'),
319 (291,
320 (4, ''),
321 (5, ''),
322 (264,
323 (265,
324 (266,
325 (272,
326 (275,
327 (1, 'return'),
328 (313,
329 (292,
330 (293,
331 (294,
332 (295,
333 (297,
334 (298,
335 (299,
336 (300,
337 (301,
338 (302, (303, (304, (305, (2, '1')))))))))))))))))),
339 (264,
340 (265,
341 (266,
342 (272,
343 (276,
344 (1, 'yield'),
345 (313,
346 (292,
347 (293,
348 (294,
349 (295,
350 (297,
351 (298,
352 (299,
353 (300,
354 (301,
355 (302,
356 (303, (304, (305, (2, '1')))))))))))))))))),
357 (4, ''))),
358 (6, ''))))),
359 (4, ''),
360 (0, ''))))
361 self.check_bad_tree(tree, "def f():\n return 1\n yield 1")
362
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]363 def test_print_chevron_comma(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000[diff] [blame]364 # Illegal input: print >>fp,
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]365 tree = \
366 (257,
367 (264,
368 (265,
369 (266,
370 (268,
371 (1, 'print'),
372 (35, '>>'),
373 (290,
374 (291,
375 (292,
376 (293,
377 (295,
378 (296,
379 (297,
380 (298, (299, (300, (301, (302, (303, (1, 'fp')))))))))))))),
381 (12, ','))),
382 (4, ''))),
383 (0, ''))
384 self.check_bad_tree(tree, "print >>fp,")
385
386 def test_a_comma_comma_c(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000[diff] [blame]387 # Illegal input: a,,c
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]388 tree = \
389 (258,
390 (311,
391 (290,
392 (291,
393 (292,
394 (293,
395 (295,
396 (296,
397 (297,
398 (298, (299, (300, (301, (302, (303, (1, 'a')))))))))))))),
399 (12, ','),
400 (12, ','),
401 (290,
402 (291,
403 (292,
404 (293,
405 (295,
406 (296,
407 (297,
408 (298, (299, (300, (301, (302, (303, (1, 'c'))))))))))))))),
409 (4, ''),
410 (0, ''))
411 self.check_bad_tree(tree, "a,,c")
412
413 def test_illegal_operator(self):
Guido van Rossum32c2ae72002-08-22 19:45:32 +0000[diff] [blame]414 # Illegal input: a $= b
Fred Drake58422e52001-06-04 03:56:24 +0000[diff] [blame]415 tree = \
416 (257,
417 (264,
418 (265,
419 (266,
420 (267,
421 (312,
422 (291,
423 (292,
424 (293,
425 (294,
426 (296,
427 (297,
428 (298,
429 (299,
430 (300, (301, (302, (303, (304, (1, 'a'))))))))))))))),
431 (268, (37, '$=')),
432 (312,
433 (291,
434 (292,
435 (293,
436 (294,
437 (296,
438 (297,
439 (298,
440 (299,
441 (300, (301, (302, (303, (304, (1, 'b'))))))))))))))))),
442 (4, ''))),
443 (0, ''))
444 self.check_bad_tree(tree, "a $= b")
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]445
Neal Norwitz9caf9c02003-02-10 01:54:06 +0000[diff] [blame]446 def test_malformed_global(self):
447 #doesn't have global keyword in ast
448 tree = (257,
449 (264,
450 (265,
451 (266,
452 (282, (1, 'foo'))), (4, ''))),
453 (4, ''),
Tim Petersf2715e02003-02-19 02:35:07 +0000[diff] [blame]454 (0, ''))
Neal Norwitz9caf9c02003-02-10 01:54:06 +0000[diff] [blame]455 self.check_bad_tree(tree, "malformed global ast")
Fred Drake79ca79d2000-08-21 22:30:53 +0000[diff] [blame]456
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000[diff] [blame]457
458class CompileTestCase(unittest.TestCase):
459
460 # These tests are very minimal. :-(
461
462 def test_compile_expr(self):
463 st = parser.expr('2 + 3')
464 code = parser.compilest(st)
465 self.assertEquals(eval(code), 5)
466
467 def test_compile_suite(self):
468 st = parser.suite('x = 2; y = x + 3')
469 code = parser.compilest(st)
470 globs = {}
471 exec code in globs
472 self.assertEquals(globs['y'], 5)
473
474 def test_compile_error(self):
475 st = parser.suite('1 = 3 + 4')
476 self.assertRaises(SyntaxError, parser.compilest, st)
477
Guido van Rossumb6ac23c2007-07-18 17:19:14 +0000[diff] [blame]478 def test_compile_badunicode(self):
479 st = parser.suite('a = u"\U12345678"')
480 self.assertRaises(SyntaxError, parser.compilest, st)
481 st = parser.suite('a = u"\u1"')
482 self.assertRaises(SyntaxError, parser.compilest, st)
483
Facundo Batistafc2d0102008-02-23 12:01:13 +0000[diff] [blame]484class ParserStackLimitTestCase(unittest.TestCase):
485 """try to push the parser to/over it's limits.
486 see http://bugs.python.org/issue1881 for a discussion
487 """
488 def _nested_expression(self, level):
489 return "["*level+"]"*level
490
491 def test_deeply_nested_list(self):
492 e = self._nested_expression(99)
493 st = parser.expr(e)
494 st.compile()
495
496 def test_trigger_memory_error(self):
497 e = self._nested_expression(100)
Martin v. Löwis66e26632008-03-18 13:16:05 +0000[diff] [blame]498 print >>sys.stderr, "Expecting 's_push: parser stack overflow' in next line"
Facundo Batistafc2d0102008-02-23 12:01:13 +0000[diff] [blame]499 self.assertRaises(MemoryError, parser.expr, e)
500
Fred Drake2e2be372001-09-20 21:33:42 +0000[diff] [blame]501def test_main():
Walter Dörwald21d3a322003-05-01 17:45:56 +0000[diff] [blame]502 test_support.run_unittest(
503 RoundtripLegalSyntaxTestCase,
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000[diff] [blame]504 IllegalSyntaxTestCase,
505 CompileTestCase,
Facundo Batistafc2d0102008-02-23 12:01:13 +0000[diff] [blame]506 ParserStackLimitTestCase,
Walter Dörwald21d3a322003-05-01 17:45:56 +0000[diff] [blame]507 )
Fred Drake2e2be372001-09-20 21:33:42 +0000[diff] [blame]508
509
510if __name__ == "__main__":
511 test_main()