Fix 'gather' rules in the python parser generator (GH-22021)
Currently, empty sequences in gather rules make the conditional for
gather rules fail as empty sequences evaluate as "False". We need to
explicitly check for "None" (the failure condition) to avoid false
negatives.
diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py
index 5b4e964..bcfee3f 100644
--- a/Lib/test/test_peg_generator/test_pegen.py
+++ b/Lib/test/test_peg_generator/test_pegen.py
@@ -74,7 +74,7 @@
"Rule('term', 'int', Rhs([Alt([NamedItem(None, NameLeaf('NUMBER'))])]))"
)
- def test_repeat_with_separator_rules(self) -> None:
+ def test_gather(self) -> None:
grammar = """
start: ','.thing+ NEWLINE
thing: NUMBER
@@ -85,6 +85,20 @@
"Rule('start', None, Rhs([Alt([NamedItem(None, Gather(StringLeaf(\"','\"), NameLeaf('thing'"
))
self.assertEqual(str(rules["thing"]), "thing: NUMBER")
+ parser_class = make_parser(grammar)
+ node = parse_string("42\n", parser_class)
+ assert node == [
+ [[TokenInfo(NUMBER, string="42", start=(1, 0), end=(1, 2), line="42\n")]],
+ TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="42\n"),
+ ]
+ node = parse_string("1, 2\n", parser_class)
+ assert node == [
+ [
+ [TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1, 2\n")],
+ [TokenInfo(NUMBER, string="2", start=(1, 3), end=(1, 4), line="1, 2\n")],
+ ],
+ TokenInfo(NEWLINE, string="\n", start=(1, 4), end=(1, 5), line="1, 2\n"),
+ ]
def test_expr_grammar(self) -> None:
grammar = """
diff --git a/Tools/peg_generator/pegen/python_generator.py b/Tools/peg_generator/pegen/python_generator.py
index 45a7597..b786de7 100644
--- a/Tools/peg_generator/pegen/python_generator.py
+++ b/Tools/peg_generator/pegen/python_generator.py
@@ -217,6 +217,9 @@
else:
self.print("and")
self.visit(item)
+ if is_gather:
+ self.print("is not None")
+
self.print("):")
with self.indent():
action = node.action