[3.9] bpo-42123: Run the parser two times and only enable invalid rules on the second run (GH-22111) (GH-23011)
* Implement running the parser a second time for the errors messages
The first parser run is only responsible for detecting whether
there is a `SyntaxError` or not. If there isn't the AST gets returned.
Otherwise, the parser is run a second time with all the `invalid_*`
rules enabled so that all the customized error messages get produced.
(cherry picked from commit bca701403253379409dece03053dbd739c0bd059)
diff --git a/Parser/pegen/pegen.c b/Parser/pegen/pegen.c
index a7643fc..78891af 100644
--- a/Parser/pegen/pegen.c
+++ b/Parser/pegen/pegen.c
@@ -1101,15 +1101,28 @@
p->feature_version = feature_version;
p->known_err_token = NULL;
p->level = 0;
+ p->call_invalid_rules = 0;
return p;
}
+static void
+reset_parser_state(Parser *p)
+{
+ for (int i = 0; i < p->fill; i++) {
+ p->tokens[i]->memo = NULL;
+ }
+ p->mark = 0;
+ p->call_invalid_rules = 1;
+}
+
void *
_PyPegen_run_parser(Parser *p)
{
void *res = _PyPegen_parse(p);
if (res == NULL) {
+ reset_parser_state(p);
+ _PyPegen_parse(p);
if (PyErr_Occurred()) {
return NULL;
}