bpo-40334: Catch E_EOF error, when the tokenizer returns ERRORTOKEN (GH-19743)
An E_EOF error was only being caught after the parser exited before this commit. There are some cases though, where the tokenizer returns ERRORTOKEN *and* has set an E_EOF error (like when EOF directly follows a line continuation character) which weren't correctly handled before.
diff --git a/Parser/pegen/pegen.c b/Parser/pegen/pegen.c
index d75267b..6f78d8c 100644
--- a/Parser/pegen/pegen.c
+++ b/Parser/pegen/pegen.c
@@ -344,13 +344,16 @@
break;
case E_BADPREFIX:
return tokenizer_error_with_col_offset(p,
- PyExc_SyntaxError, "invalid string prefix");
+ errtype, "invalid string prefix");
case E_EOFS:
return tokenizer_error_with_col_offset(p,
- PyExc_SyntaxError, "EOF while scanning triple-quoted string literal");
+ errtype, "EOF while scanning triple-quoted string literal");
case E_EOLS:
return tokenizer_error_with_col_offset(p,
- PyExc_SyntaxError, "EOL while scanning string literal");
+ errtype, "EOL while scanning string literal");
+ case E_EOF:
+ return tokenizer_error_with_col_offset(p,
+ errtype, "unexpected EOF while parsing");
case E_DEDENT:
return tokenizer_error_with_col_offset(p,
PyExc_IndentationError, "unindent does not match any outer indentation level");