blob: f841ace28e7172833af19e6e7ea0fb9e0cf96107 [file] [log] [blame]
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001#include <Python.h>
Victor Stinnereec8e612021-03-18 14:57:49 +01002#include "pycore_ast.h" // _PyAST_Validate()
Pablo Galindoc5fc1562020-04-22 23:29:27 +01003#include <errcode.h>
Pablo Galindo1ed83ad2020-06-11 17:30:46 +01004#include "tokenizer.h"
Pablo Galindoc5fc1562020-04-22 23:29:27 +01005
6#include "pegen.h"
Pablo Galindo1ed83ad2020-06-11 17:30:46 +01007#include "string_parser.h"
Pablo Galindoc5fc1562020-04-22 23:29:27 +01008
Guido van Rossumc001c092020-04-30 12:12:19 -07009PyObject *
10_PyPegen_new_type_comment(Parser *p, char *s)
11{
12 PyObject *res = PyUnicode_DecodeUTF8(s, strlen(s), NULL);
13 if (res == NULL) {
14 return NULL;
15 }
Victor Stinner8370e072021-03-24 02:23:01 +010016 if (_PyArena_AddPyObject(p->arena, res) < 0) {
Guido van Rossumc001c092020-04-30 12:12:19 -070017 Py_DECREF(res);
18 return NULL;
19 }
20 return res;
21}
22
23arg_ty
24_PyPegen_add_type_comment_to_arg(Parser *p, arg_ty a, Token *tc)
25{
26 if (tc == NULL) {
27 return a;
28 }
29 char *bytes = PyBytes_AsString(tc->bytes);
30 if (bytes == NULL) {
31 return NULL;
32 }
33 PyObject *tco = _PyPegen_new_type_comment(p, bytes);
34 if (tco == NULL) {
35 return NULL;
36 }
Victor Stinnerd27f8d22021-04-07 21:34:22 +020037 return _PyAST_arg(a->arg, a->annotation, tco,
38 a->lineno, a->col_offset, a->end_lineno, a->end_col_offset,
39 p->arena);
Guido van Rossumc001c092020-04-30 12:12:19 -070040}
41
Pablo Galindoc5fc1562020-04-22 23:29:27 +010042static int
43init_normalization(Parser *p)
44{
Lysandros Nikolaouebebb642020-04-23 18:36:06 +030045 if (p->normalize) {
46 return 1;
47 }
Pablo Galindoc5fc1562020-04-22 23:29:27 +010048 PyObject *m = PyImport_ImportModuleNoBlock("unicodedata");
49 if (!m)
50 {
51 return 0;
52 }
53 p->normalize = PyObject_GetAttrString(m, "normalize");
54 Py_DECREF(m);
55 if (!p->normalize)
56 {
57 return 0;
58 }
59 return 1;
60}
61
Pablo Galindo2b74c832020-04-27 18:02:07 +010062/* Checks if the NOTEQUAL token is valid given the current parser flags
630 indicates success and nonzero indicates failure (an exception may be set) */
64int
Pablo Galindo06f8c332020-10-30 23:48:42 +000065_PyPegen_check_barry_as_flufl(Parser *p, Token* t) {
Pablo Galindo2b74c832020-04-27 18:02:07 +010066 assert(t->bytes != NULL);
67 assert(t->type == NOTEQUAL);
68
69 char* tok_str = PyBytes_AS_STRING(t->bytes);
Pablo Galindofb61c422020-06-15 14:23:43 +010070 if (p->flags & PyPARSE_BARRY_AS_BDFL && strcmp(tok_str, "<>") != 0) {
Pablo Galindo2b74c832020-04-27 18:02:07 +010071 RAISE_SYNTAX_ERROR("with Barry as BDFL, use '<>' instead of '!='");
72 return -1;
Pablo Galindofb61c422020-06-15 14:23:43 +010073 }
74 if (!(p->flags & PyPARSE_BARRY_AS_BDFL)) {
Pablo Galindo2b74c832020-04-27 18:02:07 +010075 return strcmp(tok_str, "!=");
76 }
77 return 0;
78}
79
Pablo Galindoc5fc1562020-04-22 23:29:27 +010080PyObject *
81_PyPegen_new_identifier(Parser *p, char *n)
82{
83 PyObject *id = PyUnicode_DecodeUTF8(n, strlen(n), NULL);
84 if (!id) {
85 goto error;
86 }
87 /* PyUnicode_DecodeUTF8 should always return a ready string. */
88 assert(PyUnicode_IS_READY(id));
89 /* Check whether there are non-ASCII characters in the
90 identifier; if so, normalize to NFKC. */
91 if (!PyUnicode_IS_ASCII(id))
92 {
93 PyObject *id2;
Lysandros Nikolaouebebb642020-04-23 18:36:06 +030094 if (!init_normalization(p))
Pablo Galindoc5fc1562020-04-22 23:29:27 +010095 {
96 Py_DECREF(id);
97 goto error;
98 }
99 PyObject *form = PyUnicode_InternFromString("NFKC");
100 if (form == NULL)
101 {
102 Py_DECREF(id);
103 goto error;
104 }
105 PyObject *args[2] = {form, id};
106 id2 = _PyObject_FastCall(p->normalize, args, 2);
107 Py_DECREF(id);
108 Py_DECREF(form);
109 if (!id2) {
110 goto error;
111 }
112 if (!PyUnicode_Check(id2))
113 {
114 PyErr_Format(PyExc_TypeError,
115 "unicodedata.normalize() must return a string, not "
116 "%.200s",
117 _PyType_Name(Py_TYPE(id2)));
118 Py_DECREF(id2);
119 goto error;
120 }
121 id = id2;
122 }
123 PyUnicode_InternInPlace(&id);
Victor Stinner8370e072021-03-24 02:23:01 +0100124 if (_PyArena_AddPyObject(p->arena, id) < 0)
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100125 {
126 Py_DECREF(id);
127 goto error;
128 }
129 return id;
130
131error:
132 p->error_indicator = 1;
133 return NULL;
134}
135
136static PyObject *
137_create_dummy_identifier(Parser *p)
138{
139 return _PyPegen_new_identifier(p, "");
140}
141
142static inline Py_ssize_t
Pablo Galindo51c58962020-06-16 16:49:43 +0100143byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset)
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100144{
145 const char *str = PyUnicode_AsUTF8(line);
Lysandros Nikolaouebebb642020-04-23 18:36:06 +0300146 if (!str) {
147 return 0;
148 }
Pablo Galindo123ff262021-03-22 16:24:39 +0000149 Py_ssize_t len = strlen(str);
150 if (col_offset > len) {
151 col_offset = len;
152 }
153 assert(col_offset >= 0);
Batuhan Taskaya76c1b4d2020-05-01 16:13:43 +0300154 PyObject *text = PyUnicode_DecodeUTF8(str, col_offset, "replace");
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100155 if (!text) {
156 return 0;
157 }
158 Py_ssize_t size = PyUnicode_GET_LENGTH(text);
159 Py_DECREF(text);
160 return size;
161}
162
163const char *
164_PyPegen_get_expr_name(expr_ty e)
165{
Pablo Galindo9f495902020-06-08 02:57:00 +0100166 assert(e != NULL);
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100167 switch (e->kind) {
168 case Attribute_kind:
169 return "attribute";
170 case Subscript_kind:
171 return "subscript";
172 case Starred_kind:
173 return "starred";
174 case Name_kind:
175 return "name";
176 case List_kind:
177 return "list";
178 case Tuple_kind:
179 return "tuple";
180 case Lambda_kind:
181 return "lambda";
182 case Call_kind:
183 return "function call";
184 case BoolOp_kind:
185 case BinOp_kind:
186 case UnaryOp_kind:
187 return "operator";
188 case GeneratorExp_kind:
189 return "generator expression";
190 case Yield_kind:
191 case YieldFrom_kind:
192 return "yield expression";
193 case Await_kind:
194 return "await expression";
195 case ListComp_kind:
196 return "list comprehension";
197 case SetComp_kind:
198 return "set comprehension";
199 case DictComp_kind:
200 return "dict comprehension";
201 case Dict_kind:
202 return "dict display";
203 case Set_kind:
204 return "set display";
205 case JoinedStr_kind:
206 case FormattedValue_kind:
207 return "f-string expression";
208 case Constant_kind: {
209 PyObject *value = e->v.Constant.value;
210 if (value == Py_None) {
211 return "None";
212 }
213 if (value == Py_False) {
214 return "False";
215 }
216 if (value == Py_True) {
217 return "True";
218 }
219 if (value == Py_Ellipsis) {
220 return "Ellipsis";
221 }
222 return "literal";
223 }
224 case Compare_kind:
225 return "comparison";
226 case IfExp_kind:
227 return "conditional expression";
228 case NamedExpr_kind:
229 return "named expression";
230 default:
231 PyErr_Format(PyExc_SystemError,
232 "unexpected expression in assignment %d (line %d)",
233 e->kind, e->lineno);
234 return NULL;
235 }
236}
237
Lysandros Nikolaouebebb642020-04-23 18:36:06 +0300238static int
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100239raise_decode_error(Parser *p)
240{
Lysandros Nikolaouebebb642020-04-23 18:36:06 +0300241 assert(PyErr_Occurred());
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100242 const char *errtype = NULL;
243 if (PyErr_ExceptionMatches(PyExc_UnicodeError)) {
244 errtype = "unicode error";
245 }
246 else if (PyErr_ExceptionMatches(PyExc_ValueError)) {
247 errtype = "value error";
248 }
249 if (errtype) {
Pablo Galindofb61c422020-06-15 14:23:43 +0100250 PyObject *type;
251 PyObject *value;
252 PyObject *tback;
253 PyObject *errstr;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100254 PyErr_Fetch(&type, &value, &tback);
255 errstr = PyObject_Str(value);
256 if (errstr) {
257 RAISE_SYNTAX_ERROR("(%s) %U", errtype, errstr);
258 Py_DECREF(errstr);
259 }
260 else {
261 PyErr_Clear();
262 RAISE_SYNTAX_ERROR("(%s) unknown error", errtype);
263 }
264 Py_XDECREF(type);
265 Py_XDECREF(value);
266 Py_XDECREF(tback);
267 }
Lysandros Nikolaouebebb642020-04-23 18:36:06 +0300268
269 return -1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100270}
271
Pablo Galindod6d63712021-01-19 23:59:33 +0000272static inline void
273raise_unclosed_parentheses_error(Parser *p) {
274 int error_lineno = p->tok->parenlinenostack[p->tok->level-1];
275 int error_col = p->tok->parencolstack[p->tok->level-1];
276 RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError,
277 error_lineno, error_col,
278 "'%c' was never closed",
279 p->tok->parenstack[p->tok->level-1]);
280}
281
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100282static void
283raise_tokenizer_init_error(PyObject *filename)
284{
285 if (!(PyErr_ExceptionMatches(PyExc_LookupError)
286 || PyErr_ExceptionMatches(PyExc_ValueError)
287 || PyErr_ExceptionMatches(PyExc_UnicodeDecodeError))) {
288 return;
289 }
Lysandros Nikolaouebebb642020-04-23 18:36:06 +0300290 PyObject *errstr = NULL;
291 PyObject *tuple = NULL;
Pablo Galindofb61c422020-06-15 14:23:43 +0100292 PyObject *type;
293 PyObject *value;
294 PyObject *tback;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100295 PyErr_Fetch(&type, &value, &tback);
296 errstr = PyObject_Str(value);
Lysandros Nikolaouebebb642020-04-23 18:36:06 +0300297 if (!errstr) {
298 goto error;
299 }
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100300
Lysandros Nikolaouebebb642020-04-23 18:36:06 +0300301 PyObject *tmp = Py_BuildValue("(OiiO)", filename, 0, -1, Py_None);
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100302 if (!tmp) {
303 goto error;
304 }
305
Lysandros Nikolaouebebb642020-04-23 18:36:06 +0300306 tuple = PyTuple_Pack(2, errstr, tmp);
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100307 Py_DECREF(tmp);
308 if (!value) {
309 goto error;
310 }
Lysandros Nikolaouebebb642020-04-23 18:36:06 +0300311 PyErr_SetObject(PyExc_SyntaxError, tuple);
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100312
313error:
314 Py_XDECREF(type);
315 Py_XDECREF(value);
316 Py_XDECREF(tback);
Lysandros Nikolaouebebb642020-04-23 18:36:06 +0300317 Py_XDECREF(errstr);
318 Py_XDECREF(tuple);
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100319}
320
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100321static int
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100322tokenizer_error(Parser *p)
323{
324 if (PyErr_Occurred()) {
325 return -1;
326 }
327
328 const char *msg = NULL;
329 PyObject* errtype = PyExc_SyntaxError;
Pablo Galindo96eeff52021-03-22 17:28:11 +0000330 Py_ssize_t col_offset = -1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100331 switch (p->tok->done) {
332 case E_TOKEN:
333 msg = "invalid token";
334 break;
Lysandros Nikolaoud55133f2020-04-28 03:23:35 +0300335 case E_EOF:
Pablo Galindod6d63712021-01-19 23:59:33 +0000336 if (p->tok->level) {
337 raise_unclosed_parentheses_error(p);
338 } else {
339 RAISE_SYNTAX_ERROR("unexpected EOF while parsing");
340 }
Batuhan Taskaya76c1b4d2020-05-01 16:13:43 +0300341 return -1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100342 case E_DEDENT:
Batuhan Taskaya76c1b4d2020-05-01 16:13:43 +0300343 RAISE_INDENTATION_ERROR("unindent does not match any outer indentation level");
344 return -1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100345 case E_INTR:
346 if (!PyErr_Occurred()) {
347 PyErr_SetNone(PyExc_KeyboardInterrupt);
348 }
349 return -1;
350 case E_NOMEM:
351 PyErr_NoMemory();
352 return -1;
353 case E_TABSPACE:
354 errtype = PyExc_TabError;
355 msg = "inconsistent use of tabs and spaces in indentation";
356 break;
357 case E_TOODEEP:
358 errtype = PyExc_IndentationError;
359 msg = "too many levels of indentation";
360 break;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100361 case E_LINECONT:
Pablo Galindo96eeff52021-03-22 17:28:11 +0000362 col_offset = strlen(strtok(p->tok->buf, "\n")) - 1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100363 msg = "unexpected character after line continuation character";
364 break;
365 default:
366 msg = "unknown parsing error";
367 }
368
Pablo Galindo96eeff52021-03-22 17:28:11 +0000369 RAISE_ERROR_KNOWN_LOCATION(p, errtype, p->tok->lineno, col_offset, msg);
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100370 return -1;
371}
372
373void *
Lysandros Nikolaoua15c9b32020-05-13 22:36:27 +0300374_PyPegen_raise_error(Parser *p, PyObject *errtype, const char *errmsg, ...)
375{
376 Token *t = p->known_err_token != NULL ? p->known_err_token : p->tokens[p->fill - 1];
Pablo Galindo51c58962020-06-16 16:49:43 +0100377 Py_ssize_t col_offset;
Lysandros Nikolaoua15c9b32020-05-13 22:36:27 +0300378 if (t->col_offset == -1) {
379 col_offset = Py_SAFE_DOWNCAST(p->tok->cur - p->tok->buf,
380 intptr_t, int);
381 } else {
382 col_offset = t->col_offset + 1;
383 }
384
385 va_list va;
386 va_start(va, errmsg);
387 _PyPegen_raise_error_known_location(p, errtype, t->lineno,
388 col_offset, errmsg, va);
389 va_end(va);
390
391 return NULL;
392}
393
Lysandros Nikolaoue5fe5092021-01-14 23:36:30 +0200394static PyObject *
395get_error_line(Parser *p, Py_ssize_t lineno)
396{
Pablo Galindo123ff262021-03-22 16:24:39 +0000397 /* If the file descriptor is interactive, the source lines of the current
398 * (multi-line) statement are stored in p->tok->interactive_src_start.
399 * If not, we're parsing from a string, which means that the whole source
400 * is stored in p->tok->str. */
Lysandros Nikolaoue5fe5092021-01-14 23:36:30 +0200401 assert(p->tok->fp == NULL || p->tok->fp == stdin);
402
Pablo Galindocd8dcbc2021-03-14 04:38:40 +0100403 char *cur_line = p->tok->fp_interactive ? p->tok->interactive_src_start : p->tok->str;
404
Lysandros Nikolaoue5fe5092021-01-14 23:36:30 +0200405 for (int i = 0; i < lineno - 1; i++) {
406 cur_line = strchr(cur_line, '\n') + 1;
407 }
408
409 char *next_newline;
410 if ((next_newline = strchr(cur_line, '\n')) == NULL) { // This is the last line
411 next_newline = cur_line + strlen(cur_line);
412 }
413 return PyUnicode_DecodeUTF8(cur_line, next_newline - cur_line, "replace");
414}
415
Lysandros Nikolaoua15c9b32020-05-13 22:36:27 +0300416void *
417_PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
Pablo Galindo51c58962020-06-16 16:49:43 +0100418 Py_ssize_t lineno, Py_ssize_t col_offset,
Lysandros Nikolaoua15c9b32020-05-13 22:36:27 +0300419 const char *errmsg, va_list va)
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100420{
421 PyObject *value = NULL;
422 PyObject *errstr = NULL;
Lysandros Nikolaoua15c9b32020-05-13 22:36:27 +0300423 PyObject *error_line = NULL;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100424 PyObject *tmp = NULL;
Lysandros Nikolaou7f06af62020-05-04 03:20:09 +0300425 p->error_indicator = 1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100426
Lysandros Nikolaou2e0a9202020-06-26 14:24:05 +0300427 if (p->start_rule == Py_fstring_input) {
428 const char *fstring_msg = "f-string: ";
429 Py_ssize_t len = strlen(fstring_msg) + strlen(errmsg);
430
Lysandros Nikolaou6dcbc242020-06-27 20:47:00 +0300431 char *new_errmsg = PyMem_Malloc(len + 1); // Lengths of both strings plus NULL character
Lysandros Nikolaou2e0a9202020-06-26 14:24:05 +0300432 if (!new_errmsg) {
433 return (void *) PyErr_NoMemory();
434 }
435
436 // Copy both strings into new buffer
437 memcpy(new_errmsg, fstring_msg, strlen(fstring_msg));
438 memcpy(new_errmsg + strlen(fstring_msg), errmsg, strlen(errmsg));
439 new_errmsg[len] = 0;
440 errmsg = new_errmsg;
441 }
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100442 errstr = PyUnicode_FromFormatV(errmsg, va);
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100443 if (!errstr) {
444 goto error;
445 }
446
Pablo Galindocd8dcbc2021-03-14 04:38:40 +0100447 if (p->tok->fp_interactive) {
448 error_line = get_error_line(p, lineno);
449 }
450 else if (p->start_rule == Py_file_input) {
Lysandros Nikolaou861efc62020-06-20 15:57:27 +0300451 error_line = PyErr_ProgramTextObject(p->tok->filename, (int) lineno);
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100452 }
453
Lysandros Nikolaoua15c9b32020-05-13 22:36:27 +0300454 if (!error_line) {
Lysandros Nikolaoue5fe5092021-01-14 23:36:30 +0200455 /* PyErr_ProgramTextObject was not called or returned NULL. If it was not called,
456 then we need to find the error line from some other source, because
457 p->start_rule != Py_file_input. If it returned NULL, then it either unexpectedly
458 failed or we're parsing from a string or the REPL. There's a third edge case where
459 we're actually parsing from a file, which has an E_EOF SyntaxError and in that case
460 `PyErr_ProgramTextObject` fails because lineno points to last_file_line + 1, which
461 does not physically exist */
462 assert(p->tok->fp == NULL || p->tok->fp == stdin || p->tok->done == E_EOF);
463
Pablo Galindo40901512021-01-31 22:48:23 +0000464 if (p->tok->lineno <= lineno) {
Lysandros Nikolaoue5fe5092021-01-14 23:36:30 +0200465 Py_ssize_t size = p->tok->inp - p->tok->buf;
466 error_line = PyUnicode_DecodeUTF8(p->tok->buf, size, "replace");
467 }
468 else {
469 error_line = get_error_line(p, lineno);
470 }
Lysandros Nikolaoua15c9b32020-05-13 22:36:27 +0300471 if (!error_line) {
472 goto error;
Batuhan Taskaya76c1b4d2020-05-01 16:13:43 +0300473 }
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100474 }
475
Lysandros Nikolaou1f0f4ab2020-06-28 02:41:48 +0300476 if (p->start_rule == Py_fstring_input) {
477 col_offset -= p->starting_col_offset;
478 }
Pablo Galindo51c58962020-06-16 16:49:43 +0100479 Py_ssize_t col_number = col_offset;
480
481 if (p->tok->encoding != NULL) {
482 col_number = byte_offset_to_character_offset(error_line, col_offset);
483 }
Lysandros Nikolaoua15c9b32020-05-13 22:36:27 +0300484
485 tmp = Py_BuildValue("(OiiN)", p->tok->filename, lineno, col_number, error_line);
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100486 if (!tmp) {
487 goto error;
488 }
489 value = PyTuple_Pack(2, errstr, tmp);
490 Py_DECREF(tmp);
491 if (!value) {
492 goto error;
493 }
494 PyErr_SetObject(errtype, value);
495
496 Py_DECREF(errstr);
497 Py_DECREF(value);
Lysandros Nikolaou2e0a9202020-06-26 14:24:05 +0300498 if (p->start_rule == Py_fstring_input) {
Lysandros Nikolaou6dcbc242020-06-27 20:47:00 +0300499 PyMem_Free((void *)errmsg);
Lysandros Nikolaou2e0a9202020-06-26 14:24:05 +0300500 }
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100501 return NULL;
502
503error:
504 Py_XDECREF(errstr);
Lysandros Nikolaoua15c9b32020-05-13 22:36:27 +0300505 Py_XDECREF(error_line);
Lysandros Nikolaou2e0a9202020-06-26 14:24:05 +0300506 if (p->start_rule == Py_fstring_input) {
Lysandros Nikolaou6dcbc242020-06-27 20:47:00 +0300507 PyMem_Free((void *)errmsg);
Lysandros Nikolaou2e0a9202020-06-26 14:24:05 +0300508 }
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100509 return NULL;
510}
511
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100512#if 0
513static const char *
514token_name(int type)
515{
516 if (0 <= type && type <= N_TOKENS) {
517 return _PyParser_TokenNames[type];
518 }
519 return "<Huh?>";
520}
521#endif
522
523// Here, mark is the start of the node, while p->mark is the end.
524// If node==NULL, they should be the same.
525int
526_PyPegen_insert_memo(Parser *p, int mark, int type, void *node)
527{
528 // Insert in front
Victor Stinner8370e072021-03-24 02:23:01 +0100529 Memo *m = _PyArena_Malloc(p->arena, sizeof(Memo));
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100530 if (m == NULL) {
531 return -1;
532 }
533 m->type = type;
534 m->node = node;
535 m->mark = p->mark;
536 m->next = p->tokens[mark]->memo;
537 p->tokens[mark]->memo = m;
538 return 0;
539}
540
541// Like _PyPegen_insert_memo(), but updates an existing node if found.
542int
543_PyPegen_update_memo(Parser *p, int mark, int type, void *node)
544{
545 for (Memo *m = p->tokens[mark]->memo; m != NULL; m = m->next) {
546 if (m->type == type) {
547 // Update existing node.
548 m->node = node;
549 m->mark = p->mark;
550 return 0;
551 }
552 }
553 // Insert new node.
554 return _PyPegen_insert_memo(p, mark, type, node);
555}
556
557// Return dummy NAME.
558void *
559_PyPegen_dummy_name(Parser *p, ...)
560{
561 static void *cache = NULL;
562
563 if (cache != NULL) {
564 return cache;
565 }
566
567 PyObject *id = _create_dummy_identifier(p);
568 if (!id) {
569 return NULL;
570 }
Victor Stinnerd27f8d22021-04-07 21:34:22 +0200571 cache = _PyAST_Name(id, Load, 1, 0, 1, 0, p->arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100572 return cache;
573}
574
575static int
576_get_keyword_or_name_type(Parser *p, const char *name, int name_len)
577{
Lysandros Nikolaou782f44b2020-07-07 01:42:21 +0300578 assert(name_len > 0);
Pablo Galindo1ac0cbc2020-07-06 20:31:16 +0100579 if (name_len >= p->n_keyword_lists ||
580 p->keywords[name_len] == NULL ||
581 p->keywords[name_len]->type == -1) {
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100582 return NAME;
583 }
Pablo Galindo1ac0cbc2020-07-06 20:31:16 +0100584 for (KeywordToken *k = p->keywords[name_len]; k != NULL && k->type != -1; k++) {
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100585 if (strncmp(k->str, name, name_len) == 0) {
586 return k->type;
587 }
588 }
589 return NAME;
590}
591
Guido van Rossumc001c092020-04-30 12:12:19 -0700592static int
593growable_comment_array_init(growable_comment_array *arr, size_t initial_size) {
594 assert(initial_size > 0);
595 arr->items = PyMem_Malloc(initial_size * sizeof(*arr->items));
596 arr->size = initial_size;
597 arr->num_items = 0;
598
599 return arr->items != NULL;
600}
601
602static int
603growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) {
604 if (arr->num_items >= arr->size) {
605 size_t new_size = arr->size * 2;
606 void *new_items_array = PyMem_Realloc(arr->items, new_size * sizeof(*arr->items));
607 if (!new_items_array) {
608 return 0;
609 }
610 arr->items = new_items_array;
611 arr->size = new_size;
612 }
613
614 arr->items[arr->num_items].lineno = lineno;
615 arr->items[arr->num_items].comment = comment; // Take ownership
616 arr->num_items++;
617 return 1;
618}
619
620static void
621growable_comment_array_deallocate(growable_comment_array *arr) {
622 for (unsigned i = 0; i < arr->num_items; i++) {
623 PyMem_Free(arr->items[i].comment);
624 }
625 PyMem_Free(arr->items);
626}
627
Pablo Galindod00a4492021-04-09 01:32:25 +0100628static int
629initialize_token(Parser *p, Token *token, const char *start, const char *end, int token_type) {
630 assert(token != NULL);
631
632 token->type = (token_type == NAME) ? _get_keyword_or_name_type(p, start, (int)(end - start)) : token_type;
633 token->bytes = PyBytes_FromStringAndSize(start, end - start);
634 if (token->bytes == NULL) {
635 return -1;
636 }
637
638 if (_PyArena_AddPyObject(p->arena, token->bytes) < 0) {
639 Py_DECREF(token->bytes);
640 return -1;
641 }
642
643 const char *line_start = token_type == STRING ? p->tok->multi_line_start : p->tok->line_start;
644 int lineno = token_type == STRING ? p->tok->first_lineno : p->tok->lineno;
645 int end_lineno = p->tok->lineno;
646
647 int col_offset = (start != NULL && start >= line_start) ? (int)(start - line_start) : -1;
648 int end_col_offset = (end != NULL && end >= p->tok->line_start) ? (int)(end - p->tok->line_start) : -1;
649
650 token->lineno = p->starting_lineno + lineno;
651 token->col_offset = p->tok->lineno == 1 ? p->starting_col_offset + col_offset : col_offset;
652 token->end_lineno = p->starting_lineno + end_lineno;
653 token->end_col_offset = p->tok->lineno == 1 ? p->starting_col_offset + end_col_offset : end_col_offset;
654
655 p->fill += 1;
656
657 if (token_type == ERRORTOKEN && p->tok->done == E_DECODE) {
658 return raise_decode_error(p);
659 }
660
661 return (token_type == ERRORTOKEN ? tokenizer_error(p) : 0);
662}
663
664static int
665_resize_tokens_array(Parser *p) {
666 int newsize = p->size * 2;
667 Token **new_tokens = PyMem_Realloc(p->tokens, newsize * sizeof(Token *));
668 if (new_tokens == NULL) {
669 PyErr_NoMemory();
670 return -1;
671 }
672 p->tokens = new_tokens;
673
674 for (int i = p->size; i < newsize; i++) {
675 p->tokens[i] = PyMem_Calloc(1, sizeof(Token));
676 if (p->tokens[i] == NULL) {
677 p->size = i; // Needed, in order to cleanup correctly after parser fails
678 PyErr_NoMemory();
679 return -1;
680 }
681 }
682 p->size = newsize;
683 return 0;
684}
685
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100686int
687_PyPegen_fill_token(Parser *p)
688{
Pablo Galindofb61c422020-06-15 14:23:43 +0100689 const char *start;
690 const char *end;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100691 int type = PyTokenizer_Get(p->tok, &start, &end);
Guido van Rossumc001c092020-04-30 12:12:19 -0700692
693 // Record and skip '# type: ignore' comments
694 while (type == TYPE_IGNORE) {
695 Py_ssize_t len = end - start;
696 char *tag = PyMem_Malloc(len + 1);
697 if (tag == NULL) {
698 PyErr_NoMemory();
699 return -1;
700 }
701 strncpy(tag, start, len);
702 tag[len] = '\0';
703 // Ownership of tag passes to the growable array
704 if (!growable_comment_array_add(&p->type_ignore_comments, p->tok->lineno, tag)) {
705 PyErr_NoMemory();
706 return -1;
707 }
708 type = PyTokenizer_Get(p->tok, &start, &end);
709 }
710
Pablo Galindod00a4492021-04-09 01:32:25 +0100711 // If we have reached the end and we are in single input mode we need to insert a newline and reset the parsing
712 if (p->start_rule == Py_single_input && type == ENDMARKER && p->parsing_started) {
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100713 type = NEWLINE; /* Add an extra newline */
714 p->parsing_started = 0;
715
Pablo Galindob94dbd72020-04-27 18:35:58 +0100716 if (p->tok->indent && !(p->flags & PyPARSE_DONT_IMPLY_DEDENT)) {
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100717 p->tok->pendin = -p->tok->indent;
718 p->tok->indent = 0;
719 }
720 }
721 else {
722 p->parsing_started = 1;
723 }
724
Pablo Galindod00a4492021-04-09 01:32:25 +0100725 // Check if we are at the limit of the token array capacity and resize if needed
726 if ((p->fill == p->size) && (_resize_tokens_array(p) != 0)) {
727 return -1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100728 }
729
730 Token *t = p->tokens[p->fill];
Pablo Galindod00a4492021-04-09 01:32:25 +0100731 return initialize_token(p, t, start, end, type);
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100732}
733
Pablo Galindo58bafe42021-04-09 01:17:31 +0100734
735#if defined(Py_DEBUG)
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100736// Instrumentation to count the effectiveness of memoization.
737// The array counts the number of tokens skipped by memoization,
738// indexed by type.
739
740#define NSTATISTICS 2000
741static long memo_statistics[NSTATISTICS];
742
743void
744_PyPegen_clear_memo_statistics()
745{
746 for (int i = 0; i < NSTATISTICS; i++) {
747 memo_statistics[i] = 0;
748 }
749}
750
751PyObject *
752_PyPegen_get_memo_statistics()
753{
754 PyObject *ret = PyList_New(NSTATISTICS);
755 if (ret == NULL) {
756 return NULL;
757 }
758 for (int i = 0; i < NSTATISTICS; i++) {
759 PyObject *value = PyLong_FromLong(memo_statistics[i]);
760 if (value == NULL) {
761 Py_DECREF(ret);
762 return NULL;
763 }
764 // PyList_SetItem borrows a reference to value.
765 if (PyList_SetItem(ret, i, value) < 0) {
766 Py_DECREF(ret);
767 return NULL;
768 }
769 }
770 return ret;
771}
Pablo Galindo58bafe42021-04-09 01:17:31 +0100772#endif
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100773
774int // bool
775_PyPegen_is_memoized(Parser *p, int type, void *pres)
776{
777 if (p->mark == p->fill) {
778 if (_PyPegen_fill_token(p) < 0) {
Lysandros Nikolaouebebb642020-04-23 18:36:06 +0300779 p->error_indicator = 1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100780 return -1;
781 }
782 }
783
784 Token *t = p->tokens[p->mark];
785
786 for (Memo *m = t->memo; m != NULL; m = m->next) {
787 if (m->type == type) {
Pablo Galindo58bafe42021-04-09 01:17:31 +0100788#if defined(PY_DEBUG)
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100789 if (0 <= type && type < NSTATISTICS) {
790 long count = m->mark - p->mark;
791 // A memoized negative result counts for one.
792 if (count <= 0) {
793 count = 1;
794 }
795 memo_statistics[type] += count;
796 }
Pablo Galindo58bafe42021-04-09 01:17:31 +0100797#endif
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100798 p->mark = m->mark;
799 *(void **)(pres) = m->node;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100800 return 1;
801 }
802 }
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100803 return 0;
804}
805
Pablo Galindo1df5a9e2020-04-23 12:42:13 +0100806int
807_PyPegen_lookahead_with_name(int positive, expr_ty (func)(Parser *), Parser *p)
808{
809 int mark = p->mark;
810 void *res = func(p);
811 p->mark = mark;
812 return (res != NULL) == positive;
813}
814
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100815int
Pablo Galindo404b23b2020-05-27 00:15:52 +0100816_PyPegen_lookahead_with_string(int positive, expr_ty (func)(Parser *, const char*), Parser *p, const char* arg)
817{
818 int mark = p->mark;
819 void *res = func(p, arg);
820 p->mark = mark;
821 return (res != NULL) == positive;
822}
823
824int
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100825_PyPegen_lookahead_with_int(int positive, Token *(func)(Parser *, int), Parser *p, int arg)
826{
827 int mark = p->mark;
828 void *res = func(p, arg);
829 p->mark = mark;
830 return (res != NULL) == positive;
831}
832
833int
834_PyPegen_lookahead(int positive, void *(func)(Parser *), Parser *p)
835{
836 int mark = p->mark;
Pablo Galindo1df5a9e2020-04-23 12:42:13 +0100837 void *res = (void*)func(p);
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100838 p->mark = mark;
839 return (res != NULL) == positive;
840}
841
842Token *
843_PyPegen_expect_token(Parser *p, int type)
844{
845 if (p->mark == p->fill) {
846 if (_PyPegen_fill_token(p) < 0) {
Lysandros Nikolaouebebb642020-04-23 18:36:06 +0300847 p->error_indicator = 1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100848 return NULL;
849 }
850 }
851 Token *t = p->tokens[p->mark];
852 if (t->type != type) {
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100853 return NULL;
854 }
855 p->mark += 1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100856 return t;
857}
858
Pablo Galindo58fb1562021-02-02 19:54:22 +0000859Token *
860_PyPegen_expect_forced_token(Parser *p, int type, const char* expected) {
861
862 if (p->error_indicator == 1) {
863 return NULL;
864 }
865
866 if (p->mark == p->fill) {
867 if (_PyPegen_fill_token(p) < 0) {
868 p->error_indicator = 1;
869 return NULL;
870 }
871 }
872 Token *t = p->tokens[p->mark];
873 if (t->type != type) {
874 RAISE_SYNTAX_ERROR_KNOWN_LOCATION(t, "expected '%s'", expected);
875 return NULL;
876 }
877 p->mark += 1;
878 return t;
879}
880
Guido van Rossumb45af1a2020-05-26 10:58:44 -0700881expr_ty
882_PyPegen_expect_soft_keyword(Parser *p, const char *keyword)
883{
884 if (p->mark == p->fill) {
885 if (_PyPegen_fill_token(p) < 0) {
886 p->error_indicator = 1;
887 return NULL;
888 }
889 }
890 Token *t = p->tokens[p->mark];
891 if (t->type != NAME) {
892 return NULL;
893 }
Lysandros Nikolaou526e23f2020-05-27 19:04:11 +0300894 char *s = PyBytes_AsString(t->bytes);
Guido van Rossumb45af1a2020-05-26 10:58:44 -0700895 if (!s) {
Lysandros Nikolaou526e23f2020-05-27 19:04:11 +0300896 p->error_indicator = 1;
Guido van Rossumb45af1a2020-05-26 10:58:44 -0700897 return NULL;
898 }
899 if (strcmp(s, keyword) != 0) {
900 return NULL;
901 }
Lysandros Nikolaou526e23f2020-05-27 19:04:11 +0300902 return _PyPegen_name_token(p);
Guido van Rossumb45af1a2020-05-26 10:58:44 -0700903}
904
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100905Token *
906_PyPegen_get_last_nonnwhitespace_token(Parser *p)
907{
908 assert(p->mark >= 0);
909 Token *token = NULL;
910 for (int m = p->mark - 1; m >= 0; m--) {
911 token = p->tokens[m];
912 if (token->type != ENDMARKER && (token->type < NEWLINE || token->type > DEDENT)) {
913 break;
914 }
915 }
916 return token;
917}
918
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100919expr_ty
920_PyPegen_name_token(Parser *p)
921{
922 Token *t = _PyPegen_expect_token(p, NAME);
923 if (t == NULL) {
924 return NULL;
925 }
926 char* s = PyBytes_AsString(t->bytes);
927 if (!s) {
Lysandros Nikolaou526e23f2020-05-27 19:04:11 +0300928 p->error_indicator = 1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100929 return NULL;
930 }
931 PyObject *id = _PyPegen_new_identifier(p, s);
932 if (id == NULL) {
Lysandros Nikolaou526e23f2020-05-27 19:04:11 +0300933 p->error_indicator = 1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100934 return NULL;
935 }
Victor Stinnerd27f8d22021-04-07 21:34:22 +0200936 return _PyAST_Name(id, Load, t->lineno, t->col_offset, t->end_lineno,
937 t->end_col_offset, p->arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100938}
939
940void *
941_PyPegen_string_token(Parser *p)
942{
943 return _PyPegen_expect_token(p, STRING);
944}
945
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100946static PyObject *
947parsenumber_raw(const char *s)
948{
949 const char *end;
950 long x;
951 double dx;
952 Py_complex compl;
953 int imflag;
954
955 assert(s != NULL);
956 errno = 0;
957 end = s + strlen(s) - 1;
958 imflag = *end == 'j' || *end == 'J';
959 if (s[0] == '0') {
960 x = (long)PyOS_strtoul(s, (char **)&end, 0);
961 if (x < 0 && errno == 0) {
962 return PyLong_FromString(s, (char **)0, 0);
963 }
964 }
Pablo Galindofb61c422020-06-15 14:23:43 +0100965 else {
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100966 x = PyOS_strtol(s, (char **)&end, 0);
Pablo Galindofb61c422020-06-15 14:23:43 +0100967 }
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100968 if (*end == '\0') {
Pablo Galindofb61c422020-06-15 14:23:43 +0100969 if (errno != 0) {
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100970 return PyLong_FromString(s, (char **)0, 0);
Pablo Galindofb61c422020-06-15 14:23:43 +0100971 }
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100972 return PyLong_FromLong(x);
973 }
974 /* XXX Huge floats may silently fail */
975 if (imflag) {
976 compl.real = 0.;
977 compl.imag = PyOS_string_to_double(s, (char **)&end, NULL);
Pablo Galindofb61c422020-06-15 14:23:43 +0100978 if (compl.imag == -1.0 && PyErr_Occurred()) {
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100979 return NULL;
Pablo Galindofb61c422020-06-15 14:23:43 +0100980 }
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100981 return PyComplex_FromCComplex(compl);
982 }
Pablo Galindofb61c422020-06-15 14:23:43 +0100983 dx = PyOS_string_to_double(s, NULL, NULL);
984 if (dx == -1.0 && PyErr_Occurred()) {
985 return NULL;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100986 }
Pablo Galindofb61c422020-06-15 14:23:43 +0100987 return PyFloat_FromDouble(dx);
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100988}
989
990static PyObject *
991parsenumber(const char *s)
992{
Pablo Galindofb61c422020-06-15 14:23:43 +0100993 char *dup;
994 char *end;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100995 PyObject *res = NULL;
996
997 assert(s != NULL);
998
999 if (strchr(s, '_') == NULL) {
1000 return parsenumber_raw(s);
1001 }
1002 /* Create a duplicate without underscores. */
1003 dup = PyMem_Malloc(strlen(s) + 1);
1004 if (dup == NULL) {
1005 return PyErr_NoMemory();
1006 }
1007 end = dup;
1008 for (; *s; s++) {
1009 if (*s != '_') {
1010 *end++ = *s;
1011 }
1012 }
1013 *end = '\0';
1014 res = parsenumber_raw(dup);
1015 PyMem_Free(dup);
1016 return res;
1017}
1018
1019expr_ty
1020_PyPegen_number_token(Parser *p)
1021{
1022 Token *t = _PyPegen_expect_token(p, NUMBER);
1023 if (t == NULL) {
1024 return NULL;
1025 }
1026
1027 char *num_raw = PyBytes_AsString(t->bytes);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001028 if (num_raw == NULL) {
Lysandros Nikolaou526e23f2020-05-27 19:04:11 +03001029 p->error_indicator = 1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001030 return NULL;
1031 }
1032
Lysandros Nikolaou3e0a6f32020-05-01 06:27:52 +03001033 if (p->feature_version < 6 && strchr(num_raw, '_') != NULL) {
1034 p->error_indicator = 1;
Shantanuc3f00142020-05-04 01:13:30 -07001035 return RAISE_SYNTAX_ERROR("Underscores in numeric literals are only supported "
Lysandros Nikolaou3e0a6f32020-05-01 06:27:52 +03001036 "in Python 3.6 and greater");
1037 }
1038
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001039 PyObject *c = parsenumber(num_raw);
1040
1041 if (c == NULL) {
Lysandros Nikolaou526e23f2020-05-27 19:04:11 +03001042 p->error_indicator = 1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001043 return NULL;
1044 }
1045
Victor Stinner8370e072021-03-24 02:23:01 +01001046 if (_PyArena_AddPyObject(p->arena, c) < 0) {
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001047 Py_DECREF(c);
Lysandros Nikolaou526e23f2020-05-27 19:04:11 +03001048 p->error_indicator = 1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001049 return NULL;
1050 }
1051
Victor Stinnerd27f8d22021-04-07 21:34:22 +02001052 return _PyAST_Constant(c, NULL, t->lineno, t->col_offset, t->end_lineno,
1053 t->end_col_offset, p->arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001054}
1055
Lysandros Nikolaou6d650872020-04-29 04:42:27 +03001056static int // bool
1057newline_in_string(Parser *p, const char *cur)
1058{
Pablo Galindo2e6593d2020-06-06 00:52:27 +01001059 for (const char *c = cur; c >= p->tok->buf; c--) {
1060 if (*c == '\'' || *c == '"') {
Lysandros Nikolaou6d650872020-04-29 04:42:27 +03001061 return 1;
1062 }
1063 }
1064 return 0;
1065}
1066
1067/* Check that the source for a single input statement really is a single
1068 statement by looking at what is left in the buffer after parsing.
1069 Trailing whitespace and comments are OK. */
1070static int // bool
1071bad_single_statement(Parser *p)
1072{
1073 const char *cur = strchr(p->tok->buf, '\n');
1074
1075 /* Newlines are allowed if preceded by a line continuation character
1076 or if they appear inside a string. */
Pablo Galindoe68c6782020-10-25 23:03:41 +00001077 if (!cur || (cur != p->tok->buf && *(cur - 1) == '\\')
1078 || newline_in_string(p, cur)) {
Lysandros Nikolaou6d650872020-04-29 04:42:27 +03001079 return 0;
1080 }
1081 char c = *cur;
1082
1083 for (;;) {
1084 while (c == ' ' || c == '\t' || c == '\n' || c == '\014') {
1085 c = *++cur;
1086 }
1087
1088 if (!c) {
1089 return 0;
1090 }
1091
1092 if (c != '#') {
1093 return 1;
1094 }
1095
1096 /* Suck up comment. */
1097 while (c && c != '\n') {
1098 c = *++cur;
1099 }
1100 }
1101}
1102
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001103void
1104_PyPegen_Parser_Free(Parser *p)
1105{
1106 Py_XDECREF(p->normalize);
1107 for (int i = 0; i < p->size; i++) {
1108 PyMem_Free(p->tokens[i]);
1109 }
1110 PyMem_Free(p->tokens);
Guido van Rossumc001c092020-04-30 12:12:19 -07001111 growable_comment_array_deallocate(&p->type_ignore_comments);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001112 PyMem_Free(p);
1113}
1114
Pablo Galindo2b74c832020-04-27 18:02:07 +01001115static int
1116compute_parser_flags(PyCompilerFlags *flags)
1117{
1118 int parser_flags = 0;
1119 if (!flags) {
1120 return 0;
1121 }
1122 if (flags->cf_flags & PyCF_DONT_IMPLY_DEDENT) {
1123 parser_flags |= PyPARSE_DONT_IMPLY_DEDENT;
1124 }
1125 if (flags->cf_flags & PyCF_IGNORE_COOKIE) {
1126 parser_flags |= PyPARSE_IGNORE_COOKIE;
1127 }
1128 if (flags->cf_flags & CO_FUTURE_BARRY_AS_BDFL) {
1129 parser_flags |= PyPARSE_BARRY_AS_BDFL;
1130 }
1131 if (flags->cf_flags & PyCF_TYPE_COMMENTS) {
1132 parser_flags |= PyPARSE_TYPE_COMMENTS;
1133 }
Guido van Rossum9d197c72020-06-27 17:33:49 -07001134 if ((flags->cf_flags & PyCF_ONLY_AST) && flags->cf_feature_version < 7) {
Lysandros Nikolaou3e0a6f32020-05-01 06:27:52 +03001135 parser_flags |= PyPARSE_ASYNC_HACKS;
1136 }
Pablo Galindo2b74c832020-04-27 18:02:07 +01001137 return parser_flags;
1138}
1139
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001140Parser *
Pablo Galindo2b74c832020-04-27 18:02:07 +01001141_PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags,
Lysandros Nikolaou3e0a6f32020-05-01 06:27:52 +03001142 int feature_version, int *errcode, PyArena *arena)
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001143{
1144 Parser *p = PyMem_Malloc(sizeof(Parser));
1145 if (p == NULL) {
Lysandros Nikolaouebebb642020-04-23 18:36:06 +03001146 return (Parser *) PyErr_NoMemory();
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001147 }
1148 assert(tok != NULL);
Guido van Rossumd9d6ead2020-05-01 09:42:32 -07001149 tok->type_comments = (flags & PyPARSE_TYPE_COMMENTS) > 0;
1150 tok->async_hacks = (flags & PyPARSE_ASYNC_HACKS) > 0;
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001151 p->tok = tok;
1152 p->keywords = NULL;
1153 p->n_keyword_lists = -1;
1154 p->tokens = PyMem_Malloc(sizeof(Token *));
1155 if (!p->tokens) {
1156 PyMem_Free(p);
Lysandros Nikolaouebebb642020-04-23 18:36:06 +03001157 return (Parser *) PyErr_NoMemory();
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001158 }
Guido van Rossumc001c092020-04-30 12:12:19 -07001159 p->tokens[0] = PyMem_Calloc(1, sizeof(Token));
Lysandros Nikolaouebebb642020-04-23 18:36:06 +03001160 if (!p->tokens) {
1161 PyMem_Free(p->tokens);
1162 PyMem_Free(p);
1163 return (Parser *) PyErr_NoMemory();
1164 }
Guido van Rossumc001c092020-04-30 12:12:19 -07001165 if (!growable_comment_array_init(&p->type_ignore_comments, 10)) {
1166 PyMem_Free(p->tokens[0]);
1167 PyMem_Free(p->tokens);
1168 PyMem_Free(p);
1169 return (Parser *) PyErr_NoMemory();
1170 }
1171
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001172 p->mark = 0;
1173 p->fill = 0;
1174 p->size = 1;
1175
1176 p->errcode = errcode;
1177 p->arena = arena;
1178 p->start_rule = start_rule;
1179 p->parsing_started = 0;
1180 p->normalize = NULL;
1181 p->error_indicator = 0;
1182
1183 p->starting_lineno = 0;
1184 p->starting_col_offset = 0;
Pablo Galindo2b74c832020-04-27 18:02:07 +01001185 p->flags = flags;
Lysandros Nikolaou3e0a6f32020-05-01 06:27:52 +03001186 p->feature_version = feature_version;
Lysandros Nikolaou2f37c352020-05-07 13:37:51 +03001187 p->known_err_token = NULL;
Pablo Galindo800a35c62020-05-25 18:38:45 +01001188 p->level = 0;
Lysandros Nikolaoubca70142020-10-27 00:42:04 +02001189 p->call_invalid_rules = 0;
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001190
1191 return p;
1192}
1193
Lysandros Nikolaoubca70142020-10-27 00:42:04 +02001194static void
1195reset_parser_state(Parser *p)
1196{
1197 for (int i = 0; i < p->fill; i++) {
1198 p->tokens[i]->memo = NULL;
1199 }
1200 p->mark = 0;
1201 p->call_invalid_rules = 1;
1202}
1203
Pablo Galindod6d63712021-01-19 23:59:33 +00001204static int
1205_PyPegen_check_tokenizer_errors(Parser *p) {
1206 // Tokenize the whole input to see if there are any tokenization
1207 // errors such as mistmatching parentheses. These will get priority
1208 // over generic syntax errors only if the line number of the error is
1209 // before the one that we had for the generic error.
1210
1211 // We don't want to tokenize to the end for interactive input
1212 if (p->tok->prompt != NULL) {
1213 return 0;
1214 }
1215
Pablo Galindod6d63712021-01-19 23:59:33 +00001216 Token *current_token = p->known_err_token != NULL ? p->known_err_token : p->tokens[p->fill - 1];
1217 Py_ssize_t current_err_line = current_token->lineno;
1218
Pablo Galindod6d63712021-01-19 23:59:33 +00001219 for (;;) {
1220 const char *start;
1221 const char *end;
1222 switch (PyTokenizer_Get(p->tok, &start, &end)) {
1223 case ERRORTOKEN:
1224 if (p->tok->level != 0) {
1225 int error_lineno = p->tok->parenlinenostack[p->tok->level-1];
1226 if (current_err_line > error_lineno) {
1227 raise_unclosed_parentheses_error(p);
1228 return -1;
1229 }
1230 }
1231 break;
1232 case ENDMARKER:
1233 break;
1234 default:
1235 continue;
1236 }
1237 break;
1238 }
1239
Pablo Galindod6d63712021-01-19 23:59:33 +00001240 return 0;
1241}
1242
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001243void *
1244_PyPegen_run_parser(Parser *p)
1245{
1246 void *res = _PyPegen_parse(p);
1247 if (res == NULL) {
Lysandros Nikolaoubca70142020-10-27 00:42:04 +02001248 reset_parser_state(p);
1249 _PyPegen_parse(p);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001250 if (PyErr_Occurred()) {
1251 return NULL;
1252 }
1253 if (p->fill == 0) {
1254 RAISE_SYNTAX_ERROR("error at start before reading any input");
1255 }
Pablo Galindocd8dcbc2021-03-14 04:38:40 +01001256 else if (p->tok->done == E_EOF) {
Pablo Galindod6d63712021-01-19 23:59:33 +00001257 if (p->tok->level) {
1258 raise_unclosed_parentheses_error(p);
1259 } else {
1260 RAISE_SYNTAX_ERROR("unexpected EOF while parsing");
1261 }
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001262 }
1263 else {
1264 if (p->tokens[p->fill-1]->type == INDENT) {
1265 RAISE_INDENTATION_ERROR("unexpected indent");
1266 }
1267 else if (p->tokens[p->fill-1]->type == DEDENT) {
1268 RAISE_INDENTATION_ERROR("unexpected unindent");
1269 }
1270 else {
1271 RAISE_SYNTAX_ERROR("invalid syntax");
Pablo Galindoc3f167d2021-01-20 19:11:56 +00001272 // _PyPegen_check_tokenizer_errors will override the existing
1273 // generic SyntaxError we just raised if errors are found.
1274 _PyPegen_check_tokenizer_errors(p);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001275 }
1276 }
1277 return NULL;
1278 }
1279
Lysandros Nikolaou6d650872020-04-29 04:42:27 +03001280 if (p->start_rule == Py_single_input && bad_single_statement(p)) {
1281 p->tok->done = E_BADSINGLE; // This is not necessary for now, but might be in the future
1282 return RAISE_SYNTAX_ERROR("multiple statements found while compiling a single statement");
1283 }
1284
Victor Stinnere0bf70d2021-03-18 02:46:06 +01001285 // test_peg_generator defines _Py_TEST_PEGEN to not call PyAST_Validate()
1286#if defined(Py_DEBUG) && !defined(_Py_TEST_PEGEN)
Pablo Galindo13322262020-07-27 23:46:59 +01001287 if (p->start_rule == Py_single_input ||
1288 p->start_rule == Py_file_input ||
1289 p->start_rule == Py_eval_input)
1290 {
Victor Stinnereec8e612021-03-18 14:57:49 +01001291 if (!_PyAST_Validate(res)) {
Batuhan Taskaya3af4b582020-10-30 14:48:41 +03001292 return NULL;
1293 }
Pablo Galindo13322262020-07-27 23:46:59 +01001294 }
1295#endif
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001296 return res;
1297}
1298
1299mod_ty
1300_PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filename_ob,
1301 const char *enc, const char *ps1, const char *ps2,
Pablo Galindo2b74c832020-04-27 18:02:07 +01001302 PyCompilerFlags *flags, int *errcode, PyArena *arena)
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001303{
1304 struct tok_state *tok = PyTokenizer_FromFile(fp, enc, ps1, ps2);
1305 if (tok == NULL) {
1306 if (PyErr_Occurred()) {
1307 raise_tokenizer_init_error(filename_ob);
1308 return NULL;
1309 }
1310 return NULL;
1311 }
Pablo Galindocd8dcbc2021-03-14 04:38:40 +01001312 if (!tok->fp || ps1 != NULL || ps2 != NULL ||
1313 PyUnicode_CompareWithASCIIString(filename_ob, "<stdin>") == 0) {
1314 tok->fp_interactive = 1;
1315 }
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001316 // This transfers the ownership to the tokenizer
1317 tok->filename = filename_ob;
1318 Py_INCREF(filename_ob);
1319
1320 // From here on we need to clean up even if there's an error
1321 mod_ty result = NULL;
1322
Pablo Galindo2b74c832020-04-27 18:02:07 +01001323 int parser_flags = compute_parser_flags(flags);
Lysandros Nikolaou3e0a6f32020-05-01 06:27:52 +03001324 Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, PY_MINOR_VERSION,
1325 errcode, arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001326 if (p == NULL) {
1327 goto error;
1328 }
1329
1330 result = _PyPegen_run_parser(p);
1331 _PyPegen_Parser_Free(p);
1332
1333error:
1334 PyTokenizer_Free(tok);
1335 return result;
1336}
1337
1338mod_ty
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001339_PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filename_ob,
Pablo Galindo2b74c832020-04-27 18:02:07 +01001340 PyCompilerFlags *flags, PyArena *arena)
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001341{
1342 int exec_input = start_rule == Py_file_input;
1343
1344 struct tok_state *tok;
Pablo Galindo2b74c832020-04-27 18:02:07 +01001345 if (flags == NULL || flags->cf_flags & PyCF_IGNORE_COOKIE) {
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001346 tok = PyTokenizer_FromUTF8(str, exec_input);
1347 } else {
1348 tok = PyTokenizer_FromString(str, exec_input);
1349 }
1350 if (tok == NULL) {
1351 if (PyErr_Occurred()) {
1352 raise_tokenizer_init_error(filename_ob);
1353 }
1354 return NULL;
1355 }
1356 // This transfers the ownership to the tokenizer
1357 tok->filename = filename_ob;
1358 Py_INCREF(filename_ob);
1359
1360 // We need to clear up from here on
1361 mod_ty result = NULL;
1362
Pablo Galindo2b74c832020-04-27 18:02:07 +01001363 int parser_flags = compute_parser_flags(flags);
Guido van Rossum9d197c72020-06-27 17:33:49 -07001364 int feature_version = flags && (flags->cf_flags & PyCF_ONLY_AST) ?
1365 flags->cf_feature_version : PY_MINOR_VERSION;
Lysandros Nikolaou3e0a6f32020-05-01 06:27:52 +03001366 Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, feature_version,
1367 NULL, arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001368 if (p == NULL) {
1369 goto error;
1370 }
1371
1372 result = _PyPegen_run_parser(p);
1373 _PyPegen_Parser_Free(p);
1374
1375error:
1376 PyTokenizer_Free(tok);
1377 return result;
1378}
1379
Pablo Galindoa5634c42020-09-16 19:42:00 +01001380asdl_stmt_seq*
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001381_PyPegen_interactive_exit(Parser *p)
1382{
1383 if (p->errcode) {
1384 *(p->errcode) = E_EOF;
1385 }
1386 return NULL;
1387}
1388
1389/* Creates a single-element asdl_seq* that contains a */
1390asdl_seq *
1391_PyPegen_singleton_seq(Parser *p, void *a)
1392{
1393 assert(a != NULL);
Pablo Galindoa5634c42020-09-16 19:42:00 +01001394 asdl_seq *seq = (asdl_seq*)_Py_asdl_generic_seq_new(1, p->arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001395 if (!seq) {
1396 return NULL;
1397 }
Pablo Galindoa5634c42020-09-16 19:42:00 +01001398 asdl_seq_SET_UNTYPED(seq, 0, a);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001399 return seq;
1400}
1401
1402/* Creates a copy of seq and prepends a to it */
1403asdl_seq *
1404_PyPegen_seq_insert_in_front(Parser *p, void *a, asdl_seq *seq)
1405{
1406 assert(a != NULL);
1407 if (!seq) {
1408 return _PyPegen_singleton_seq(p, a);
1409 }
1410
Pablo Galindoa5634c42020-09-16 19:42:00 +01001411 asdl_seq *new_seq = (asdl_seq*)_Py_asdl_generic_seq_new(asdl_seq_LEN(seq) + 1, p->arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001412 if (!new_seq) {
1413 return NULL;
1414 }
1415
Pablo Galindoa5634c42020-09-16 19:42:00 +01001416 asdl_seq_SET_UNTYPED(new_seq, 0, a);
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001417 for (Py_ssize_t i = 1, l = asdl_seq_LEN(new_seq); i < l; i++) {
Pablo Galindoa5634c42020-09-16 19:42:00 +01001418 asdl_seq_SET_UNTYPED(new_seq, i, asdl_seq_GET_UNTYPED(seq, i - 1));
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001419 }
1420 return new_seq;
1421}
1422
Guido van Rossumc001c092020-04-30 12:12:19 -07001423/* Creates a copy of seq and appends a to it */
1424asdl_seq *
1425_PyPegen_seq_append_to_end(Parser *p, asdl_seq *seq, void *a)
1426{
1427 assert(a != NULL);
1428 if (!seq) {
1429 return _PyPegen_singleton_seq(p, a);
1430 }
1431
Pablo Galindoa5634c42020-09-16 19:42:00 +01001432 asdl_seq *new_seq = (asdl_seq*)_Py_asdl_generic_seq_new(asdl_seq_LEN(seq) + 1, p->arena);
Guido van Rossumc001c092020-04-30 12:12:19 -07001433 if (!new_seq) {
1434 return NULL;
1435 }
1436
1437 for (Py_ssize_t i = 0, l = asdl_seq_LEN(new_seq); i + 1 < l; i++) {
Pablo Galindoa5634c42020-09-16 19:42:00 +01001438 asdl_seq_SET_UNTYPED(new_seq, i, asdl_seq_GET_UNTYPED(seq, i));
Guido van Rossumc001c092020-04-30 12:12:19 -07001439 }
Pablo Galindoa5634c42020-09-16 19:42:00 +01001440 asdl_seq_SET_UNTYPED(new_seq, asdl_seq_LEN(new_seq) - 1, a);
Guido van Rossumc001c092020-04-30 12:12:19 -07001441 return new_seq;
1442}
1443
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001444static Py_ssize_t
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001445_get_flattened_seq_size(asdl_seq *seqs)
1446{
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001447 Py_ssize_t size = 0;
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001448 for (Py_ssize_t i = 0, l = asdl_seq_LEN(seqs); i < l; i++) {
Pablo Galindoa5634c42020-09-16 19:42:00 +01001449 asdl_seq *inner_seq = asdl_seq_GET_UNTYPED(seqs, i);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001450 size += asdl_seq_LEN(inner_seq);
1451 }
1452 return size;
1453}
1454
1455/* Flattens an asdl_seq* of asdl_seq*s */
1456asdl_seq *
1457_PyPegen_seq_flatten(Parser *p, asdl_seq *seqs)
1458{
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001459 Py_ssize_t flattened_seq_size = _get_flattened_seq_size(seqs);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001460 assert(flattened_seq_size > 0);
1461
Pablo Galindoa5634c42020-09-16 19:42:00 +01001462 asdl_seq *flattened_seq = (asdl_seq*)_Py_asdl_generic_seq_new(flattened_seq_size, p->arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001463 if (!flattened_seq) {
1464 return NULL;
1465 }
1466
1467 int flattened_seq_idx = 0;
1468 for (Py_ssize_t i = 0, l = asdl_seq_LEN(seqs); i < l; i++) {
Pablo Galindoa5634c42020-09-16 19:42:00 +01001469 asdl_seq *inner_seq = asdl_seq_GET_UNTYPED(seqs, i);
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001470 for (Py_ssize_t j = 0, li = asdl_seq_LEN(inner_seq); j < li; j++) {
Pablo Galindoa5634c42020-09-16 19:42:00 +01001471 asdl_seq_SET_UNTYPED(flattened_seq, flattened_seq_idx++, asdl_seq_GET_UNTYPED(inner_seq, j));
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001472 }
1473 }
1474 assert(flattened_seq_idx == flattened_seq_size);
1475
1476 return flattened_seq;
1477}
1478
1479/* Creates a new name of the form <first_name>.<second_name> */
1480expr_ty
1481_PyPegen_join_names_with_dot(Parser *p, expr_ty first_name, expr_ty second_name)
1482{
1483 assert(first_name != NULL && second_name != NULL);
1484 PyObject *first_identifier = first_name->v.Name.id;
1485 PyObject *second_identifier = second_name->v.Name.id;
1486
1487 if (PyUnicode_READY(first_identifier) == -1) {
1488 return NULL;
1489 }
1490 if (PyUnicode_READY(second_identifier) == -1) {
1491 return NULL;
1492 }
1493 const char *first_str = PyUnicode_AsUTF8(first_identifier);
1494 if (!first_str) {
1495 return NULL;
1496 }
1497 const char *second_str = PyUnicode_AsUTF8(second_identifier);
1498 if (!second_str) {
1499 return NULL;
1500 }
Pablo Galindo9f27dd32020-04-24 01:13:33 +01001501 Py_ssize_t len = strlen(first_str) + strlen(second_str) + 1; // +1 for the dot
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001502
1503 PyObject *str = PyBytes_FromStringAndSize(NULL, len);
1504 if (!str) {
1505 return NULL;
1506 }
1507
1508 char *s = PyBytes_AS_STRING(str);
1509 if (!s) {
1510 return NULL;
1511 }
1512
1513 strcpy(s, first_str);
1514 s += strlen(first_str);
1515 *s++ = '.';
1516 strcpy(s, second_str);
1517 s += strlen(second_str);
1518 *s = '\0';
1519
1520 PyObject *uni = PyUnicode_DecodeUTF8(PyBytes_AS_STRING(str), PyBytes_GET_SIZE(str), NULL);
1521 Py_DECREF(str);
1522 if (!uni) {
1523 return NULL;
1524 }
1525 PyUnicode_InternInPlace(&uni);
Victor Stinner8370e072021-03-24 02:23:01 +01001526 if (_PyArena_AddPyObject(p->arena, uni) < 0) {
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001527 Py_DECREF(uni);
1528 return NULL;
1529 }
1530
Victor Stinnerd27f8d22021-04-07 21:34:22 +02001531 return _PyAST_Name(uni, Load, EXTRA_EXPR(first_name, second_name));
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001532}
1533
1534/* Counts the total number of dots in seq's tokens */
1535int
1536_PyPegen_seq_count_dots(asdl_seq *seq)
1537{
1538 int number_of_dots = 0;
1539 for (Py_ssize_t i = 0, l = asdl_seq_LEN(seq); i < l; i++) {
Pablo Galindoa5634c42020-09-16 19:42:00 +01001540 Token *current_expr = asdl_seq_GET_UNTYPED(seq, i);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001541 switch (current_expr->type) {
1542 case ELLIPSIS:
1543 number_of_dots += 3;
1544 break;
1545 case DOT:
1546 number_of_dots += 1;
1547 break;
1548 default:
Lysandros Nikolaouebebb642020-04-23 18:36:06 +03001549 Py_UNREACHABLE();
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001550 }
1551 }
1552
1553 return number_of_dots;
1554}
1555
1556/* Creates an alias with '*' as the identifier name */
1557alias_ty
Matthew Suozzo75a06f02021-04-10 16:56:28 -04001558_PyPegen_alias_for_star(Parser *p, int lineno, int col_offset, int end_lineno,
1559 int end_col_offset, PyArena *arena) {
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001560 PyObject *str = PyUnicode_InternFromString("*");
1561 if (!str) {
1562 return NULL;
1563 }
Victor Stinner8370e072021-03-24 02:23:01 +01001564 if (_PyArena_AddPyObject(p->arena, str) < 0) {
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001565 Py_DECREF(str);
1566 return NULL;
1567 }
Matthew Suozzo75a06f02021-04-10 16:56:28 -04001568 return _PyAST_alias(str, NULL, lineno, col_offset, end_lineno, end_col_offset, arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001569}
1570
1571/* Creates a new asdl_seq* with the identifiers of all the names in seq */
Pablo Galindoa5634c42020-09-16 19:42:00 +01001572asdl_identifier_seq *
1573_PyPegen_map_names_to_ids(Parser *p, asdl_expr_seq *seq)
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001574{
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001575 Py_ssize_t len = asdl_seq_LEN(seq);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001576 assert(len > 0);
1577
Pablo Galindoa5634c42020-09-16 19:42:00 +01001578 asdl_identifier_seq *new_seq = _Py_asdl_identifier_seq_new(len, p->arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001579 if (!new_seq) {
1580 return NULL;
1581 }
1582 for (Py_ssize_t i = 0; i < len; i++) {
1583 expr_ty e = asdl_seq_GET(seq, i);
1584 asdl_seq_SET(new_seq, i, e->v.Name.id);
1585 }
1586 return new_seq;
1587}
1588
1589/* Constructs a CmpopExprPair */
1590CmpopExprPair *
1591_PyPegen_cmpop_expr_pair(Parser *p, cmpop_ty cmpop, expr_ty expr)
1592{
1593 assert(expr != NULL);
Victor Stinner8370e072021-03-24 02:23:01 +01001594 CmpopExprPair *a = _PyArena_Malloc(p->arena, sizeof(CmpopExprPair));
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001595 if (!a) {
1596 return NULL;
1597 }
1598 a->cmpop = cmpop;
1599 a->expr = expr;
1600 return a;
1601}
1602
1603asdl_int_seq *
1604_PyPegen_get_cmpops(Parser *p, asdl_seq *seq)
1605{
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001606 Py_ssize_t len = asdl_seq_LEN(seq);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001607 assert(len > 0);
1608
1609 asdl_int_seq *new_seq = _Py_asdl_int_seq_new(len, p->arena);
1610 if (!new_seq) {
1611 return NULL;
1612 }
1613 for (Py_ssize_t i = 0; i < len; i++) {
Pablo Galindoa5634c42020-09-16 19:42:00 +01001614 CmpopExprPair *pair = asdl_seq_GET_UNTYPED(seq, i);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001615 asdl_seq_SET(new_seq, i, pair->cmpop);
1616 }
1617 return new_seq;
1618}
1619
Pablo Galindoa5634c42020-09-16 19:42:00 +01001620asdl_expr_seq *
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001621_PyPegen_get_exprs(Parser *p, asdl_seq *seq)
1622{
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001623 Py_ssize_t len = asdl_seq_LEN(seq);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001624 assert(len > 0);
1625
Pablo Galindoa5634c42020-09-16 19:42:00 +01001626 asdl_expr_seq *new_seq = _Py_asdl_expr_seq_new(len, p->arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001627 if (!new_seq) {
1628 return NULL;
1629 }
1630 for (Py_ssize_t i = 0; i < len; i++) {
Pablo Galindoa5634c42020-09-16 19:42:00 +01001631 CmpopExprPair *pair = asdl_seq_GET_UNTYPED(seq, i);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001632 asdl_seq_SET(new_seq, i, pair->expr);
1633 }
1634 return new_seq;
1635}
1636
1637/* Creates an asdl_seq* where all the elements have been changed to have ctx as context */
Pablo Galindoa5634c42020-09-16 19:42:00 +01001638static asdl_expr_seq *
1639_set_seq_context(Parser *p, asdl_expr_seq *seq, expr_context_ty ctx)
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001640{
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001641 Py_ssize_t len = asdl_seq_LEN(seq);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001642 if (len == 0) {
1643 return NULL;
1644 }
1645
Pablo Galindoa5634c42020-09-16 19:42:00 +01001646 asdl_expr_seq *new_seq = _Py_asdl_expr_seq_new(len, p->arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001647 if (!new_seq) {
1648 return NULL;
1649 }
1650 for (Py_ssize_t i = 0; i < len; i++) {
1651 expr_ty e = asdl_seq_GET(seq, i);
1652 asdl_seq_SET(new_seq, i, _PyPegen_set_expr_context(p, e, ctx));
1653 }
1654 return new_seq;
1655}
1656
1657static expr_ty
1658_set_name_context(Parser *p, expr_ty e, expr_context_ty ctx)
1659{
Victor Stinnerd27f8d22021-04-07 21:34:22 +02001660 return _PyAST_Name(e->v.Name.id, ctx, EXTRA_EXPR(e, e));
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001661}
1662
1663static expr_ty
1664_set_tuple_context(Parser *p, expr_ty e, expr_context_ty ctx)
1665{
Victor Stinnerd27f8d22021-04-07 21:34:22 +02001666 return _PyAST_Tuple(
Pablo Galindoa5634c42020-09-16 19:42:00 +01001667 _set_seq_context(p, e->v.Tuple.elts, ctx),
1668 ctx,
1669 EXTRA_EXPR(e, e));
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001670}
1671
1672static expr_ty
1673_set_list_context(Parser *p, expr_ty e, expr_context_ty ctx)
1674{
Victor Stinnerd27f8d22021-04-07 21:34:22 +02001675 return _PyAST_List(
Pablo Galindoa5634c42020-09-16 19:42:00 +01001676 _set_seq_context(p, e->v.List.elts, ctx),
1677 ctx,
1678 EXTRA_EXPR(e, e));
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001679}
1680
1681static expr_ty
1682_set_subscript_context(Parser *p, expr_ty e, expr_context_ty ctx)
1683{
Victor Stinnerd27f8d22021-04-07 21:34:22 +02001684 return _PyAST_Subscript(e->v.Subscript.value, e->v.Subscript.slice,
1685 ctx, EXTRA_EXPR(e, e));
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001686}
1687
1688static expr_ty
1689_set_attribute_context(Parser *p, expr_ty e, expr_context_ty ctx)
1690{
Victor Stinnerd27f8d22021-04-07 21:34:22 +02001691 return _PyAST_Attribute(e->v.Attribute.value, e->v.Attribute.attr,
1692 ctx, EXTRA_EXPR(e, e));
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001693}
1694
1695static expr_ty
1696_set_starred_context(Parser *p, expr_ty e, expr_context_ty ctx)
1697{
Victor Stinnerd27f8d22021-04-07 21:34:22 +02001698 return _PyAST_Starred(_PyPegen_set_expr_context(p, e->v.Starred.value, ctx),
1699 ctx, EXTRA_EXPR(e, e));
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001700}
1701
1702/* Creates an `expr_ty` equivalent to `expr` but with `ctx` as context */
1703expr_ty
1704_PyPegen_set_expr_context(Parser *p, expr_ty expr, expr_context_ty ctx)
1705{
1706 assert(expr != NULL);
1707
1708 expr_ty new = NULL;
1709 switch (expr->kind) {
1710 case Name_kind:
1711 new = _set_name_context(p, expr, ctx);
1712 break;
1713 case Tuple_kind:
1714 new = _set_tuple_context(p, expr, ctx);
1715 break;
1716 case List_kind:
1717 new = _set_list_context(p, expr, ctx);
1718 break;
1719 case Subscript_kind:
1720 new = _set_subscript_context(p, expr, ctx);
1721 break;
1722 case Attribute_kind:
1723 new = _set_attribute_context(p, expr, ctx);
1724 break;
1725 case Starred_kind:
1726 new = _set_starred_context(p, expr, ctx);
1727 break;
1728 default:
1729 new = expr;
1730 }
1731 return new;
1732}
1733
1734/* Constructs a KeyValuePair that is used when parsing a dict's key value pairs */
1735KeyValuePair *
1736_PyPegen_key_value_pair(Parser *p, expr_ty key, expr_ty value)
1737{
Victor Stinner8370e072021-03-24 02:23:01 +01001738 KeyValuePair *a = _PyArena_Malloc(p->arena, sizeof(KeyValuePair));
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001739 if (!a) {
1740 return NULL;
1741 }
1742 a->key = key;
1743 a->value = value;
1744 return a;
1745}
1746
1747/* Extracts all keys from an asdl_seq* of KeyValuePair*'s */
Pablo Galindoa5634c42020-09-16 19:42:00 +01001748asdl_expr_seq *
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001749_PyPegen_get_keys(Parser *p, asdl_seq *seq)
1750{
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001751 Py_ssize_t len = asdl_seq_LEN(seq);
Pablo Galindoa5634c42020-09-16 19:42:00 +01001752 asdl_expr_seq *new_seq = _Py_asdl_expr_seq_new(len, p->arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001753 if (!new_seq) {
1754 return NULL;
1755 }
1756 for (Py_ssize_t i = 0; i < len; i++) {
Pablo Galindoa5634c42020-09-16 19:42:00 +01001757 KeyValuePair *pair = asdl_seq_GET_UNTYPED(seq, i);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001758 asdl_seq_SET(new_seq, i, pair->key);
1759 }
1760 return new_seq;
1761}
1762
1763/* Extracts all values from an asdl_seq* of KeyValuePair*'s */
Pablo Galindoa5634c42020-09-16 19:42:00 +01001764asdl_expr_seq *
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001765_PyPegen_get_values(Parser *p, asdl_seq *seq)
1766{
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001767 Py_ssize_t len = asdl_seq_LEN(seq);
Pablo Galindoa5634c42020-09-16 19:42:00 +01001768 asdl_expr_seq *new_seq = _Py_asdl_expr_seq_new(len, p->arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001769 if (!new_seq) {
1770 return NULL;
1771 }
1772 for (Py_ssize_t i = 0; i < len; i++) {
Pablo Galindoa5634c42020-09-16 19:42:00 +01001773 KeyValuePair *pair = asdl_seq_GET_UNTYPED(seq, i);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001774 asdl_seq_SET(new_seq, i, pair->value);
1775 }
1776 return new_seq;
1777}
1778
1779/* Constructs a NameDefaultPair */
1780NameDefaultPair *
Guido van Rossumc001c092020-04-30 12:12:19 -07001781_PyPegen_name_default_pair(Parser *p, arg_ty arg, expr_ty value, Token *tc)
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001782{
Victor Stinner8370e072021-03-24 02:23:01 +01001783 NameDefaultPair *a = _PyArena_Malloc(p->arena, sizeof(NameDefaultPair));
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001784 if (!a) {
1785 return NULL;
1786 }
Guido van Rossumc001c092020-04-30 12:12:19 -07001787 a->arg = _PyPegen_add_type_comment_to_arg(p, arg, tc);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001788 a->value = value;
1789 return a;
1790}
1791
1792/* Constructs a SlashWithDefault */
1793SlashWithDefault *
Pablo Galindoa5634c42020-09-16 19:42:00 +01001794_PyPegen_slash_with_default(Parser *p, asdl_arg_seq *plain_names, asdl_seq *names_with_defaults)
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001795{
Victor Stinner8370e072021-03-24 02:23:01 +01001796 SlashWithDefault *a = _PyArena_Malloc(p->arena, sizeof(SlashWithDefault));
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001797 if (!a) {
1798 return NULL;
1799 }
1800 a->plain_names = plain_names;
1801 a->names_with_defaults = names_with_defaults;
1802 return a;
1803}
1804
1805/* Constructs a StarEtc */
1806StarEtc *
1807_PyPegen_star_etc(Parser *p, arg_ty vararg, asdl_seq *kwonlyargs, arg_ty kwarg)
1808{
Victor Stinner8370e072021-03-24 02:23:01 +01001809 StarEtc *a = _PyArena_Malloc(p->arena, sizeof(StarEtc));
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001810 if (!a) {
1811 return NULL;
1812 }
1813 a->vararg = vararg;
1814 a->kwonlyargs = kwonlyargs;
1815 a->kwarg = kwarg;
1816 return a;
1817}
1818
1819asdl_seq *
1820_PyPegen_join_sequences(Parser *p, asdl_seq *a, asdl_seq *b)
1821{
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001822 Py_ssize_t first_len = asdl_seq_LEN(a);
1823 Py_ssize_t second_len = asdl_seq_LEN(b);
Pablo Galindoa5634c42020-09-16 19:42:00 +01001824 asdl_seq *new_seq = (asdl_seq*)_Py_asdl_generic_seq_new(first_len + second_len, p->arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001825 if (!new_seq) {
1826 return NULL;
1827 }
1828
1829 int k = 0;
1830 for (Py_ssize_t i = 0; i < first_len; i++) {
Pablo Galindoa5634c42020-09-16 19:42:00 +01001831 asdl_seq_SET_UNTYPED(new_seq, k++, asdl_seq_GET_UNTYPED(a, i));
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001832 }
1833 for (Py_ssize_t i = 0; i < second_len; i++) {
Pablo Galindoa5634c42020-09-16 19:42:00 +01001834 asdl_seq_SET_UNTYPED(new_seq, k++, asdl_seq_GET_UNTYPED(b, i));
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001835 }
1836
1837 return new_seq;
1838}
1839
Pablo Galindoa5634c42020-09-16 19:42:00 +01001840static asdl_arg_seq*
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001841_get_names(Parser *p, asdl_seq *names_with_defaults)
1842{
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001843 Py_ssize_t len = asdl_seq_LEN(names_with_defaults);
Pablo Galindoa5634c42020-09-16 19:42:00 +01001844 asdl_arg_seq *seq = _Py_asdl_arg_seq_new(len, p->arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001845 if (!seq) {
1846 return NULL;
1847 }
1848 for (Py_ssize_t i = 0; i < len; i++) {
Pablo Galindoa5634c42020-09-16 19:42:00 +01001849 NameDefaultPair *pair = asdl_seq_GET_UNTYPED(names_with_defaults, i);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001850 asdl_seq_SET(seq, i, pair->arg);
1851 }
1852 return seq;
1853}
1854
Pablo Galindoa5634c42020-09-16 19:42:00 +01001855static asdl_expr_seq *
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001856_get_defaults(Parser *p, asdl_seq *names_with_defaults)
1857{
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001858 Py_ssize_t len = asdl_seq_LEN(names_with_defaults);
Pablo Galindoa5634c42020-09-16 19:42:00 +01001859 asdl_expr_seq *seq = _Py_asdl_expr_seq_new(len, p->arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001860 if (!seq) {
1861 return NULL;
1862 }
1863 for (Py_ssize_t i = 0; i < len; i++) {
Pablo Galindoa5634c42020-09-16 19:42:00 +01001864 NameDefaultPair *pair = asdl_seq_GET_UNTYPED(names_with_defaults, i);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001865 asdl_seq_SET(seq, i, pair->value);
1866 }
1867 return seq;
1868}
1869
Pablo Galindo4f642da2021-04-09 00:48:53 +01001870static int
1871_make_posonlyargs(Parser *p,
1872 asdl_arg_seq *slash_without_default,
1873 SlashWithDefault *slash_with_default,
1874 asdl_arg_seq **posonlyargs) {
1875 if (slash_without_default != NULL) {
1876 *posonlyargs = slash_without_default;
1877 }
1878 else if (slash_with_default != NULL) {
1879 asdl_arg_seq *slash_with_default_names =
1880 _get_names(p, slash_with_default->names_with_defaults);
1881 if (!slash_with_default_names) {
1882 return -1;
1883 }
1884 *posonlyargs = (asdl_arg_seq*)_PyPegen_join_sequences(
1885 p,
1886 (asdl_seq*)slash_with_default->plain_names,
1887 (asdl_seq*)slash_with_default_names);
1888 }
1889 else {
1890 *posonlyargs = _Py_asdl_arg_seq_new(0, p->arena);
1891 }
1892 return *posonlyargs == NULL ? -1 : 0;
1893}
1894
1895static int
1896_make_posargs(Parser *p,
1897 asdl_arg_seq *plain_names,
1898 asdl_seq *names_with_default,
1899 asdl_arg_seq **posargs) {
1900 if (plain_names != NULL && names_with_default != NULL) {
1901 asdl_arg_seq *names_with_default_names = _get_names(p, names_with_default);
1902 if (!names_with_default_names) {
1903 return -1;
1904 }
1905 *posargs = (asdl_arg_seq*)_PyPegen_join_sequences(
1906 p,(asdl_seq*)plain_names, (asdl_seq*)names_with_default_names);
1907 }
1908 else if (plain_names == NULL && names_with_default != NULL) {
1909 *posargs = _get_names(p, names_with_default);
1910 }
1911 else if (plain_names != NULL && names_with_default == NULL) {
1912 *posargs = plain_names;
1913 }
1914 else {
1915 *posargs = _Py_asdl_arg_seq_new(0, p->arena);
1916 }
1917 return *posargs == NULL ? -1 : 0;
1918}
1919
1920static int
1921_make_posdefaults(Parser *p,
1922 SlashWithDefault *slash_with_default,
1923 asdl_seq *names_with_default,
1924 asdl_expr_seq **posdefaults) {
1925 if (slash_with_default != NULL && names_with_default != NULL) {
1926 asdl_expr_seq *slash_with_default_values =
1927 _get_defaults(p, slash_with_default->names_with_defaults);
1928 if (!slash_with_default_values) {
1929 return -1;
1930 }
1931 asdl_expr_seq *names_with_default_values = _get_defaults(p, names_with_default);
1932 if (!names_with_default_values) {
1933 return -1;
1934 }
1935 *posdefaults = (asdl_expr_seq*)_PyPegen_join_sequences(
1936 p,
1937 (asdl_seq*)slash_with_default_values,
1938 (asdl_seq*)names_with_default_values);
1939 }
1940 else if (slash_with_default == NULL && names_with_default != NULL) {
1941 *posdefaults = _get_defaults(p, names_with_default);
1942 }
1943 else if (slash_with_default != NULL && names_with_default == NULL) {
1944 *posdefaults = _get_defaults(p, slash_with_default->names_with_defaults);
1945 }
1946 else {
1947 *posdefaults = _Py_asdl_expr_seq_new(0, p->arena);
1948 }
1949 return *posdefaults == NULL ? -1 : 0;
1950}
1951
1952static int
1953_make_kwargs(Parser *p, StarEtc *star_etc,
1954 asdl_arg_seq **kwonlyargs,
1955 asdl_expr_seq **kwdefaults) {
1956 if (star_etc != NULL && star_etc->kwonlyargs != NULL) {
1957 *kwonlyargs = _get_names(p, star_etc->kwonlyargs);
1958 }
1959 else {
1960 *kwonlyargs = _Py_asdl_arg_seq_new(0, p->arena);
1961 }
1962
1963 if (*kwonlyargs == NULL) {
1964 return -1;
1965 }
1966
1967 if (star_etc != NULL && star_etc->kwonlyargs != NULL) {
1968 *kwdefaults = _get_defaults(p, star_etc->kwonlyargs);
1969 }
1970 else {
1971 *kwdefaults = _Py_asdl_expr_seq_new(0, p->arena);
1972 }
1973
1974 if (*kwdefaults == NULL) {
1975 return -1;
1976 }
1977
1978 return 0;
1979}
1980
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001981/* Constructs an arguments_ty object out of all the parsed constructs in the parameters rule */
1982arguments_ty
Pablo Galindoa5634c42020-09-16 19:42:00 +01001983_PyPegen_make_arguments(Parser *p, asdl_arg_seq *slash_without_default,
1984 SlashWithDefault *slash_with_default, asdl_arg_seq *plain_names,
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001985 asdl_seq *names_with_default, StarEtc *star_etc)
1986{
Pablo Galindoa5634c42020-09-16 19:42:00 +01001987 asdl_arg_seq *posonlyargs;
Pablo Galindo4f642da2021-04-09 00:48:53 +01001988 if (_make_posonlyargs(p, slash_without_default, slash_with_default, &posonlyargs) == -1) {
1989 return NULL;
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001990 }
1991
Pablo Galindoa5634c42020-09-16 19:42:00 +01001992 asdl_arg_seq *posargs;
Pablo Galindo4f642da2021-04-09 00:48:53 +01001993 if (_make_posargs(p, plain_names, names_with_default, &posargs) == -1) {
1994 return NULL;
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001995 }
1996
Pablo Galindoa5634c42020-09-16 19:42:00 +01001997 asdl_expr_seq *posdefaults;
Pablo Galindo4f642da2021-04-09 00:48:53 +01001998 if (_make_posdefaults(p,slash_with_default, names_with_default, &posdefaults) == -1) {
1999 return NULL;
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002000 }
2001
2002 arg_ty vararg = NULL;
2003 if (star_etc != NULL && star_etc->vararg != NULL) {
2004 vararg = star_etc->vararg;
2005 }
2006
Pablo Galindoa5634c42020-09-16 19:42:00 +01002007 asdl_arg_seq *kwonlyargs;
Pablo Galindoa5634c42020-09-16 19:42:00 +01002008 asdl_expr_seq *kwdefaults;
Pablo Galindo4f642da2021-04-09 00:48:53 +01002009 if (_make_kwargs(p, star_etc, &kwonlyargs, &kwdefaults) == -1) {
2010 return NULL;
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002011 }
2012
2013 arg_ty kwarg = NULL;
2014 if (star_etc != NULL && star_etc->kwarg != NULL) {
2015 kwarg = star_etc->kwarg;
2016 }
2017
Victor Stinnerd27f8d22021-04-07 21:34:22 +02002018 return _PyAST_arguments(posonlyargs, posargs, vararg, kwonlyargs,
2019 kwdefaults, kwarg, posdefaults, p->arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002020}
2021
Pablo Galindo4f642da2021-04-09 00:48:53 +01002022
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002023/* Constructs an empty arguments_ty object, that gets used when a function accepts no
2024 * arguments. */
2025arguments_ty
2026_PyPegen_empty_arguments(Parser *p)
2027{
Pablo Galindoa5634c42020-09-16 19:42:00 +01002028 asdl_arg_seq *posonlyargs = _Py_asdl_arg_seq_new(0, p->arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002029 if (!posonlyargs) {
2030 return NULL;
2031 }
Pablo Galindoa5634c42020-09-16 19:42:00 +01002032 asdl_arg_seq *posargs = _Py_asdl_arg_seq_new(0, p->arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002033 if (!posargs) {
2034 return NULL;
2035 }
Pablo Galindoa5634c42020-09-16 19:42:00 +01002036 asdl_expr_seq *posdefaults = _Py_asdl_expr_seq_new(0, p->arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002037 if (!posdefaults) {
2038 return NULL;
2039 }
Pablo Galindoa5634c42020-09-16 19:42:00 +01002040 asdl_arg_seq *kwonlyargs = _Py_asdl_arg_seq_new(0, p->arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002041 if (!kwonlyargs) {
2042 return NULL;
2043 }
Pablo Galindoa5634c42020-09-16 19:42:00 +01002044 asdl_expr_seq *kwdefaults = _Py_asdl_expr_seq_new(0, p->arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002045 if (!kwdefaults) {
2046 return NULL;
2047 }
2048
Victor Stinnerd27f8d22021-04-07 21:34:22 +02002049 return _PyAST_arguments(posonlyargs, posargs, NULL, kwonlyargs,
2050 kwdefaults, NULL, posdefaults, p->arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002051}
2052
2053/* Encapsulates the value of an operator_ty into an AugOperator struct */
2054AugOperator *
2055_PyPegen_augoperator(Parser *p, operator_ty kind)
2056{
Victor Stinner8370e072021-03-24 02:23:01 +01002057 AugOperator *a = _PyArena_Malloc(p->arena, sizeof(AugOperator));
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002058 if (!a) {
2059 return NULL;
2060 }
2061 a->kind = kind;
2062 return a;
2063}
2064
2065/* Construct a FunctionDef equivalent to function_def, but with decorators */
2066stmt_ty
Pablo Galindoa5634c42020-09-16 19:42:00 +01002067_PyPegen_function_def_decorators(Parser *p, asdl_expr_seq *decorators, stmt_ty function_def)
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002068{
2069 assert(function_def != NULL);
2070 if (function_def->kind == AsyncFunctionDef_kind) {
Victor Stinnerd27f8d22021-04-07 21:34:22 +02002071 return _PyAST_AsyncFunctionDef(
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002072 function_def->v.FunctionDef.name, function_def->v.FunctionDef.args,
2073 function_def->v.FunctionDef.body, decorators, function_def->v.FunctionDef.returns,
2074 function_def->v.FunctionDef.type_comment, function_def->lineno,
2075 function_def->col_offset, function_def->end_lineno, function_def->end_col_offset,
2076 p->arena);
2077 }
2078
Victor Stinnerd27f8d22021-04-07 21:34:22 +02002079 return _PyAST_FunctionDef(
2080 function_def->v.FunctionDef.name, function_def->v.FunctionDef.args,
2081 function_def->v.FunctionDef.body, decorators,
2082 function_def->v.FunctionDef.returns,
2083 function_def->v.FunctionDef.type_comment, function_def->lineno,
2084 function_def->col_offset, function_def->end_lineno,
2085 function_def->end_col_offset, p->arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002086}
2087
2088/* Construct a ClassDef equivalent to class_def, but with decorators */
2089stmt_ty
Pablo Galindoa5634c42020-09-16 19:42:00 +01002090_PyPegen_class_def_decorators(Parser *p, asdl_expr_seq *decorators, stmt_ty class_def)
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002091{
2092 assert(class_def != NULL);
Victor Stinnerd27f8d22021-04-07 21:34:22 +02002093 return _PyAST_ClassDef(
2094 class_def->v.ClassDef.name, class_def->v.ClassDef.bases,
2095 class_def->v.ClassDef.keywords, class_def->v.ClassDef.body, decorators,
2096 class_def->lineno, class_def->col_offset, class_def->end_lineno,
2097 class_def->end_col_offset, p->arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002098}
2099
2100/* Construct a KeywordOrStarred */
2101KeywordOrStarred *
2102_PyPegen_keyword_or_starred(Parser *p, void *element, int is_keyword)
2103{
Victor Stinner8370e072021-03-24 02:23:01 +01002104 KeywordOrStarred *a = _PyArena_Malloc(p->arena, sizeof(KeywordOrStarred));
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002105 if (!a) {
2106 return NULL;
2107 }
2108 a->element = element;
2109 a->is_keyword = is_keyword;
2110 return a;
2111}
2112
2113/* Get the number of starred expressions in an asdl_seq* of KeywordOrStarred*s */
2114static int
2115_seq_number_of_starred_exprs(asdl_seq *seq)
2116{
2117 int n = 0;
2118 for (Py_ssize_t i = 0, l = asdl_seq_LEN(seq); i < l; i++) {
Pablo Galindoa5634c42020-09-16 19:42:00 +01002119 KeywordOrStarred *k = asdl_seq_GET_UNTYPED(seq, i);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002120 if (!k->is_keyword) {
2121 n++;
2122 }
2123 }
2124 return n;
2125}
2126
2127/* Extract the starred expressions of an asdl_seq* of KeywordOrStarred*s */
Pablo Galindoa5634c42020-09-16 19:42:00 +01002128asdl_expr_seq *
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002129_PyPegen_seq_extract_starred_exprs(Parser *p, asdl_seq *kwargs)
2130{
2131 int new_len = _seq_number_of_starred_exprs(kwargs);
2132 if (new_len == 0) {
2133 return NULL;
2134 }
Pablo Galindoa5634c42020-09-16 19:42:00 +01002135 asdl_expr_seq *new_seq = _Py_asdl_expr_seq_new(new_len, p->arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002136 if (!new_seq) {
2137 return NULL;
2138 }
2139
2140 int idx = 0;
2141 for (Py_ssize_t i = 0, len = asdl_seq_LEN(kwargs); i < len; i++) {
Pablo Galindoa5634c42020-09-16 19:42:00 +01002142 KeywordOrStarred *k = asdl_seq_GET_UNTYPED(kwargs, i);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002143 if (!k->is_keyword) {
2144 asdl_seq_SET(new_seq, idx++, k->element);
2145 }
2146 }
2147 return new_seq;
2148}
2149
2150/* Return a new asdl_seq* with only the keywords in kwargs */
Pablo Galindoa5634c42020-09-16 19:42:00 +01002151asdl_keyword_seq*
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002152_PyPegen_seq_delete_starred_exprs(Parser *p, asdl_seq *kwargs)
2153{
Pablo Galindoee40e4b2020-04-23 03:43:08 +01002154 Py_ssize_t len = asdl_seq_LEN(kwargs);
2155 Py_ssize_t new_len = len - _seq_number_of_starred_exprs(kwargs);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002156 if (new_len == 0) {
2157 return NULL;
2158 }
Pablo Galindoa5634c42020-09-16 19:42:00 +01002159 asdl_keyword_seq *new_seq = _Py_asdl_keyword_seq_new(new_len, p->arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002160 if (!new_seq) {
2161 return NULL;
2162 }
2163
2164 int idx = 0;
2165 for (Py_ssize_t i = 0; i < len; i++) {
Pablo Galindoa5634c42020-09-16 19:42:00 +01002166 KeywordOrStarred *k = asdl_seq_GET_UNTYPED(kwargs, i);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002167 if (k->is_keyword) {
2168 asdl_seq_SET(new_seq, idx++, k->element);
2169 }
2170 }
2171 return new_seq;
2172}
2173
2174expr_ty
2175_PyPegen_concatenate_strings(Parser *p, asdl_seq *strings)
2176{
Pablo Galindoee40e4b2020-04-23 03:43:08 +01002177 Py_ssize_t len = asdl_seq_LEN(strings);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002178 assert(len > 0);
2179
Pablo Galindoa5634c42020-09-16 19:42:00 +01002180 Token *first = asdl_seq_GET_UNTYPED(strings, 0);
2181 Token *last = asdl_seq_GET_UNTYPED(strings, len - 1);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002182
2183 int bytesmode = 0;
2184 PyObject *bytes_str = NULL;
2185
2186 FstringParser state;
2187 _PyPegen_FstringParser_Init(&state);
2188
2189 for (Py_ssize_t i = 0; i < len; i++) {
Pablo Galindoa5634c42020-09-16 19:42:00 +01002190 Token *t = asdl_seq_GET_UNTYPED(strings, i);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002191
2192 int this_bytesmode;
2193 int this_rawmode;
2194 PyObject *s;
2195 const char *fstr;
2196 Py_ssize_t fstrlen = -1;
2197
Lysandros Nikolaou2f37c352020-05-07 13:37:51 +03002198 if (_PyPegen_parsestr(p, &this_bytesmode, &this_rawmode, &s, &fstr, &fstrlen, t) != 0) {
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002199 goto error;
2200 }
2201
2202 /* Check that we are not mixing bytes with unicode. */
2203 if (i != 0 && bytesmode != this_bytesmode) {
2204 RAISE_SYNTAX_ERROR("cannot mix bytes and nonbytes literals");
2205 Py_XDECREF(s);
2206 goto error;
2207 }
2208 bytesmode = this_bytesmode;
2209
2210 if (fstr != NULL) {
2211 assert(s == NULL && !bytesmode);
2212
2213 int result = _PyPegen_FstringParser_ConcatFstring(p, &state, &fstr, fstr + fstrlen,
2214 this_rawmode, 0, first, t, last);
2215 if (result < 0) {
2216 goto error;
2217 }
2218 }
2219 else {
2220 /* String or byte string. */
2221 assert(s != NULL && fstr == NULL);
2222 assert(bytesmode ? PyBytes_CheckExact(s) : PyUnicode_CheckExact(s));
2223
2224 if (bytesmode) {
2225 if (i == 0) {
2226 bytes_str = s;
2227 }
2228 else {
2229 PyBytes_ConcatAndDel(&bytes_str, s);
2230 if (!bytes_str) {
2231 goto error;
2232 }
2233 }
2234 }
2235 else {
2236 /* This is a regular string. Concatenate it. */
2237 if (_PyPegen_FstringParser_ConcatAndDel(&state, s) < 0) {
2238 goto error;
2239 }
2240 }
2241 }
2242 }
2243
2244 if (bytesmode) {
Victor Stinner8370e072021-03-24 02:23:01 +01002245 if (_PyArena_AddPyObject(p->arena, bytes_str) < 0) {
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002246 goto error;
2247 }
Victor Stinnerd27f8d22021-04-07 21:34:22 +02002248 return _PyAST_Constant(bytes_str, NULL, first->lineno,
2249 first->col_offset, last->end_lineno,
2250 last->end_col_offset, p->arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002251 }
2252
2253 return _PyPegen_FstringParser_Finish(p, &state, first, last);
2254
2255error:
2256 Py_XDECREF(bytes_str);
2257 _PyPegen_FstringParser_Dealloc(&state);
2258 if (PyErr_Occurred()) {
2259 raise_decode_error(p);
2260 }
2261 return NULL;
2262}
Guido van Rossumc001c092020-04-30 12:12:19 -07002263
2264mod_ty
Pablo Galindoa5634c42020-09-16 19:42:00 +01002265_PyPegen_make_module(Parser *p, asdl_stmt_seq *a) {
2266 asdl_type_ignore_seq *type_ignores = NULL;
Guido van Rossumc001c092020-04-30 12:12:19 -07002267 Py_ssize_t num = p->type_ignore_comments.num_items;
2268 if (num > 0) {
2269 // Turn the raw (comment, lineno) pairs into TypeIgnore objects in the arena
Pablo Galindoa5634c42020-09-16 19:42:00 +01002270 type_ignores = _Py_asdl_type_ignore_seq_new(num, p->arena);
Guido van Rossumc001c092020-04-30 12:12:19 -07002271 if (type_ignores == NULL) {
2272 return NULL;
2273 }
2274 for (int i = 0; i < num; i++) {
2275 PyObject *tag = _PyPegen_new_type_comment(p, p->type_ignore_comments.items[i].comment);
2276 if (tag == NULL) {
2277 return NULL;
2278 }
Victor Stinnerd27f8d22021-04-07 21:34:22 +02002279 type_ignore_ty ti = _PyAST_TypeIgnore(p->type_ignore_comments.items[i].lineno,
2280 tag, p->arena);
Guido van Rossumc001c092020-04-30 12:12:19 -07002281 if (ti == NULL) {
2282 return NULL;
2283 }
2284 asdl_seq_SET(type_ignores, i, ti);
2285 }
2286 }
Victor Stinnerd27f8d22021-04-07 21:34:22 +02002287 return _PyAST_Module(a, type_ignores, p->arena);
Guido van Rossumc001c092020-04-30 12:12:19 -07002288}
Pablo Galindo16ab0702020-05-15 02:04:52 +01002289
2290// Error reporting helpers
2291
2292expr_ty
Lysandros Nikolaou01ece632020-06-19 02:10:43 +03002293_PyPegen_get_invalid_target(expr_ty e, TARGETS_TYPE targets_type)
Pablo Galindo16ab0702020-05-15 02:04:52 +01002294{
2295 if (e == NULL) {
2296 return NULL;
2297 }
2298
2299#define VISIT_CONTAINER(CONTAINER, TYPE) do { \
Pablo Galindo58bafe42021-04-09 01:17:31 +01002300 Py_ssize_t len = asdl_seq_LEN((CONTAINER)->v.TYPE.elts);\
Pablo Galindo16ab0702020-05-15 02:04:52 +01002301 for (Py_ssize_t i = 0; i < len; i++) {\
Pablo Galindo58bafe42021-04-09 01:17:31 +01002302 expr_ty other = asdl_seq_GET((CONTAINER)->v.TYPE.elts, i);\
Lysandros Nikolaou01ece632020-06-19 02:10:43 +03002303 expr_ty child = _PyPegen_get_invalid_target(other, targets_type);\
Pablo Galindo16ab0702020-05-15 02:04:52 +01002304 if (child != NULL) {\
2305 return child;\
2306 }\
2307 }\
2308 } while (0)
2309
2310 // We only need to visit List and Tuple nodes recursively as those
2311 // are the only ones that can contain valid names in targets when
2312 // they are parsed as expressions. Any other kind of expression
2313 // that is a container (like Sets or Dicts) is directly invalid and
2314 // we don't need to visit it recursively.
2315
2316 switch (e->kind) {
Lysandros Nikolaou01ece632020-06-19 02:10:43 +03002317 case List_kind:
Pablo Galindo16ab0702020-05-15 02:04:52 +01002318 VISIT_CONTAINER(e, List);
2319 return NULL;
Lysandros Nikolaou01ece632020-06-19 02:10:43 +03002320 case Tuple_kind:
Pablo Galindo16ab0702020-05-15 02:04:52 +01002321 VISIT_CONTAINER(e, Tuple);
2322 return NULL;
Pablo Galindo16ab0702020-05-15 02:04:52 +01002323 case Starred_kind:
Lysandros Nikolaou01ece632020-06-19 02:10:43 +03002324 if (targets_type == DEL_TARGETS) {
2325 return e;
2326 }
2327 return _PyPegen_get_invalid_target(e->v.Starred.value, targets_type);
2328 case Compare_kind:
2329 // This is needed, because the `a in b` in `for a in b` gets parsed
2330 // as a comparison, and so we need to search the left side of the comparison
2331 // for invalid targets.
2332 if (targets_type == FOR_TARGETS) {
2333 cmpop_ty cmpop = (cmpop_ty) asdl_seq_GET(e->v.Compare.ops, 0);
2334 if (cmpop == In) {
2335 return _PyPegen_get_invalid_target(e->v.Compare.left, targets_type);
2336 }
2337 return NULL;
2338 }
2339 return e;
Pablo Galindo16ab0702020-05-15 02:04:52 +01002340 case Name_kind:
2341 case Subscript_kind:
2342 case Attribute_kind:
2343 return NULL;
2344 default:
2345 return e;
2346 }
Lysandros Nikolaou75b863a2020-05-18 22:14:47 +03002347}
2348
2349void *_PyPegen_arguments_parsing_error(Parser *p, expr_ty e) {
2350 int kwarg_unpacking = 0;
2351 for (Py_ssize_t i = 0, l = asdl_seq_LEN(e->v.Call.keywords); i < l; i++) {
2352 keyword_ty keyword = asdl_seq_GET(e->v.Call.keywords, i);
2353 if (!keyword->arg) {
2354 kwarg_unpacking = 1;
2355 }
2356 }
2357
2358 const char *msg = NULL;
2359 if (kwarg_unpacking) {
2360 msg = "positional argument follows keyword argument unpacking";
2361 } else {
2362 msg = "positional argument follows keyword argument";
2363 }
2364
2365 return RAISE_SYNTAX_ERROR(msg);
2366}
Lysandros Nikolaouae145832020-05-22 03:56:52 +03002367
2368void *
2369_PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args)
2370{
2371 /* The rule that calls this function is 'args for_if_clauses'.
2372 For the input f(L, x for x in y), L and x are in args and
2373 the for is parsed as a for_if_clause. We have to check if
2374 len <= 1, so that input like dict((a, b) for a, b in x)
2375 gets successfully parsed and then we pass the last
2376 argument (x in the above example) as the location of the
2377 error */
2378 Py_ssize_t len = asdl_seq_LEN(args->v.Call.args);
2379 if (len <= 1) {
2380 return NULL;
2381 }
2382
2383 return RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
2384 (expr_ty) asdl_seq_GET(args->v.Call.args, len - 1),
2385 "Generator expression must be parenthesized"
2386 );
2387}
Pablo Galindo4a97b152020-09-02 17:44:19 +01002388
2389
Pablo Galindoa5634c42020-09-16 19:42:00 +01002390expr_ty _PyPegen_collect_call_seqs(Parser *p, asdl_expr_seq *a, asdl_seq *b,
Pablo Galindo315a61f2020-09-03 15:29:32 +01002391 int lineno, int col_offset, int end_lineno,
2392 int end_col_offset, PyArena *arena) {
Pablo Galindo4a97b152020-09-02 17:44:19 +01002393 Py_ssize_t args_len = asdl_seq_LEN(a);
2394 Py_ssize_t total_len = args_len;
2395
2396 if (b == NULL) {
Victor Stinnerd27f8d22021-04-07 21:34:22 +02002397 return _PyAST_Call(_PyPegen_dummy_name(p), a, NULL, lineno, col_offset,
Pablo Galindo315a61f2020-09-03 15:29:32 +01002398 end_lineno, end_col_offset, arena);
Pablo Galindo4a97b152020-09-02 17:44:19 +01002399
2400 }
2401
Pablo Galindoa5634c42020-09-16 19:42:00 +01002402 asdl_expr_seq *starreds = _PyPegen_seq_extract_starred_exprs(p, b);
2403 asdl_keyword_seq *keywords = _PyPegen_seq_delete_starred_exprs(p, b);
Pablo Galindo4a97b152020-09-02 17:44:19 +01002404
2405 if (starreds) {
2406 total_len += asdl_seq_LEN(starreds);
2407 }
2408
Pablo Galindoa5634c42020-09-16 19:42:00 +01002409 asdl_expr_seq *args = _Py_asdl_expr_seq_new(total_len, arena);
Pablo Galindo4a97b152020-09-02 17:44:19 +01002410
2411 Py_ssize_t i = 0;
2412 for (i = 0; i < args_len; i++) {
2413 asdl_seq_SET(args, i, asdl_seq_GET(a, i));
2414 }
2415 for (; i < total_len; i++) {
2416 asdl_seq_SET(args, i, asdl_seq_GET(starreds, i - args_len));
2417 }
2418
Victor Stinnerd27f8d22021-04-07 21:34:22 +02002419 return _PyAST_Call(_PyPegen_dummy_name(p), args, keywords, lineno,
2420 col_offset, end_lineno, end_col_offset, arena);
Pablo Galindo4a97b152020-09-02 17:44:19 +01002421}