blob: 2507bc4b38280b06e6bcf2c55d0b3331d712b3ee [file] [log] [blame]
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001#include <Python.h>
2#include <errcode.h>
Pablo Galindo1ed83ad2020-06-11 17:30:46 +01003#include "tokenizer.h"
Pablo Galindoc5fc1562020-04-22 23:29:27 +01004
5#include "pegen.h"
Pablo Galindo1ed83ad2020-06-11 17:30:46 +01006#include "string_parser.h"
Pablo Galindo13322262020-07-27 23:46:59 +01007#include "ast.h"
Pablo Galindoc5fc1562020-04-22 23:29:27 +01008
Guido van Rossumc001c092020-04-30 12:12:19 -07009PyObject *
10_PyPegen_new_type_comment(Parser *p, char *s)
11{
12 PyObject *res = PyUnicode_DecodeUTF8(s, strlen(s), NULL);
13 if (res == NULL) {
14 return NULL;
15 }
16 if (PyArena_AddPyObject(p->arena, res) < 0) {
17 Py_DECREF(res);
18 return NULL;
19 }
20 return res;
21}
22
23arg_ty
24_PyPegen_add_type_comment_to_arg(Parser *p, arg_ty a, Token *tc)
25{
26 if (tc == NULL) {
27 return a;
28 }
29 char *bytes = PyBytes_AsString(tc->bytes);
30 if (bytes == NULL) {
31 return NULL;
32 }
33 PyObject *tco = _PyPegen_new_type_comment(p, bytes);
34 if (tco == NULL) {
35 return NULL;
36 }
37 return arg(a->arg, a->annotation, tco,
38 a->lineno, a->col_offset, a->end_lineno, a->end_col_offset,
39 p->arena);
40}
41
Pablo Galindoc5fc1562020-04-22 23:29:27 +010042static int
43init_normalization(Parser *p)
44{
Lysandros Nikolaouebebb642020-04-23 18:36:06 +030045 if (p->normalize) {
46 return 1;
47 }
Pablo Galindoc5fc1562020-04-22 23:29:27 +010048 PyObject *m = PyImport_ImportModuleNoBlock("unicodedata");
49 if (!m)
50 {
51 return 0;
52 }
53 p->normalize = PyObject_GetAttrString(m, "normalize");
54 Py_DECREF(m);
55 if (!p->normalize)
56 {
57 return 0;
58 }
59 return 1;
60}
61
Pablo Galindo2b74c832020-04-27 18:02:07 +010062/* Checks if the NOTEQUAL token is valid given the current parser flags
630 indicates success and nonzero indicates failure (an exception may be set) */
64int
65_PyPegen_check_barry_as_flufl(Parser *p) {
66 Token *t = p->tokens[p->fill - 1];
67 assert(t->bytes != NULL);
68 assert(t->type == NOTEQUAL);
69
70 char* tok_str = PyBytes_AS_STRING(t->bytes);
Pablo Galindofb61c422020-06-15 14:23:43 +010071 if (p->flags & PyPARSE_BARRY_AS_BDFL && strcmp(tok_str, "<>") != 0) {
Pablo Galindo2b74c832020-04-27 18:02:07 +010072 RAISE_SYNTAX_ERROR("with Barry as BDFL, use '<>' instead of '!='");
73 return -1;
Pablo Galindofb61c422020-06-15 14:23:43 +010074 }
75 if (!(p->flags & PyPARSE_BARRY_AS_BDFL)) {
Pablo Galindo2b74c832020-04-27 18:02:07 +010076 return strcmp(tok_str, "!=");
77 }
78 return 0;
79}
80
Pablo Galindoc5fc1562020-04-22 23:29:27 +010081PyObject *
82_PyPegen_new_identifier(Parser *p, char *n)
83{
84 PyObject *id = PyUnicode_DecodeUTF8(n, strlen(n), NULL);
85 if (!id) {
86 goto error;
87 }
88 /* PyUnicode_DecodeUTF8 should always return a ready string. */
89 assert(PyUnicode_IS_READY(id));
90 /* Check whether there are non-ASCII characters in the
91 identifier; if so, normalize to NFKC. */
92 if (!PyUnicode_IS_ASCII(id))
93 {
94 PyObject *id2;
Lysandros Nikolaouebebb642020-04-23 18:36:06 +030095 if (!init_normalization(p))
Pablo Galindoc5fc1562020-04-22 23:29:27 +010096 {
97 Py_DECREF(id);
98 goto error;
99 }
100 PyObject *form = PyUnicode_InternFromString("NFKC");
101 if (form == NULL)
102 {
103 Py_DECREF(id);
104 goto error;
105 }
106 PyObject *args[2] = {form, id};
107 id2 = _PyObject_FastCall(p->normalize, args, 2);
108 Py_DECREF(id);
109 Py_DECREF(form);
110 if (!id2) {
111 goto error;
112 }
113 if (!PyUnicode_Check(id2))
114 {
115 PyErr_Format(PyExc_TypeError,
116 "unicodedata.normalize() must return a string, not "
117 "%.200s",
118 _PyType_Name(Py_TYPE(id2)));
119 Py_DECREF(id2);
120 goto error;
121 }
122 id = id2;
123 }
124 PyUnicode_InternInPlace(&id);
125 if (PyArena_AddPyObject(p->arena, id) < 0)
126 {
127 Py_DECREF(id);
128 goto error;
129 }
130 return id;
131
132error:
133 p->error_indicator = 1;
134 return NULL;
135}
136
137static PyObject *
138_create_dummy_identifier(Parser *p)
139{
140 return _PyPegen_new_identifier(p, "");
141}
142
143static inline Py_ssize_t
Pablo Galindo51c58962020-06-16 16:49:43 +0100144byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset)
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100145{
146 const char *str = PyUnicode_AsUTF8(line);
Lysandros Nikolaouebebb642020-04-23 18:36:06 +0300147 if (!str) {
148 return 0;
149 }
Pablo Galindo51c58962020-06-16 16:49:43 +0100150 assert(col_offset >= 0 && (unsigned long)col_offset <= strlen(str));
Batuhan Taskaya76c1b4d2020-05-01 16:13:43 +0300151 PyObject *text = PyUnicode_DecodeUTF8(str, col_offset, "replace");
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100152 if (!text) {
153 return 0;
154 }
155 Py_ssize_t size = PyUnicode_GET_LENGTH(text);
156 Py_DECREF(text);
157 return size;
158}
159
160const char *
161_PyPegen_get_expr_name(expr_ty e)
162{
Pablo Galindo9f495902020-06-08 02:57:00 +0100163 assert(e != NULL);
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100164 switch (e->kind) {
165 case Attribute_kind:
166 return "attribute";
167 case Subscript_kind:
168 return "subscript";
169 case Starred_kind:
170 return "starred";
171 case Name_kind:
172 return "name";
173 case List_kind:
174 return "list";
175 case Tuple_kind:
176 return "tuple";
177 case Lambda_kind:
178 return "lambda";
179 case Call_kind:
180 return "function call";
181 case BoolOp_kind:
182 case BinOp_kind:
183 case UnaryOp_kind:
184 return "operator";
185 case GeneratorExp_kind:
186 return "generator expression";
187 case Yield_kind:
188 case YieldFrom_kind:
189 return "yield expression";
190 case Await_kind:
191 return "await expression";
192 case ListComp_kind:
193 return "list comprehension";
194 case SetComp_kind:
195 return "set comprehension";
196 case DictComp_kind:
197 return "dict comprehension";
198 case Dict_kind:
199 return "dict display";
200 case Set_kind:
201 return "set display";
202 case JoinedStr_kind:
203 case FormattedValue_kind:
204 return "f-string expression";
205 case Constant_kind: {
206 PyObject *value = e->v.Constant.value;
207 if (value == Py_None) {
208 return "None";
209 }
210 if (value == Py_False) {
211 return "False";
212 }
213 if (value == Py_True) {
214 return "True";
215 }
216 if (value == Py_Ellipsis) {
217 return "Ellipsis";
218 }
219 return "literal";
220 }
221 case Compare_kind:
222 return "comparison";
223 case IfExp_kind:
224 return "conditional expression";
225 case NamedExpr_kind:
226 return "named expression";
227 default:
228 PyErr_Format(PyExc_SystemError,
229 "unexpected expression in assignment %d (line %d)",
230 e->kind, e->lineno);
231 return NULL;
232 }
233}
234
Lysandros Nikolaouebebb642020-04-23 18:36:06 +0300235static int
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100236raise_decode_error(Parser *p)
237{
Lysandros Nikolaouebebb642020-04-23 18:36:06 +0300238 assert(PyErr_Occurred());
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100239 const char *errtype = NULL;
240 if (PyErr_ExceptionMatches(PyExc_UnicodeError)) {
241 errtype = "unicode error";
242 }
243 else if (PyErr_ExceptionMatches(PyExc_ValueError)) {
244 errtype = "value error";
245 }
246 if (errtype) {
Pablo Galindofb61c422020-06-15 14:23:43 +0100247 PyObject *type;
248 PyObject *value;
249 PyObject *tback;
250 PyObject *errstr;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100251 PyErr_Fetch(&type, &value, &tback);
252 errstr = PyObject_Str(value);
253 if (errstr) {
254 RAISE_SYNTAX_ERROR("(%s) %U", errtype, errstr);
255 Py_DECREF(errstr);
256 }
257 else {
258 PyErr_Clear();
259 RAISE_SYNTAX_ERROR("(%s) unknown error", errtype);
260 }
261 Py_XDECREF(type);
262 Py_XDECREF(value);
263 Py_XDECREF(tback);
264 }
Lysandros Nikolaouebebb642020-04-23 18:36:06 +0300265
266 return -1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100267}
268
269static void
270raise_tokenizer_init_error(PyObject *filename)
271{
272 if (!(PyErr_ExceptionMatches(PyExc_LookupError)
273 || PyErr_ExceptionMatches(PyExc_ValueError)
274 || PyErr_ExceptionMatches(PyExc_UnicodeDecodeError))) {
275 return;
276 }
Lysandros Nikolaouebebb642020-04-23 18:36:06 +0300277 PyObject *errstr = NULL;
278 PyObject *tuple = NULL;
Pablo Galindofb61c422020-06-15 14:23:43 +0100279 PyObject *type;
280 PyObject *value;
281 PyObject *tback;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100282 PyErr_Fetch(&type, &value, &tback);
283 errstr = PyObject_Str(value);
Lysandros Nikolaouebebb642020-04-23 18:36:06 +0300284 if (!errstr) {
285 goto error;
286 }
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100287
Lysandros Nikolaouebebb642020-04-23 18:36:06 +0300288 PyObject *tmp = Py_BuildValue("(OiiO)", filename, 0, -1, Py_None);
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100289 if (!tmp) {
290 goto error;
291 }
292
Lysandros Nikolaouebebb642020-04-23 18:36:06 +0300293 tuple = PyTuple_Pack(2, errstr, tmp);
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100294 Py_DECREF(tmp);
295 if (!value) {
296 goto error;
297 }
Lysandros Nikolaouebebb642020-04-23 18:36:06 +0300298 PyErr_SetObject(PyExc_SyntaxError, tuple);
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100299
300error:
301 Py_XDECREF(type);
302 Py_XDECREF(value);
303 Py_XDECREF(tback);
Lysandros Nikolaouebebb642020-04-23 18:36:06 +0300304 Py_XDECREF(errstr);
305 Py_XDECREF(tuple);
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100306}
307
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100308static int
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100309tokenizer_error(Parser *p)
310{
311 if (PyErr_Occurred()) {
312 return -1;
313 }
314
315 const char *msg = NULL;
316 PyObject* errtype = PyExc_SyntaxError;
317 switch (p->tok->done) {
318 case E_TOKEN:
319 msg = "invalid token";
320 break;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100321 case E_EOFS:
Batuhan Taskaya76c1b4d2020-05-01 16:13:43 +0300322 RAISE_SYNTAX_ERROR("EOF while scanning triple-quoted string literal");
323 return -1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100324 case E_EOLS:
Batuhan Taskaya76c1b4d2020-05-01 16:13:43 +0300325 RAISE_SYNTAX_ERROR("EOL while scanning string literal");
326 return -1;
Lysandros Nikolaoud55133f2020-04-28 03:23:35 +0300327 case E_EOF:
Batuhan Taskaya76c1b4d2020-05-01 16:13:43 +0300328 RAISE_SYNTAX_ERROR("unexpected EOF while parsing");
329 return -1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100330 case E_DEDENT:
Batuhan Taskaya76c1b4d2020-05-01 16:13:43 +0300331 RAISE_INDENTATION_ERROR("unindent does not match any outer indentation level");
332 return -1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100333 case E_INTR:
334 if (!PyErr_Occurred()) {
335 PyErr_SetNone(PyExc_KeyboardInterrupt);
336 }
337 return -1;
338 case E_NOMEM:
339 PyErr_NoMemory();
340 return -1;
341 case E_TABSPACE:
342 errtype = PyExc_TabError;
343 msg = "inconsistent use of tabs and spaces in indentation";
344 break;
345 case E_TOODEEP:
346 errtype = PyExc_IndentationError;
347 msg = "too many levels of indentation";
348 break;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100349 case E_LINECONT:
350 msg = "unexpected character after line continuation character";
351 break;
352 default:
353 msg = "unknown parsing error";
354 }
355
356 PyErr_Format(errtype, msg);
357 // There is no reliable column information for this error
358 PyErr_SyntaxLocationObject(p->tok->filename, p->tok->lineno, 0);
359
360 return -1;
361}
362
363void *
Lysandros Nikolaoua15c9b32020-05-13 22:36:27 +0300364_PyPegen_raise_error(Parser *p, PyObject *errtype, const char *errmsg, ...)
365{
366 Token *t = p->known_err_token != NULL ? p->known_err_token : p->tokens[p->fill - 1];
Pablo Galindo51c58962020-06-16 16:49:43 +0100367 Py_ssize_t col_offset;
Lysandros Nikolaoua15c9b32020-05-13 22:36:27 +0300368 if (t->col_offset == -1) {
369 col_offset = Py_SAFE_DOWNCAST(p->tok->cur - p->tok->buf,
370 intptr_t, int);
371 } else {
372 col_offset = t->col_offset + 1;
373 }
374
375 va_list va;
376 va_start(va, errmsg);
377 _PyPegen_raise_error_known_location(p, errtype, t->lineno,
378 col_offset, errmsg, va);
379 va_end(va);
380
381 return NULL;
382}
383
Lysandros Nikolaoua15c9b32020-05-13 22:36:27 +0300384void *
385_PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
Pablo Galindo51c58962020-06-16 16:49:43 +0100386 Py_ssize_t lineno, Py_ssize_t col_offset,
Lysandros Nikolaoua15c9b32020-05-13 22:36:27 +0300387 const char *errmsg, va_list va)
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100388{
389 PyObject *value = NULL;
390 PyObject *errstr = NULL;
Lysandros Nikolaoua15c9b32020-05-13 22:36:27 +0300391 PyObject *error_line = NULL;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100392 PyObject *tmp = NULL;
Lysandros Nikolaou7f06af62020-05-04 03:20:09 +0300393 p->error_indicator = 1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100394
Lysandros Nikolaou2e0a9202020-06-26 14:24:05 +0300395 if (p->start_rule == Py_fstring_input) {
396 const char *fstring_msg = "f-string: ";
397 Py_ssize_t len = strlen(fstring_msg) + strlen(errmsg);
398
Lysandros Nikolaou6dcbc242020-06-27 20:47:00 +0300399 char *new_errmsg = PyMem_Malloc(len + 1); // Lengths of both strings plus NULL character
Lysandros Nikolaou2e0a9202020-06-26 14:24:05 +0300400 if (!new_errmsg) {
401 return (void *) PyErr_NoMemory();
402 }
403
404 // Copy both strings into new buffer
405 memcpy(new_errmsg, fstring_msg, strlen(fstring_msg));
406 memcpy(new_errmsg + strlen(fstring_msg), errmsg, strlen(errmsg));
407 new_errmsg[len] = 0;
408 errmsg = new_errmsg;
409 }
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100410 errstr = PyUnicode_FromFormatV(errmsg, va);
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100411 if (!errstr) {
412 goto error;
413 }
414
415 if (p->start_rule == Py_file_input) {
Lysandros Nikolaou861efc62020-06-20 15:57:27 +0300416 error_line = PyErr_ProgramTextObject(p->tok->filename, (int) lineno);
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100417 }
418
Lysandros Nikolaoua15c9b32020-05-13 22:36:27 +0300419 if (!error_line) {
Pablo Galindobcc30362020-05-14 21:11:48 +0100420 Py_ssize_t size = p->tok->inp - p->tok->buf;
Pablo Galindobcc30362020-05-14 21:11:48 +0100421 error_line = PyUnicode_DecodeUTF8(p->tok->buf, size, "replace");
Lysandros Nikolaoua15c9b32020-05-13 22:36:27 +0300422 if (!error_line) {
423 goto error;
Batuhan Taskaya76c1b4d2020-05-01 16:13:43 +0300424 }
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100425 }
426
Lysandros Nikolaou1f0f4ab2020-06-28 02:41:48 +0300427 if (p->start_rule == Py_fstring_input) {
428 col_offset -= p->starting_col_offset;
429 }
Pablo Galindo51c58962020-06-16 16:49:43 +0100430 Py_ssize_t col_number = col_offset;
431
432 if (p->tok->encoding != NULL) {
433 col_number = byte_offset_to_character_offset(error_line, col_offset);
434 }
Lysandros Nikolaoua15c9b32020-05-13 22:36:27 +0300435
436 tmp = Py_BuildValue("(OiiN)", p->tok->filename, lineno, col_number, error_line);
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100437 if (!tmp) {
438 goto error;
439 }
440 value = PyTuple_Pack(2, errstr, tmp);
441 Py_DECREF(tmp);
442 if (!value) {
443 goto error;
444 }
445 PyErr_SetObject(errtype, value);
446
447 Py_DECREF(errstr);
448 Py_DECREF(value);
Lysandros Nikolaou2e0a9202020-06-26 14:24:05 +0300449 if (p->start_rule == Py_fstring_input) {
Lysandros Nikolaou6dcbc242020-06-27 20:47:00 +0300450 PyMem_Free((void *)errmsg);
Lysandros Nikolaou2e0a9202020-06-26 14:24:05 +0300451 }
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100452 return NULL;
453
454error:
455 Py_XDECREF(errstr);
Lysandros Nikolaoua15c9b32020-05-13 22:36:27 +0300456 Py_XDECREF(error_line);
Lysandros Nikolaou2e0a9202020-06-26 14:24:05 +0300457 if (p->start_rule == Py_fstring_input) {
Lysandros Nikolaou6dcbc242020-06-27 20:47:00 +0300458 PyMem_Free((void *)errmsg);
Lysandros Nikolaou2e0a9202020-06-26 14:24:05 +0300459 }
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100460 return NULL;
461}
462
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100463#if 0
464static const char *
465token_name(int type)
466{
467 if (0 <= type && type <= N_TOKENS) {
468 return _PyParser_TokenNames[type];
469 }
470 return "<Huh?>";
471}
472#endif
473
474// Here, mark is the start of the node, while p->mark is the end.
475// If node==NULL, they should be the same.
476int
477_PyPegen_insert_memo(Parser *p, int mark, int type, void *node)
478{
479 // Insert in front
480 Memo *m = PyArena_Malloc(p->arena, sizeof(Memo));
481 if (m == NULL) {
482 return -1;
483 }
484 m->type = type;
485 m->node = node;
486 m->mark = p->mark;
487 m->next = p->tokens[mark]->memo;
488 p->tokens[mark]->memo = m;
489 return 0;
490}
491
492// Like _PyPegen_insert_memo(), but updates an existing node if found.
493int
494_PyPegen_update_memo(Parser *p, int mark, int type, void *node)
495{
496 for (Memo *m = p->tokens[mark]->memo; m != NULL; m = m->next) {
497 if (m->type == type) {
498 // Update existing node.
499 m->node = node;
500 m->mark = p->mark;
501 return 0;
502 }
503 }
504 // Insert new node.
505 return _PyPegen_insert_memo(p, mark, type, node);
506}
507
508// Return dummy NAME.
509void *
510_PyPegen_dummy_name(Parser *p, ...)
511{
512 static void *cache = NULL;
513
514 if (cache != NULL) {
515 return cache;
516 }
517
518 PyObject *id = _create_dummy_identifier(p);
519 if (!id) {
520 return NULL;
521 }
522 cache = Name(id, Load, 1, 0, 1, 0, p->arena);
523 return cache;
524}
525
526static int
527_get_keyword_or_name_type(Parser *p, const char *name, int name_len)
528{
Lysandros Nikolaou782f44b2020-07-07 01:42:21 +0300529 assert(name_len > 0);
Pablo Galindo1ac0cbc2020-07-06 20:31:16 +0100530 if (name_len >= p->n_keyword_lists ||
531 p->keywords[name_len] == NULL ||
532 p->keywords[name_len]->type == -1) {
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100533 return NAME;
534 }
Pablo Galindo1ac0cbc2020-07-06 20:31:16 +0100535 for (KeywordToken *k = p->keywords[name_len]; k != NULL && k->type != -1; k++) {
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100536 if (strncmp(k->str, name, name_len) == 0) {
537 return k->type;
538 }
539 }
540 return NAME;
541}
542
Guido van Rossumc001c092020-04-30 12:12:19 -0700543static int
544growable_comment_array_init(growable_comment_array *arr, size_t initial_size) {
545 assert(initial_size > 0);
546 arr->items = PyMem_Malloc(initial_size * sizeof(*arr->items));
547 arr->size = initial_size;
548 arr->num_items = 0;
549
550 return arr->items != NULL;
551}
552
553static int
554growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) {
555 if (arr->num_items >= arr->size) {
556 size_t new_size = arr->size * 2;
557 void *new_items_array = PyMem_Realloc(arr->items, new_size * sizeof(*arr->items));
558 if (!new_items_array) {
559 return 0;
560 }
561 arr->items = new_items_array;
562 arr->size = new_size;
563 }
564
565 arr->items[arr->num_items].lineno = lineno;
566 arr->items[arr->num_items].comment = comment; // Take ownership
567 arr->num_items++;
568 return 1;
569}
570
571static void
572growable_comment_array_deallocate(growable_comment_array *arr) {
573 for (unsigned i = 0; i < arr->num_items; i++) {
574 PyMem_Free(arr->items[i].comment);
575 }
576 PyMem_Free(arr->items);
577}
578
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100579int
580_PyPegen_fill_token(Parser *p)
581{
Pablo Galindofb61c422020-06-15 14:23:43 +0100582 const char *start;
583 const char *end;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100584 int type = PyTokenizer_Get(p->tok, &start, &end);
Guido van Rossumc001c092020-04-30 12:12:19 -0700585
586 // Record and skip '# type: ignore' comments
587 while (type == TYPE_IGNORE) {
588 Py_ssize_t len = end - start;
589 char *tag = PyMem_Malloc(len + 1);
590 if (tag == NULL) {
591 PyErr_NoMemory();
592 return -1;
593 }
594 strncpy(tag, start, len);
595 tag[len] = '\0';
596 // Ownership of tag passes to the growable array
597 if (!growable_comment_array_add(&p->type_ignore_comments, p->tok->lineno, tag)) {
598 PyErr_NoMemory();
599 return -1;
600 }
601 type = PyTokenizer_Get(p->tok, &start, &end);
602 }
603
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100604 if (type == ENDMARKER && p->start_rule == Py_single_input && p->parsing_started) {
605 type = NEWLINE; /* Add an extra newline */
606 p->parsing_started = 0;
607
Pablo Galindob94dbd72020-04-27 18:35:58 +0100608 if (p->tok->indent && !(p->flags & PyPARSE_DONT_IMPLY_DEDENT)) {
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100609 p->tok->pendin = -p->tok->indent;
610 p->tok->indent = 0;
611 }
612 }
613 else {
614 p->parsing_started = 1;
615 }
616
617 if (p->fill == p->size) {
618 int newsize = p->size * 2;
Lysandros Nikolaouebebb642020-04-23 18:36:06 +0300619 Token **new_tokens = PyMem_Realloc(p->tokens, newsize * sizeof(Token *));
620 if (new_tokens == NULL) {
621 PyErr_NoMemory();
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100622 return -1;
623 }
Pablo Galindofb61c422020-06-15 14:23:43 +0100624 p->tokens = new_tokens;
625
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100626 for (int i = p->size; i < newsize; i++) {
627 p->tokens[i] = PyMem_Malloc(sizeof(Token));
Lysandros Nikolaouebebb642020-04-23 18:36:06 +0300628 if (p->tokens[i] == NULL) {
629 p->size = i; // Needed, in order to cleanup correctly after parser fails
630 PyErr_NoMemory();
631 return -1;
632 }
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100633 memset(p->tokens[i], '\0', sizeof(Token));
634 }
635 p->size = newsize;
636 }
637
638 Token *t = p->tokens[p->fill];
639 t->type = (type == NAME) ? _get_keyword_or_name_type(p, start, (int)(end - start)) : type;
640 t->bytes = PyBytes_FromStringAndSize(start, end - start);
641 if (t->bytes == NULL) {
642 return -1;
643 }
644 PyArena_AddPyObject(p->arena, t->bytes);
645
646 int lineno = type == STRING ? p->tok->first_lineno : p->tok->lineno;
647 const char *line_start = type == STRING ? p->tok->multi_line_start : p->tok->line_start;
Pablo Galindo22081342020-04-29 02:04:06 +0100648 int end_lineno = p->tok->lineno;
Pablo Galindofb61c422020-06-15 14:23:43 +0100649 int col_offset = -1;
650 int end_col_offset = -1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100651 if (start != NULL && start >= line_start) {
Pablo Galindo22081342020-04-29 02:04:06 +0100652 col_offset = (int)(start - line_start);
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100653 }
654 if (end != NULL && end >= p->tok->line_start) {
Pablo Galindo22081342020-04-29 02:04:06 +0100655 end_col_offset = (int)(end - p->tok->line_start);
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100656 }
657
658 t->lineno = p->starting_lineno + lineno;
659 t->col_offset = p->tok->lineno == 1 ? p->starting_col_offset + col_offset : col_offset;
660 t->end_lineno = p->starting_lineno + end_lineno;
661 t->end_col_offset = p->tok->lineno == 1 ? p->starting_col_offset + end_col_offset : end_col_offset;
662
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100663 p->fill += 1;
Batuhan Taskaya76c1b4d2020-05-01 16:13:43 +0300664
665 if (type == ERRORTOKEN) {
666 if (p->tok->done == E_DECODE) {
667 return raise_decode_error(p);
668 }
Pablo Galindofb61c422020-06-15 14:23:43 +0100669 return tokenizer_error(p);
670
Batuhan Taskaya76c1b4d2020-05-01 16:13:43 +0300671 }
672
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100673 return 0;
674}
675
676// Instrumentation to count the effectiveness of memoization.
677// The array counts the number of tokens skipped by memoization,
678// indexed by type.
679
680#define NSTATISTICS 2000
681static long memo_statistics[NSTATISTICS];
682
683void
684_PyPegen_clear_memo_statistics()
685{
686 for (int i = 0; i < NSTATISTICS; i++) {
687 memo_statistics[i] = 0;
688 }
689}
690
691PyObject *
692_PyPegen_get_memo_statistics()
693{
694 PyObject *ret = PyList_New(NSTATISTICS);
695 if (ret == NULL) {
696 return NULL;
697 }
698 for (int i = 0; i < NSTATISTICS; i++) {
699 PyObject *value = PyLong_FromLong(memo_statistics[i]);
700 if (value == NULL) {
701 Py_DECREF(ret);
702 return NULL;
703 }
704 // PyList_SetItem borrows a reference to value.
705 if (PyList_SetItem(ret, i, value) < 0) {
706 Py_DECREF(ret);
707 return NULL;
708 }
709 }
710 return ret;
711}
712
713int // bool
714_PyPegen_is_memoized(Parser *p, int type, void *pres)
715{
716 if (p->mark == p->fill) {
717 if (_PyPegen_fill_token(p) < 0) {
Lysandros Nikolaouebebb642020-04-23 18:36:06 +0300718 p->error_indicator = 1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100719 return -1;
720 }
721 }
722
723 Token *t = p->tokens[p->mark];
724
725 for (Memo *m = t->memo; m != NULL; m = m->next) {
726 if (m->type == type) {
727 if (0 <= type && type < NSTATISTICS) {
728 long count = m->mark - p->mark;
729 // A memoized negative result counts for one.
730 if (count <= 0) {
731 count = 1;
732 }
733 memo_statistics[type] += count;
734 }
735 p->mark = m->mark;
736 *(void **)(pres) = m->node;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100737 return 1;
738 }
739 }
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100740 return 0;
741}
742
Pablo Galindo1df5a9e2020-04-23 12:42:13 +0100743
744int
745_PyPegen_lookahead_with_name(int positive, expr_ty (func)(Parser *), Parser *p)
746{
747 int mark = p->mark;
748 void *res = func(p);
749 p->mark = mark;
750 return (res != NULL) == positive;
751}
752
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100753int
Pablo Galindo404b23b2020-05-27 00:15:52 +0100754_PyPegen_lookahead_with_string(int positive, expr_ty (func)(Parser *, const char*), Parser *p, const char* arg)
755{
756 int mark = p->mark;
757 void *res = func(p, arg);
758 p->mark = mark;
759 return (res != NULL) == positive;
760}
761
762int
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100763_PyPegen_lookahead_with_int(int positive, Token *(func)(Parser *, int), Parser *p, int arg)
764{
765 int mark = p->mark;
766 void *res = func(p, arg);
767 p->mark = mark;
768 return (res != NULL) == positive;
769}
770
771int
772_PyPegen_lookahead(int positive, void *(func)(Parser *), Parser *p)
773{
774 int mark = p->mark;
Pablo Galindo1df5a9e2020-04-23 12:42:13 +0100775 void *res = (void*)func(p);
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100776 p->mark = mark;
777 return (res != NULL) == positive;
778}
779
780Token *
781_PyPegen_expect_token(Parser *p, int type)
782{
783 if (p->mark == p->fill) {
784 if (_PyPegen_fill_token(p) < 0) {
Lysandros Nikolaouebebb642020-04-23 18:36:06 +0300785 p->error_indicator = 1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100786 return NULL;
787 }
788 }
789 Token *t = p->tokens[p->mark];
790 if (t->type != type) {
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100791 return NULL;
792 }
793 p->mark += 1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100794 return t;
795}
796
Guido van Rossumb45af1a2020-05-26 10:58:44 -0700797expr_ty
798_PyPegen_expect_soft_keyword(Parser *p, const char *keyword)
799{
800 if (p->mark == p->fill) {
801 if (_PyPegen_fill_token(p) < 0) {
802 p->error_indicator = 1;
803 return NULL;
804 }
805 }
806 Token *t = p->tokens[p->mark];
807 if (t->type != NAME) {
808 return NULL;
809 }
Lysandros Nikolaou526e23f2020-05-27 19:04:11 +0300810 char *s = PyBytes_AsString(t->bytes);
Guido van Rossumb45af1a2020-05-26 10:58:44 -0700811 if (!s) {
Lysandros Nikolaou526e23f2020-05-27 19:04:11 +0300812 p->error_indicator = 1;
Guido van Rossumb45af1a2020-05-26 10:58:44 -0700813 return NULL;
814 }
815 if (strcmp(s, keyword) != 0) {
816 return NULL;
817 }
Lysandros Nikolaou526e23f2020-05-27 19:04:11 +0300818 return _PyPegen_name_token(p);
Guido van Rossumb45af1a2020-05-26 10:58:44 -0700819}
820
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100821Token *
822_PyPegen_get_last_nonnwhitespace_token(Parser *p)
823{
824 assert(p->mark >= 0);
825 Token *token = NULL;
826 for (int m = p->mark - 1; m >= 0; m--) {
827 token = p->tokens[m];
828 if (token->type != ENDMARKER && (token->type < NEWLINE || token->type > DEDENT)) {
829 break;
830 }
831 }
832 return token;
833}
834
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100835expr_ty
836_PyPegen_name_token(Parser *p)
837{
838 Token *t = _PyPegen_expect_token(p, NAME);
839 if (t == NULL) {
840 return NULL;
841 }
842 char* s = PyBytes_AsString(t->bytes);
843 if (!s) {
Lysandros Nikolaou526e23f2020-05-27 19:04:11 +0300844 p->error_indicator = 1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100845 return NULL;
846 }
847 PyObject *id = _PyPegen_new_identifier(p, s);
848 if (id == NULL) {
Lysandros Nikolaou526e23f2020-05-27 19:04:11 +0300849 p->error_indicator = 1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100850 return NULL;
851 }
852 return Name(id, Load, t->lineno, t->col_offset, t->end_lineno, t->end_col_offset,
853 p->arena);
854}
855
856void *
857_PyPegen_string_token(Parser *p)
858{
859 return _PyPegen_expect_token(p, STRING);
860}
861
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100862static PyObject *
863parsenumber_raw(const char *s)
864{
865 const char *end;
866 long x;
867 double dx;
868 Py_complex compl;
869 int imflag;
870
871 assert(s != NULL);
872 errno = 0;
873 end = s + strlen(s) - 1;
874 imflag = *end == 'j' || *end == 'J';
875 if (s[0] == '0') {
876 x = (long)PyOS_strtoul(s, (char **)&end, 0);
877 if (x < 0 && errno == 0) {
878 return PyLong_FromString(s, (char **)0, 0);
879 }
880 }
Pablo Galindofb61c422020-06-15 14:23:43 +0100881 else {
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100882 x = PyOS_strtol(s, (char **)&end, 0);
Pablo Galindofb61c422020-06-15 14:23:43 +0100883 }
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100884 if (*end == '\0') {
Pablo Galindofb61c422020-06-15 14:23:43 +0100885 if (errno != 0) {
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100886 return PyLong_FromString(s, (char **)0, 0);
Pablo Galindofb61c422020-06-15 14:23:43 +0100887 }
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100888 return PyLong_FromLong(x);
889 }
890 /* XXX Huge floats may silently fail */
891 if (imflag) {
892 compl.real = 0.;
893 compl.imag = PyOS_string_to_double(s, (char **)&end, NULL);
Pablo Galindofb61c422020-06-15 14:23:43 +0100894 if (compl.imag == -1.0 && PyErr_Occurred()) {
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100895 return NULL;
Pablo Galindofb61c422020-06-15 14:23:43 +0100896 }
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100897 return PyComplex_FromCComplex(compl);
898 }
Pablo Galindofb61c422020-06-15 14:23:43 +0100899 dx = PyOS_string_to_double(s, NULL, NULL);
900 if (dx == -1.0 && PyErr_Occurred()) {
901 return NULL;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100902 }
Pablo Galindofb61c422020-06-15 14:23:43 +0100903 return PyFloat_FromDouble(dx);
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100904}
905
906static PyObject *
907parsenumber(const char *s)
908{
Pablo Galindofb61c422020-06-15 14:23:43 +0100909 char *dup;
910 char *end;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100911 PyObject *res = NULL;
912
913 assert(s != NULL);
914
915 if (strchr(s, '_') == NULL) {
916 return parsenumber_raw(s);
917 }
918 /* Create a duplicate without underscores. */
919 dup = PyMem_Malloc(strlen(s) + 1);
920 if (dup == NULL) {
921 return PyErr_NoMemory();
922 }
923 end = dup;
924 for (; *s; s++) {
925 if (*s != '_') {
926 *end++ = *s;
927 }
928 }
929 *end = '\0';
930 res = parsenumber_raw(dup);
931 PyMem_Free(dup);
932 return res;
933}
934
935expr_ty
936_PyPegen_number_token(Parser *p)
937{
938 Token *t = _PyPegen_expect_token(p, NUMBER);
939 if (t == NULL) {
940 return NULL;
941 }
942
943 char *num_raw = PyBytes_AsString(t->bytes);
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100944 if (num_raw == NULL) {
Lysandros Nikolaou526e23f2020-05-27 19:04:11 +0300945 p->error_indicator = 1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100946 return NULL;
947 }
948
Lysandros Nikolaou3e0a6f32020-05-01 06:27:52 +0300949 if (p->feature_version < 6 && strchr(num_raw, '_') != NULL) {
950 p->error_indicator = 1;
Shantanuc3f00142020-05-04 01:13:30 -0700951 return RAISE_SYNTAX_ERROR("Underscores in numeric literals are only supported "
Lysandros Nikolaou3e0a6f32020-05-01 06:27:52 +0300952 "in Python 3.6 and greater");
953 }
954
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100955 PyObject *c = parsenumber(num_raw);
956
957 if (c == NULL) {
Lysandros Nikolaou526e23f2020-05-27 19:04:11 +0300958 p->error_indicator = 1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100959 return NULL;
960 }
961
962 if (PyArena_AddPyObject(p->arena, c) < 0) {
963 Py_DECREF(c);
Lysandros Nikolaou526e23f2020-05-27 19:04:11 +0300964 p->error_indicator = 1;
Pablo Galindoc5fc1562020-04-22 23:29:27 +0100965 return NULL;
966 }
967
968 return Constant(c, NULL, t->lineno, t->col_offset, t->end_lineno, t->end_col_offset,
969 p->arena);
970}
971
Lysandros Nikolaou6d650872020-04-29 04:42:27 +0300972static int // bool
973newline_in_string(Parser *p, const char *cur)
974{
Pablo Galindo2e6593d2020-06-06 00:52:27 +0100975 for (const char *c = cur; c >= p->tok->buf; c--) {
976 if (*c == '\'' || *c == '"') {
Lysandros Nikolaou6d650872020-04-29 04:42:27 +0300977 return 1;
978 }
979 }
980 return 0;
981}
982
983/* Check that the source for a single input statement really is a single
984 statement by looking at what is left in the buffer after parsing.
985 Trailing whitespace and comments are OK. */
986static int // bool
987bad_single_statement(Parser *p)
988{
989 const char *cur = strchr(p->tok->buf, '\n');
990
991 /* Newlines are allowed if preceded by a line continuation character
992 or if they appear inside a string. */
993 if (!cur || *(cur - 1) == '\\' || newline_in_string(p, cur)) {
994 return 0;
995 }
996 char c = *cur;
997
998 for (;;) {
999 while (c == ' ' || c == '\t' || c == '\n' || c == '\014') {
1000 c = *++cur;
1001 }
1002
1003 if (!c) {
1004 return 0;
1005 }
1006
1007 if (c != '#') {
1008 return 1;
1009 }
1010
1011 /* Suck up comment. */
1012 while (c && c != '\n') {
1013 c = *++cur;
1014 }
1015 }
1016}
1017
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001018void
1019_PyPegen_Parser_Free(Parser *p)
1020{
1021 Py_XDECREF(p->normalize);
1022 for (int i = 0; i < p->size; i++) {
1023 PyMem_Free(p->tokens[i]);
1024 }
1025 PyMem_Free(p->tokens);
Guido van Rossumc001c092020-04-30 12:12:19 -07001026 growable_comment_array_deallocate(&p->type_ignore_comments);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001027 PyMem_Free(p);
1028}
1029
Pablo Galindo2b74c832020-04-27 18:02:07 +01001030static int
1031compute_parser_flags(PyCompilerFlags *flags)
1032{
1033 int parser_flags = 0;
1034 if (!flags) {
1035 return 0;
1036 }
1037 if (flags->cf_flags & PyCF_DONT_IMPLY_DEDENT) {
1038 parser_flags |= PyPARSE_DONT_IMPLY_DEDENT;
1039 }
1040 if (flags->cf_flags & PyCF_IGNORE_COOKIE) {
1041 parser_flags |= PyPARSE_IGNORE_COOKIE;
1042 }
1043 if (flags->cf_flags & CO_FUTURE_BARRY_AS_BDFL) {
1044 parser_flags |= PyPARSE_BARRY_AS_BDFL;
1045 }
1046 if (flags->cf_flags & PyCF_TYPE_COMMENTS) {
1047 parser_flags |= PyPARSE_TYPE_COMMENTS;
1048 }
Guido van Rossum9d197c72020-06-27 17:33:49 -07001049 if ((flags->cf_flags & PyCF_ONLY_AST) && flags->cf_feature_version < 7) {
Lysandros Nikolaou3e0a6f32020-05-01 06:27:52 +03001050 parser_flags |= PyPARSE_ASYNC_HACKS;
1051 }
Pablo Galindo2b74c832020-04-27 18:02:07 +01001052 return parser_flags;
1053}
1054
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001055Parser *
Pablo Galindo2b74c832020-04-27 18:02:07 +01001056_PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags,
Lysandros Nikolaou3e0a6f32020-05-01 06:27:52 +03001057 int feature_version, int *errcode, PyArena *arena)
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001058{
1059 Parser *p = PyMem_Malloc(sizeof(Parser));
1060 if (p == NULL) {
Lysandros Nikolaouebebb642020-04-23 18:36:06 +03001061 return (Parser *) PyErr_NoMemory();
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001062 }
1063 assert(tok != NULL);
Guido van Rossumd9d6ead2020-05-01 09:42:32 -07001064 tok->type_comments = (flags & PyPARSE_TYPE_COMMENTS) > 0;
1065 tok->async_hacks = (flags & PyPARSE_ASYNC_HACKS) > 0;
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001066 p->tok = tok;
1067 p->keywords = NULL;
1068 p->n_keyword_lists = -1;
1069 p->tokens = PyMem_Malloc(sizeof(Token *));
1070 if (!p->tokens) {
1071 PyMem_Free(p);
Lysandros Nikolaouebebb642020-04-23 18:36:06 +03001072 return (Parser *) PyErr_NoMemory();
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001073 }
Guido van Rossumc001c092020-04-30 12:12:19 -07001074 p->tokens[0] = PyMem_Calloc(1, sizeof(Token));
Lysandros Nikolaouebebb642020-04-23 18:36:06 +03001075 if (!p->tokens) {
1076 PyMem_Free(p->tokens);
1077 PyMem_Free(p);
1078 return (Parser *) PyErr_NoMemory();
1079 }
Guido van Rossumc001c092020-04-30 12:12:19 -07001080 if (!growable_comment_array_init(&p->type_ignore_comments, 10)) {
1081 PyMem_Free(p->tokens[0]);
1082 PyMem_Free(p->tokens);
1083 PyMem_Free(p);
1084 return (Parser *) PyErr_NoMemory();
1085 }
1086
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001087 p->mark = 0;
1088 p->fill = 0;
1089 p->size = 1;
1090
1091 p->errcode = errcode;
1092 p->arena = arena;
1093 p->start_rule = start_rule;
1094 p->parsing_started = 0;
1095 p->normalize = NULL;
1096 p->error_indicator = 0;
1097
1098 p->starting_lineno = 0;
1099 p->starting_col_offset = 0;
Pablo Galindo2b74c832020-04-27 18:02:07 +01001100 p->flags = flags;
Lysandros Nikolaou3e0a6f32020-05-01 06:27:52 +03001101 p->feature_version = feature_version;
Lysandros Nikolaou2f37c352020-05-07 13:37:51 +03001102 p->known_err_token = NULL;
Pablo Galindo800a35c62020-05-25 18:38:45 +01001103 p->level = 0;
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001104
1105 return p;
1106}
1107
1108void *
1109_PyPegen_run_parser(Parser *p)
1110{
1111 void *res = _PyPegen_parse(p);
1112 if (res == NULL) {
1113 if (PyErr_Occurred()) {
1114 return NULL;
1115 }
1116 if (p->fill == 0) {
1117 RAISE_SYNTAX_ERROR("error at start before reading any input");
1118 }
1119 else if (p->tok->done == E_EOF) {
1120 RAISE_SYNTAX_ERROR("unexpected EOF while parsing");
1121 }
1122 else {
1123 if (p->tokens[p->fill-1]->type == INDENT) {
1124 RAISE_INDENTATION_ERROR("unexpected indent");
1125 }
1126 else if (p->tokens[p->fill-1]->type == DEDENT) {
1127 RAISE_INDENTATION_ERROR("unexpected unindent");
1128 }
1129 else {
1130 RAISE_SYNTAX_ERROR("invalid syntax");
1131 }
1132 }
1133 return NULL;
1134 }
1135
Lysandros Nikolaou6d650872020-04-29 04:42:27 +03001136 if (p->start_rule == Py_single_input && bad_single_statement(p)) {
1137 p->tok->done = E_BADSINGLE; // This is not necessary for now, but might be in the future
1138 return RAISE_SYNTAX_ERROR("multiple statements found while compiling a single statement");
1139 }
1140
Pablo Galindo13322262020-07-27 23:46:59 +01001141#if defined(Py_DEBUG) && defined(Py_BUILD_CORE)
1142 if (p->start_rule == Py_single_input ||
1143 p->start_rule == Py_file_input ||
1144 p->start_rule == Py_eval_input)
1145 {
1146 assert(PyAST_Validate(res));
1147 }
1148#endif
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001149 return res;
1150}
1151
1152mod_ty
1153_PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filename_ob,
1154 const char *enc, const char *ps1, const char *ps2,
Pablo Galindo2b74c832020-04-27 18:02:07 +01001155 PyCompilerFlags *flags, int *errcode, PyArena *arena)
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001156{
1157 struct tok_state *tok = PyTokenizer_FromFile(fp, enc, ps1, ps2);
1158 if (tok == NULL) {
1159 if (PyErr_Occurred()) {
1160 raise_tokenizer_init_error(filename_ob);
1161 return NULL;
1162 }
1163 return NULL;
1164 }
1165 // This transfers the ownership to the tokenizer
1166 tok->filename = filename_ob;
1167 Py_INCREF(filename_ob);
1168
1169 // From here on we need to clean up even if there's an error
1170 mod_ty result = NULL;
1171
Pablo Galindo2b74c832020-04-27 18:02:07 +01001172 int parser_flags = compute_parser_flags(flags);
Lysandros Nikolaou3e0a6f32020-05-01 06:27:52 +03001173 Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, PY_MINOR_VERSION,
1174 errcode, arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001175 if (p == NULL) {
1176 goto error;
1177 }
1178
1179 result = _PyPegen_run_parser(p);
1180 _PyPegen_Parser_Free(p);
1181
1182error:
1183 PyTokenizer_Free(tok);
1184 return result;
1185}
1186
1187mod_ty
1188_PyPegen_run_parser_from_file(const char *filename, int start_rule,
Pablo Galindo2b74c832020-04-27 18:02:07 +01001189 PyObject *filename_ob, PyCompilerFlags *flags, PyArena *arena)
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001190{
1191 FILE *fp = fopen(filename, "rb");
1192 if (fp == NULL) {
1193 PyErr_SetFromErrnoWithFilename(PyExc_OSError, filename);
1194 return NULL;
1195 }
1196
1197 mod_ty result = _PyPegen_run_parser_from_file_pointer(fp, start_rule, filename_ob,
Pablo Galindo2b74c832020-04-27 18:02:07 +01001198 NULL, NULL, NULL, flags, NULL, arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001199
1200 fclose(fp);
1201 return result;
1202}
1203
1204mod_ty
1205_PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filename_ob,
Pablo Galindo2b74c832020-04-27 18:02:07 +01001206 PyCompilerFlags *flags, PyArena *arena)
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001207{
1208 int exec_input = start_rule == Py_file_input;
1209
1210 struct tok_state *tok;
Pablo Galindo2b74c832020-04-27 18:02:07 +01001211 if (flags == NULL || flags->cf_flags & PyCF_IGNORE_COOKIE) {
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001212 tok = PyTokenizer_FromUTF8(str, exec_input);
1213 } else {
1214 tok = PyTokenizer_FromString(str, exec_input);
1215 }
1216 if (tok == NULL) {
1217 if (PyErr_Occurred()) {
1218 raise_tokenizer_init_error(filename_ob);
1219 }
1220 return NULL;
1221 }
1222 // This transfers the ownership to the tokenizer
1223 tok->filename = filename_ob;
1224 Py_INCREF(filename_ob);
1225
1226 // We need to clear up from here on
1227 mod_ty result = NULL;
1228
Pablo Galindo2b74c832020-04-27 18:02:07 +01001229 int parser_flags = compute_parser_flags(flags);
Guido van Rossum9d197c72020-06-27 17:33:49 -07001230 int feature_version = flags && (flags->cf_flags & PyCF_ONLY_AST) ?
1231 flags->cf_feature_version : PY_MINOR_VERSION;
Lysandros Nikolaou3e0a6f32020-05-01 06:27:52 +03001232 Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, feature_version,
1233 NULL, arena);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001234 if (p == NULL) {
1235 goto error;
1236 }
1237
1238 result = _PyPegen_run_parser(p);
1239 _PyPegen_Parser_Free(p);
1240
1241error:
1242 PyTokenizer_Free(tok);
1243 return result;
1244}
1245
1246void *
1247_PyPegen_interactive_exit(Parser *p)
1248{
1249 if (p->errcode) {
1250 *(p->errcode) = E_EOF;
1251 }
1252 return NULL;
1253}
1254
1255/* Creates a single-element asdl_seq* that contains a */
1256asdl_seq *
1257_PyPegen_singleton_seq(Parser *p, void *a)
1258{
1259 assert(a != NULL);
1260 asdl_seq *seq = _Py_asdl_seq_new(1, p->arena);
1261 if (!seq) {
1262 return NULL;
1263 }
1264 asdl_seq_SET(seq, 0, a);
1265 return seq;
1266}
1267
1268/* Creates a copy of seq and prepends a to it */
1269asdl_seq *
1270_PyPegen_seq_insert_in_front(Parser *p, void *a, asdl_seq *seq)
1271{
1272 assert(a != NULL);
1273 if (!seq) {
1274 return _PyPegen_singleton_seq(p, a);
1275 }
1276
1277 asdl_seq *new_seq = _Py_asdl_seq_new(asdl_seq_LEN(seq) + 1, p->arena);
1278 if (!new_seq) {
1279 return NULL;
1280 }
1281
1282 asdl_seq_SET(new_seq, 0, a);
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001283 for (Py_ssize_t i = 1, l = asdl_seq_LEN(new_seq); i < l; i++) {
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001284 asdl_seq_SET(new_seq, i, asdl_seq_GET(seq, i - 1));
1285 }
1286 return new_seq;
1287}
1288
Guido van Rossumc001c092020-04-30 12:12:19 -07001289/* Creates a copy of seq and appends a to it */
1290asdl_seq *
1291_PyPegen_seq_append_to_end(Parser *p, asdl_seq *seq, void *a)
1292{
1293 assert(a != NULL);
1294 if (!seq) {
1295 return _PyPegen_singleton_seq(p, a);
1296 }
1297
1298 asdl_seq *new_seq = _Py_asdl_seq_new(asdl_seq_LEN(seq) + 1, p->arena);
1299 if (!new_seq) {
1300 return NULL;
1301 }
1302
1303 for (Py_ssize_t i = 0, l = asdl_seq_LEN(new_seq); i + 1 < l; i++) {
1304 asdl_seq_SET(new_seq, i, asdl_seq_GET(seq, i));
1305 }
1306 asdl_seq_SET(new_seq, asdl_seq_LEN(new_seq) - 1, a);
1307 return new_seq;
1308}
1309
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001310static Py_ssize_t
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001311_get_flattened_seq_size(asdl_seq *seqs)
1312{
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001313 Py_ssize_t size = 0;
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001314 for (Py_ssize_t i = 0, l = asdl_seq_LEN(seqs); i < l; i++) {
1315 asdl_seq *inner_seq = asdl_seq_GET(seqs, i);
1316 size += asdl_seq_LEN(inner_seq);
1317 }
1318 return size;
1319}
1320
1321/* Flattens an asdl_seq* of asdl_seq*s */
1322asdl_seq *
1323_PyPegen_seq_flatten(Parser *p, asdl_seq *seqs)
1324{
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001325 Py_ssize_t flattened_seq_size = _get_flattened_seq_size(seqs);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001326 assert(flattened_seq_size > 0);
1327
1328 asdl_seq *flattened_seq = _Py_asdl_seq_new(flattened_seq_size, p->arena);
1329 if (!flattened_seq) {
1330 return NULL;
1331 }
1332
1333 int flattened_seq_idx = 0;
1334 for (Py_ssize_t i = 0, l = asdl_seq_LEN(seqs); i < l; i++) {
1335 asdl_seq *inner_seq = asdl_seq_GET(seqs, i);
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001336 for (Py_ssize_t j = 0, li = asdl_seq_LEN(inner_seq); j < li; j++) {
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001337 asdl_seq_SET(flattened_seq, flattened_seq_idx++, asdl_seq_GET(inner_seq, j));
1338 }
1339 }
1340 assert(flattened_seq_idx == flattened_seq_size);
1341
1342 return flattened_seq;
1343}
1344
1345/* Creates a new name of the form <first_name>.<second_name> */
1346expr_ty
1347_PyPegen_join_names_with_dot(Parser *p, expr_ty first_name, expr_ty second_name)
1348{
1349 assert(first_name != NULL && second_name != NULL);
1350 PyObject *first_identifier = first_name->v.Name.id;
1351 PyObject *second_identifier = second_name->v.Name.id;
1352
1353 if (PyUnicode_READY(first_identifier) == -1) {
1354 return NULL;
1355 }
1356 if (PyUnicode_READY(second_identifier) == -1) {
1357 return NULL;
1358 }
1359 const char *first_str = PyUnicode_AsUTF8(first_identifier);
1360 if (!first_str) {
1361 return NULL;
1362 }
1363 const char *second_str = PyUnicode_AsUTF8(second_identifier);
1364 if (!second_str) {
1365 return NULL;
1366 }
Pablo Galindo9f27dd32020-04-24 01:13:33 +01001367 Py_ssize_t len = strlen(first_str) + strlen(second_str) + 1; // +1 for the dot
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001368
1369 PyObject *str = PyBytes_FromStringAndSize(NULL, len);
1370 if (!str) {
1371 return NULL;
1372 }
1373
1374 char *s = PyBytes_AS_STRING(str);
1375 if (!s) {
1376 return NULL;
1377 }
1378
1379 strcpy(s, first_str);
1380 s += strlen(first_str);
1381 *s++ = '.';
1382 strcpy(s, second_str);
1383 s += strlen(second_str);
1384 *s = '\0';
1385
1386 PyObject *uni = PyUnicode_DecodeUTF8(PyBytes_AS_STRING(str), PyBytes_GET_SIZE(str), NULL);
1387 Py_DECREF(str);
1388 if (!uni) {
1389 return NULL;
1390 }
1391 PyUnicode_InternInPlace(&uni);
1392 if (PyArena_AddPyObject(p->arena, uni) < 0) {
1393 Py_DECREF(uni);
1394 return NULL;
1395 }
1396
1397 return _Py_Name(uni, Load, EXTRA_EXPR(first_name, second_name));
1398}
1399
1400/* Counts the total number of dots in seq's tokens */
1401int
1402_PyPegen_seq_count_dots(asdl_seq *seq)
1403{
1404 int number_of_dots = 0;
1405 for (Py_ssize_t i = 0, l = asdl_seq_LEN(seq); i < l; i++) {
1406 Token *current_expr = asdl_seq_GET(seq, i);
1407 switch (current_expr->type) {
1408 case ELLIPSIS:
1409 number_of_dots += 3;
1410 break;
1411 case DOT:
1412 number_of_dots += 1;
1413 break;
1414 default:
Lysandros Nikolaouebebb642020-04-23 18:36:06 +03001415 Py_UNREACHABLE();
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001416 }
1417 }
1418
1419 return number_of_dots;
1420}
1421
1422/* Creates an alias with '*' as the identifier name */
1423alias_ty
1424_PyPegen_alias_for_star(Parser *p)
1425{
1426 PyObject *str = PyUnicode_InternFromString("*");
1427 if (!str) {
1428 return NULL;
1429 }
1430 if (PyArena_AddPyObject(p->arena, str) < 0) {
1431 Py_DECREF(str);
1432 return NULL;
1433 }
1434 return alias(str, NULL, p->arena);
1435}
1436
1437/* Creates a new asdl_seq* with the identifiers of all the names in seq */
1438asdl_seq *
1439_PyPegen_map_names_to_ids(Parser *p, asdl_seq *seq)
1440{
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001441 Py_ssize_t len = asdl_seq_LEN(seq);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001442 assert(len > 0);
1443
1444 asdl_seq *new_seq = _Py_asdl_seq_new(len, p->arena);
1445 if (!new_seq) {
1446 return NULL;
1447 }
1448 for (Py_ssize_t i = 0; i < len; i++) {
1449 expr_ty e = asdl_seq_GET(seq, i);
1450 asdl_seq_SET(new_seq, i, e->v.Name.id);
1451 }
1452 return new_seq;
1453}
1454
1455/* Constructs a CmpopExprPair */
1456CmpopExprPair *
1457_PyPegen_cmpop_expr_pair(Parser *p, cmpop_ty cmpop, expr_ty expr)
1458{
1459 assert(expr != NULL);
1460 CmpopExprPair *a = PyArena_Malloc(p->arena, sizeof(CmpopExprPair));
1461 if (!a) {
1462 return NULL;
1463 }
1464 a->cmpop = cmpop;
1465 a->expr = expr;
1466 return a;
1467}
1468
1469asdl_int_seq *
1470_PyPegen_get_cmpops(Parser *p, asdl_seq *seq)
1471{
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001472 Py_ssize_t len = asdl_seq_LEN(seq);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001473 assert(len > 0);
1474
1475 asdl_int_seq *new_seq = _Py_asdl_int_seq_new(len, p->arena);
1476 if (!new_seq) {
1477 return NULL;
1478 }
1479 for (Py_ssize_t i = 0; i < len; i++) {
1480 CmpopExprPair *pair = asdl_seq_GET(seq, i);
1481 asdl_seq_SET(new_seq, i, pair->cmpop);
1482 }
1483 return new_seq;
1484}
1485
1486asdl_seq *
1487_PyPegen_get_exprs(Parser *p, asdl_seq *seq)
1488{
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001489 Py_ssize_t len = asdl_seq_LEN(seq);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001490 assert(len > 0);
1491
1492 asdl_seq *new_seq = _Py_asdl_seq_new(len, p->arena);
1493 if (!new_seq) {
1494 return NULL;
1495 }
1496 for (Py_ssize_t i = 0; i < len; i++) {
1497 CmpopExprPair *pair = asdl_seq_GET(seq, i);
1498 asdl_seq_SET(new_seq, i, pair->expr);
1499 }
1500 return new_seq;
1501}
1502
1503/* Creates an asdl_seq* where all the elements have been changed to have ctx as context */
1504static asdl_seq *
1505_set_seq_context(Parser *p, asdl_seq *seq, expr_context_ty ctx)
1506{
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001507 Py_ssize_t len = asdl_seq_LEN(seq);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001508 if (len == 0) {
1509 return NULL;
1510 }
1511
1512 asdl_seq *new_seq = _Py_asdl_seq_new(len, p->arena);
1513 if (!new_seq) {
1514 return NULL;
1515 }
1516 for (Py_ssize_t i = 0; i < len; i++) {
1517 expr_ty e = asdl_seq_GET(seq, i);
1518 asdl_seq_SET(new_seq, i, _PyPegen_set_expr_context(p, e, ctx));
1519 }
1520 return new_seq;
1521}
1522
1523static expr_ty
1524_set_name_context(Parser *p, expr_ty e, expr_context_ty ctx)
1525{
1526 return _Py_Name(e->v.Name.id, ctx, EXTRA_EXPR(e, e));
1527}
1528
1529static expr_ty
1530_set_tuple_context(Parser *p, expr_ty e, expr_context_ty ctx)
1531{
1532 return _Py_Tuple(_set_seq_context(p, e->v.Tuple.elts, ctx), ctx, EXTRA_EXPR(e, e));
1533}
1534
1535static expr_ty
1536_set_list_context(Parser *p, expr_ty e, expr_context_ty ctx)
1537{
1538 return _Py_List(_set_seq_context(p, e->v.List.elts, ctx), ctx, EXTRA_EXPR(e, e));
1539}
1540
1541static expr_ty
1542_set_subscript_context(Parser *p, expr_ty e, expr_context_ty ctx)
1543{
1544 return _Py_Subscript(e->v.Subscript.value, e->v.Subscript.slice, ctx, EXTRA_EXPR(e, e));
1545}
1546
1547static expr_ty
1548_set_attribute_context(Parser *p, expr_ty e, expr_context_ty ctx)
1549{
1550 return _Py_Attribute(e->v.Attribute.value, e->v.Attribute.attr, ctx, EXTRA_EXPR(e, e));
1551}
1552
1553static expr_ty
1554_set_starred_context(Parser *p, expr_ty e, expr_context_ty ctx)
1555{
1556 return _Py_Starred(_PyPegen_set_expr_context(p, e->v.Starred.value, ctx), ctx, EXTRA_EXPR(e, e));
1557}
1558
1559/* Creates an `expr_ty` equivalent to `expr` but with `ctx` as context */
1560expr_ty
1561_PyPegen_set_expr_context(Parser *p, expr_ty expr, expr_context_ty ctx)
1562{
1563 assert(expr != NULL);
1564
1565 expr_ty new = NULL;
1566 switch (expr->kind) {
1567 case Name_kind:
1568 new = _set_name_context(p, expr, ctx);
1569 break;
1570 case Tuple_kind:
1571 new = _set_tuple_context(p, expr, ctx);
1572 break;
1573 case List_kind:
1574 new = _set_list_context(p, expr, ctx);
1575 break;
1576 case Subscript_kind:
1577 new = _set_subscript_context(p, expr, ctx);
1578 break;
1579 case Attribute_kind:
1580 new = _set_attribute_context(p, expr, ctx);
1581 break;
1582 case Starred_kind:
1583 new = _set_starred_context(p, expr, ctx);
1584 break;
1585 default:
1586 new = expr;
1587 }
1588 return new;
1589}
1590
1591/* Constructs a KeyValuePair that is used when parsing a dict's key value pairs */
1592KeyValuePair *
1593_PyPegen_key_value_pair(Parser *p, expr_ty key, expr_ty value)
1594{
1595 KeyValuePair *a = PyArena_Malloc(p->arena, sizeof(KeyValuePair));
1596 if (!a) {
1597 return NULL;
1598 }
1599 a->key = key;
1600 a->value = value;
1601 return a;
1602}
1603
1604/* Extracts all keys from an asdl_seq* of KeyValuePair*'s */
1605asdl_seq *
1606_PyPegen_get_keys(Parser *p, asdl_seq *seq)
1607{
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001608 Py_ssize_t len = asdl_seq_LEN(seq);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001609 asdl_seq *new_seq = _Py_asdl_seq_new(len, p->arena);
1610 if (!new_seq) {
1611 return NULL;
1612 }
1613 for (Py_ssize_t i = 0; i < len; i++) {
1614 KeyValuePair *pair = asdl_seq_GET(seq, i);
1615 asdl_seq_SET(new_seq, i, pair->key);
1616 }
1617 return new_seq;
1618}
1619
1620/* Extracts all values from an asdl_seq* of KeyValuePair*'s */
1621asdl_seq *
1622_PyPegen_get_values(Parser *p, asdl_seq *seq)
1623{
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001624 Py_ssize_t len = asdl_seq_LEN(seq);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001625 asdl_seq *new_seq = _Py_asdl_seq_new(len, p->arena);
1626 if (!new_seq) {
1627 return NULL;
1628 }
1629 for (Py_ssize_t i = 0; i < len; i++) {
1630 KeyValuePair *pair = asdl_seq_GET(seq, i);
1631 asdl_seq_SET(new_seq, i, pair->value);
1632 }
1633 return new_seq;
1634}
1635
1636/* Constructs a NameDefaultPair */
1637NameDefaultPair *
Guido van Rossumc001c092020-04-30 12:12:19 -07001638_PyPegen_name_default_pair(Parser *p, arg_ty arg, expr_ty value, Token *tc)
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001639{
1640 NameDefaultPair *a = PyArena_Malloc(p->arena, sizeof(NameDefaultPair));
1641 if (!a) {
1642 return NULL;
1643 }
Guido van Rossumc001c092020-04-30 12:12:19 -07001644 a->arg = _PyPegen_add_type_comment_to_arg(p, arg, tc);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001645 a->value = value;
1646 return a;
1647}
1648
1649/* Constructs a SlashWithDefault */
1650SlashWithDefault *
1651_PyPegen_slash_with_default(Parser *p, asdl_seq *plain_names, asdl_seq *names_with_defaults)
1652{
1653 SlashWithDefault *a = PyArena_Malloc(p->arena, sizeof(SlashWithDefault));
1654 if (!a) {
1655 return NULL;
1656 }
1657 a->plain_names = plain_names;
1658 a->names_with_defaults = names_with_defaults;
1659 return a;
1660}
1661
1662/* Constructs a StarEtc */
1663StarEtc *
1664_PyPegen_star_etc(Parser *p, arg_ty vararg, asdl_seq *kwonlyargs, arg_ty kwarg)
1665{
1666 StarEtc *a = PyArena_Malloc(p->arena, sizeof(StarEtc));
1667 if (!a) {
1668 return NULL;
1669 }
1670 a->vararg = vararg;
1671 a->kwonlyargs = kwonlyargs;
1672 a->kwarg = kwarg;
1673 return a;
1674}
1675
1676asdl_seq *
1677_PyPegen_join_sequences(Parser *p, asdl_seq *a, asdl_seq *b)
1678{
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001679 Py_ssize_t first_len = asdl_seq_LEN(a);
1680 Py_ssize_t second_len = asdl_seq_LEN(b);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001681 asdl_seq *new_seq = _Py_asdl_seq_new(first_len + second_len, p->arena);
1682 if (!new_seq) {
1683 return NULL;
1684 }
1685
1686 int k = 0;
1687 for (Py_ssize_t i = 0; i < first_len; i++) {
1688 asdl_seq_SET(new_seq, k++, asdl_seq_GET(a, i));
1689 }
1690 for (Py_ssize_t i = 0; i < second_len; i++) {
1691 asdl_seq_SET(new_seq, k++, asdl_seq_GET(b, i));
1692 }
1693
1694 return new_seq;
1695}
1696
1697static asdl_seq *
1698_get_names(Parser *p, asdl_seq *names_with_defaults)
1699{
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001700 Py_ssize_t len = asdl_seq_LEN(names_with_defaults);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001701 asdl_seq *seq = _Py_asdl_seq_new(len, p->arena);
1702 if (!seq) {
1703 return NULL;
1704 }
1705 for (Py_ssize_t i = 0; i < len; i++) {
1706 NameDefaultPair *pair = asdl_seq_GET(names_with_defaults, i);
1707 asdl_seq_SET(seq, i, pair->arg);
1708 }
1709 return seq;
1710}
1711
1712static asdl_seq *
1713_get_defaults(Parser *p, asdl_seq *names_with_defaults)
1714{
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001715 Py_ssize_t len = asdl_seq_LEN(names_with_defaults);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001716 asdl_seq *seq = _Py_asdl_seq_new(len, p->arena);
1717 if (!seq) {
1718 return NULL;
1719 }
1720 for (Py_ssize_t i = 0; i < len; i++) {
1721 NameDefaultPair *pair = asdl_seq_GET(names_with_defaults, i);
1722 asdl_seq_SET(seq, i, pair->value);
1723 }
1724 return seq;
1725}
1726
1727/* Constructs an arguments_ty object out of all the parsed constructs in the parameters rule */
1728arguments_ty
1729_PyPegen_make_arguments(Parser *p, asdl_seq *slash_without_default,
1730 SlashWithDefault *slash_with_default, asdl_seq *plain_names,
1731 asdl_seq *names_with_default, StarEtc *star_etc)
1732{
1733 asdl_seq *posonlyargs;
1734 if (slash_without_default != NULL) {
1735 posonlyargs = slash_without_default;
1736 }
1737 else if (slash_with_default != NULL) {
1738 asdl_seq *slash_with_default_names =
1739 _get_names(p, slash_with_default->names_with_defaults);
1740 if (!slash_with_default_names) {
1741 return NULL;
1742 }
1743 posonlyargs = _PyPegen_join_sequences(p, slash_with_default->plain_names, slash_with_default_names);
1744 if (!posonlyargs) {
1745 return NULL;
1746 }
1747 }
1748 else {
1749 posonlyargs = _Py_asdl_seq_new(0, p->arena);
1750 if (!posonlyargs) {
1751 return NULL;
1752 }
1753 }
1754
1755 asdl_seq *posargs;
1756 if (plain_names != NULL && names_with_default != NULL) {
1757 asdl_seq *names_with_default_names = _get_names(p, names_with_default);
1758 if (!names_with_default_names) {
1759 return NULL;
1760 }
1761 posargs = _PyPegen_join_sequences(p, plain_names, names_with_default_names);
1762 if (!posargs) {
1763 return NULL;
1764 }
1765 }
1766 else if (plain_names == NULL && names_with_default != NULL) {
1767 posargs = _get_names(p, names_with_default);
1768 if (!posargs) {
1769 return NULL;
1770 }
1771 }
1772 else if (plain_names != NULL && names_with_default == NULL) {
1773 posargs = plain_names;
1774 }
1775 else {
1776 posargs = _Py_asdl_seq_new(0, p->arena);
1777 if (!posargs) {
1778 return NULL;
1779 }
1780 }
1781
1782 asdl_seq *posdefaults;
1783 if (slash_with_default != NULL && names_with_default != NULL) {
1784 asdl_seq *slash_with_default_values =
1785 _get_defaults(p, slash_with_default->names_with_defaults);
1786 if (!slash_with_default_values) {
1787 return NULL;
1788 }
1789 asdl_seq *names_with_default_values = _get_defaults(p, names_with_default);
1790 if (!names_with_default_values) {
1791 return NULL;
1792 }
1793 posdefaults = _PyPegen_join_sequences(p, slash_with_default_values, names_with_default_values);
1794 if (!posdefaults) {
1795 return NULL;
1796 }
1797 }
1798 else if (slash_with_default == NULL && names_with_default != NULL) {
1799 posdefaults = _get_defaults(p, names_with_default);
1800 if (!posdefaults) {
1801 return NULL;
1802 }
1803 }
1804 else if (slash_with_default != NULL && names_with_default == NULL) {
1805 posdefaults = _get_defaults(p, slash_with_default->names_with_defaults);
1806 if (!posdefaults) {
1807 return NULL;
1808 }
1809 }
1810 else {
1811 posdefaults = _Py_asdl_seq_new(0, p->arena);
1812 if (!posdefaults) {
1813 return NULL;
1814 }
1815 }
1816
1817 arg_ty vararg = NULL;
1818 if (star_etc != NULL && star_etc->vararg != NULL) {
1819 vararg = star_etc->vararg;
1820 }
1821
1822 asdl_seq *kwonlyargs;
1823 if (star_etc != NULL && star_etc->kwonlyargs != NULL) {
1824 kwonlyargs = _get_names(p, star_etc->kwonlyargs);
1825 if (!kwonlyargs) {
1826 return NULL;
1827 }
1828 }
1829 else {
1830 kwonlyargs = _Py_asdl_seq_new(0, p->arena);
1831 if (!kwonlyargs) {
1832 return NULL;
1833 }
1834 }
1835
1836 asdl_seq *kwdefaults;
1837 if (star_etc != NULL && star_etc->kwonlyargs != NULL) {
1838 kwdefaults = _get_defaults(p, star_etc->kwonlyargs);
1839 if (!kwdefaults) {
1840 return NULL;
1841 }
1842 }
1843 else {
1844 kwdefaults = _Py_asdl_seq_new(0, p->arena);
1845 if (!kwdefaults) {
1846 return NULL;
1847 }
1848 }
1849
1850 arg_ty kwarg = NULL;
1851 if (star_etc != NULL && star_etc->kwarg != NULL) {
1852 kwarg = star_etc->kwarg;
1853 }
1854
1855 return _Py_arguments(posonlyargs, posargs, vararg, kwonlyargs, kwdefaults, kwarg,
1856 posdefaults, p->arena);
1857}
1858
1859/* Constructs an empty arguments_ty object, that gets used when a function accepts no
1860 * arguments. */
1861arguments_ty
1862_PyPegen_empty_arguments(Parser *p)
1863{
1864 asdl_seq *posonlyargs = _Py_asdl_seq_new(0, p->arena);
1865 if (!posonlyargs) {
1866 return NULL;
1867 }
1868 asdl_seq *posargs = _Py_asdl_seq_new(0, p->arena);
1869 if (!posargs) {
1870 return NULL;
1871 }
1872 asdl_seq *posdefaults = _Py_asdl_seq_new(0, p->arena);
1873 if (!posdefaults) {
1874 return NULL;
1875 }
1876 asdl_seq *kwonlyargs = _Py_asdl_seq_new(0, p->arena);
1877 if (!kwonlyargs) {
1878 return NULL;
1879 }
1880 asdl_seq *kwdefaults = _Py_asdl_seq_new(0, p->arena);
1881 if (!kwdefaults) {
1882 return NULL;
1883 }
1884
1885 return _Py_arguments(posonlyargs, posargs, NULL, kwonlyargs, kwdefaults, NULL, kwdefaults,
1886 p->arena);
1887}
1888
1889/* Encapsulates the value of an operator_ty into an AugOperator struct */
1890AugOperator *
1891_PyPegen_augoperator(Parser *p, operator_ty kind)
1892{
1893 AugOperator *a = PyArena_Malloc(p->arena, sizeof(AugOperator));
1894 if (!a) {
1895 return NULL;
1896 }
1897 a->kind = kind;
1898 return a;
1899}
1900
1901/* Construct a FunctionDef equivalent to function_def, but with decorators */
1902stmt_ty
1903_PyPegen_function_def_decorators(Parser *p, asdl_seq *decorators, stmt_ty function_def)
1904{
1905 assert(function_def != NULL);
1906 if (function_def->kind == AsyncFunctionDef_kind) {
1907 return _Py_AsyncFunctionDef(
1908 function_def->v.FunctionDef.name, function_def->v.FunctionDef.args,
1909 function_def->v.FunctionDef.body, decorators, function_def->v.FunctionDef.returns,
1910 function_def->v.FunctionDef.type_comment, function_def->lineno,
1911 function_def->col_offset, function_def->end_lineno, function_def->end_col_offset,
1912 p->arena);
1913 }
1914
1915 return _Py_FunctionDef(function_def->v.FunctionDef.name, function_def->v.FunctionDef.args,
1916 function_def->v.FunctionDef.body, decorators,
1917 function_def->v.FunctionDef.returns,
1918 function_def->v.FunctionDef.type_comment, function_def->lineno,
1919 function_def->col_offset, function_def->end_lineno,
1920 function_def->end_col_offset, p->arena);
1921}
1922
1923/* Construct a ClassDef equivalent to class_def, but with decorators */
1924stmt_ty
1925_PyPegen_class_def_decorators(Parser *p, asdl_seq *decorators, stmt_ty class_def)
1926{
1927 assert(class_def != NULL);
1928 return _Py_ClassDef(class_def->v.ClassDef.name, class_def->v.ClassDef.bases,
1929 class_def->v.ClassDef.keywords, class_def->v.ClassDef.body, decorators,
1930 class_def->lineno, class_def->col_offset, class_def->end_lineno,
1931 class_def->end_col_offset, p->arena);
1932}
1933
1934/* Construct a KeywordOrStarred */
1935KeywordOrStarred *
1936_PyPegen_keyword_or_starred(Parser *p, void *element, int is_keyword)
1937{
1938 KeywordOrStarred *a = PyArena_Malloc(p->arena, sizeof(KeywordOrStarred));
1939 if (!a) {
1940 return NULL;
1941 }
1942 a->element = element;
1943 a->is_keyword = is_keyword;
1944 return a;
1945}
1946
1947/* Get the number of starred expressions in an asdl_seq* of KeywordOrStarred*s */
1948static int
1949_seq_number_of_starred_exprs(asdl_seq *seq)
1950{
1951 int n = 0;
1952 for (Py_ssize_t i = 0, l = asdl_seq_LEN(seq); i < l; i++) {
1953 KeywordOrStarred *k = asdl_seq_GET(seq, i);
1954 if (!k->is_keyword) {
1955 n++;
1956 }
1957 }
1958 return n;
1959}
1960
1961/* Extract the starred expressions of an asdl_seq* of KeywordOrStarred*s */
1962asdl_seq *
1963_PyPegen_seq_extract_starred_exprs(Parser *p, asdl_seq *kwargs)
1964{
1965 int new_len = _seq_number_of_starred_exprs(kwargs);
1966 if (new_len == 0) {
1967 return NULL;
1968 }
1969 asdl_seq *new_seq = _Py_asdl_seq_new(new_len, p->arena);
1970 if (!new_seq) {
1971 return NULL;
1972 }
1973
1974 int idx = 0;
1975 for (Py_ssize_t i = 0, len = asdl_seq_LEN(kwargs); i < len; i++) {
1976 KeywordOrStarred *k = asdl_seq_GET(kwargs, i);
1977 if (!k->is_keyword) {
1978 asdl_seq_SET(new_seq, idx++, k->element);
1979 }
1980 }
1981 return new_seq;
1982}
1983
1984/* Return a new asdl_seq* with only the keywords in kwargs */
1985asdl_seq *
1986_PyPegen_seq_delete_starred_exprs(Parser *p, asdl_seq *kwargs)
1987{
Pablo Galindoee40e4b2020-04-23 03:43:08 +01001988 Py_ssize_t len = asdl_seq_LEN(kwargs);
1989 Py_ssize_t new_len = len - _seq_number_of_starred_exprs(kwargs);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01001990 if (new_len == 0) {
1991 return NULL;
1992 }
1993 asdl_seq *new_seq = _Py_asdl_seq_new(new_len, p->arena);
1994 if (!new_seq) {
1995 return NULL;
1996 }
1997
1998 int idx = 0;
1999 for (Py_ssize_t i = 0; i < len; i++) {
2000 KeywordOrStarred *k = asdl_seq_GET(kwargs, i);
2001 if (k->is_keyword) {
2002 asdl_seq_SET(new_seq, idx++, k->element);
2003 }
2004 }
2005 return new_seq;
2006}
2007
2008expr_ty
2009_PyPegen_concatenate_strings(Parser *p, asdl_seq *strings)
2010{
Pablo Galindoee40e4b2020-04-23 03:43:08 +01002011 Py_ssize_t len = asdl_seq_LEN(strings);
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002012 assert(len > 0);
2013
2014 Token *first = asdl_seq_GET(strings, 0);
2015 Token *last = asdl_seq_GET(strings, len - 1);
2016
2017 int bytesmode = 0;
2018 PyObject *bytes_str = NULL;
2019
2020 FstringParser state;
2021 _PyPegen_FstringParser_Init(&state);
2022
2023 for (Py_ssize_t i = 0; i < len; i++) {
2024 Token *t = asdl_seq_GET(strings, i);
2025
2026 int this_bytesmode;
2027 int this_rawmode;
2028 PyObject *s;
2029 const char *fstr;
2030 Py_ssize_t fstrlen = -1;
2031
Lysandros Nikolaou2f37c352020-05-07 13:37:51 +03002032 if (_PyPegen_parsestr(p, &this_bytesmode, &this_rawmode, &s, &fstr, &fstrlen, t) != 0) {
Pablo Galindoc5fc1562020-04-22 23:29:27 +01002033 goto error;
2034 }
2035
2036 /* Check that we are not mixing bytes with unicode. */
2037 if (i != 0 && bytesmode != this_bytesmode) {
2038 RAISE_SYNTAX_ERROR("cannot mix bytes and nonbytes literals");
2039 Py_XDECREF(s);
2040 goto error;
2041 }
2042 bytesmode = this_bytesmode;
2043
2044 if (fstr != NULL) {
2045 assert(s == NULL && !bytesmode);
2046
2047 int result = _PyPegen_FstringParser_ConcatFstring(p, &state, &fstr, fstr + fstrlen,
2048 this_rawmode, 0, first, t, last);
2049 if (result < 0) {
2050 goto error;
2051 }
2052 }
2053 else {
2054 /* String or byte string. */
2055 assert(s != NULL && fstr == NULL);
2056 assert(bytesmode ? PyBytes_CheckExact(s) : PyUnicode_CheckExact(s));
2057
2058 if (bytesmode) {
2059 if (i == 0) {
2060 bytes_str = s;
2061 }
2062 else {
2063 PyBytes_ConcatAndDel(&bytes_str, s);
2064 if (!bytes_str) {
2065 goto error;
2066 }
2067 }
2068 }
2069 else {
2070 /* This is a regular string. Concatenate it. */
2071 if (_PyPegen_FstringParser_ConcatAndDel(&state, s) < 0) {
2072 goto error;
2073 }
2074 }
2075 }
2076 }
2077
2078 if (bytesmode) {
2079 if (PyArena_AddPyObject(p->arena, bytes_str) < 0) {
2080 goto error;
2081 }
2082 return Constant(bytes_str, NULL, first->lineno, first->col_offset, last->end_lineno,
2083 last->end_col_offset, p->arena);
2084 }
2085
2086 return _PyPegen_FstringParser_Finish(p, &state, first, last);
2087
2088error:
2089 Py_XDECREF(bytes_str);
2090 _PyPegen_FstringParser_Dealloc(&state);
2091 if (PyErr_Occurred()) {
2092 raise_decode_error(p);
2093 }
2094 return NULL;
2095}
Guido van Rossumc001c092020-04-30 12:12:19 -07002096
2097mod_ty
2098_PyPegen_make_module(Parser *p, asdl_seq *a) {
2099 asdl_seq *type_ignores = NULL;
2100 Py_ssize_t num = p->type_ignore_comments.num_items;
2101 if (num > 0) {
2102 // Turn the raw (comment, lineno) pairs into TypeIgnore objects in the arena
2103 type_ignores = _Py_asdl_seq_new(num, p->arena);
2104 if (type_ignores == NULL) {
2105 return NULL;
2106 }
2107 for (int i = 0; i < num; i++) {
2108 PyObject *tag = _PyPegen_new_type_comment(p, p->type_ignore_comments.items[i].comment);
2109 if (tag == NULL) {
2110 return NULL;
2111 }
2112 type_ignore_ty ti = TypeIgnore(p->type_ignore_comments.items[i].lineno, tag, p->arena);
2113 if (ti == NULL) {
2114 return NULL;
2115 }
2116 asdl_seq_SET(type_ignores, i, ti);
2117 }
2118 }
2119 return Module(a, type_ignores, p->arena);
2120}
Pablo Galindo16ab0702020-05-15 02:04:52 +01002121
2122// Error reporting helpers
2123
2124expr_ty
Lysandros Nikolaou01ece632020-06-19 02:10:43 +03002125_PyPegen_get_invalid_target(expr_ty e, TARGETS_TYPE targets_type)
Pablo Galindo16ab0702020-05-15 02:04:52 +01002126{
2127 if (e == NULL) {
2128 return NULL;
2129 }
2130
2131#define VISIT_CONTAINER(CONTAINER, TYPE) do { \
2132 Py_ssize_t len = asdl_seq_LEN(CONTAINER->v.TYPE.elts);\
2133 for (Py_ssize_t i = 0; i < len; i++) {\
2134 expr_ty other = asdl_seq_GET(CONTAINER->v.TYPE.elts, i);\
Lysandros Nikolaou01ece632020-06-19 02:10:43 +03002135 expr_ty child = _PyPegen_get_invalid_target(other, targets_type);\
Pablo Galindo16ab0702020-05-15 02:04:52 +01002136 if (child != NULL) {\
2137 return child;\
2138 }\
2139 }\
2140 } while (0)
2141
2142 // We only need to visit List and Tuple nodes recursively as those
2143 // are the only ones that can contain valid names in targets when
2144 // they are parsed as expressions. Any other kind of expression
2145 // that is a container (like Sets or Dicts) is directly invalid and
2146 // we don't need to visit it recursively.
2147
2148 switch (e->kind) {
Lysandros Nikolaou01ece632020-06-19 02:10:43 +03002149 case List_kind:
Pablo Galindo16ab0702020-05-15 02:04:52 +01002150 VISIT_CONTAINER(e, List);
2151 return NULL;
Lysandros Nikolaou01ece632020-06-19 02:10:43 +03002152 case Tuple_kind:
Pablo Galindo16ab0702020-05-15 02:04:52 +01002153 VISIT_CONTAINER(e, Tuple);
2154 return NULL;
Pablo Galindo16ab0702020-05-15 02:04:52 +01002155 case Starred_kind:
Lysandros Nikolaou01ece632020-06-19 02:10:43 +03002156 if (targets_type == DEL_TARGETS) {
2157 return e;
2158 }
2159 return _PyPegen_get_invalid_target(e->v.Starred.value, targets_type);
2160 case Compare_kind:
2161 // This is needed, because the `a in b` in `for a in b` gets parsed
2162 // as a comparison, and so we need to search the left side of the comparison
2163 // for invalid targets.
2164 if (targets_type == FOR_TARGETS) {
2165 cmpop_ty cmpop = (cmpop_ty) asdl_seq_GET(e->v.Compare.ops, 0);
2166 if (cmpop == In) {
2167 return _PyPegen_get_invalid_target(e->v.Compare.left, targets_type);
2168 }
2169 return NULL;
2170 }
2171 return e;
Pablo Galindo16ab0702020-05-15 02:04:52 +01002172 case Name_kind:
2173 case Subscript_kind:
2174 case Attribute_kind:
2175 return NULL;
2176 default:
2177 return e;
2178 }
Lysandros Nikolaou75b863a2020-05-18 22:14:47 +03002179}
2180
2181void *_PyPegen_arguments_parsing_error(Parser *p, expr_ty e) {
2182 int kwarg_unpacking = 0;
2183 for (Py_ssize_t i = 0, l = asdl_seq_LEN(e->v.Call.keywords); i < l; i++) {
2184 keyword_ty keyword = asdl_seq_GET(e->v.Call.keywords, i);
2185 if (!keyword->arg) {
2186 kwarg_unpacking = 1;
2187 }
2188 }
2189
2190 const char *msg = NULL;
2191 if (kwarg_unpacking) {
2192 msg = "positional argument follows keyword argument unpacking";
2193 } else {
2194 msg = "positional argument follows keyword argument";
2195 }
2196
2197 return RAISE_SYNTAX_ERROR(msg);
2198}
Lysandros Nikolaouae145832020-05-22 03:56:52 +03002199
2200void *
2201_PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args)
2202{
2203 /* The rule that calls this function is 'args for_if_clauses'.
2204 For the input f(L, x for x in y), L and x are in args and
2205 the for is parsed as a for_if_clause. We have to check if
2206 len <= 1, so that input like dict((a, b) for a, b in x)
2207 gets successfully parsed and then we pass the last
2208 argument (x in the above example) as the location of the
2209 error */
2210 Py_ssize_t len = asdl_seq_LEN(args->v.Call.args);
2211 if (len <= 1) {
2212 return NULL;
2213 }
2214
2215 return RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
2216 (expr_ty) asdl_seq_GET(args->v.Call.args, len - 1),
2217 "Generator expression must be parenthesized"
2218 );
2219}
Pablo Galindo4a97b152020-09-02 17:44:19 +01002220
2221
2222expr_ty _PyPegen_collect_call_seqs(Parser *p, asdl_seq *a, asdl_seq *b) {
2223 Py_ssize_t args_len = asdl_seq_LEN(a);
2224 Py_ssize_t total_len = args_len;
2225
2226 if (b == NULL) {
2227 expr_ty first = asdl_seq_GET(a, 0);
2228 expr_ty last = asdl_seq_GET(a, args_len - 1);
2229 return _Py_Call(_PyPegen_dummy_name(p), a, NULL, EXTRA_EXPR(first, last));
2230
2231 }
2232
2233 asdl_seq *starreds = _PyPegen_seq_extract_starred_exprs(p, b);
2234 asdl_seq *keywords = _PyPegen_seq_delete_starred_exprs(p, b);
2235
2236 if (starreds) {
2237 total_len += asdl_seq_LEN(starreds);
2238 }
2239
2240 asdl_seq *args = _Py_asdl_seq_new(total_len, p->arena);
2241
2242 Py_ssize_t i = 0;
2243 for (i = 0; i < args_len; i++) {
2244 asdl_seq_SET(args, i, asdl_seq_GET(a, i));
2245 }
2246 for (; i < total_len; i++) {
2247 asdl_seq_SET(args, i, asdl_seq_GET(starreds, i - args_len));
2248 }
2249
2250 expr_ty first = asdl_seq_GET(args, 0);
2251 expr_ty last = asdl_seq_GET(b, asdl_seq_LEN(b)-1);
2252
2253 return _Py_Call(_PyPegen_dummy_name(p), args, keywords, EXTRA_EXPR(first, last));
2254}