Guido van Rossum | f70e43a | 1991-02-19 12:39:46 +0000 | [diff] [blame] | 1 | |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 2 | /* Parser-tokenizer link implementation */ |
| 3 | |
Pablo Galindo | f2cf1e3 | 2019-04-13 17:05:14 +0100 | [diff] [blame] | 4 | #include "Python.h" |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 5 | #include "tokenizer.h" |
| 6 | #include "node.h" |
| 7 | #include "grammar.h" |
| 8 | #include "parser.h" |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 9 | #include "parsetok.h" |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 10 | #include "errcode.h" |
Martin v. Löwis | 00f1e3f | 2002-08-04 17:29:52 +0000 | [diff] [blame] | 11 | #include "graminit.h" |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 12 | |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 13 | |
| 14 | /* Forward */ |
Christian Heimes | 4d6ec85 | 2008-03-26 22:34:47 +0000 | [diff] [blame] | 15 | static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int *); |
Victor Stinner | 14e461d | 2013-08-26 22:28:21 +0200 | [diff] [blame] | 16 | static int initerr(perrdetail *err_ret, PyObject * filename); |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 17 | |
Guido van Rossum | dcfcd14 | 2019-01-31 03:40:27 -0800 | [diff] [blame] | 18 | typedef struct { |
Michael J. Sullivan | 933e150 | 2019-05-22 07:54:20 -0700 | [diff] [blame] | 19 | struct { |
| 20 | int lineno; |
| 21 | char *comment; |
| 22 | } *items; |
Guido van Rossum | dcfcd14 | 2019-01-31 03:40:27 -0800 | [diff] [blame] | 23 | size_t size; |
| 24 | size_t num_items; |
Michael J. Sullivan | 933e150 | 2019-05-22 07:54:20 -0700 | [diff] [blame] | 25 | } growable_comment_array; |
Guido van Rossum | dcfcd14 | 2019-01-31 03:40:27 -0800 | [diff] [blame] | 26 | |
| 27 | static int |
Michael J. Sullivan | 933e150 | 2019-05-22 07:54:20 -0700 | [diff] [blame] | 28 | growable_comment_array_init(growable_comment_array *arr, size_t initial_size) { |
Guido van Rossum | dcfcd14 | 2019-01-31 03:40:27 -0800 | [diff] [blame] | 29 | assert(initial_size > 0); |
| 30 | arr->items = malloc(initial_size * sizeof(*arr->items)); |
| 31 | arr->size = initial_size; |
| 32 | arr->num_items = 0; |
| 33 | |
| 34 | return arr->items != NULL; |
| 35 | } |
| 36 | |
| 37 | static int |
Michael J. Sullivan | 933e150 | 2019-05-22 07:54:20 -0700 | [diff] [blame] | 38 | growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) { |
Guido van Rossum | dcfcd14 | 2019-01-31 03:40:27 -0800 | [diff] [blame] | 39 | if (arr->num_items >= arr->size) { |
| 40 | arr->size *= 2; |
| 41 | arr->items = realloc(arr->items, arr->size * sizeof(*arr->items)); |
| 42 | if (!arr->items) { |
| 43 | return 0; |
| 44 | } |
| 45 | } |
| 46 | |
Michael J. Sullivan | 933e150 | 2019-05-22 07:54:20 -0700 | [diff] [blame] | 47 | arr->items[arr->num_items].lineno = lineno; |
| 48 | arr->items[arr->num_items].comment = comment; |
Guido van Rossum | dcfcd14 | 2019-01-31 03:40:27 -0800 | [diff] [blame] | 49 | arr->num_items++; |
| 50 | return 1; |
| 51 | } |
| 52 | |
| 53 | static void |
Michael J. Sullivan | 933e150 | 2019-05-22 07:54:20 -0700 | [diff] [blame] | 54 | growable_comment_array_deallocate(growable_comment_array *arr) { |
| 55 | for (unsigned i = 0; i < arr->num_items; i++) { |
| 56 | PyObject_FREE(arr->items[i].comment); |
| 57 | } |
Guido van Rossum | dcfcd14 | 2019-01-31 03:40:27 -0800 | [diff] [blame] | 58 | free(arr->items); |
| 59 | } |
| 60 | |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 61 | /* Parse input coming from a string. Return error code, print some errors. */ |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 62 | node * |
Martin v. Löwis | 95292d6 | 2002-12-11 14:04:59 +0000 | [diff] [blame] | 63 | PyParser_ParseString(const char *s, grammar *g, int start, perrdetail *err_ret) |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 64 | { |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 65 | return PyParser_ParseStringFlagsFilename(s, NULL, g, start, err_ret, 0); |
Tim Peters | fe2127d | 2001-07-16 05:37:24 +0000 | [diff] [blame] | 66 | } |
| 67 | |
| 68 | node * |
Martin v. Löwis | 95292d6 | 2002-12-11 14:04:59 +0000 | [diff] [blame] | 69 | PyParser_ParseStringFlags(const char *s, grammar *g, int start, |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 70 | perrdetail *err_ret, int flags) |
Tim Peters | fe2127d | 2001-07-16 05:37:24 +0000 | [diff] [blame] | 71 | { |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 72 | return PyParser_ParseStringFlagsFilename(s, NULL, |
| 73 | g, start, err_ret, flags); |
Thomas Heller | 6b17abf | 2002-07-09 09:23:27 +0000 | [diff] [blame] | 74 | } |
| 75 | |
| 76 | node * |
Martin v. Löwis | 95292d6 | 2002-12-11 14:04:59 +0000 | [diff] [blame] | 77 | PyParser_ParseStringFlagsFilename(const char *s, const char *filename, |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 78 | grammar *g, int start, |
| 79 | perrdetail *err_ret, int flags) |
Thomas Heller | 6b17abf | 2002-07-09 09:23:27 +0000 | [diff] [blame] | 80 | { |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 81 | int iflags = flags; |
| 82 | return PyParser_ParseStringFlagsFilenameEx(s, filename, g, start, |
| 83 | err_ret, &iflags); |
Christian Heimes | 4d6ec85 | 2008-03-26 22:34:47 +0000 | [diff] [blame] | 84 | } |
| 85 | |
| 86 | node * |
Victor Stinner | 14e461d | 2013-08-26 22:28:21 +0200 | [diff] [blame] | 87 | PyParser_ParseStringObject(const char *s, PyObject *filename, |
| 88 | grammar *g, int start, |
| 89 | perrdetail *err_ret, int *flags) |
Christian Heimes | 4d6ec85 | 2008-03-26 22:34:47 +0000 | [diff] [blame] | 90 | { |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 91 | struct tok_state *tok; |
| 92 | int exec_input = start == file_input; |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 93 | |
Victor Stinner | 7f2fee3 | 2011-04-05 00:39:01 +0200 | [diff] [blame] | 94 | if (initerr(err_ret, filename) < 0) |
| 95 | return NULL; |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 96 | |
Steve Dower | b82e17e | 2019-05-23 08:45:22 -0700 | [diff] [blame] | 97 | if (PySys_Audit("compile", "yO", s, err_ret->filename) < 0) { |
| 98 | err_ret->error = E_ERROR; |
| 99 | return NULL; |
| 100 | } |
| 101 | |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 102 | if (*flags & PyPARSE_IGNORE_COOKIE) |
| 103 | tok = PyTokenizer_FromUTF8(s, exec_input); |
| 104 | else |
| 105 | tok = PyTokenizer_FromString(s, exec_input); |
| 106 | if (tok == NULL) { |
| 107 | err_ret->error = PyErr_Occurred() ? E_DECODE : E_NOMEM; |
| 108 | return NULL; |
| 109 | } |
Guido van Rossum | dcfcd14 | 2019-01-31 03:40:27 -0800 | [diff] [blame] | 110 | if (*flags & PyPARSE_TYPE_COMMENTS) { |
| 111 | tok->type_comments = 1; |
| 112 | } |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 113 | |
Victor Stinner | 7f2fee3 | 2011-04-05 00:39:01 +0200 | [diff] [blame] | 114 | Py_INCREF(err_ret->filename); |
| 115 | tok->filename = err_ret->filename; |
Guido van Rossum | 495da29 | 2019-03-07 12:38:08 -0800 | [diff] [blame] | 116 | if (*flags & PyPARSE_ASYNC_HACKS) |
| 117 | tok->async_hacks = 1; |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 118 | return parsetok(tok, g, start, err_ret, flags); |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 119 | } |
| 120 | |
Victor Stinner | 14e461d | 2013-08-26 22:28:21 +0200 | [diff] [blame] | 121 | node * |
| 122 | PyParser_ParseStringFlagsFilenameEx(const char *s, const char *filename_str, |
| 123 | grammar *g, int start, |
| 124 | perrdetail *err_ret, int *flags) |
| 125 | { |
| 126 | node *n; |
| 127 | PyObject *filename = NULL; |
Victor Stinner | 14e461d | 2013-08-26 22:28:21 +0200 | [diff] [blame] | 128 | if (filename_str != NULL) { |
| 129 | filename = PyUnicode_DecodeFSDefault(filename_str); |
| 130 | if (filename == NULL) { |
| 131 | err_ret->error = E_ERROR; |
| 132 | return NULL; |
| 133 | } |
| 134 | } |
Victor Stinner | 14e461d | 2013-08-26 22:28:21 +0200 | [diff] [blame] | 135 | n = PyParser_ParseStringObject(s, filename, g, start, err_ret, flags); |
Victor Stinner | 14e461d | 2013-08-26 22:28:21 +0200 | [diff] [blame] | 136 | Py_XDECREF(filename); |
Victor Stinner | 14e461d | 2013-08-26 22:28:21 +0200 | [diff] [blame] | 137 | return n; |
| 138 | } |
| 139 | |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 140 | /* Parse input coming from a file. Return error code, print some errors. */ |
| 141 | |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 142 | node * |
Martin v. Löwis | 95292d6 | 2002-12-11 14:04:59 +0000 | [diff] [blame] | 143 | PyParser_ParseFile(FILE *fp, const char *filename, grammar *g, int start, |
Serhiy Storchaka | c679227 | 2013-10-19 21:03:34 +0300 | [diff] [blame] | 144 | const char *ps1, const char *ps2, |
| 145 | perrdetail *err_ret) |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 146 | { |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 147 | return PyParser_ParseFileFlags(fp, filename, NULL, |
| 148 | g, start, ps1, ps2, err_ret, 0); |
Tim Peters | fe2127d | 2001-07-16 05:37:24 +0000 | [diff] [blame] | 149 | } |
| 150 | |
| 151 | node * |
Christian Heimes | 4d6ec85 | 2008-03-26 22:34:47 +0000 | [diff] [blame] | 152 | PyParser_ParseFileFlags(FILE *fp, const char *filename, const char *enc, |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 153 | grammar *g, int start, |
Serhiy Storchaka | c679227 | 2013-10-19 21:03:34 +0300 | [diff] [blame] | 154 | const char *ps1, const char *ps2, |
| 155 | perrdetail *err_ret, int flags) |
Tim Peters | fe2127d | 2001-07-16 05:37:24 +0000 | [diff] [blame] | 156 | { |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 157 | int iflags = flags; |
| 158 | return PyParser_ParseFileFlagsEx(fp, filename, enc, g, start, ps1, |
| 159 | ps2, err_ret, &iflags); |
Christian Heimes | 4d6ec85 | 2008-03-26 22:34:47 +0000 | [diff] [blame] | 160 | } |
| 161 | |
| 162 | node * |
Victor Stinner | 14e461d | 2013-08-26 22:28:21 +0200 | [diff] [blame] | 163 | PyParser_ParseFileObject(FILE *fp, PyObject *filename, |
| 164 | const char *enc, grammar *g, int start, |
Serhiy Storchaka | c679227 | 2013-10-19 21:03:34 +0300 | [diff] [blame] | 165 | const char *ps1, const char *ps2, |
| 166 | perrdetail *err_ret, int *flags) |
Christian Heimes | 4d6ec85 | 2008-03-26 22:34:47 +0000 | [diff] [blame] | 167 | { |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 168 | struct tok_state *tok; |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 169 | |
Victor Stinner | 7f2fee3 | 2011-04-05 00:39:01 +0200 | [diff] [blame] | 170 | if (initerr(err_ret, filename) < 0) |
| 171 | return NULL; |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 172 | |
Steve Dower | b82e17e | 2019-05-23 08:45:22 -0700 | [diff] [blame] | 173 | if (PySys_Audit("compile", "OO", Py_None, err_ret->filename) < 0) { |
| 174 | return NULL; |
| 175 | } |
| 176 | |
Serhiy Storchaka | c679227 | 2013-10-19 21:03:34 +0300 | [diff] [blame] | 177 | if ((tok = PyTokenizer_FromFile(fp, enc, ps1, ps2)) == NULL) { |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 178 | err_ret->error = E_NOMEM; |
| 179 | return NULL; |
| 180 | } |
Guido van Rossum | dcfcd14 | 2019-01-31 03:40:27 -0800 | [diff] [blame] | 181 | if (*flags & PyPARSE_TYPE_COMMENTS) { |
| 182 | tok->type_comments = 1; |
| 183 | } |
Victor Stinner | 7f2fee3 | 2011-04-05 00:39:01 +0200 | [diff] [blame] | 184 | Py_INCREF(err_ret->filename); |
| 185 | tok->filename = err_ret->filename; |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 186 | return parsetok(tok, g, start, err_ret, flags); |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 187 | } |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 188 | |
Victor Stinner | 14e461d | 2013-08-26 22:28:21 +0200 | [diff] [blame] | 189 | node * |
| 190 | PyParser_ParseFileFlagsEx(FILE *fp, const char *filename, |
| 191 | const char *enc, grammar *g, int start, |
Serhiy Storchaka | c679227 | 2013-10-19 21:03:34 +0300 | [diff] [blame] | 192 | const char *ps1, const char *ps2, |
| 193 | perrdetail *err_ret, int *flags) |
Victor Stinner | 14e461d | 2013-08-26 22:28:21 +0200 | [diff] [blame] | 194 | { |
| 195 | node *n; |
| 196 | PyObject *fileobj = NULL; |
Victor Stinner | 14e461d | 2013-08-26 22:28:21 +0200 | [diff] [blame] | 197 | if (filename != NULL) { |
| 198 | fileobj = PyUnicode_DecodeFSDefault(filename); |
| 199 | if (fileobj == NULL) { |
| 200 | err_ret->error = E_ERROR; |
| 201 | return NULL; |
| 202 | } |
| 203 | } |
Victor Stinner | 14e461d | 2013-08-26 22:28:21 +0200 | [diff] [blame] | 204 | n = PyParser_ParseFileObject(fp, fileobj, enc, g, |
| 205 | start, ps1, ps2, err_ret, flags); |
Victor Stinner | 14e461d | 2013-08-26 22:28:21 +0200 | [diff] [blame] | 206 | Py_XDECREF(fileobj); |
Victor Stinner | 14e461d | 2013-08-26 22:28:21 +0200 | [diff] [blame] | 207 | return n; |
| 208 | } |
| 209 | |
Neal Norwitz | e4993c7 | 2006-03-16 06:01:25 +0000 | [diff] [blame] | 210 | #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD |
Brett Cannon | e3944a5 | 2009-04-01 05:08:41 +0000 | [diff] [blame] | 211 | #if 0 |
Serhiy Storchaka | 2d06e84 | 2015-12-25 19:53:18 +0200 | [diff] [blame] | 212 | static const char with_msg[] = |
Thomas Wouters | 34aa7ba | 2006-02-28 19:02:24 +0000 | [diff] [blame] | 213 | "%s:%d: Warning: 'with' will become a reserved keyword in Python 2.6\n"; |
| 214 | |
Serhiy Storchaka | 2d06e84 | 2015-12-25 19:53:18 +0200 | [diff] [blame] | 215 | static const char as_msg[] = |
Thomas Wouters | 34aa7ba | 2006-02-28 19:02:24 +0000 | [diff] [blame] | 216 | "%s:%d: Warning: 'as' will become a reserved keyword in Python 2.6\n"; |
| 217 | |
| 218 | static void |
| 219 | warn(const char *msg, const char *filename, int lineno) |
| 220 | { |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 221 | if (filename == NULL) |
| 222 | filename = "<string>"; |
| 223 | PySys_WriteStderr(msg, filename, lineno); |
Thomas Wouters | 34aa7ba | 2006-02-28 19:02:24 +0000 | [diff] [blame] | 224 | } |
Neal Norwitz | fc85c92 | 2006-03-17 05:44:46 +0000 | [diff] [blame] | 225 | #endif |
Brett Cannon | e3944a5 | 2009-04-01 05:08:41 +0000 | [diff] [blame] | 226 | #endif |
Guido van Rossum | da62ecc | 2001-07-17 16:53:11 +0000 | [diff] [blame] | 227 | |
Thomas Wouters | 89f507f | 2006-12-13 04:49:30 +0000 | [diff] [blame] | 228 | /* Parse input coming from the given tokenizer structure. |
| 229 | Return error code. */ |
| 230 | |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 231 | static node * |
Tim Peters | fe2127d | 2001-07-16 05:37:24 +0000 | [diff] [blame] | 232 | parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 233 | int *flags) |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 234 | { |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 235 | parser_state *ps; |
| 236 | node *n; |
Brett Cannon | b94767f | 2011-02-22 20:15:44 +0000 | [diff] [blame] | 237 | int started = 0; |
Ivan Levkivskyi | 9932a22 | 2019-01-22 11:18:22 +0000 | [diff] [blame] | 238 | int col_offset, end_col_offset; |
Michael J. Sullivan | 933e150 | 2019-05-22 07:54:20 -0700 | [diff] [blame] | 239 | growable_comment_array type_ignores; |
Guido van Rossum | dcfcd14 | 2019-01-31 03:40:27 -0800 | [diff] [blame] | 240 | |
Michael J. Sullivan | 933e150 | 2019-05-22 07:54:20 -0700 | [diff] [blame] | 241 | if (!growable_comment_array_init(&type_ignores, 10)) { |
Guido van Rossum | dcfcd14 | 2019-01-31 03:40:27 -0800 | [diff] [blame] | 242 | err_ret->error = E_NOMEM; |
| 243 | PyTokenizer_Free(tok); |
| 244 | return NULL; |
| 245 | } |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 246 | |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 247 | if ((ps = PyParser_New(g, start)) == NULL) { |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 248 | err_ret->error = E_NOMEM; |
| 249 | PyTokenizer_Free(tok); |
| 250 | return NULL; |
| 251 | } |
Thomas Wouters | 34aa7ba | 2006-02-28 19:02:24 +0000 | [diff] [blame] | 252 | #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 253 | if (*flags & PyPARSE_BARRY_AS_BDFL) |
| 254 | ps->p_flags |= CO_FUTURE_BARRY_AS_BDFL; |
Guido van Rossum | dcfcd14 | 2019-01-31 03:40:27 -0800 | [diff] [blame] | 255 | if (*flags & PyPARSE_TYPE_COMMENTS) |
| 256 | ps->p_flags |= PyCF_TYPE_COMMENTS; |
Neil Schemenauer | c24ea08 | 2002-03-22 23:53:36 +0000 | [diff] [blame] | 257 | #endif |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 258 | |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 259 | for (;;) { |
| 260 | char *a, *b; |
| 261 | int type; |
| 262 | size_t len; |
| 263 | char *str; |
Ammar Askar | 025eb98 | 2018-09-24 17:12:49 -0400 | [diff] [blame] | 264 | col_offset = -1; |
Anthony Sottile | 995d9b9 | 2019-01-12 20:05:13 -0800 | [diff] [blame] | 265 | int lineno; |
| 266 | const char *line_start; |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 267 | |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 268 | type = PyTokenizer_Get(tok, &a, &b); |
| 269 | if (type == ERRORTOKEN) { |
| 270 | err_ret->error = tok->done; |
| 271 | break; |
| 272 | } |
| 273 | if (type == ENDMARKER && started) { |
| 274 | type = NEWLINE; /* Add an extra newline */ |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 275 | started = 0; |
| 276 | /* Add the right number of dedent tokens, |
| 277 | except if a certain flag is given -- |
| 278 | codeop.py uses this. */ |
| 279 | if (tok->indent && |
| 280 | !(*flags & PyPARSE_DONT_IMPLY_DEDENT)) |
| 281 | { |
| 282 | tok->pendin = -tok->indent; |
| 283 | tok->indent = 0; |
| 284 | } |
| 285 | } |
| 286 | else |
| 287 | started = 1; |
Zackery Spytz | 7c4ab2a | 2018-08-15 00:27:26 -0600 | [diff] [blame] | 288 | len = (a != NULL && b != NULL) ? b - a : 0; |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 289 | str = (char *) PyObject_MALLOC(len + 1); |
| 290 | if (str == NULL) { |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 291 | err_ret->error = E_NOMEM; |
| 292 | break; |
| 293 | } |
| 294 | if (len > 0) |
| 295 | strncpy(str, a, len); |
| 296 | str[len] = '\0'; |
Guido van Rossum | da62ecc | 2001-07-17 16:53:11 +0000 | [diff] [blame] | 297 | |
Thomas Wouters | 34aa7ba | 2006-02-28 19:02:24 +0000 | [diff] [blame] | 298 | #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 299 | if (type == NOTEQUAL) { |
| 300 | if (!(ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) && |
| 301 | strcmp(str, "!=")) { |
Antoine Pitrou | 9ec2593 | 2011-11-13 01:01:23 +0100 | [diff] [blame] | 302 | PyObject_FREE(str); |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 303 | err_ret->error = E_SYNTAX; |
| 304 | break; |
| 305 | } |
| 306 | else if ((ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) && |
| 307 | strcmp(str, "<>")) { |
Antoine Pitrou | 9ec2593 | 2011-11-13 01:01:23 +0100 | [diff] [blame] | 308 | PyObject_FREE(str); |
Serhiy Storchaka | aba24ff | 2018-07-23 23:41:11 +0300 | [diff] [blame] | 309 | err_ret->expected = NOTEQUAL; |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 310 | err_ret->error = E_SYNTAX; |
| 311 | break; |
| 312 | } |
| 313 | } |
Neil Schemenauer | c24ea08 | 2002-03-22 23:53:36 +0000 | [diff] [blame] | 314 | #endif |
Anthony Sottile | 995d9b9 | 2019-01-12 20:05:13 -0800 | [diff] [blame] | 315 | |
| 316 | /* Nodes of type STRING, especially multi line strings |
| 317 | must be handled differently in order to get both |
| 318 | the starting line number and the column offset right. |
| 319 | (cf. issue 16806) */ |
| 320 | lineno = type == STRING ? tok->first_lineno : tok->lineno; |
| 321 | line_start = type == STRING ? tok->multi_line_start : tok->line_start; |
| 322 | if (a != NULL && a >= line_start) { |
| 323 | col_offset = Py_SAFE_DOWNCAST(a - line_start, |
Benjamin Peterson | ca47063 | 2016-09-06 13:47:26 -0700 | [diff] [blame] | 324 | intptr_t, int); |
Zackery Spytz | 3e26e42 | 2018-08-20 21:11:40 -0600 | [diff] [blame] | 325 | } |
| 326 | else { |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 327 | col_offset = -1; |
Zackery Spytz | 3e26e42 | 2018-08-20 21:11:40 -0600 | [diff] [blame] | 328 | } |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 329 | |
Ivan Levkivskyi | 9932a22 | 2019-01-22 11:18:22 +0000 | [diff] [blame] | 330 | if (b != NULL && b >= tok->line_start) { |
| 331 | end_col_offset = Py_SAFE_DOWNCAST(b - tok->line_start, |
| 332 | intptr_t, int); |
| 333 | } |
| 334 | else { |
| 335 | end_col_offset = -1; |
| 336 | } |
Guido van Rossum | dcfcd14 | 2019-01-31 03:40:27 -0800 | [diff] [blame] | 337 | |
| 338 | if (type == TYPE_IGNORE) { |
Michael J. Sullivan | 933e150 | 2019-05-22 07:54:20 -0700 | [diff] [blame] | 339 | if (!growable_comment_array_add(&type_ignores, tok->lineno, str)) { |
Guido van Rossum | dcfcd14 | 2019-01-31 03:40:27 -0800 | [diff] [blame] | 340 | err_ret->error = E_NOMEM; |
| 341 | break; |
| 342 | } |
| 343 | continue; |
| 344 | } |
| 345 | |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 346 | if ((err_ret->error = |
| 347 | PyParser_AddToken(ps, (int)type, str, |
Ivan Levkivskyi | 9932a22 | 2019-01-22 11:18:22 +0000 | [diff] [blame] | 348 | lineno, col_offset, tok->lineno, end_col_offset, |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 349 | &(err_ret->expected))) != E_OK) { |
| 350 | if (err_ret->error != E_DONE) { |
| 351 | PyObject_FREE(str); |
| 352 | err_ret->token = type; |
| 353 | } |
| 354 | break; |
| 355 | } |
| 356 | } |
| 357 | |
| 358 | if (err_ret->error == E_DONE) { |
| 359 | n = ps->p_tree; |
| 360 | ps->p_tree = NULL; |
Meador Inge | fa21bf0 | 2012-01-19 01:08:41 -0600 | [diff] [blame] | 361 | |
Guido van Rossum | dcfcd14 | 2019-01-31 03:40:27 -0800 | [diff] [blame] | 362 | if (n->n_type == file_input) { |
| 363 | /* Put type_ignore nodes in the ENDMARKER of file_input. */ |
| 364 | int num; |
| 365 | node *ch; |
| 366 | size_t i; |
| 367 | |
| 368 | num = NCH(n); |
| 369 | ch = CHILD(n, num - 1); |
| 370 | REQ(ch, ENDMARKER); |
| 371 | |
| 372 | for (i = 0; i < type_ignores.num_items; i++) { |
Michael J. Sullivan | 933e150 | 2019-05-22 07:54:20 -0700 | [diff] [blame] | 373 | int res = PyNode_AddChild(ch, TYPE_IGNORE, type_ignores.items[i].comment, |
| 374 | type_ignores.items[i].lineno, 0, |
| 375 | type_ignores.items[i].lineno, 0); |
| 376 | if (res != 0) { |
| 377 | err_ret->error = res; |
| 378 | PyNode_Free(n); |
| 379 | n = NULL; |
| 380 | break; |
| 381 | } |
| 382 | type_ignores.items[i].comment = NULL; |
Guido van Rossum | dcfcd14 | 2019-01-31 03:40:27 -0800 | [diff] [blame] | 383 | } |
| 384 | } |
Guido van Rossum | dcfcd14 | 2019-01-31 03:40:27 -0800 | [diff] [blame] | 385 | |
Meador Inge | fa21bf0 | 2012-01-19 01:08:41 -0600 | [diff] [blame] | 386 | /* Check that the source for a single input statement really |
| 387 | is a single statement by looking at what is left in the |
| 388 | buffer after parsing. Trailing whitespace and comments |
| 389 | are OK. */ |
Michael J. Sullivan | 933e150 | 2019-05-22 07:54:20 -0700 | [diff] [blame] | 390 | if (err_ret->error == E_DONE && start == single_input) { |
Meador Inge | fa21bf0 | 2012-01-19 01:08:41 -0600 | [diff] [blame] | 391 | char *cur = tok->cur; |
| 392 | char c = *tok->cur; |
| 393 | |
Benjamin Peterson | cff9237 | 2012-01-19 17:46:13 -0500 | [diff] [blame] | 394 | for (;;) { |
| 395 | while (c == ' ' || c == '\t' || c == '\n' || c == '\014') |
| 396 | c = *++cur; |
Meador Inge | fa21bf0 | 2012-01-19 01:08:41 -0600 | [diff] [blame] | 397 | |
Benjamin Peterson | cff9237 | 2012-01-19 17:46:13 -0500 | [diff] [blame] | 398 | if (!c) |
| 399 | break; |
| 400 | |
| 401 | if (c != '#') { |
| 402 | err_ret->error = E_BADSINGLE; |
| 403 | PyNode_Free(n); |
| 404 | n = NULL; |
| 405 | break; |
| 406 | } |
| 407 | |
| 408 | /* Suck up comment. */ |
| 409 | while (c && c != '\n') |
| 410 | c = *++cur; |
Meador Inge | fa21bf0 | 2012-01-19 01:08:41 -0600 | [diff] [blame] | 411 | } |
| 412 | } |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 413 | } |
| 414 | else |
| 415 | n = NULL; |
Christian Heimes | b1b3efc | 2008-03-26 23:24:27 +0000 | [diff] [blame] | 416 | |
Michael J. Sullivan | 933e150 | 2019-05-22 07:54:20 -0700 | [diff] [blame] | 417 | growable_comment_array_deallocate(&type_ignores); |
Pablo Galindo | b9d2e97 | 2019-02-13 00:45:53 +0000 | [diff] [blame] | 418 | |
Christian Heimes | 4d6ec85 | 2008-03-26 22:34:47 +0000 | [diff] [blame] | 419 | #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 420 | *flags = ps->p_flags; |
Christian Heimes | 4d6ec85 | 2008-03-26 22:34:47 +0000 | [diff] [blame] | 421 | #endif |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 422 | PyParser_Delete(ps); |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 423 | |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 424 | if (n == NULL) { |
Benjamin Peterson | 758888d | 2011-05-30 11:12:38 -0500 | [diff] [blame] | 425 | if (tok->done == E_EOF) |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 426 | err_ret->error = E_EOF; |
| 427 | err_ret->lineno = tok->lineno; |
| 428 | if (tok->buf != NULL) { |
| 429 | size_t len; |
| 430 | assert(tok->cur - tok->buf < INT_MAX); |
Ammar Askar | 025eb98 | 2018-09-24 17:12:49 -0400 | [diff] [blame] | 431 | /* if we've managed to parse a token, point the offset to its start, |
| 432 | * else use the current reading position of the tokenizer |
| 433 | */ |
| 434 | err_ret->offset = col_offset != -1 ? col_offset + 1 : ((int)(tok->cur - tok->buf)); |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 435 | len = tok->inp - tok->buf; |
| 436 | err_ret->text = (char *) PyObject_MALLOC(len + 1); |
| 437 | if (err_ret->text != NULL) { |
| 438 | if (len > 0) |
| 439 | strncpy(err_ret->text, tok->buf, len); |
| 440 | err_ret->text[len] = '\0'; |
| 441 | } |
| 442 | } |
| 443 | } else if (tok->encoding != NULL) { |
| 444 | /* 'nodes->n_str' uses PyObject_*, while 'tok->encoding' was |
| 445 | * allocated using PyMem_ |
| 446 | */ |
| 447 | node* r = PyNode_New(encoding_decl); |
| 448 | if (r) |
| 449 | r->n_str = PyObject_MALLOC(strlen(tok->encoding)+1); |
| 450 | if (!r || !r->n_str) { |
| 451 | err_ret->error = E_NOMEM; |
| 452 | if (r) |
| 453 | PyObject_FREE(r); |
| 454 | n = NULL; |
| 455 | goto done; |
| 456 | } |
| 457 | strcpy(r->n_str, tok->encoding); |
| 458 | PyMem_FREE(tok->encoding); |
| 459 | tok->encoding = NULL; |
| 460 | r->n_nchildren = 1; |
| 461 | r->n_child = n; |
| 462 | n = r; |
| 463 | } |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 464 | |
Thomas Wouters | 00ee7ba | 2006-08-21 19:07:27 +0000 | [diff] [blame] | 465 | done: |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 466 | PyTokenizer_Free(tok); |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 467 | |
Ivan Levkivskyi | 9932a22 | 2019-01-22 11:18:22 +0000 | [diff] [blame] | 468 | if (n != NULL) { |
| 469 | _PyNode_FinalizeEndPos(n); |
| 470 | } |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 471 | return n; |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 472 | } |
Guido van Rossum | 0c156a5 | 2001-10-20 14:27:56 +0000 | [diff] [blame] | 473 | |
Victor Stinner | 7f2fee3 | 2011-04-05 00:39:01 +0200 | [diff] [blame] | 474 | static int |
Victor Stinner | 14e461d | 2013-08-26 22:28:21 +0200 | [diff] [blame] | 475 | initerr(perrdetail *err_ret, PyObject *filename) |
Guido van Rossum | 0c156a5 | 2001-10-20 14:27:56 +0000 | [diff] [blame] | 476 | { |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 477 | err_ret->error = E_OK; |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 478 | err_ret->lineno = 0; |
| 479 | err_ret->offset = 0; |
| 480 | err_ret->text = NULL; |
| 481 | err_ret->token = -1; |
| 482 | err_ret->expected = -1; |
Victor Stinner | 14e461d | 2013-08-26 22:28:21 +0200 | [diff] [blame] | 483 | if (filename) { |
| 484 | Py_INCREF(filename); |
| 485 | err_ret->filename = filename; |
| 486 | } |
| 487 | else { |
Victor Stinner | 7f2fee3 | 2011-04-05 00:39:01 +0200 | [diff] [blame] | 488 | err_ret->filename = PyUnicode_FromString("<string>"); |
Victor Stinner | 14e461d | 2013-08-26 22:28:21 +0200 | [diff] [blame] | 489 | if (err_ret->filename == NULL) { |
| 490 | err_ret->error = E_ERROR; |
| 491 | return -1; |
| 492 | } |
Victor Stinner | 7f2fee3 | 2011-04-05 00:39:01 +0200 | [diff] [blame] | 493 | } |
Victor Stinner | 7f2fee3 | 2011-04-05 00:39:01 +0200 | [diff] [blame] | 494 | return 0; |
Guido van Rossum | 0c156a5 | 2001-10-20 14:27:56 +0000 | [diff] [blame] | 495 | } |