blob: 7beb735d1565e8e1bc42d0cd9dfa92821412381e [file] [log] [blame]
Guido van Rossumf70e43a1991-02-19 12:39:46 +00001
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00002/* Parser-tokenizer link implementation */
3
Guido van Rossum3f5da241990-12-20 15:06:42 +00004#include "pgenheaders.h"
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00005#include "tokenizer.h"
6#include "node.h"
7#include "grammar.h"
8#include "parser.h"
Guido van Rossum3f5da241990-12-20 15:06:42 +00009#include "parsetok.h"
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000010#include "errcode.h"
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +000011#include "graminit.h"
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000012
Guido van Rossum3f5da241990-12-20 15:06:42 +000013
14/* Forward */
Christian Heimes4d6ec852008-03-26 22:34:47 +000015static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int *);
Victor Stinner7f2fee32011-04-05 00:39:01 +020016static int initerr(perrdetail *err_ret, const char* filename);
Guido van Rossum3f5da241990-12-20 15:06:42 +000017
18/* Parse input coming from a string. Return error code, print some errors. */
Guido van Rossumbd0389d1994-08-29 12:25:45 +000019node *
Martin v. Löwis95292d62002-12-11 14:04:59 +000020PyParser_ParseString(const char *s, grammar *g, int start, perrdetail *err_ret)
Guido van Rossum3f5da241990-12-20 15:06:42 +000021{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000022 return PyParser_ParseStringFlagsFilename(s, NULL, g, start, err_ret, 0);
Tim Petersfe2127d2001-07-16 05:37:24 +000023}
24
25node *
Martin v. Löwis95292d62002-12-11 14:04:59 +000026PyParser_ParseStringFlags(const char *s, grammar *g, int start,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000027 perrdetail *err_ret, int flags)
Tim Petersfe2127d2001-07-16 05:37:24 +000028{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000029 return PyParser_ParseStringFlagsFilename(s, NULL,
30 g, start, err_ret, flags);
Thomas Heller6b17abf2002-07-09 09:23:27 +000031}
32
33node *
Martin v. Löwis95292d62002-12-11 14:04:59 +000034PyParser_ParseStringFlagsFilename(const char *s, const char *filename,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000035 grammar *g, int start,
36 perrdetail *err_ret, int flags)
Thomas Heller6b17abf2002-07-09 09:23:27 +000037{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000038 int iflags = flags;
39 return PyParser_ParseStringFlagsFilenameEx(s, filename, g, start,
40 err_ret, &iflags);
Christian Heimes4d6ec852008-03-26 22:34:47 +000041}
42
43node *
44PyParser_ParseStringFlagsFilenameEx(const char *s, const char *filename,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000045 grammar *g, int start,
46 perrdetail *err_ret, int *flags)
Christian Heimes4d6ec852008-03-26 22:34:47 +000047{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000048 struct tok_state *tok;
49 int exec_input = start == file_input;
Guido van Rossumbd0389d1994-08-29 12:25:45 +000050
Victor Stinner7f2fee32011-04-05 00:39:01 +020051 if (initerr(err_ret, filename) < 0)
52 return NULL;
Guido van Rossumbd0389d1994-08-29 12:25:45 +000053
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000054 if (*flags & PyPARSE_IGNORE_COOKIE)
55 tok = PyTokenizer_FromUTF8(s, exec_input);
56 else
57 tok = PyTokenizer_FromString(s, exec_input);
58 if (tok == NULL) {
59 err_ret->error = PyErr_Occurred() ? E_DECODE : E_NOMEM;
60 return NULL;
61 }
Guido van Rossumbd0389d1994-08-29 12:25:45 +000062
Victor Stinner7f2fee32011-04-05 00:39:01 +020063#ifndef PGEN
64 Py_INCREF(err_ret->filename);
65 tok->filename = err_ret->filename;
66#endif
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000067 return parsetok(tok, g, start, err_ret, flags);
Guido van Rossum3f5da241990-12-20 15:06:42 +000068}
69
Guido van Rossum3f5da241990-12-20 15:06:42 +000070/* Parse input coming from a file. Return error code, print some errors. */
71
Guido van Rossumbd0389d1994-08-29 12:25:45 +000072node *
Martin v. Löwis95292d62002-12-11 14:04:59 +000073PyParser_ParseFile(FILE *fp, const char *filename, grammar *g, int start,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000074 char *ps1, char *ps2, perrdetail *err_ret)
Guido van Rossum3f5da241990-12-20 15:06:42 +000075{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000076 return PyParser_ParseFileFlags(fp, filename, NULL,
77 g, start, ps1, ps2, err_ret, 0);
Tim Petersfe2127d2001-07-16 05:37:24 +000078}
79
80node *
Christian Heimes4d6ec852008-03-26 22:34:47 +000081PyParser_ParseFileFlags(FILE *fp, const char *filename, const char *enc,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000082 grammar *g, int start,
83 char *ps1, char *ps2, perrdetail *err_ret, int flags)
Tim Petersfe2127d2001-07-16 05:37:24 +000084{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000085 int iflags = flags;
86 return PyParser_ParseFileFlagsEx(fp, filename, enc, g, start, ps1,
87 ps2, err_ret, &iflags);
Christian Heimes4d6ec852008-03-26 22:34:47 +000088}
89
90node *
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000091PyParser_ParseFileFlagsEx(FILE *fp, const char *filename,
92 const char *enc, grammar *g, int start,
93 char *ps1, char *ps2, perrdetail *err_ret, int *flags)
Christian Heimes4d6ec852008-03-26 22:34:47 +000094{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000095 struct tok_state *tok;
Guido van Rossumbd0389d1994-08-29 12:25:45 +000096
Victor Stinner7f2fee32011-04-05 00:39:01 +020097 if (initerr(err_ret, filename) < 0)
98 return NULL;
Guido van Rossumbd0389d1994-08-29 12:25:45 +000099
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000100 if ((tok = PyTokenizer_FromFile(fp, (char *)enc, ps1, ps2)) == NULL) {
101 err_ret->error = E_NOMEM;
102 return NULL;
103 }
Victor Stinner7f2fee32011-04-05 00:39:01 +0200104#ifndef PGEN
105 Py_INCREF(err_ret->filename);
106 tok->filename = err_ret->filename;
107#endif
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000108 return parsetok(tok, g, start, err_ret, flags);
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000109}
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000110
Neal Norwitze4993c72006-03-16 06:01:25 +0000111#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
Brett Cannone3944a52009-04-01 05:08:41 +0000112#if 0
Thomas Wouters34aa7ba2006-02-28 19:02:24 +0000113static char with_msg[] =
114"%s:%d: Warning: 'with' will become a reserved keyword in Python 2.6\n";
115
116static char as_msg[] =
117"%s:%d: Warning: 'as' will become a reserved keyword in Python 2.6\n";
118
119static void
120warn(const char *msg, const char *filename, int lineno)
121{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000122 if (filename == NULL)
123 filename = "<string>";
124 PySys_WriteStderr(msg, filename, lineno);
Thomas Wouters34aa7ba2006-02-28 19:02:24 +0000125}
Neal Norwitzfc85c922006-03-17 05:44:46 +0000126#endif
Brett Cannone3944a52009-04-01 05:08:41 +0000127#endif
Guido van Rossumda62ecc2001-07-17 16:53:11 +0000128
Thomas Wouters89f507f2006-12-13 04:49:30 +0000129/* Parse input coming from the given tokenizer structure.
130 Return error code. */
131
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000132static node *
Tim Petersfe2127d2001-07-16 05:37:24 +0000133parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000134 int *flags)
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000135{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000136 parser_state *ps;
137 node *n;
Brett Cannonb94767f2011-02-22 20:15:44 +0000138 int started = 0;
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000139
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000140 if ((ps = PyParser_New(g, start)) == NULL) {
141 fprintf(stderr, "no mem for new parser\n");
142 err_ret->error = E_NOMEM;
143 PyTokenizer_Free(tok);
144 return NULL;
145 }
Thomas Wouters34aa7ba2006-02-28 19:02:24 +0000146#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000147 if (*flags & PyPARSE_BARRY_AS_BDFL)
148 ps->p_flags |= CO_FUTURE_BARRY_AS_BDFL;
Neil Schemenauerc24ea082002-03-22 23:53:36 +0000149#endif
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000150
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000151 for (;;) {
152 char *a, *b;
153 int type;
154 size_t len;
155 char *str;
156 int col_offset;
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000157
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000158 type = PyTokenizer_Get(tok, &a, &b);
159 if (type == ERRORTOKEN) {
160 err_ret->error = tok->done;
161 break;
162 }
163 if (type == ENDMARKER && started) {
164 type = NEWLINE; /* Add an extra newline */
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000165 started = 0;
166 /* Add the right number of dedent tokens,
167 except if a certain flag is given --
168 codeop.py uses this. */
169 if (tok->indent &&
170 !(*flags & PyPARSE_DONT_IMPLY_DEDENT))
171 {
172 tok->pendin = -tok->indent;
173 tok->indent = 0;
174 }
175 }
176 else
177 started = 1;
178 len = b - a; /* XXX this may compute NULL - NULL */
179 str = (char *) PyObject_MALLOC(len + 1);
180 if (str == NULL) {
181 fprintf(stderr, "no mem for next token\n");
182 err_ret->error = E_NOMEM;
183 break;
184 }
185 if (len > 0)
186 strncpy(str, a, len);
187 str[len] = '\0';
Guido van Rossumda62ecc2001-07-17 16:53:11 +0000188
Thomas Wouters34aa7ba2006-02-28 19:02:24 +0000189#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000190 if (type == NOTEQUAL) {
191 if (!(ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) &&
192 strcmp(str, "!=")) {
Antoine Pitrou9ec25932011-11-13 01:01:23 +0100193 PyObject_FREE(str);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000194 err_ret->error = E_SYNTAX;
195 break;
196 }
197 else if ((ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) &&
198 strcmp(str, "<>")) {
Antoine Pitrou9ec25932011-11-13 01:01:23 +0100199 PyObject_FREE(str);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000200 err_ret->text = "with Barry as BDFL, use '<>' "
201 "instead of '!='";
202 err_ret->error = E_SYNTAX;
203 break;
204 }
205 }
Neil Schemenauerc24ea082002-03-22 23:53:36 +0000206#endif
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000207 if (a >= tok->line_start)
208 col_offset = a - tok->line_start;
209 else
210 col_offset = -1;
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000211
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000212 if ((err_ret->error =
213 PyParser_AddToken(ps, (int)type, str,
214 tok->lineno, col_offset,
215 &(err_ret->expected))) != E_OK) {
216 if (err_ret->error != E_DONE) {
217 PyObject_FREE(str);
218 err_ret->token = type;
219 }
220 break;
221 }
222 }
223
224 if (err_ret->error == E_DONE) {
225 n = ps->p_tree;
226 ps->p_tree = NULL;
Meador Ingefa21bf02012-01-19 01:08:41 -0600227
Benjamin Peterson79c1f962012-01-19 08:48:11 -0500228#ifndef PGEN
Meador Ingefa21bf02012-01-19 01:08:41 -0600229 /* Check that the source for a single input statement really
230 is a single statement by looking at what is left in the
231 buffer after parsing. Trailing whitespace and comments
232 are OK. */
233 if (start == single_input) {
234 char *cur = tok->cur;
235 char c = *tok->cur;
236
Benjamin Petersoncff92372012-01-19 17:46:13 -0500237 for (;;) {
238 while (c == ' ' || c == '\t' || c == '\n' || c == '\014')
239 c = *++cur;
Meador Ingefa21bf02012-01-19 01:08:41 -0600240
Benjamin Petersoncff92372012-01-19 17:46:13 -0500241 if (!c)
242 break;
243
244 if (c != '#') {
245 err_ret->error = E_BADSINGLE;
246 PyNode_Free(n);
247 n = NULL;
248 break;
249 }
250
251 /* Suck up comment. */
252 while (c && c != '\n')
253 c = *++cur;
Meador Ingefa21bf02012-01-19 01:08:41 -0600254 }
255 }
Benjamin Peterson79c1f962012-01-19 08:48:11 -0500256#endif
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000257 }
258 else
259 n = NULL;
Christian Heimesb1b3efc2008-03-26 23:24:27 +0000260
Christian Heimes4d6ec852008-03-26 22:34:47 +0000261#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000262 *flags = ps->p_flags;
Christian Heimes4d6ec852008-03-26 22:34:47 +0000263#endif
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000264 PyParser_Delete(ps);
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000265
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000266 if (n == NULL) {
Benjamin Peterson758888d2011-05-30 11:12:38 -0500267 if (tok->done == E_EOF)
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000268 err_ret->error = E_EOF;
269 err_ret->lineno = tok->lineno;
270 if (tok->buf != NULL) {
271 size_t len;
272 assert(tok->cur - tok->buf < INT_MAX);
273 err_ret->offset = (int)(tok->cur - tok->buf);
274 len = tok->inp - tok->buf;
275 err_ret->text = (char *) PyObject_MALLOC(len + 1);
276 if (err_ret->text != NULL) {
277 if (len > 0)
278 strncpy(err_ret->text, tok->buf, len);
279 err_ret->text[len] = '\0';
280 }
281 }
282 } else if (tok->encoding != NULL) {
283 /* 'nodes->n_str' uses PyObject_*, while 'tok->encoding' was
284 * allocated using PyMem_
285 */
286 node* r = PyNode_New(encoding_decl);
287 if (r)
288 r->n_str = PyObject_MALLOC(strlen(tok->encoding)+1);
289 if (!r || !r->n_str) {
290 err_ret->error = E_NOMEM;
291 if (r)
292 PyObject_FREE(r);
293 n = NULL;
294 goto done;
295 }
296 strcpy(r->n_str, tok->encoding);
297 PyMem_FREE(tok->encoding);
298 tok->encoding = NULL;
299 r->n_nchildren = 1;
300 r->n_child = n;
301 n = r;
302 }
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000303
Thomas Wouters00ee7ba2006-08-21 19:07:27 +0000304done:
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000305 PyTokenizer_Free(tok);
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000306
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000307 return n;
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000308}
Guido van Rossum0c156a52001-10-20 14:27:56 +0000309
Victor Stinner7f2fee32011-04-05 00:39:01 +0200310static int
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000311initerr(perrdetail *err_ret, const char *filename)
Guido van Rossum0c156a52001-10-20 14:27:56 +0000312{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000313 err_ret->error = E_OK;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000314 err_ret->lineno = 0;
315 err_ret->offset = 0;
316 err_ret->text = NULL;
317 err_ret->token = -1;
318 err_ret->expected = -1;
Victor Stinner7f2fee32011-04-05 00:39:01 +0200319#ifndef PGEN
320 if (filename)
321 err_ret->filename = PyUnicode_DecodeFSDefault(filename);
322 else
323 err_ret->filename = PyUnicode_FromString("<string>");
324 if (err_ret->filename == NULL) {
325 err_ret->error = E_ERROR;
326 return -1;
327 }
328#endif
329 return 0;
Guido van Rossum0c156a52001-10-20 14:27:56 +0000330}