blob: ff4ca70922f4092e4f24b64c0b2c8b97c2083797 [file] [log] [blame]
Guido van Rossumf70e43a1991-02-19 12:39:46 +00001
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00002/* Parser-tokenizer link implementation */
3
Guido van Rossum3f5da241990-12-20 15:06:42 +00004#include "pgenheaders.h"
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00005#include "tokenizer.h"
6#include "node.h"
7#include "grammar.h"
8#include "parser.h"
Guido van Rossum3f5da241990-12-20 15:06:42 +00009#include "parsetok.h"
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000010#include "errcode.h"
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +000011#include "graminit.h"
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000012
Guido van Rossum3f5da241990-12-20 15:06:42 +000013
14/* Forward */
Christian Heimes4d6ec852008-03-26 22:34:47 +000015static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int *);
Martin v. Löwis95292d62002-12-11 14:04:59 +000016static void initerr(perrdetail *err_ret, const char* filename);
Guido van Rossum3f5da241990-12-20 15:06:42 +000017
18/* Parse input coming from a string. Return error code, print some errors. */
Guido van Rossumbd0389d1994-08-29 12:25:45 +000019node *
Martin v. Löwis95292d62002-12-11 14:04:59 +000020PyParser_ParseString(const char *s, grammar *g, int start, perrdetail *err_ret)
Guido van Rossum3f5da241990-12-20 15:06:42 +000021{
Jeremy Hylton3e0055f2005-10-20 19:59:25 +000022 return PyParser_ParseStringFlagsFilename(s, NULL, g, start, err_ret, 0);
Tim Petersfe2127d2001-07-16 05:37:24 +000023}
24
25node *
Martin v. Löwis95292d62002-12-11 14:04:59 +000026PyParser_ParseStringFlags(const char *s, grammar *g, int start,
Tim Petersfe2127d2001-07-16 05:37:24 +000027 perrdetail *err_ret, int flags)
28{
Thomas Heller6b17abf2002-07-09 09:23:27 +000029 return PyParser_ParseStringFlagsFilename(s, NULL,
Neal Norwitz4ded4b52004-07-08 01:54:07 +000030 g, start, err_ret, flags);
Thomas Heller6b17abf2002-07-09 09:23:27 +000031}
32
33node *
Martin v. Löwis95292d62002-12-11 14:04:59 +000034PyParser_ParseStringFlagsFilename(const char *s, const char *filename,
Thomas Heller6b17abf2002-07-09 09:23:27 +000035 grammar *g, int start,
36 perrdetail *err_ret, int flags)
37{
Christian Heimes4d6ec852008-03-26 22:34:47 +000038 int iflags = flags;
39 return PyParser_ParseStringFlagsFilenameEx(s, filename, g, start,
40 err_ret, &iflags);
41}
42
43node *
44PyParser_ParseStringFlagsFilenameEx(const char *s, const char *filename,
45 grammar *g, int start,
46 perrdetail *err_ret, int *flags)
47{
Guido van Rossumbd0389d1994-08-29 12:25:45 +000048 struct tok_state *tok;
Benjamin Petersonaeaa5922009-11-13 00:17:59 +000049 int exec_input = start == file_input;
Guido van Rossumbd0389d1994-08-29 12:25:45 +000050
Thomas Heller6b17abf2002-07-09 09:23:27 +000051 initerr(err_ret, filename);
Guido van Rossumbd0389d1994-08-29 12:25:45 +000052
Benjamin Petersonf5b52242009-03-02 23:31:26 +000053 if (*flags & PyPARSE_IGNORE_COOKIE)
Benjamin Petersonaeaa5922009-11-13 00:17:59 +000054 tok = PyTokenizer_FromUTF8(s, exec_input);
Benjamin Petersonf5b52242009-03-02 23:31:26 +000055 else
Benjamin Petersonaeaa5922009-11-13 00:17:59 +000056 tok = PyTokenizer_FromString(s, exec_input);
Benjamin Petersonf5b52242009-03-02 23:31:26 +000057 if (tok == NULL) {
Neal Norwitz40d37812005-10-02 01:48:49 +000058 err_ret->error = PyErr_Occurred() ? E_DECODE : E_NOMEM;
Guido van Rossumbd0389d1994-08-29 12:25:45 +000059 return NULL;
Guido van Rossum3f5da241990-12-20 15:06:42 +000060 }
Guido van Rossumbd0389d1994-08-29 12:25:45 +000061
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +000062 tok->filename = filename ? filename : "<string>";
Tim Petersfe2127d2001-07-16 05:37:24 +000063 return parsetok(tok, g, start, err_ret, flags);
Guido van Rossum3f5da241990-12-20 15:06:42 +000064}
65
Guido van Rossum3f5da241990-12-20 15:06:42 +000066/* Parse input coming from a file. Return error code, print some errors. */
67
Guido van Rossumbd0389d1994-08-29 12:25:45 +000068node *
Martin v. Löwis95292d62002-12-11 14:04:59 +000069PyParser_ParseFile(FILE *fp, const char *filename, grammar *g, int start,
Thomas Wouters23c9e002000-07-22 19:20:54 +000070 char *ps1, char *ps2, perrdetail *err_ret)
Guido van Rossum3f5da241990-12-20 15:06:42 +000071{
Martin v. Löwis85bcc662007-09-04 09:18:06 +000072 return PyParser_ParseFileFlags(fp, filename, NULL,
73 g, start, ps1, ps2, err_ret, 0);
Tim Petersfe2127d2001-07-16 05:37:24 +000074}
75
76node *
Christian Heimes4d6ec852008-03-26 22:34:47 +000077PyParser_ParseFileFlags(FILE *fp, const char *filename, const char *enc,
Martin v. Löwis85bcc662007-09-04 09:18:06 +000078 grammar *g, int start,
Tim Petersfe2127d2001-07-16 05:37:24 +000079 char *ps1, char *ps2, perrdetail *err_ret, int flags)
80{
Christian Heimes4d6ec852008-03-26 22:34:47 +000081 int iflags = flags;
82 return PyParser_ParseFileFlagsEx(fp, filename, enc, g, start, ps1,
83 ps2, err_ret, &iflags);
84}
85
86node *
87PyParser_ParseFileFlagsEx(FILE *fp, const char *filename,
88 const char *enc, grammar *g, int start,
89 char *ps1, char *ps2, perrdetail *err_ret, int *flags)
90{
Guido van Rossumbd0389d1994-08-29 12:25:45 +000091 struct tok_state *tok;
92
Martin v. Löwis16eff6f2002-01-05 21:40:08 +000093 initerr(err_ret, filename);
Guido van Rossumbd0389d1994-08-29 12:25:45 +000094
Brett Cannonfdc1a562007-09-05 20:35:46 +000095 if ((tok = PyTokenizer_FromFile(fp, (char *)enc, ps1, ps2)) == NULL) {
Guido van Rossumbd0389d1994-08-29 12:25:45 +000096 err_ret->error = E_NOMEM;
97 return NULL;
Guido van Rossum3f5da241990-12-20 15:06:42 +000098 }
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +000099 tok->filename = filename;
Tim Petersfe2127d2001-07-16 05:37:24 +0000100 return parsetok(tok, g, start, err_ret, flags);
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000101}
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000102
Neal Norwitze4993c72006-03-16 06:01:25 +0000103#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
Brett Cannone3944a52009-04-01 05:08:41 +0000104#if 0
Thomas Wouters34aa7ba2006-02-28 19:02:24 +0000105static char with_msg[] =
106"%s:%d: Warning: 'with' will become a reserved keyword in Python 2.6\n";
107
108static char as_msg[] =
109"%s:%d: Warning: 'as' will become a reserved keyword in Python 2.6\n";
110
111static void
112warn(const char *msg, const char *filename, int lineno)
113{
114 if (filename == NULL)
115 filename = "<string>";
116 PySys_WriteStderr(msg, filename, lineno);
117}
Neal Norwitzfc85c922006-03-17 05:44:46 +0000118#endif
Brett Cannone3944a52009-04-01 05:08:41 +0000119#endif
Guido van Rossumda62ecc2001-07-17 16:53:11 +0000120
Thomas Wouters89f507f2006-12-13 04:49:30 +0000121/* Parse input coming from the given tokenizer structure.
122 Return error code. */
123
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000124static node *
Tim Petersfe2127d2001-07-16 05:37:24 +0000125parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
Christian Heimes4d6ec852008-03-26 22:34:47 +0000126 int *flags)
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000127{
128 parser_state *ps;
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000129 node *n;
Thomas Wouters34aa7ba2006-02-28 19:02:24 +0000130 int started = 0, handling_import = 0, handling_with = 0;
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000131
Guido van Rossum86bea461997-04-29 21:03:06 +0000132 if ((ps = PyParser_New(g, start)) == NULL) {
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000133 fprintf(stderr, "no mem for new parser\n");
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000134 err_ret->error = E_NOMEM;
Neal Norwitz56c65612005-11-15 04:54:23 +0000135 PyTokenizer_Free(tok);
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000136 return NULL;
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000137 }
Thomas Wouters34aa7ba2006-02-28 19:02:24 +0000138#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
Brett Cannone3944a52009-04-01 05:08:41 +0000139 if (*flags & PyPARSE_BARRY_AS_BDFL)
140 ps->p_flags |= CO_FUTURE_BARRY_AS_BDFL;
Neil Schemenauerc24ea082002-03-22 23:53:36 +0000141#endif
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000142
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000143 for (;;) {
144 char *a, *b;
145 int type;
Guido van Rossum6da34342000-06-28 22:00:02 +0000146 size_t len;
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000147 char *str;
Martin v. Löwis49c5da12006-03-01 22:49:05 +0000148 int col_offset;
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000149
Guido van Rossum86bea461997-04-29 21:03:06 +0000150 type = PyTokenizer_Get(tok, &a, &b);
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000151 if (type == ERRORTOKEN) {
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000152 err_ret->error = tok->done;
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000153 break;
154 }
Guido van Rossumd8b1d371992-03-04 16:40:44 +0000155 if (type == ENDMARKER && started) {
156 type = NEWLINE; /* Add an extra newline */
Thomas Wouters34aa7ba2006-02-28 19:02:24 +0000157 handling_with = handling_import = 0;
Guido van Rossumd8b1d371992-03-04 16:40:44 +0000158 started = 0;
Guido van Rossum4b499dd32003-02-13 22:07:59 +0000159 /* Add the right number of dedent tokens,
160 except if a certain flag is given --
161 codeop.py uses this. */
162 if (tok->indent &&
Christian Heimes4d6ec852008-03-26 22:34:47 +0000163 !(*flags & PyPARSE_DONT_IMPLY_DEDENT))
Guido van Rossum4b499dd32003-02-13 22:07:59 +0000164 {
165 tok->pendin = -tok->indent;
166 tok->indent = 0;
167 }
Guido van Rossumd8b1d371992-03-04 16:40:44 +0000168 }
169 else
170 started = 1;
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000171 len = b - a; /* XXX this may compute NULL - NULL */
Andrew MacIntyre80d4e2a2002-08-04 06:28:21 +0000172 str = (char *) PyObject_MALLOC(len + 1);
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000173 if (str == NULL) {
174 fprintf(stderr, "no mem for next token\n");
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000175 err_ret->error = E_NOMEM;
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000176 break;
177 }
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000178 if (len > 0)
179 strncpy(str, a, len);
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000180 str[len] = '\0';
Guido van Rossumda62ecc2001-07-17 16:53:11 +0000181
Thomas Wouters34aa7ba2006-02-28 19:02:24 +0000182#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
Brett Cannone3944a52009-04-01 05:08:41 +0000183 if (type == NOTEQUAL) {
184 if (!(ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) &&
185 strcmp(str, "!=")) {
186 err_ret->error = E_SYNTAX;
187 break;
188 }
189 else if ((ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) &&
190 strcmp(str, "<>")) {
191 err_ret->text = "with Barry as BDFL, use '<>' "
192 "instead of '!='";
193 err_ret->error = E_SYNTAX;
194 break;
195 }
Thomas Wouters34aa7ba2006-02-28 19:02:24 +0000196 }
Neil Schemenauerc24ea082002-03-22 23:53:36 +0000197#endif
Martin v. Löwis49c5da12006-03-01 22:49:05 +0000198 if (a >= tok->line_start)
199 col_offset = a - tok->line_start;
200 else
201 col_offset = -1;
202
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000203 if ((err_ret->error =
Guido van Rossum45aecf42006-03-15 04:58:47 +0000204 PyParser_AddToken(ps, (int)type, str,
205 tok->lineno, col_offset,
Fred Drake85f36392000-07-11 17:53:00 +0000206 &(err_ret->expected))) != E_OK) {
Thomas Wouters477c8d52006-05-27 19:21:47 +0000207 if (err_ret->error != E_DONE) {
Andrew MacIntyre80d4e2a2002-08-04 06:28:21 +0000208 PyObject_FREE(str);
Thomas Wouters477c8d52006-05-27 19:21:47 +0000209 err_ret->token = type;
210 }
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000211 break;
Guido van Rossumff0ec521997-07-27 01:52:50 +0000212 }
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000213 }
214
215 if (err_ret->error == E_DONE) {
216 n = ps->p_tree;
217 ps->p_tree = NULL;
218 }
219 else
220 n = NULL;
Christian Heimesb1b3efc2008-03-26 23:24:27 +0000221
Christian Heimes4d6ec852008-03-26 22:34:47 +0000222#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
223 *flags = ps->p_flags;
224#endif
Guido van Rossum86bea461997-04-29 21:03:06 +0000225 PyParser_Delete(ps);
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000226
227 if (n == NULL) {
228 if (tok->lineno <= 1 && tok->done == E_EOF)
229 err_ret->error = E_EOF;
230 err_ret->lineno = tok->lineno;
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000231 if (tok->buf != NULL) {
Martin v. Löwisa87c4452006-02-15 19:56:18 +0000232 size_t len;
Martin v. Löwis18e16552006-02-15 17:27:45 +0000233 assert(tok->cur - tok->buf < INT_MAX);
234 err_ret->offset = (int)(tok->cur - tok->buf);
Martin v. Löwisa87c4452006-02-15 19:56:18 +0000235 len = tok->inp - tok->buf;
Martin v. Löwis25931462008-03-17 20:43:42 +0000236 err_ret->text = (char *) PyObject_MALLOC(len + 1);
237 if (err_ret->text != NULL) {
238 if (len > 0)
239 strncpy(err_ret->text, tok->buf, len);
240 err_ret->text[len] = '\0';
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000241 }
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000242 }
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000243 } else if (tok->encoding != NULL) {
Kristján Valur Jónssonae4cfb12009-09-28 13:45:02 +0000244 /* 'nodes->n_str' uses PyObject_*, while 'tok->encoding' was
245 * allocated using PyMem_
246 */
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000247 node* r = PyNode_New(encoding_decl);
Kristján Valur Jónssonae4cfb12009-09-28 13:45:02 +0000248 if (r)
249 r->n_str = PyObject_MALLOC(strlen(tok->encoding)+1);
250 if (!r || !r->n_str) {
Thomas Wouters00ee7ba2006-08-21 19:07:27 +0000251 err_ret->error = E_NOMEM;
Kristján Valur Jónssonae4cfb12009-09-28 13:45:02 +0000252 if (r)
253 PyObject_FREE(r);
Thomas Wouters00ee7ba2006-08-21 19:07:27 +0000254 n = NULL;
255 goto done;
256 }
Kristján Valur Jónssonae4cfb12009-09-28 13:45:02 +0000257 strcpy(r->n_str, tok->encoding);
258 PyMem_FREE(tok->encoding);
259 tok->encoding = NULL;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000260 r->n_nchildren = 1;
261 r->n_child = n;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000262 n = r;
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000263 }
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000264
Thomas Wouters00ee7ba2006-08-21 19:07:27 +0000265done:
Guido van Rossum86bea461997-04-29 21:03:06 +0000266 PyTokenizer_Free(tok);
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000267
268 return n;
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000269}
Guido van Rossum0c156a52001-10-20 14:27:56 +0000270
271static void
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000272initerr(perrdetail *err_ret, const char *filename)
Guido van Rossum0c156a52001-10-20 14:27:56 +0000273{
274 err_ret->error = E_OK;
Martin v. Löwis16eff6f2002-01-05 21:40:08 +0000275 err_ret->filename = filename;
Guido van Rossum0c156a52001-10-20 14:27:56 +0000276 err_ret->lineno = 0;
277 err_ret->offset = 0;
278 err_ret->text = NULL;
279 err_ret->token = -1;
280 err_ret->expected = -1;
281}