blob: 0b3e957e0ca913f9f8dad3a7952dfaa65c7d0afe [file] [log] [blame]
Guido van Rossumf70e43a1991-02-19 12:39:46 +00001
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00002/* Parser-tokenizer link implementation */
3
Guido van Rossum3f5da241990-12-20 15:06:42 +00004#include "pgenheaders.h"
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00005#include "tokenizer.h"
6#include "node.h"
7#include "grammar.h"
8#include "parser.h"
Guido van Rossum3f5da241990-12-20 15:06:42 +00009#include "parsetok.h"
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000010#include "errcode.h"
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +000011#include "graminit.h"
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000012
Guido van Rossum6135df61998-04-10 19:35:06 +000013int Py_TabcheckFlag;
14
Guido van Rossum3f5da241990-12-20 15:06:42 +000015
16/* Forward */
Tim Petersfe2127d2001-07-16 05:37:24 +000017static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int);
Martin v. Löwis95292d62002-12-11 14:04:59 +000018static void initerr(perrdetail *err_ret, const char* filename);
Guido van Rossum3f5da241990-12-20 15:06:42 +000019
20/* Parse input coming from a string. Return error code, print some errors. */
Guido van Rossumbd0389d1994-08-29 12:25:45 +000021node *
Martin v. Löwis95292d62002-12-11 14:04:59 +000022PyParser_ParseString(const char *s, grammar *g, int start, perrdetail *err_ret)
Guido van Rossum3f5da241990-12-20 15:06:42 +000023{
Tim Petersfe2127d2001-07-16 05:37:24 +000024 return PyParser_ParseStringFlags(s, g, start, err_ret, 0);
25}
26
27node *
Martin v. Löwis95292d62002-12-11 14:04:59 +000028PyParser_ParseStringFlags(const char *s, grammar *g, int start,
Tim Petersfe2127d2001-07-16 05:37:24 +000029 perrdetail *err_ret, int flags)
30{
Thomas Heller6b17abf2002-07-09 09:23:27 +000031 return PyParser_ParseStringFlagsFilename(s, NULL,
32 g, start, err_ret, 0);
33}
34
35node *
Martin v. Löwis95292d62002-12-11 14:04:59 +000036PyParser_ParseStringFlagsFilename(const char *s, const char *filename,
Thomas Heller6b17abf2002-07-09 09:23:27 +000037 grammar *g, int start,
38 perrdetail *err_ret, int flags)
39{
Guido van Rossumbd0389d1994-08-29 12:25:45 +000040 struct tok_state *tok;
41
Thomas Heller6b17abf2002-07-09 09:23:27 +000042 initerr(err_ret, filename);
Guido van Rossumbd0389d1994-08-29 12:25:45 +000043
Guido van Rossum86bea461997-04-29 21:03:06 +000044 if ((tok = PyTokenizer_FromString(s)) == NULL) {
Guido van Rossumbd0389d1994-08-29 12:25:45 +000045 err_ret->error = E_NOMEM;
46 return NULL;
Guido van Rossum3f5da241990-12-20 15:06:42 +000047 }
Guido van Rossumbd0389d1994-08-29 12:25:45 +000048
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +000049 tok->filename = filename ? filename : "<string>";
Guido van Rossum89ce4541998-12-21 18:32:40 +000050 if (Py_TabcheckFlag || Py_VerboseFlag) {
Guido van Rossum89ce4541998-12-21 18:32:40 +000051 tok->altwarning = (tok->filename != NULL);
52 if (Py_TabcheckFlag >= 2)
53 tok->alterror++;
54 }
55
Tim Petersfe2127d2001-07-16 05:37:24 +000056 return parsetok(tok, g, start, err_ret, flags);
Guido van Rossum3f5da241990-12-20 15:06:42 +000057}
58
59
60/* Parse input coming from a file. Return error code, print some errors. */
61
Guido van Rossumbd0389d1994-08-29 12:25:45 +000062node *
Martin v. Löwis95292d62002-12-11 14:04:59 +000063PyParser_ParseFile(FILE *fp, const char *filename, grammar *g, int start,
Thomas Wouters23c9e002000-07-22 19:20:54 +000064 char *ps1, char *ps2, perrdetail *err_ret)
Guido van Rossum3f5da241990-12-20 15:06:42 +000065{
Tim Petersfe2127d2001-07-16 05:37:24 +000066 return PyParser_ParseFileFlags(fp, filename, g, start, ps1, ps2,
67 err_ret, 0);
68}
69
70node *
Martin v. Löwis95292d62002-12-11 14:04:59 +000071PyParser_ParseFileFlags(FILE *fp, const char *filename, grammar *g, int start,
Tim Petersfe2127d2001-07-16 05:37:24 +000072 char *ps1, char *ps2, perrdetail *err_ret, int flags)
73{
Guido van Rossumbd0389d1994-08-29 12:25:45 +000074 struct tok_state *tok;
75
Martin v. Löwis16eff6f2002-01-05 21:40:08 +000076 initerr(err_ret, filename);
Guido van Rossumbd0389d1994-08-29 12:25:45 +000077
Guido van Rossum86bea461997-04-29 21:03:06 +000078 if ((tok = PyTokenizer_FromFile(fp, ps1, ps2)) == NULL) {
Guido van Rossumbd0389d1994-08-29 12:25:45 +000079 err_ret->error = E_NOMEM;
80 return NULL;
Guido van Rossum3f5da241990-12-20 15:06:42 +000081 }
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +000082 tok->filename = filename;
Guido van Rossum6135df61998-04-10 19:35:06 +000083 if (Py_TabcheckFlag || Py_VerboseFlag) {
Guido van Rossum6135df61998-04-10 19:35:06 +000084 tok->altwarning = (filename != NULL);
85 if (Py_TabcheckFlag >= 2)
86 tok->alterror++;
87 }
Guido van Rossumbd0389d1994-08-29 12:25:45 +000088
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000089
Tim Petersfe2127d2001-07-16 05:37:24 +000090 return parsetok(tok, g, start, err_ret, flags);
Guido van Rossumbd0389d1994-08-29 12:25:45 +000091}
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000092
93/* Parse input coming from the given tokenizer structure.
94 Return error code. */
95
Neil Schemenauerc24ea082002-03-22 23:53:36 +000096#if 0 /* future keyword */
Guido van Rossumda62ecc2001-07-17 16:53:11 +000097static char yield_msg[] =
98"%s:%d: Warning: 'yield' will become a reserved keyword in the future\n";
Neil Schemenauerc24ea082002-03-22 23:53:36 +000099#endif
Guido van Rossumda62ecc2001-07-17 16:53:11 +0000100
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000101static node *
Tim Petersfe2127d2001-07-16 05:37:24 +0000102parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
103 int flags)
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000104{
105 parser_state *ps;
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000106 node *n;
Guido van Rossumd8b1d371992-03-04 16:40:44 +0000107 int started = 0;
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000108
Guido van Rossum86bea461997-04-29 21:03:06 +0000109 if ((ps = PyParser_New(g, start)) == NULL) {
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000110 fprintf(stderr, "no mem for new parser\n");
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000111 err_ret->error = E_NOMEM;
112 return NULL;
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000113 }
Neil Schemenauerc24ea082002-03-22 23:53:36 +0000114#if 0 /* future keyword */
Tim Petersfe2127d2001-07-16 05:37:24 +0000115 if (flags & PyPARSE_YIELD_IS_KEYWORD)
116 ps->p_generators = 1;
Neil Schemenauerc24ea082002-03-22 23:53:36 +0000117#endif
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000118
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000119 for (;;) {
120 char *a, *b;
121 int type;
Guido van Rossum6da34342000-06-28 22:00:02 +0000122 size_t len;
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000123 char *str;
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000124
Guido van Rossum86bea461997-04-29 21:03:06 +0000125 type = PyTokenizer_Get(tok, &a, &b);
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000126 if (type == ERRORTOKEN) {
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000127 err_ret->error = tok->done;
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000128 break;
129 }
Guido van Rossumd8b1d371992-03-04 16:40:44 +0000130 if (type == ENDMARKER && started) {
131 type = NEWLINE; /* Add an extra newline */
132 started = 0;
Guido van Rossum4b499dd32003-02-13 22:07:59 +0000133 /* Add the right number of dedent tokens,
134 except if a certain flag is given --
135 codeop.py uses this. */
136 if (tok->indent &&
137 !(flags & PyPARSE_DONT_IMPLY_DEDENT))
138 {
139 tok->pendin = -tok->indent;
140 tok->indent = 0;
141 }
Guido van Rossumd8b1d371992-03-04 16:40:44 +0000142 }
143 else
144 started = 1;
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000145 len = b - a; /* XXX this may compute NULL - NULL */
Andrew MacIntyre80d4e2a2002-08-04 06:28:21 +0000146 str = (char *) PyObject_MALLOC(len + 1);
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000147 if (str == NULL) {
148 fprintf(stderr, "no mem for next token\n");
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000149 err_ret->error = E_NOMEM;
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000150 break;
151 }
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000152 if (len > 0)
153 strncpy(str, a, len);
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000154 str[len] = '\0';
Guido van Rossumda62ecc2001-07-17 16:53:11 +0000155
Neil Schemenauerc24ea082002-03-22 23:53:36 +0000156#if 0 /* future keyword */
Guido van Rossumda62ecc2001-07-17 16:53:11 +0000157 /* Warn about yield as NAME */
158 if (type == NAME && !ps->p_generators &&
159 len == 5 && str[0] == 'y' && strcmp(str, "yield") == 0)
160 PySys_WriteStderr(yield_msg,
161 err_ret->filename==NULL ?
162 "<string>" : err_ret->filename,
163 tok->lineno);
Neil Schemenauerc24ea082002-03-22 23:53:36 +0000164#endif
Guido van Rossumda62ecc2001-07-17 16:53:11 +0000165
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000166 if ((err_ret->error =
Fred Drake85f36392000-07-11 17:53:00 +0000167 PyParser_AddToken(ps, (int)type, str, tok->lineno,
168 &(err_ret->expected))) != E_OK) {
Guido van Rossumff0ec521997-07-27 01:52:50 +0000169 if (err_ret->error != E_DONE)
Andrew MacIntyre80d4e2a2002-08-04 06:28:21 +0000170 PyObject_FREE(str);
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000171 break;
Guido van Rossumff0ec521997-07-27 01:52:50 +0000172 }
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000173 }
174
175 if (err_ret->error == E_DONE) {
176 n = ps->p_tree;
177 ps->p_tree = NULL;
178 }
179 else
180 n = NULL;
181
Guido van Rossum86bea461997-04-29 21:03:06 +0000182 PyParser_Delete(ps);
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000183
184 if (n == NULL) {
185 if (tok->lineno <= 1 && tok->done == E_EOF)
186 err_ret->error = E_EOF;
187 err_ret->lineno = tok->lineno;
188 err_ret->offset = tok->cur - tok->buf;
189 if (tok->buf != NULL) {
Guido van Rossum6da34342000-06-28 22:00:02 +0000190 size_t len = tok->inp - tok->buf;
Andrew MacIntyre80d4e2a2002-08-04 06:28:21 +0000191 err_ret->text = (char *) PyObject_MALLOC(len + 1);
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000192 if (err_ret->text != NULL) {
Guido van Rossumec498271995-01-20 16:59:12 +0000193 if (len > 0)
194 strncpy(err_ret->text, tok->buf, len);
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000195 err_ret->text[len] = '\0';
196 }
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000197 }
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000198 } else if (tok->encoding != NULL) {
199 node* r = PyNode_New(encoding_decl);
200 r->n_str = tok->encoding;
201 r->n_nchildren = 1;
202 r->n_child = n;
203 tok->encoding = NULL;
204 n = r;
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000205 }
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000206
Guido van Rossum86bea461997-04-29 21:03:06 +0000207 PyTokenizer_Free(tok);
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000208
209 return n;
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000210}
Guido van Rossum0c156a52001-10-20 14:27:56 +0000211
212static void
Martin v. Löwis95292d62002-12-11 14:04:59 +0000213initerr(perrdetail *err_ret, const char* filename)
Guido van Rossum0c156a52001-10-20 14:27:56 +0000214{
215 err_ret->error = E_OK;
Martin v. Löwis16eff6f2002-01-05 21:40:08 +0000216 err_ret->filename = filename;
Guido van Rossum0c156a52001-10-20 14:27:56 +0000217 err_ret->lineno = 0;
218 err_ret->offset = 0;
219 err_ret->text = NULL;
220 err_ret->token = -1;
221 err_ret->expected = -1;
222}