blob: cd3887dff9cb5c8134391e80d5f36a3dc5fca883 [file] [log] [blame]
Guido van Rossumf70e43a1991-02-19 12:39:46 +00001
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00002/* Parser-tokenizer link implementation */
3
Guido van Rossum3f5da241990-12-20 15:06:42 +00004#include "pgenheaders.h"
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00005#include "tokenizer.h"
6#include "node.h"
7#include "grammar.h"
8#include "parser.h"
Guido van Rossum3f5da241990-12-20 15:06:42 +00009#include "parsetok.h"
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000010#include "errcode.h"
11
Guido van Rossum6135df61998-04-10 19:35:06 +000012int Py_TabcheckFlag;
13
Guido van Rossum3f5da241990-12-20 15:06:42 +000014
15/* Forward */
Tim Petersfe2127d2001-07-16 05:37:24 +000016static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int);
Martin v. Löwis16eff6f2002-01-05 21:40:08 +000017static void initerr(perrdetail *err_ret, char* filename);
Guido van Rossum3f5da241990-12-20 15:06:42 +000018
19/* Parse input coming from a string. Return error code, print some errors. */
Guido van Rossumbd0389d1994-08-29 12:25:45 +000020node *
Thomas Wouters23c9e002000-07-22 19:20:54 +000021PyParser_ParseString(char *s, grammar *g, int start, perrdetail *err_ret)
Guido van Rossum3f5da241990-12-20 15:06:42 +000022{
Tim Petersfe2127d2001-07-16 05:37:24 +000023 return PyParser_ParseStringFlags(s, g, start, err_ret, 0);
24}
25
26node *
27PyParser_ParseStringFlags(char *s, grammar *g, int start,
28 perrdetail *err_ret, int flags)
29{
Thomas Heller6b17abf2002-07-09 09:23:27 +000030 return PyParser_ParseStringFlagsFilename(s, NULL,
31 g, start, err_ret, 0);
32}
33
34node *
35PyParser_ParseStringFlagsFilename(char *s, char *filename,
36 grammar *g, int start,
37 perrdetail *err_ret, int flags)
38{
Guido van Rossumbd0389d1994-08-29 12:25:45 +000039 struct tok_state *tok;
40
Thomas Heller6b17abf2002-07-09 09:23:27 +000041 initerr(err_ret, filename);
Guido van Rossumbd0389d1994-08-29 12:25:45 +000042
Guido van Rossum86bea461997-04-29 21:03:06 +000043 if ((tok = PyTokenizer_FromString(s)) == NULL) {
Guido van Rossumbd0389d1994-08-29 12:25:45 +000044 err_ret->error = E_NOMEM;
45 return NULL;
Guido van Rossum3f5da241990-12-20 15:06:42 +000046 }
Guido van Rossumbd0389d1994-08-29 12:25:45 +000047
Guido van Rossum89ce4541998-12-21 18:32:40 +000048 if (Py_TabcheckFlag || Py_VerboseFlag) {
Thomas Heller6b17abf2002-07-09 09:23:27 +000049 tok->filename = filename ? filename : "<string>";
Guido van Rossum89ce4541998-12-21 18:32:40 +000050 tok->altwarning = (tok->filename != NULL);
51 if (Py_TabcheckFlag >= 2)
52 tok->alterror++;
53 }
54
Tim Petersfe2127d2001-07-16 05:37:24 +000055 return parsetok(tok, g, start, err_ret, flags);
Guido van Rossum3f5da241990-12-20 15:06:42 +000056}
57
58
59/* Parse input coming from a file. Return error code, print some errors. */
60
Guido van Rossumbd0389d1994-08-29 12:25:45 +000061node *
Thomas Wouters23c9e002000-07-22 19:20:54 +000062PyParser_ParseFile(FILE *fp, char *filename, grammar *g, int start,
63 char *ps1, char *ps2, perrdetail *err_ret)
Guido van Rossum3f5da241990-12-20 15:06:42 +000064{
Tim Petersfe2127d2001-07-16 05:37:24 +000065 return PyParser_ParseFileFlags(fp, filename, g, start, ps1, ps2,
66 err_ret, 0);
67}
68
69node *
70PyParser_ParseFileFlags(FILE *fp, char *filename, grammar *g, int start,
71 char *ps1, char *ps2, perrdetail *err_ret, int flags)
72{
Guido van Rossumbd0389d1994-08-29 12:25:45 +000073 struct tok_state *tok;
74
Martin v. Löwis16eff6f2002-01-05 21:40:08 +000075 initerr(err_ret, filename);
Guido van Rossumbd0389d1994-08-29 12:25:45 +000076
Guido van Rossum86bea461997-04-29 21:03:06 +000077 if ((tok = PyTokenizer_FromFile(fp, ps1, ps2)) == NULL) {
Guido van Rossumbd0389d1994-08-29 12:25:45 +000078 err_ret->error = E_NOMEM;
79 return NULL;
Guido van Rossum3f5da241990-12-20 15:06:42 +000080 }
Guido van Rossum6135df61998-04-10 19:35:06 +000081 if (Py_TabcheckFlag || Py_VerboseFlag) {
82 tok->filename = filename;
83 tok->altwarning = (filename != NULL);
84 if (Py_TabcheckFlag >= 2)
85 tok->alterror++;
86 }
Guido van Rossumbd0389d1994-08-29 12:25:45 +000087
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000088
Tim Petersfe2127d2001-07-16 05:37:24 +000089 return parsetok(tok, g, start, err_ret, flags);
Guido van Rossumbd0389d1994-08-29 12:25:45 +000090}
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000091
92/* Parse input coming from the given tokenizer structure.
93 Return error code. */
94
Neil Schemenauerc24ea082002-03-22 23:53:36 +000095#if 0 /* future keyword */
Guido van Rossumda62ecc2001-07-17 16:53:11 +000096static char yield_msg[] =
97"%s:%d: Warning: 'yield' will become a reserved keyword in the future\n";
Neil Schemenauerc24ea082002-03-22 23:53:36 +000098#endif
Guido van Rossumda62ecc2001-07-17 16:53:11 +000099
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000100static node *
Tim Petersfe2127d2001-07-16 05:37:24 +0000101parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
102 int flags)
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000103{
104 parser_state *ps;
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000105 node *n;
Guido van Rossumd8b1d371992-03-04 16:40:44 +0000106 int started = 0;
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000107
Guido van Rossum86bea461997-04-29 21:03:06 +0000108 if ((ps = PyParser_New(g, start)) == NULL) {
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000109 fprintf(stderr, "no mem for new parser\n");
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000110 err_ret->error = E_NOMEM;
111 return NULL;
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000112 }
Neil Schemenauerc24ea082002-03-22 23:53:36 +0000113#if 0 /* future keyword */
Tim Petersfe2127d2001-07-16 05:37:24 +0000114 if (flags & PyPARSE_YIELD_IS_KEYWORD)
115 ps->p_generators = 1;
Neil Schemenauerc24ea082002-03-22 23:53:36 +0000116#endif
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000117
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000118 for (;;) {
119 char *a, *b;
120 int type;
Guido van Rossum6da34342000-06-28 22:00:02 +0000121 size_t len;
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000122 char *str;
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000123
Guido van Rossum86bea461997-04-29 21:03:06 +0000124 type = PyTokenizer_Get(tok, &a, &b);
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000125 if (type == ERRORTOKEN) {
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000126 err_ret->error = tok->done;
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000127 break;
128 }
Guido van Rossumd8b1d371992-03-04 16:40:44 +0000129 if (type == ENDMARKER && started) {
130 type = NEWLINE; /* Add an extra newline */
131 started = 0;
132 }
133 else
134 started = 1;
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000135 len = b - a; /* XXX this may compute NULL - NULL */
Guido van Rossum86bea461997-04-29 21:03:06 +0000136 str = PyMem_NEW(char, len + 1);
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000137 if (str == NULL) {
138 fprintf(stderr, "no mem for next token\n");
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000139 err_ret->error = E_NOMEM;
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000140 break;
141 }
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000142 if (len > 0)
143 strncpy(str, a, len);
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000144 str[len] = '\0';
Guido van Rossumda62ecc2001-07-17 16:53:11 +0000145
Neil Schemenauerc24ea082002-03-22 23:53:36 +0000146#if 0 /* future keyword */
Guido van Rossumda62ecc2001-07-17 16:53:11 +0000147 /* Warn about yield as NAME */
148 if (type == NAME && !ps->p_generators &&
149 len == 5 && str[0] == 'y' && strcmp(str, "yield") == 0)
150 PySys_WriteStderr(yield_msg,
151 err_ret->filename==NULL ?
152 "<string>" : err_ret->filename,
153 tok->lineno);
Neil Schemenauerc24ea082002-03-22 23:53:36 +0000154#endif
Guido van Rossumda62ecc2001-07-17 16:53:11 +0000155
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000156 if ((err_ret->error =
Fred Drake85f36392000-07-11 17:53:00 +0000157 PyParser_AddToken(ps, (int)type, str, tok->lineno,
158 &(err_ret->expected))) != E_OK) {
Guido van Rossumff0ec521997-07-27 01:52:50 +0000159 if (err_ret->error != E_DONE)
160 PyMem_DEL(str);
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000161 break;
Guido van Rossumff0ec521997-07-27 01:52:50 +0000162 }
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000163 }
164
165 if (err_ret->error == E_DONE) {
166 n = ps->p_tree;
167 ps->p_tree = NULL;
168 }
169 else
170 n = NULL;
171
Guido van Rossum86bea461997-04-29 21:03:06 +0000172 PyParser_Delete(ps);
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000173
174 if (n == NULL) {
175 if (tok->lineno <= 1 && tok->done == E_EOF)
176 err_ret->error = E_EOF;
177 err_ret->lineno = tok->lineno;
178 err_ret->offset = tok->cur - tok->buf;
179 if (tok->buf != NULL) {
Guido van Rossum6da34342000-06-28 22:00:02 +0000180 size_t len = tok->inp - tok->buf;
Guido van Rossumb18618d2000-05-03 23:44:39 +0000181 err_ret->text = PyMem_NEW(char, len + 1);
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000182 if (err_ret->text != NULL) {
Guido van Rossumec498271995-01-20 16:59:12 +0000183 if (len > 0)
184 strncpy(err_ret->text, tok->buf, len);
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000185 err_ret->text[len] = '\0';
186 }
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000187 }
188 }
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000189
Guido van Rossum86bea461997-04-29 21:03:06 +0000190 PyTokenizer_Free(tok);
Guido van Rossumbd0389d1994-08-29 12:25:45 +0000191
192 return n;
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000193}
Guido van Rossum0c156a52001-10-20 14:27:56 +0000194
195static void
Martin v. Löwis16eff6f2002-01-05 21:40:08 +0000196initerr(perrdetail *err_ret, char* filename)
Guido van Rossum0c156a52001-10-20 14:27:56 +0000197{
198 err_ret->error = E_OK;
Martin v. Löwis16eff6f2002-01-05 21:40:08 +0000199 err_ret->filename = filename;
Guido van Rossum0c156a52001-10-20 14:27:56 +0000200 err_ret->lineno = 0;
201 err_ret->offset = 0;
202 err_ret->text = NULL;
203 err_ret->token = -1;
204 err_ret->expected = -1;
205}