Guido van Rossum | f70e43a | 1991-02-19 12:39:46 +0000 | [diff] [blame] | 1 | |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 2 | /* Parser-tokenizer link implementation */ |
| 3 | |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 4 | #include "pgenheaders.h" |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 5 | #include "tokenizer.h" |
| 6 | #include "node.h" |
| 7 | #include "grammar.h" |
| 8 | #include "parser.h" |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 9 | #include "parsetok.h" |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 10 | #include "errcode.h" |
| 11 | |
Guido van Rossum | 6135df6 | 1998-04-10 19:35:06 +0000 | [diff] [blame] | 12 | int Py_TabcheckFlag; |
| 13 | |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 14 | |
| 15 | /* Forward */ |
Tim Peters | fe2127d | 2001-07-16 05:37:24 +0000 | [diff] [blame] | 16 | static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int); |
Martin v. Löwis | 16eff6f | 2002-01-05 21:40:08 +0000 | [diff] [blame] | 17 | static void initerr(perrdetail *err_ret, char* filename); |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 18 | |
| 19 | /* Parse input coming from a string. Return error code, print some errors. */ |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 20 | node * |
Thomas Wouters | 23c9e00 | 2000-07-22 19:20:54 +0000 | [diff] [blame] | 21 | PyParser_ParseString(char *s, grammar *g, int start, perrdetail *err_ret) |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 22 | { |
Tim Peters | fe2127d | 2001-07-16 05:37:24 +0000 | [diff] [blame] | 23 | return PyParser_ParseStringFlags(s, g, start, err_ret, 0); |
| 24 | } |
| 25 | |
| 26 | node * |
| 27 | PyParser_ParseStringFlags(char *s, grammar *g, int start, |
| 28 | perrdetail *err_ret, int flags) |
| 29 | { |
Thomas Heller | 6b17abf | 2002-07-09 09:23:27 +0000 | [diff] [blame] | 30 | return PyParser_ParseStringFlagsFilename(s, NULL, |
| 31 | g, start, err_ret, 0); |
| 32 | } |
| 33 | |
| 34 | node * |
| 35 | PyParser_ParseStringFlagsFilename(char *s, char *filename, |
| 36 | grammar *g, int start, |
| 37 | perrdetail *err_ret, int flags) |
| 38 | { |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 39 | struct tok_state *tok; |
| 40 | |
Thomas Heller | 6b17abf | 2002-07-09 09:23:27 +0000 | [diff] [blame] | 41 | initerr(err_ret, filename); |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 42 | |
Guido van Rossum | 86bea46 | 1997-04-29 21:03:06 +0000 | [diff] [blame] | 43 | if ((tok = PyTokenizer_FromString(s)) == NULL) { |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 44 | err_ret->error = E_NOMEM; |
| 45 | return NULL; |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 46 | } |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 47 | |
Guido van Rossum | 89ce454 | 1998-12-21 18:32:40 +0000 | [diff] [blame] | 48 | if (Py_TabcheckFlag || Py_VerboseFlag) { |
Thomas Heller | 6b17abf | 2002-07-09 09:23:27 +0000 | [diff] [blame] | 49 | tok->filename = filename ? filename : "<string>"; |
Guido van Rossum | 89ce454 | 1998-12-21 18:32:40 +0000 | [diff] [blame] | 50 | tok->altwarning = (tok->filename != NULL); |
| 51 | if (Py_TabcheckFlag >= 2) |
| 52 | tok->alterror++; |
| 53 | } |
| 54 | |
Tim Peters | fe2127d | 2001-07-16 05:37:24 +0000 | [diff] [blame] | 55 | return parsetok(tok, g, start, err_ret, flags); |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 56 | } |
| 57 | |
| 58 | |
| 59 | /* Parse input coming from a file. Return error code, print some errors. */ |
| 60 | |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 61 | node * |
Thomas Wouters | 23c9e00 | 2000-07-22 19:20:54 +0000 | [diff] [blame] | 62 | PyParser_ParseFile(FILE *fp, char *filename, grammar *g, int start, |
| 63 | char *ps1, char *ps2, perrdetail *err_ret) |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 64 | { |
Tim Peters | fe2127d | 2001-07-16 05:37:24 +0000 | [diff] [blame] | 65 | return PyParser_ParseFileFlags(fp, filename, g, start, ps1, ps2, |
| 66 | err_ret, 0); |
| 67 | } |
| 68 | |
| 69 | node * |
| 70 | PyParser_ParseFileFlags(FILE *fp, char *filename, grammar *g, int start, |
| 71 | char *ps1, char *ps2, perrdetail *err_ret, int flags) |
| 72 | { |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 73 | struct tok_state *tok; |
| 74 | |
Martin v. Löwis | 16eff6f | 2002-01-05 21:40:08 +0000 | [diff] [blame] | 75 | initerr(err_ret, filename); |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 76 | |
Guido van Rossum | 86bea46 | 1997-04-29 21:03:06 +0000 | [diff] [blame] | 77 | if ((tok = PyTokenizer_FromFile(fp, ps1, ps2)) == NULL) { |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 78 | err_ret->error = E_NOMEM; |
| 79 | return NULL; |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 80 | } |
Guido van Rossum | 6135df6 | 1998-04-10 19:35:06 +0000 | [diff] [blame] | 81 | if (Py_TabcheckFlag || Py_VerboseFlag) { |
| 82 | tok->filename = filename; |
| 83 | tok->altwarning = (filename != NULL); |
| 84 | if (Py_TabcheckFlag >= 2) |
| 85 | tok->alterror++; |
| 86 | } |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 87 | |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 88 | |
Tim Peters | fe2127d | 2001-07-16 05:37:24 +0000 | [diff] [blame] | 89 | return parsetok(tok, g, start, err_ret, flags); |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 90 | } |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 91 | |
| 92 | /* Parse input coming from the given tokenizer structure. |
| 93 | Return error code. */ |
| 94 | |
Neil Schemenauer | c24ea08 | 2002-03-22 23:53:36 +0000 | [diff] [blame] | 95 | #if 0 /* future keyword */ |
Guido van Rossum | da62ecc | 2001-07-17 16:53:11 +0000 | [diff] [blame] | 96 | static char yield_msg[] = |
| 97 | "%s:%d: Warning: 'yield' will become a reserved keyword in the future\n"; |
Neil Schemenauer | c24ea08 | 2002-03-22 23:53:36 +0000 | [diff] [blame] | 98 | #endif |
Guido van Rossum | da62ecc | 2001-07-17 16:53:11 +0000 | [diff] [blame] | 99 | |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 100 | static node * |
Tim Peters | fe2127d | 2001-07-16 05:37:24 +0000 | [diff] [blame] | 101 | parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, |
| 102 | int flags) |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 103 | { |
| 104 | parser_state *ps; |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 105 | node *n; |
Guido van Rossum | d8b1d37 | 1992-03-04 16:40:44 +0000 | [diff] [blame] | 106 | int started = 0; |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 107 | |
Guido van Rossum | 86bea46 | 1997-04-29 21:03:06 +0000 | [diff] [blame] | 108 | if ((ps = PyParser_New(g, start)) == NULL) { |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 109 | fprintf(stderr, "no mem for new parser\n"); |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 110 | err_ret->error = E_NOMEM; |
| 111 | return NULL; |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 112 | } |
Neil Schemenauer | c24ea08 | 2002-03-22 23:53:36 +0000 | [diff] [blame] | 113 | #if 0 /* future keyword */ |
Tim Peters | fe2127d | 2001-07-16 05:37:24 +0000 | [diff] [blame] | 114 | if (flags & PyPARSE_YIELD_IS_KEYWORD) |
| 115 | ps->p_generators = 1; |
Neil Schemenauer | c24ea08 | 2002-03-22 23:53:36 +0000 | [diff] [blame] | 116 | #endif |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 117 | |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 118 | for (;;) { |
| 119 | char *a, *b; |
| 120 | int type; |
Guido van Rossum | 6da3434 | 2000-06-28 22:00:02 +0000 | [diff] [blame] | 121 | size_t len; |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 122 | char *str; |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 123 | |
Guido van Rossum | 86bea46 | 1997-04-29 21:03:06 +0000 | [diff] [blame] | 124 | type = PyTokenizer_Get(tok, &a, &b); |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 125 | if (type == ERRORTOKEN) { |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 126 | err_ret->error = tok->done; |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 127 | break; |
| 128 | } |
Guido van Rossum | d8b1d37 | 1992-03-04 16:40:44 +0000 | [diff] [blame] | 129 | if (type == ENDMARKER && started) { |
| 130 | type = NEWLINE; /* Add an extra newline */ |
| 131 | started = 0; |
| 132 | } |
| 133 | else |
| 134 | started = 1; |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 135 | len = b - a; /* XXX this may compute NULL - NULL */ |
Guido van Rossum | 86bea46 | 1997-04-29 21:03:06 +0000 | [diff] [blame] | 136 | str = PyMem_NEW(char, len + 1); |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 137 | if (str == NULL) { |
| 138 | fprintf(stderr, "no mem for next token\n"); |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 139 | err_ret->error = E_NOMEM; |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 140 | break; |
| 141 | } |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 142 | if (len > 0) |
| 143 | strncpy(str, a, len); |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 144 | str[len] = '\0'; |
Guido van Rossum | da62ecc | 2001-07-17 16:53:11 +0000 | [diff] [blame] | 145 | |
Neil Schemenauer | c24ea08 | 2002-03-22 23:53:36 +0000 | [diff] [blame] | 146 | #if 0 /* future keyword */ |
Guido van Rossum | da62ecc | 2001-07-17 16:53:11 +0000 | [diff] [blame] | 147 | /* Warn about yield as NAME */ |
| 148 | if (type == NAME && !ps->p_generators && |
| 149 | len == 5 && str[0] == 'y' && strcmp(str, "yield") == 0) |
| 150 | PySys_WriteStderr(yield_msg, |
| 151 | err_ret->filename==NULL ? |
| 152 | "<string>" : err_ret->filename, |
| 153 | tok->lineno); |
Neil Schemenauer | c24ea08 | 2002-03-22 23:53:36 +0000 | [diff] [blame] | 154 | #endif |
Guido van Rossum | da62ecc | 2001-07-17 16:53:11 +0000 | [diff] [blame] | 155 | |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 156 | if ((err_ret->error = |
Fred Drake | 85f3639 | 2000-07-11 17:53:00 +0000 | [diff] [blame] | 157 | PyParser_AddToken(ps, (int)type, str, tok->lineno, |
| 158 | &(err_ret->expected))) != E_OK) { |
Guido van Rossum | ff0ec52 | 1997-07-27 01:52:50 +0000 | [diff] [blame] | 159 | if (err_ret->error != E_DONE) |
| 160 | PyMem_DEL(str); |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 161 | break; |
Guido van Rossum | ff0ec52 | 1997-07-27 01:52:50 +0000 | [diff] [blame] | 162 | } |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 163 | } |
| 164 | |
| 165 | if (err_ret->error == E_DONE) { |
| 166 | n = ps->p_tree; |
| 167 | ps->p_tree = NULL; |
| 168 | } |
| 169 | else |
| 170 | n = NULL; |
| 171 | |
Guido van Rossum | 86bea46 | 1997-04-29 21:03:06 +0000 | [diff] [blame] | 172 | PyParser_Delete(ps); |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 173 | |
| 174 | if (n == NULL) { |
| 175 | if (tok->lineno <= 1 && tok->done == E_EOF) |
| 176 | err_ret->error = E_EOF; |
| 177 | err_ret->lineno = tok->lineno; |
| 178 | err_ret->offset = tok->cur - tok->buf; |
| 179 | if (tok->buf != NULL) { |
Guido van Rossum | 6da3434 | 2000-06-28 22:00:02 +0000 | [diff] [blame] | 180 | size_t len = tok->inp - tok->buf; |
Guido van Rossum | b18618d | 2000-05-03 23:44:39 +0000 | [diff] [blame] | 181 | err_ret->text = PyMem_NEW(char, len + 1); |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 182 | if (err_ret->text != NULL) { |
Guido van Rossum | ec49827 | 1995-01-20 16:59:12 +0000 | [diff] [blame] | 183 | if (len > 0) |
| 184 | strncpy(err_ret->text, tok->buf, len); |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 185 | err_ret->text[len] = '\0'; |
| 186 | } |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 187 | } |
| 188 | } |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 189 | |
Guido van Rossum | 86bea46 | 1997-04-29 21:03:06 +0000 | [diff] [blame] | 190 | PyTokenizer_Free(tok); |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 191 | |
| 192 | return n; |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 193 | } |
Guido van Rossum | 0c156a5 | 2001-10-20 14:27:56 +0000 | [diff] [blame] | 194 | |
| 195 | static void |
Martin v. Löwis | 16eff6f | 2002-01-05 21:40:08 +0000 | [diff] [blame] | 196 | initerr(perrdetail *err_ret, char* filename) |
Guido van Rossum | 0c156a5 | 2001-10-20 14:27:56 +0000 | [diff] [blame] | 197 | { |
| 198 | err_ret->error = E_OK; |
Martin v. Löwis | 16eff6f | 2002-01-05 21:40:08 +0000 | [diff] [blame] | 199 | err_ret->filename = filename; |
Guido van Rossum | 0c156a5 | 2001-10-20 14:27:56 +0000 | [diff] [blame] | 200 | err_ret->lineno = 0; |
| 201 | err_ret->offset = 0; |
| 202 | err_ret->text = NULL; |
| 203 | err_ret->token = -1; |
| 204 | err_ret->expected = -1; |
| 205 | } |