Guido van Rossum | f70e43a | 1991-02-19 12:39:46 +0000 | [diff] [blame] | 1 | /*********************************************************** |
Guido van Rossum | b9f8d6e | 1995-01-04 19:08:09 +0000 | [diff] [blame] | 2 | Copyright 1991-1995 by Stichting Mathematisch Centrum, Amsterdam, |
| 3 | The Netherlands. |
Guido van Rossum | f70e43a | 1991-02-19 12:39:46 +0000 | [diff] [blame] | 4 | |
| 5 | All Rights Reserved |
| 6 | |
Guido van Rossum | d266eb4 | 1996-10-25 14:44:06 +0000 | [diff] [blame] | 7 | Permission to use, copy, modify, and distribute this software and its |
| 8 | documentation for any purpose and without fee is hereby granted, |
Guido van Rossum | f70e43a | 1991-02-19 12:39:46 +0000 | [diff] [blame] | 9 | provided that the above copyright notice appear in all copies and that |
Guido van Rossum | d266eb4 | 1996-10-25 14:44:06 +0000 | [diff] [blame] | 10 | both that copyright notice and this permission notice appear in |
Guido van Rossum | f70e43a | 1991-02-19 12:39:46 +0000 | [diff] [blame] | 11 | supporting documentation, and that the names of Stichting Mathematisch |
Guido van Rossum | d266eb4 | 1996-10-25 14:44:06 +0000 | [diff] [blame] | 12 | Centrum or CWI or Corporation for National Research Initiatives or |
| 13 | CNRI not be used in advertising or publicity pertaining to |
| 14 | distribution of the software without specific, written prior |
| 15 | permission. |
Guido van Rossum | f70e43a | 1991-02-19 12:39:46 +0000 | [diff] [blame] | 16 | |
Guido van Rossum | d266eb4 | 1996-10-25 14:44:06 +0000 | [diff] [blame] | 17 | While CWI is the initial source for this software, a modified version |
| 18 | is made available by the Corporation for National Research Initiatives |
| 19 | (CNRI) at the Internet address ftp://ftp.python.org. |
| 20 | |
| 21 | STICHTING MATHEMATISCH CENTRUM AND CNRI DISCLAIM ALL WARRANTIES WITH |
| 22 | REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF |
| 23 | MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH |
| 24 | CENTRUM OR CNRI BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL |
| 25 | DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR |
| 26 | PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER |
| 27 | TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR |
| 28 | PERFORMANCE OF THIS SOFTWARE. |
Guido van Rossum | f70e43a | 1991-02-19 12:39:46 +0000 | [diff] [blame] | 29 | |
| 30 | ******************************************************************/ |
| 31 | |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 32 | /* Parser-tokenizer link implementation */ |
| 33 | |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 34 | #include "pgenheaders.h" |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 35 | #include "tokenizer.h" |
| 36 | #include "node.h" |
| 37 | #include "grammar.h" |
| 38 | #include "parser.h" |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 39 | #include "parsetok.h" |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 40 | #include "errcode.h" |
| 41 | |
Guido van Rossum | 6135df6 | 1998-04-10 19:35:06 +0000 | [diff] [blame] | 42 | int Py_TabcheckFlag; |
| 43 | |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 44 | |
| 45 | /* Forward */ |
Guido van Rossum | 86bea46 | 1997-04-29 21:03:06 +0000 | [diff] [blame] | 46 | static node *parsetok Py_PROTO((struct tok_state *, grammar *, int, |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 47 | perrdetail *)); |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 48 | |
| 49 | /* Parse input coming from a string. Return error code, print some errors. */ |
| 50 | |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 51 | node * |
Guido van Rossum | 86bea46 | 1997-04-29 21:03:06 +0000 | [diff] [blame] | 52 | PyParser_ParseString(s, g, start, err_ret) |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 53 | char *s; |
| 54 | grammar *g; |
| 55 | int start; |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 56 | perrdetail *err_ret; |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 57 | { |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 58 | struct tok_state *tok; |
| 59 | |
| 60 | err_ret->error = E_OK; |
| 61 | err_ret->filename = NULL; |
| 62 | err_ret->lineno = 0; |
| 63 | err_ret->offset = 0; |
| 64 | err_ret->text = NULL; |
| 65 | |
Guido van Rossum | 86bea46 | 1997-04-29 21:03:06 +0000 | [diff] [blame] | 66 | if ((tok = PyTokenizer_FromString(s)) == NULL) { |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 67 | err_ret->error = E_NOMEM; |
| 68 | return NULL; |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 69 | } |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 70 | |
| 71 | return parsetok(tok, g, start, err_ret); |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 72 | } |
| 73 | |
| 74 | |
| 75 | /* Parse input coming from a file. Return error code, print some errors. */ |
| 76 | |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 77 | node * |
Guido van Rossum | 86bea46 | 1997-04-29 21:03:06 +0000 | [diff] [blame] | 78 | PyParser_ParseFile(fp, filename, g, start, ps1, ps2, err_ret) |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 79 | FILE *fp; |
| 80 | char *filename; |
| 81 | grammar *g; |
| 82 | int start; |
| 83 | char *ps1, *ps2; |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 84 | perrdetail *err_ret; |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 85 | { |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 86 | struct tok_state *tok; |
| 87 | |
| 88 | err_ret->error = E_OK; |
| 89 | err_ret->filename = filename; |
| 90 | err_ret->lineno = 0; |
| 91 | err_ret->offset = 0; |
| 92 | err_ret->text = NULL; |
| 93 | |
Guido van Rossum | 86bea46 | 1997-04-29 21:03:06 +0000 | [diff] [blame] | 94 | if ((tok = PyTokenizer_FromFile(fp, ps1, ps2)) == NULL) { |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 95 | err_ret->error = E_NOMEM; |
| 96 | return NULL; |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 97 | } |
Guido van Rossum | 6135df6 | 1998-04-10 19:35:06 +0000 | [diff] [blame] | 98 | if (Py_TabcheckFlag || Py_VerboseFlag) { |
| 99 | tok->filename = filename; |
| 100 | tok->altwarning = (filename != NULL); |
| 101 | if (Py_TabcheckFlag >= 2) |
| 102 | tok->alterror++; |
| 103 | } |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 104 | |
Guido van Rossum | 91ece42 | 1992-03-25 22:32:00 +0000 | [diff] [blame] | 105 | #ifdef macintosh |
| 106 | { |
| 107 | int tabsize = guesstabsize(filename); |
| 108 | if (tabsize > 0) |
| 109 | tok->tabsize = tabsize; |
| 110 | } |
| 111 | #endif |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 112 | |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 113 | return parsetok(tok, g, start, err_ret); |
| 114 | } |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 115 | |
| 116 | /* Parse input coming from the given tokenizer structure. |
| 117 | Return error code. */ |
| 118 | |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 119 | static node * |
| 120 | parsetok(tok, g, start, err_ret) |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 121 | struct tok_state *tok; |
| 122 | grammar *g; |
| 123 | int start; |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 124 | perrdetail *err_ret; |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 125 | { |
| 126 | parser_state *ps; |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 127 | node *n; |
Guido van Rossum | d8b1d37 | 1992-03-04 16:40:44 +0000 | [diff] [blame] | 128 | int started = 0; |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 129 | |
Guido van Rossum | 86bea46 | 1997-04-29 21:03:06 +0000 | [diff] [blame] | 130 | if ((ps = PyParser_New(g, start)) == NULL) { |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 131 | fprintf(stderr, "no mem for new parser\n"); |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 132 | err_ret->error = E_NOMEM; |
| 133 | return NULL; |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 134 | } |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 135 | |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 136 | for (;;) { |
| 137 | char *a, *b; |
| 138 | int type; |
| 139 | int len; |
| 140 | char *str; |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 141 | |
Guido van Rossum | 86bea46 | 1997-04-29 21:03:06 +0000 | [diff] [blame] | 142 | type = PyTokenizer_Get(tok, &a, &b); |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 143 | if (type == ERRORTOKEN) { |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 144 | err_ret->error = tok->done; |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 145 | break; |
| 146 | } |
Guido van Rossum | d8b1d37 | 1992-03-04 16:40:44 +0000 | [diff] [blame] | 147 | if (type == ENDMARKER && started) { |
| 148 | type = NEWLINE; /* Add an extra newline */ |
| 149 | started = 0; |
| 150 | } |
| 151 | else |
| 152 | started = 1; |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 153 | len = b - a; /* XXX this may compute NULL - NULL */ |
Guido van Rossum | 86bea46 | 1997-04-29 21:03:06 +0000 | [diff] [blame] | 154 | str = PyMem_NEW(char, len + 1); |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 155 | if (str == NULL) { |
| 156 | fprintf(stderr, "no mem for next token\n"); |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 157 | err_ret->error = E_NOMEM; |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 158 | break; |
| 159 | } |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 160 | if (len > 0) |
| 161 | strncpy(str, a, len); |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 162 | str[len] = '\0'; |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 163 | if ((err_ret->error = |
Guido van Rossum | 86bea46 | 1997-04-29 21:03:06 +0000 | [diff] [blame] | 164 | PyParser_AddToken(ps, (int)type, str, |
Guido van Rossum | ff0ec52 | 1997-07-27 01:52:50 +0000 | [diff] [blame] | 165 | tok->lineno)) != E_OK) { |
| 166 | if (err_ret->error != E_DONE) |
| 167 | PyMem_DEL(str); |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 168 | break; |
Guido van Rossum | ff0ec52 | 1997-07-27 01:52:50 +0000 | [diff] [blame] | 169 | } |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 170 | } |
| 171 | |
| 172 | if (err_ret->error == E_DONE) { |
| 173 | n = ps->p_tree; |
| 174 | ps->p_tree = NULL; |
| 175 | } |
| 176 | else |
| 177 | n = NULL; |
| 178 | |
Guido van Rossum | 86bea46 | 1997-04-29 21:03:06 +0000 | [diff] [blame] | 179 | PyParser_Delete(ps); |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 180 | |
| 181 | if (n == NULL) { |
| 182 | if (tok->lineno <= 1 && tok->done == E_EOF) |
| 183 | err_ret->error = E_EOF; |
| 184 | err_ret->lineno = tok->lineno; |
| 185 | err_ret->offset = tok->cur - tok->buf; |
| 186 | if (tok->buf != NULL) { |
| 187 | int len = tok->inp - tok->buf; |
| 188 | err_ret->text = malloc(len + 1); |
| 189 | if (err_ret->text != NULL) { |
Guido van Rossum | ec49827 | 1995-01-20 16:59:12 +0000 | [diff] [blame] | 190 | if (len > 0) |
| 191 | strncpy(err_ret->text, tok->buf, len); |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 192 | err_ret->text[len] = '\0'; |
| 193 | } |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 194 | } |
| 195 | } |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 196 | |
Guido van Rossum | 86bea46 | 1997-04-29 21:03:06 +0000 | [diff] [blame] | 197 | PyTokenizer_Free(tok); |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 198 | |
| 199 | return n; |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 200 | } |