Guido van Rossum | f70e43a | 1991-02-19 12:39:46 +0000 | [diff] [blame] | 1 | /*********************************************************** |
Guido van Rossum | fd71b9e | 2000-06-30 23:50:40 +0000 | [diff] [blame] | 2 | Copyright (c) 2000, BeOpen.com. |
| 3 | Copyright (c) 1995-2000, Corporation for National Research Initiatives. |
| 4 | Copyright (c) 1990-1995, Stichting Mathematisch Centrum. |
| 5 | All rights reserved. |
Guido van Rossum | f70e43a | 1991-02-19 12:39:46 +0000 | [diff] [blame] | 6 | |
Guido van Rossum | fd71b9e | 2000-06-30 23:50:40 +0000 | [diff] [blame] | 7 | See the file "Misc/COPYRIGHT" for information on usage and |
| 8 | redistribution of this file, and for a DISCLAIMER OF ALL WARRANTIES. |
Guido van Rossum | f70e43a | 1991-02-19 12:39:46 +0000 | [diff] [blame] | 9 | ******************************************************************/ |
| 10 | |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 11 | /* Parser-tokenizer link implementation */ |
| 12 | |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 13 | #include "pgenheaders.h" |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 14 | #include "tokenizer.h" |
| 15 | #include "node.h" |
| 16 | #include "grammar.h" |
| 17 | #include "parser.h" |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 18 | #include "parsetok.h" |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 19 | #include "errcode.h" |
| 20 | |
Guido van Rossum | 6135df6 | 1998-04-10 19:35:06 +0000 | [diff] [blame] | 21 | int Py_TabcheckFlag; |
| 22 | |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 23 | |
| 24 | /* Forward */ |
Tim Peters | dbd9ba6 | 2000-07-09 03:09:57 +0000 | [diff] [blame] | 25 | static node *parsetok(struct tok_state *, grammar *, int, perrdetail *); |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 26 | |
| 27 | /* Parse input coming from a string. Return error code, print some errors. */ |
| 28 | |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 29 | node * |
Thomas Wouters | 23c9e00 | 2000-07-22 19:20:54 +0000 | [diff] [blame] | 30 | PyParser_ParseString(char *s, grammar *g, int start, perrdetail *err_ret) |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 31 | { |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 32 | struct tok_state *tok; |
| 33 | |
| 34 | err_ret->error = E_OK; |
| 35 | err_ret->filename = NULL; |
| 36 | err_ret->lineno = 0; |
| 37 | err_ret->offset = 0; |
| 38 | err_ret->text = NULL; |
Barry Warsaw | 38aa14a | 2000-08-18 05:04:08 +0000 | [diff] [blame] | 39 | err_ret->token = -1; |
| 40 | err_ret->expected = -1; |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 41 | |
Guido van Rossum | 86bea46 | 1997-04-29 21:03:06 +0000 | [diff] [blame] | 42 | if ((tok = PyTokenizer_FromString(s)) == NULL) { |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 43 | err_ret->error = E_NOMEM; |
| 44 | return NULL; |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 45 | } |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 46 | |
Guido van Rossum | 89ce454 | 1998-12-21 18:32:40 +0000 | [diff] [blame] | 47 | if (Py_TabcheckFlag || Py_VerboseFlag) { |
| 48 | tok->filename = "<string>"; |
| 49 | tok->altwarning = (tok->filename != NULL); |
| 50 | if (Py_TabcheckFlag >= 2) |
| 51 | tok->alterror++; |
| 52 | } |
| 53 | |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 54 | return parsetok(tok, g, start, err_ret); |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 55 | } |
| 56 | |
| 57 | |
| 58 | /* Parse input coming from a file. Return error code, print some errors. */ |
| 59 | |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 60 | node * |
Thomas Wouters | 23c9e00 | 2000-07-22 19:20:54 +0000 | [diff] [blame] | 61 | PyParser_ParseFile(FILE *fp, char *filename, grammar *g, int start, |
| 62 | char *ps1, char *ps2, perrdetail *err_ret) |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 63 | { |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 64 | struct tok_state *tok; |
| 65 | |
| 66 | err_ret->error = E_OK; |
| 67 | err_ret->filename = filename; |
| 68 | err_ret->lineno = 0; |
| 69 | err_ret->offset = 0; |
| 70 | err_ret->text = NULL; |
| 71 | |
Guido van Rossum | 86bea46 | 1997-04-29 21:03:06 +0000 | [diff] [blame] | 72 | if ((tok = PyTokenizer_FromFile(fp, ps1, ps2)) == NULL) { |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 73 | err_ret->error = E_NOMEM; |
| 74 | return NULL; |
Guido van Rossum | 3f5da24 | 1990-12-20 15:06:42 +0000 | [diff] [blame] | 75 | } |
Guido van Rossum | 6135df6 | 1998-04-10 19:35:06 +0000 | [diff] [blame] | 76 | if (Py_TabcheckFlag || Py_VerboseFlag) { |
| 77 | tok->filename = filename; |
| 78 | tok->altwarning = (filename != NULL); |
| 79 | if (Py_TabcheckFlag >= 2) |
| 80 | tok->alterror++; |
| 81 | } |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 82 | |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 83 | |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 84 | return parsetok(tok, g, start, err_ret); |
| 85 | } |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 86 | |
| 87 | /* Parse input coming from the given tokenizer structure. |
| 88 | Return error code. */ |
| 89 | |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 90 | static node * |
Thomas Wouters | 23c9e00 | 2000-07-22 19:20:54 +0000 | [diff] [blame] | 91 | parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret) |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 92 | { |
| 93 | parser_state *ps; |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 94 | node *n; |
Guido van Rossum | d8b1d37 | 1992-03-04 16:40:44 +0000 | [diff] [blame] | 95 | int started = 0; |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 96 | |
Guido van Rossum | 86bea46 | 1997-04-29 21:03:06 +0000 | [diff] [blame] | 97 | if ((ps = PyParser_New(g, start)) == NULL) { |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 98 | fprintf(stderr, "no mem for new parser\n"); |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 99 | err_ret->error = E_NOMEM; |
| 100 | return NULL; |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 101 | } |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 102 | |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 103 | for (;;) { |
| 104 | char *a, *b; |
| 105 | int type; |
Guido van Rossum | 6da3434 | 2000-06-28 22:00:02 +0000 | [diff] [blame] | 106 | size_t len; |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 107 | char *str; |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 108 | |
Guido van Rossum | 86bea46 | 1997-04-29 21:03:06 +0000 | [diff] [blame] | 109 | type = PyTokenizer_Get(tok, &a, &b); |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 110 | if (type == ERRORTOKEN) { |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 111 | err_ret->error = tok->done; |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 112 | break; |
| 113 | } |
Guido van Rossum | d8b1d37 | 1992-03-04 16:40:44 +0000 | [diff] [blame] | 114 | if (type == ENDMARKER && started) { |
| 115 | type = NEWLINE; /* Add an extra newline */ |
| 116 | started = 0; |
| 117 | } |
| 118 | else |
| 119 | started = 1; |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 120 | len = b - a; /* XXX this may compute NULL - NULL */ |
Guido van Rossum | 86bea46 | 1997-04-29 21:03:06 +0000 | [diff] [blame] | 121 | str = PyMem_NEW(char, len + 1); |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 122 | if (str == NULL) { |
| 123 | fprintf(stderr, "no mem for next token\n"); |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 124 | err_ret->error = E_NOMEM; |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 125 | break; |
| 126 | } |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 127 | if (len > 0) |
| 128 | strncpy(str, a, len); |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 129 | str[len] = '\0'; |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 130 | if ((err_ret->error = |
Fred Drake | 85f3639 | 2000-07-11 17:53:00 +0000 | [diff] [blame] | 131 | PyParser_AddToken(ps, (int)type, str, tok->lineno, |
| 132 | &(err_ret->expected))) != E_OK) { |
Guido van Rossum | ff0ec52 | 1997-07-27 01:52:50 +0000 | [diff] [blame] | 133 | if (err_ret->error != E_DONE) |
| 134 | PyMem_DEL(str); |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 135 | break; |
Guido van Rossum | ff0ec52 | 1997-07-27 01:52:50 +0000 | [diff] [blame] | 136 | } |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 137 | } |
| 138 | |
| 139 | if (err_ret->error == E_DONE) { |
| 140 | n = ps->p_tree; |
| 141 | ps->p_tree = NULL; |
| 142 | } |
| 143 | else |
| 144 | n = NULL; |
| 145 | |
Guido van Rossum | 86bea46 | 1997-04-29 21:03:06 +0000 | [diff] [blame] | 146 | PyParser_Delete(ps); |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 147 | |
| 148 | if (n == NULL) { |
| 149 | if (tok->lineno <= 1 && tok->done == E_EOF) |
| 150 | err_ret->error = E_EOF; |
| 151 | err_ret->lineno = tok->lineno; |
| 152 | err_ret->offset = tok->cur - tok->buf; |
| 153 | if (tok->buf != NULL) { |
Guido van Rossum | 6da3434 | 2000-06-28 22:00:02 +0000 | [diff] [blame] | 154 | size_t len = tok->inp - tok->buf; |
Guido van Rossum | b18618d | 2000-05-03 23:44:39 +0000 | [diff] [blame] | 155 | err_ret->text = PyMem_NEW(char, len + 1); |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 156 | if (err_ret->text != NULL) { |
Guido van Rossum | ec49827 | 1995-01-20 16:59:12 +0000 | [diff] [blame] | 157 | if (len > 0) |
| 158 | strncpy(err_ret->text, tok->buf, len); |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 159 | err_ret->text[len] = '\0'; |
| 160 | } |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 161 | } |
| 162 | } |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 163 | |
Guido van Rossum | 86bea46 | 1997-04-29 21:03:06 +0000 | [diff] [blame] | 164 | PyTokenizer_Free(tok); |
Guido van Rossum | bd0389d | 1994-08-29 12:25:45 +0000 | [diff] [blame] | 165 | |
| 166 | return n; |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 167 | } |