* pythonrun.c: Print exception type+arg *after* stack trace instead of
before it.
* ceval.c, object.c: moved testbool() to object.c (now extern visible)
* stringobject.c: fix bugs in and rationalize string resize in formatstring()
* tokenizer.[ch]: fix non-working code for lines longer than BUFSIZ
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 22c20b0..38a9e9a 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -170,31 +170,15 @@
tok_nextc(tok)
register struct tok_state *tok;
{
- if (tok->done != E_OK)
- return EOF;
-
for (;;) {
- if (tok->cur < tok->inp)
- return *tok->cur++;
+ if (tok->cur != tok->inp)
+ return *tok->cur++; /* Fast path */
+ if (tok->done != E_OK)
+ return EOF;
if (tok->fp == NULL) {
tok->done = E_EOF;
return EOF;
}
- if (tok->inp > tok->buf && tok->inp[-1] == '\n')
- tok->inp = tok->buf;
- if (tok->inp == tok->end) {
- int n = tok->end - tok->buf;
- char *new = tok->buf;
- RESIZE(new, char, n+n);
- if (new == NULL) {
- fprintf(stderr, "tokenizer out of mem\n");
- tok->done = E_NOMEM;
- return EOF;
- }
- tok->buf = new;
- tok->inp = tok->buf + n;
- tok->end = tok->inp + n;
- }
#ifdef USE_READLINE
if (tok->prompt != NULL) {
extern char *readline PROTO((char *prompt));
@@ -211,46 +195,71 @@
(void) intrcheck(); /* Clear pending interrupt */
if (tok->nextprompt != NULL)
tok->prompt = tok->nextprompt;
- /* XXX different semantics w/o readline()! */
if (tok->buf == NULL) {
tok->done = E_EOF;
}
else {
- unsigned int n = strlen(tok->buf);
- if (n > 0)
+ tok->end = strchr(tok->buf, '\0');
+ if (tok->end > tok->buf)
add_history(tok->buf);
- /* Append the '\n' that readline()
- doesn't give us, for the tokenizer... */
- tok->buf = realloc(tok->buf, n+2);
- if (tok->buf == NULL)
- tok->done = E_NOMEM;
- else {
- tok->end = tok->buf + n;
- *tok->end++ = '\n';
- *tok->end = '\0';
- tok->inp = tok->end;
- tok->cur = tok->buf;
- }
+ /* Replace trailing '\n' by '\0'
+ (we don't need a '\0', but the
+ tokenizer wants a '\n'...) */
+ *tok->end++ = '\n';
+ tok->inp = tok->end;
+ tok->cur = tok->buf;
}
}
else
#endif
{
- tok->cur = tok->inp;
- if (tok->prompt != NULL && tok->inp == tok->buf) {
+ if (tok->prompt != NULL) {
fprintf(stderr, "%s", tok->prompt);
- tok->prompt = tok->nextprompt;
+ if (tok->nextprompt != NULL)
+ tok->prompt = tok->nextprompt;
}
- tok->done = fgets_intr(tok->inp,
- (int)(tok->end - tok->inp), tok->fp);
+ if (tok->buf == NULL) {
+ tok->buf = NEW(char, BUFSIZ);
+ if (tok->buf == NULL) {
+ tok->done = E_NOMEM;
+ return EOF;
+ }
+ tok->end = tok->buf + BUFSIZ;
+ }
+ tok->done = fgets_intr(tok->buf,
+ (int)(tok->end - tok->buf), tok->fp);
+ tok->inp = strchr(tok->buf, '\0');
+ /* Read until '\n' or EOF */
+ while (tok->inp+1==tok->end && tok->inp[-1]!='\n') {
+ int curvalid = tok->inp - tok->buf;
+ int cursize = tok->end - tok->buf;
+ int newsize = cursize + BUFSIZ;
+ char *newbuf = tok->buf;
+ RESIZE(newbuf, char, newsize);
+ if (newbuf == NULL) {
+ tok->done = E_NOMEM;
+ tok->cur = tok->inp;
+ return EOF;
+ }
+ tok->buf = newbuf;
+ tok->inp = tok->buf + curvalid;
+ tok->end = tok->buf + newsize;
+ if (fgets_intr(tok->inp,
+ (int)(tok->end - tok->inp),
+ tok->fp) != E_OK)
+ break;
+ tok->inp = strchr(tok->inp, '\0');
+ }
+ tok->cur = tok->buf;
}
if (tok->done != E_OK) {
if (tok->prompt != NULL)
fprintf(stderr, "\n");
+ tok->cur = tok->inp;
return EOF;
}
- tok->inp = strchr(tok->inp, '\0');
}
+ /*NOTREACHED*/
}
@@ -390,6 +399,7 @@
if (tok->indent+1 >= MAXINDENT) {
fprintf(stderr, "excessive indent\n");
tok->done = E_TOKEN;
+ tok->cur = tok->inp;
return ERRORTOKEN;
}
tok->pendin++;
@@ -405,6 +415,7 @@
if (col != tok->indstack[tok->indent]) {
fprintf(stderr, "inconsistent dedent\n");
tok->done = E_TOKEN;
+ tok->cur = tok->inp;
return ERRORTOKEN;
}
}
@@ -558,6 +569,7 @@
c = tok_nextc(tok);
if (c == '\n' || c == EOF) {
tok->done = E_TOKEN;
+ tok->cur = tok->inp;
return ERRORTOKEN;
}
if (c == '\\') {
@@ -565,6 +577,7 @@
*p_end = tok->cur;
if (c == '\n' || c == EOF) {
tok->done = E_TOKEN;
+ tok->cur = tok->inp;
return ERRORTOKEN;
}
continue;
@@ -581,6 +594,7 @@
c = tok_nextc(tok);
if (c != '\n') {
tok->done = E_TOKEN;
+ tok->cur = tok->inp;
return ERRORTOKEN;
}
tok->lineno++;
diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
index 636295a..9abc598 100644
--- a/Parser/tokenizer.h
+++ b/Parser/tokenizer.h
@@ -31,12 +31,13 @@
/* Tokenizer state */
struct tok_state {
/* Input state; buf <= cur <= inp <= end */
- /* NB an entire token must fit in the buffer */
- char *buf; /* Input buffer */
+ /* NB an entire line is held in the buffer */
+ char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL */
char *cur; /* Next character in buffer */
char *inp; /* End of data in buffer */
- char *end; /* End of input buffer */
- int done; /* 0 normally, 1 at EOF, -1 after error */
+ char *end; /* End of input buffer if buf != NULL */
+ int done; /* E_OK normally, E_EOF at EOF, otherwise error code */
+ /* NB If done != E_OK, cur must be == inp!!! */
FILE *fp; /* Rest of input; NULL if tokenizing a string */
int tabsize; /* Tab spacing */
int indent; /* Current indentation index */