Completely ignore lines with only a newline token on them, except
wholly empty lines interactively.
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index ccff9d1..d737e55 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -124,7 +124,7 @@
}
-/* Set up tokenizer for string */
+/* Set up tokenizer for file */
struct tok_state *
tok_setupf(fp, ps1, ps2)
@@ -309,7 +309,11 @@
char **p_start, **p_end; /* Out: point to start/end of token */
{
register int c;
-
+ int blankline;
+
+ nextline:
+ blankline = 0;
+
/* Get indentation level */
if (tok->atbol) {
register int col = 0;
@@ -325,30 +329,45 @@
break;
}
tok_backup(tok, c);
- if (col == tok->indstack[tok->indent]) {
- /* No change */
+ if (c == '#' || c == '\n') {
+ /* Lines with only whitespace and/or comments
+ shouldn't affect the indentation and are
+ not passed to the parser as NEWLINE tokens,
+ except *totally* empty lines in interactive
+ mode, which signal the end of a command group. */
+ if (col == 0 && c == '\n' && tok->prompt != NULL)
+ blankline = 0; /* Let it through */
+ else
+ blankline = 1; /* Ignore completely */
+ /* We can't jump back right here since we still
+ may need to skip to the end of a comment */
}
- else if (col > tok->indstack[tok->indent]) {
- /* Indent -- always one */
- if (tok->indent+1 >= MAXINDENT) {
- fprintf(stderr, "excessive indent\n");
- tok->done = E_TOKEN;
- return ERRORTOKEN;
+ if (!blankline) {
+ if (col == tok->indstack[tok->indent]) {
+ /* No change */
}
- tok->pendin++;
- tok->indstack[++tok->indent] = col;
- }
- else /* col < tok->indstack[tok->indent] */ {
- /* Dedent -- any number, must be consistent */
- while (tok->indent > 0 &&
- col < tok->indstack[tok->indent]) {
- tok->indent--;
- tok->pendin--;
+ else if (col > tok->indstack[tok->indent]) {
+ /* Indent -- always one */
+ if (tok->indent+1 >= MAXINDENT) {
+ fprintf(stderr, "excessive indent\n");
+ tok->done = E_TOKEN;
+ return ERRORTOKEN;
+ }
+ tok->pendin++;
+ tok->indstack[++tok->indent] = col;
}
- if (col != tok->indstack[tok->indent]) {
- fprintf(stderr, "inconsistent dedent\n");
- tok->done = E_TOKEN;
- return ERRORTOKEN;
+ else /* col < tok->indstack[tok->indent] */ {
+ /* Dedent -- any number, must be consistent */
+ while (tok->indent > 0 &&
+ col < tok->indstack[tok->indent]) {
+ tok->indent--;
+ tok->pendin--;
+ }
+ if (col != tok->indstack[tok->indent]) {
+ fprintf(stderr, "inconsistent dedent\n");
+ tok->done = E_TOKEN;
+ return ERRORTOKEN;
+ }
}
}
}
@@ -380,13 +399,14 @@
if (c == '#') {
/* Hack to allow overriding the tabsize in the file.
This is also recognized by vi, when it occurs near the
- beginning or end of the file. (Will vi never die...?) */
+ beginning or end of the file. (Will vi never die...?)
+ For Python it must be at the beginning of the file! */
int x;
/* XXX The case to (unsigned char *) is needed by THINK C 3.0 */
if (sscanf(/*(unsigned char *)*/tok->cur,
" vi:set tabsize=%d:", &x) == 1 &&
x >= 1 && x <= 40) {
- fprintf(stderr, "# vi:set tabsize=%d:\n", x);
+ /* fprintf(stderr, "# vi:set tabsize=%d:\n", x); */
tok->tabsize = x;
}
do {
@@ -411,6 +431,8 @@
/* Newline */
if (c == '\n') {
tok->atbol = 1;
+ if (blankline)
+ goto nextline;
*p_end = tok->cur - 1; /* Leave '\n' out of the string */
return NEWLINE;
}