syntax: improve REPL parsing (#98)
Previously, the REPL used a heuristic: it would consume a single line
and attempt to parse it; if that failed, it would consume lines up to
a blank line then parse the whole as a file. This was suboptimal for
various reasons: it failed to parse lines ending with an unfinished
multi-line string literal, for example, and it would prematurely
stop reading even while parentheses were open.
This change integrates the REPL with the scanner and parser (as Python
does). The REPL invokes a new parser entry point, ParseCompoundStmt,
that consumes only enough input to parse a compound statement, defined
as (a) blank line, (b) a semicolon-separated list of simple statements
all on one line, or (c) a complex statement such as def, if or for.
If the 'src' value provided to the scanner is a function of type
func() ([]byte, error), then the scanner will call it each time
it runs out of input.
Fixes #81
diff --git a/syntax/parse.go b/syntax/parse.go
index af79cc1..2361085 100644
--- a/syntax/parse.go
+++ b/syntax/parse.go
@@ -47,6 +47,38 @@
return f, nil
}
+// ParseCompoundStmt parses a single compound statement:
+// a blank line, a def, for, while, or if statement, or a
+// semicolon-separated list of simple statements followed
+// by a newline. These are the units on which the REPL operates.
+// ParseCompoundStmt does not consume any following input.
+// The parser calls the readline function each
+// time it needs a new line of input.
+func ParseCompoundStmt(filename string, readline func() ([]byte, error)) (f *File, err error) {
+ in, err := newScanner(filename, readline, false)
+ if err != nil {
+ return nil, err
+ }
+
+ p := parser{in: in}
+ defer p.in.recover(&err)
+
+ p.nextToken() // read first lookahead token
+
+ var stmts []Stmt
+ switch p.tok {
+ case DEF, IF, FOR, WHILE:
+ stmts = p.parseStmt(stmts)
+ case NEWLINE:
+ // blank line
+ default:
+ // Don't consume newline, to avoid blocking again.
+ stmts = p.parseSimpleStmt(stmts, false)
+ }
+
+ return &File{Path: filename, Stmts: stmts}, nil
+}
+
// ParseExpr parses a Starlark expression.
// See Parse for explanation of parameters.
func ParseExpr(filename string, src interface{}, mode Mode) (expr Expr, err error) {
@@ -58,6 +90,10 @@
defer p.in.recover(&err)
p.nextToken() // read first lookahead token
+
+ // TODO(adonovan): Python's eval would use the equivalent of
+ // parseExpr here, which permits an unparenthesized tuple.
+ // We should too.
expr = p.parseTest()
// A following newline (e.g. "f()\n") appears outside any brackets,
@@ -114,7 +150,7 @@
} else if p.tok == WHILE {
return append(stmts, p.parseWhileStmt())
}
- return p.parseSimpleStmt(stmts)
+ return p.parseSimpleStmt(stmts, true)
}
func (p *parser) parseDefStmt() Stmt {
@@ -219,7 +255,8 @@
}
// simple_stmt = small_stmt (SEMI small_stmt)* SEMI? NEWLINE
-func (p *parser) parseSimpleStmt(stmts []Stmt) []Stmt {
+// In REPL mode, it does not consume the NEWLINE.
+func (p *parser) parseSimpleStmt(stmts []Stmt, consumeNL bool) []Stmt {
for {
stmts = append(stmts, p.parseSmallStmt())
if p.tok != SEMI {
@@ -231,9 +268,10 @@
}
}
// EOF without NEWLINE occurs in `if x: pass`, for example.
- if p.tok != EOF {
+ if p.tok != EOF && consumeNL {
p.consume(NEWLINE)
}
+
return stmts
}
@@ -355,7 +393,7 @@
return stmts
}
- return p.parseSimpleStmt(nil)
+ return p.parseSimpleStmt(nil, true)
}
func (p *parser) parseIdent() *Ident {