syntax: support setting initial line/col for scanner (#349)
The new FilePortion type, which may be provided to the scanner,
parser, or ExecFile functions, combines a piece of text along
with its start line/column numbers, for applications that
extract a Starlark expression from the middle of a larger file.
Fixes https://github.com/google/starlark-go/issues/346
diff --git a/syntax/parse.go b/syntax/parse.go
index 0281e4b..50b8087 100644
--- a/syntax/parse.go
+++ b/syntax/parse.go
@@ -28,7 +28,7 @@
// If src != nil, ParseFile parses the source from src and the filename
// is only used when recording position information.
// The type of the argument for the src parameter must be string,
-// []byte, or io.Reader.
+// []byte, io.Reader, or FilePortion.
// If src == nil, ParseFile parses the file specified by filename.
func Parse(filename string, src interface{}, mode Mode) (f *File, err error) {
in, err := newScanner(filename, src, mode&RetainComments != 0)
diff --git a/syntax/parse_test.go b/syntax/parse_test.go
index 76f9eb3..6052e79 100644
--- a/syntax/parse_test.go
+++ b/syntax/parse_test.go
@@ -439,6 +439,27 @@
}
}
+func TestFilePortion(t *testing.T) {
+ // Imagine that the Starlark file or expression print(x.f) is extracted
+ // from the middle of a file in some hypothetical template language;
+ // see https://github.com/google/starlark-go/issues/346. For example:
+ // --
+ // {{loop x seq}}
+ // {{print(x.f)}}
+ // {{end}}
+ // --
+ fp := syntax.FilePortion{Content: []byte("print(x.f)"), FirstLine: 2, FirstCol: 4}
+ file, err := syntax.Parse("foo.template", fp, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ span := fmt.Sprint(file.Stmts[0].Span())
+ want := "foo.template:2:4 foo.template:2:14"
+ if span != want {
+ t.Errorf("wrong span: got %q, want %q", span, want)
+ }
+}
+
// dataFile is the same as starlarktest.DataFile.
// We make a copy to avoid a dependency cycle.
var dataFile = func(pkgdir, filename string) string {
diff --git a/syntax/scan.go b/syntax/scan.go
index 53d9f5c..a162264 100644
--- a/syntax/scan.go
+++ b/syntax/scan.go
@@ -182,6 +182,15 @@
WHILE: "while",
}
+// A FilePortion describes the content of a portion of a file.
+// Callers may provide a FilePortion for the src argument of Parse
+// when the desired initial line and column numbers are not (1, 1),
+// such as when an expression is parsed from within larger file.
+type FilePortion struct {
+ Content []byte
+ FirstLine, FirstCol int32
+}
+
// A Position describes the location of a rune of input.
type Position struct {
file *string // filename (indirect for compactness)
@@ -249,8 +258,12 @@
}
func newScanner(filename string, src interface{}, keepComments bool) (*scanner, error) {
+ var firstLine, firstCol int32 = 1, 1
+ if portion, ok := src.(FilePortion); ok {
+ firstLine, firstCol = portion.FirstLine, portion.FirstCol
+ }
sc := &scanner{
- pos: Position{file: &filename, Line: 1, Col: 1},
+ pos: MakePosition(&filename, firstLine, firstCol),
indentstk: make([]int, 1, 10), // []int{0} + spare capacity
lineStart: true,
keepComments: keepComments,
@@ -279,6 +292,8 @@
return nil, err
}
return data, nil
+ case FilePortion:
+ return src.Content, nil
case nil:
return ioutil.ReadFile(filename)
default: