Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 1 | // Copyright 2017 The Bazel Authors. All rights reserved. |
| 2 | // Use of this source code is governed by a BSD-style |
| 3 | // license that can be found in the LICENSE file. |
| 4 | |
| 5 | package syntax_test |
| 6 | |
| 7 | import ( |
alandonovan | 30e71c6 | 2019-01-04 13:48:12 -0500 | [diff] [blame] | 8 | "bufio" |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 9 | "bytes" |
| 10 | "fmt" |
alandonovan | 30e71c6 | 2019-01-04 13:48:12 -0500 | [diff] [blame] | 11 | "go/build" |
| 12 | "io/ioutil" |
| 13 | "path/filepath" |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 14 | "reflect" |
| 15 | "strings" |
| 16 | "testing" |
| 17 | |
Alan Donovan | 6beab7e | 2018-10-31 17:53:09 -0400 | [diff] [blame] | 18 | "go.starlark.net/internal/chunkedfile" |
| 19 | "go.starlark.net/starlarktest" |
| 20 | "go.starlark.net/syntax" |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 21 | ) |
| 22 | |
| 23 | func TestExprParseTrees(t *testing.T) { |
| 24 | for _, test := range []struct { |
| 25 | input, want string |
| 26 | }{ |
| 27 | {`print(1)`, |
| 28 | `(CallExpr Fn=print Args=(1))`}, |
Alan Donovan | ae06384 | 2017-10-10 15:46:17 -0400 | [diff] [blame] | 29 | {"print(1)\n", |
| 30 | `(CallExpr Fn=print Args=(1))`}, |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 31 | {`x + 1`, |
| 32 | `(BinaryExpr X=x Op=+ Y=1)`}, |
| 33 | {`[x for x in y]`, |
| 34 | `(Comprehension Body=x Clauses=((ForClause Vars=x X=y)))`}, |
| 35 | {`[x for x in (a if b else c)]`, |
Laurent Le Brun | 28ceca7 | 2018-02-26 15:01:53 +0100 | [diff] [blame] | 36 | `(Comprehension Body=x Clauses=((ForClause Vars=x X=(ParenExpr X=(CondExpr Cond=b True=a False=c)))))`}, |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 37 | {`x[i].f(42)`, |
| 38 | `(CallExpr Fn=(DotExpr X=(IndexExpr X=x Y=i) Name=f) Args=(42))`}, |
| 39 | {`x.f()`, |
| 40 | `(CallExpr Fn=(DotExpr X=x Name=f))`}, |
| 41 | {`x+y*z`, |
| 42 | `(BinaryExpr X=x Op=+ Y=(BinaryExpr X=y Op=* Y=z))`}, |
| 43 | {`x%y-z`, |
| 44 | `(BinaryExpr X=(BinaryExpr X=x Op=% Y=y) Op=- Y=z)`}, |
| 45 | {`a + b not in c`, |
| 46 | `(BinaryExpr X=(BinaryExpr X=a Op=+ Y=b) Op=not in Y=c)`}, |
| 47 | {`lambda x, *args, **kwargs: None`, |
alandonovan | 6ddc71c | 2019-06-04 09:08:55 -0400 | [diff] [blame] | 48 | `(LambdaExpr Params=(x (UnaryExpr Op=* X=args) (UnaryExpr Op=** X=kwargs)) Body=None)`}, |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 49 | {`{"one": 1}`, |
| 50 | `(DictExpr List=((DictEntry Key="one" Value=1)))`}, |
| 51 | {`a[i]`, |
| 52 | `(IndexExpr X=a Y=i)`}, |
| 53 | {`a[i:]`, |
| 54 | `(SliceExpr X=a Lo=i)`}, |
| 55 | {`a[:j]`, |
| 56 | `(SliceExpr X=a Hi=j)`}, |
| 57 | {`a[::]`, |
| 58 | `(SliceExpr X=a)`}, |
| 59 | {`a[::k]`, |
| 60 | `(SliceExpr X=a Step=k)`}, |
| 61 | {`[]`, |
| 62 | `(ListExpr)`}, |
| 63 | {`[1]`, |
| 64 | `(ListExpr List=(1))`}, |
| 65 | {`[1,]`, |
| 66 | `(ListExpr List=(1))`}, |
| 67 | {`[1, 2]`, |
| 68 | `(ListExpr List=(1 2))`}, |
| 69 | {`()`, |
| 70 | `(TupleExpr)`}, |
| 71 | {`(4,)`, |
Laurent Le Brun | 28ceca7 | 2018-02-26 15:01:53 +0100 | [diff] [blame] | 72 | `(ParenExpr X=(TupleExpr List=(4)))`}, |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 73 | {`(4)`, |
Laurent Le Brun | 28ceca7 | 2018-02-26 15:01:53 +0100 | [diff] [blame] | 74 | `(ParenExpr X=4)`}, |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 75 | {`(4, 5)`, |
Laurent Le Brun | 28ceca7 | 2018-02-26 15:01:53 +0100 | [diff] [blame] | 76 | `(ParenExpr X=(TupleExpr List=(4 5)))`}, |
alandonovan | f26cf18 | 2019-05-28 16:17:30 -0400 | [diff] [blame] | 77 | {`1, 2, 3`, |
| 78 | `(TupleExpr List=(1 2 3))`}, |
| 79 | {`1, 2,`, |
| 80 | `unparenthesized tuple with trailing comma`}, |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 81 | {`{}`, |
| 82 | `(DictExpr)`}, |
| 83 | {`{"a": 1}`, |
| 84 | `(DictExpr List=((DictEntry Key="a" Value=1)))`}, |
| 85 | {`{"a": 1,}`, |
| 86 | `(DictExpr List=((DictEntry Key="a" Value=1)))`}, |
| 87 | {`{"a": 1, "b": 2}`, |
| 88 | `(DictExpr List=((DictEntry Key="a" Value=1) (DictEntry Key="b" Value=2)))`}, |
| 89 | {`{x: y for (x, y) in z}`, |
Laurent Le Brun | 28ceca7 | 2018-02-26 15:01:53 +0100 | [diff] [blame] | 90 | `(Comprehension Curly Body=(DictEntry Key=x Value=y) Clauses=((ForClause Vars=(ParenExpr X=(TupleExpr List=(x y))) X=z)))`}, |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 91 | {`{x: y for a in b if c}`, |
| 92 | `(Comprehension Curly Body=(DictEntry Key=x Value=y) Clauses=((ForClause Vars=a X=b) (IfClause Cond=c)))`}, |
| 93 | {`-1 + +2`, |
| 94 | `(BinaryExpr X=(UnaryExpr Op=- X=1) Op=+ Y=(UnaryExpr Op=+ X=2))`}, |
| 95 | {`"foo" + "bar"`, |
alandonovan | 60e4b3d | 2018-04-02 12:32:39 -0400 | [diff] [blame] | 96 | `(BinaryExpr X="foo" Op=+ Y="bar")`}, |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 97 | {`-1 * 2`, // prec(unary -) > prec(binary *) |
| 98 | `(BinaryExpr X=(UnaryExpr Op=- X=1) Op=* Y=2)`}, |
| 99 | {`-x[i]`, // prec(unary -) < prec(x[i]) |
| 100 | `(UnaryExpr Op=- X=(IndexExpr X=x Y=i))`}, |
| 101 | {`a | b & c | d`, // prec(|) < prec(&) |
| 102 | `(BinaryExpr X=(BinaryExpr X=a Op=| Y=(BinaryExpr X=b Op=& Y=c)) Op=| Y=d)`}, |
| 103 | {`a or b and c or d`, |
| 104 | `(BinaryExpr X=(BinaryExpr X=a Op=or Y=(BinaryExpr X=b Op=and Y=c)) Op=or Y=d)`}, |
| 105 | {`a and b or c and d`, |
| 106 | `(BinaryExpr X=(BinaryExpr X=a Op=and Y=b) Op=or Y=(BinaryExpr X=c Op=and Y=d))`}, |
| 107 | {`f(1, x=y)`, |
| 108 | `(CallExpr Fn=f Args=(1 (BinaryExpr X=x Op== Y=y)))`}, |
| 109 | {`f(*args, **kwargs)`, |
| 110 | `(CallExpr Fn=f Args=((UnaryExpr Op=* X=args) (UnaryExpr Op=** X=kwargs)))`}, |
Alan Donovan | 5215385 | 2019-02-13 19:18:15 -0500 | [diff] [blame] | 111 | {`lambda *args, *, x=1, **kwargs: 0`, |
alandonovan | 6ddc71c | 2019-06-04 09:08:55 -0400 | [diff] [blame] | 112 | `(LambdaExpr Params=((UnaryExpr Op=* X=args) (UnaryExpr Op=*) (BinaryExpr X=x Op== Y=1) (UnaryExpr Op=** X=kwargs)) Body=0)`}, |
Alan Donovan | 5215385 | 2019-02-13 19:18:15 -0500 | [diff] [blame] | 113 | {`lambda *, a, *b: 0`, |
alandonovan | 6ddc71c | 2019-06-04 09:08:55 -0400 | [diff] [blame] | 114 | `(LambdaExpr Params=((UnaryExpr Op=*) a (UnaryExpr Op=* X=b)) Body=0)`}, |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 115 | {`a if b else c`, |
| 116 | `(CondExpr Cond=b True=a False=c)`}, |
| 117 | {`a and not b`, |
| 118 | `(BinaryExpr X=a Op=and Y=(UnaryExpr Op=not X=b))`}, |
alandonovan | f9faf3b | 2018-01-16 10:06:02 -0500 | [diff] [blame] | 119 | {`[e for x in y if cond1 if cond2]`, |
alandonovan | 7a86632 | 2018-11-21 14:57:52 -0500 | [diff] [blame] | 120 | `(Comprehension Body=e Clauses=((ForClause Vars=x X=y) (IfClause Cond=cond1) (IfClause Cond=cond2)))`}, // github.com/google/skylark/issues/53 |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 121 | } { |
Alan Donovan | e3deafe | 2018-10-23 11:05:09 -0400 | [diff] [blame] | 122 | e, err := syntax.ParseExpr("foo.star", test.input, 0) |
alandonovan | f26cf18 | 2019-05-28 16:17:30 -0400 | [diff] [blame] | 123 | var got string |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 124 | if err != nil { |
alandonovan | f26cf18 | 2019-05-28 16:17:30 -0400 | [diff] [blame] | 125 | got = stripPos(err) |
| 126 | } else { |
| 127 | got = treeString(e) |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 128 | } |
alandonovan | f26cf18 | 2019-05-28 16:17:30 -0400 | [diff] [blame] | 129 | if test.want != got { |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 130 | t.Errorf("parse `%s` = %s, want %s", test.input, got, test.want) |
| 131 | } |
| 132 | } |
| 133 | } |
| 134 | |
| 135 | func TestStmtParseTrees(t *testing.T) { |
| 136 | for _, test := range []struct { |
| 137 | input, want string |
| 138 | }{ |
| 139 | {`print(1)`, |
| 140 | `(ExprStmt X=(CallExpr Fn=print Args=(1)))`}, |
| 141 | {`return 1, 2`, |
| 142 | `(ReturnStmt Result=(TupleExpr List=(1 2)))`}, |
| 143 | {`return`, |
| 144 | `(ReturnStmt)`}, |
| 145 | {`for i in "abc": break`, |
| 146 | `(ForStmt Vars=i X="abc" Body=((BranchStmt Token=break)))`}, |
| 147 | {`for i in "abc": continue`, |
| 148 | `(ForStmt Vars=i X="abc" Body=((BranchStmt Token=continue)))`}, |
| 149 | {`for x, y in z: pass`, |
| 150 | `(ForStmt Vars=(TupleExpr List=(x y)) X=z Body=((BranchStmt Token=pass)))`}, |
| 151 | {`if True: pass`, |
| 152 | `(IfStmt Cond=True True=((BranchStmt Token=pass)))`}, |
| 153 | {`if True: break`, |
| 154 | `(IfStmt Cond=True True=((BranchStmt Token=break)))`}, |
| 155 | {`if True: continue`, |
| 156 | `(IfStmt Cond=True True=((BranchStmt Token=continue)))`}, |
| 157 | {`if True: pass |
| 158 | else: |
| 159 | pass`, |
| 160 | `(IfStmt Cond=True True=((BranchStmt Token=pass)) False=((BranchStmt Token=pass)))`}, |
| 161 | {"if a: pass\nelif b: pass\nelse: pass", |
| 162 | `(IfStmt Cond=a True=((BranchStmt Token=pass)) False=((IfStmt Cond=b True=((BranchStmt Token=pass)) False=((BranchStmt Token=pass)))))`}, |
| 163 | {`x, y = 1, 2`, |
| 164 | `(AssignStmt Op== LHS=(TupleExpr List=(x y)) RHS=(TupleExpr List=(1 2)))`}, |
| 165 | {`x[i] = 1`, |
| 166 | `(AssignStmt Op== LHS=(IndexExpr X=x Y=i) RHS=1)`}, |
| 167 | {`x.f = 1`, |
| 168 | `(AssignStmt Op== LHS=(DotExpr X=x Name=f) RHS=1)`}, |
| 169 | {`(x, y) = 1`, |
Laurent Le Brun | 28ceca7 | 2018-02-26 15:01:53 +0100 | [diff] [blame] | 170 | `(AssignStmt Op== LHS=(ParenExpr X=(TupleExpr List=(x y))) RHS=1)`}, |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 171 | {`load("", "a", b="c")`, |
| 172 | `(LoadStmt Module="" From=(a c) To=(a b))`}, |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 173 | {`if True: load("", "a", b="c")`, // load needn't be at toplevel |
| 174 | `(IfStmt Cond=True True=((LoadStmt Module="" From=(a c) To=(a b))))`}, |
| 175 | {`def f(x, *args, **kwargs): |
| 176 | pass`, |
alandonovan | 6ddc71c | 2019-06-04 09:08:55 -0400 | [diff] [blame] | 177 | `(DefStmt Name=f Params=(x (UnaryExpr Op=* X=args) (UnaryExpr Op=** X=kwargs)) Body=((BranchStmt Token=pass)))`}, |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 178 | {`def f(**kwargs, *args): pass`, |
alandonovan | 6ddc71c | 2019-06-04 09:08:55 -0400 | [diff] [blame] | 179 | `(DefStmt Name=f Params=((UnaryExpr Op=** X=kwargs) (UnaryExpr Op=* X=args)) Body=((BranchStmt Token=pass)))`}, |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 180 | {`def f(a, b, c=d): pass`, |
alandonovan | 6ddc71c | 2019-06-04 09:08:55 -0400 | [diff] [blame] | 181 | `(DefStmt Name=f Params=(a b (BinaryExpr X=c Op== Y=d)) Body=((BranchStmt Token=pass)))`}, |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 182 | {`def f(a, b=c, d): pass`, |
alandonovan | 6ddc71c | 2019-06-04 09:08:55 -0400 | [diff] [blame] | 183 | `(DefStmt Name=f Params=(a (BinaryExpr X=b Op== Y=c) d) Body=((BranchStmt Token=pass)))`}, // TODO(adonovan): fix this |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 184 | {`def f(): |
| 185 | def g(): |
| 186 | pass |
| 187 | pass |
| 188 | def h(): |
| 189 | pass`, |
alandonovan | 6ddc71c | 2019-06-04 09:08:55 -0400 | [diff] [blame] | 190 | `(DefStmt Name=f Body=((DefStmt Name=g Body=((BranchStmt Token=pass))) (BranchStmt Token=pass)))`}, |
alandonovan | 30e71c6 | 2019-01-04 13:48:12 -0500 | [diff] [blame] | 191 | {"f();g()", |
| 192 | `(ExprStmt X=(CallExpr Fn=f))`}, |
| 193 | {"f();", |
| 194 | `(ExprStmt X=(CallExpr Fn=f))`}, |
| 195 | {"f();g()\n", |
| 196 | `(ExprStmt X=(CallExpr Fn=f))`}, |
| 197 | {"f();\n", |
| 198 | `(ExprStmt X=(CallExpr Fn=f))`}, |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 199 | } { |
Alan Donovan | e3deafe | 2018-10-23 11:05:09 -0400 | [diff] [blame] | 200 | f, err := syntax.Parse("foo.star", test.input, 0) |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 201 | if err != nil { |
| 202 | t.Errorf("parse `%s` failed: %v", test.input, stripPos(err)) |
| 203 | continue |
| 204 | } |
| 205 | if got := treeString(f.Stmts[0]); test.want != got { |
| 206 | t.Errorf("parse `%s` = %s, want %s", test.input, got, test.want) |
| 207 | } |
| 208 | } |
| 209 | } |
| 210 | |
| 211 | // TestFileParseTrees tests sequences of statements, and particularly |
| 212 | // handling of indentation, newlines, line continuations, and blank lines. |
| 213 | func TestFileParseTrees(t *testing.T) { |
| 214 | for _, test := range []struct { |
| 215 | input, want string |
| 216 | }{ |
| 217 | {`x = 1 |
| 218 | print(x)`, |
| 219 | `(AssignStmt Op== LHS=x RHS=1) |
| 220 | (ExprStmt X=(CallExpr Fn=print Args=(x)))`}, |
| 221 | {"if cond:\n\tpass", |
| 222 | `(IfStmt Cond=cond True=((BranchStmt Token=pass)))`}, |
| 223 | {"if cond:\n\tpass\nelse:\n\tpass", |
| 224 | `(IfStmt Cond=cond True=((BranchStmt Token=pass)) False=((BranchStmt Token=pass)))`}, |
| 225 | {`def f(): |
| 226 | pass |
| 227 | pass |
| 228 | |
| 229 | pass`, |
alandonovan | 6ddc71c | 2019-06-04 09:08:55 -0400 | [diff] [blame] | 230 | `(DefStmt Name=f Body=((BranchStmt Token=pass))) |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 231 | (BranchStmt Token=pass) |
| 232 | (BranchStmt Token=pass)`}, |
| 233 | {`pass; pass`, |
| 234 | `(BranchStmt Token=pass) |
| 235 | (BranchStmt Token=pass)`}, |
| 236 | {"pass\npass", |
| 237 | `(BranchStmt Token=pass) |
| 238 | (BranchStmt Token=pass)`}, |
| 239 | {"pass\n\npass", |
| 240 | `(BranchStmt Token=pass) |
| 241 | (BranchStmt Token=pass)`}, |
| 242 | {`x = (1 + |
| 243 | 2)`, |
Laurent Le Brun | 28ceca7 | 2018-02-26 15:01:53 +0100 | [diff] [blame] | 244 | `(AssignStmt Op== LHS=x RHS=(ParenExpr X=(BinaryExpr X=1 Op=+ Y=2)))`}, |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 245 | {`x = 1 \ |
| 246 | + 2`, |
| 247 | `(AssignStmt Op== LHS=x RHS=(BinaryExpr X=1 Op=+ Y=2))`}, |
| 248 | } { |
Alan Donovan | e3deafe | 2018-10-23 11:05:09 -0400 | [diff] [blame] | 249 | f, err := syntax.Parse("foo.star", test.input, 0) |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 250 | if err != nil { |
| 251 | t.Errorf("parse `%s` failed: %v", test.input, stripPos(err)) |
| 252 | continue |
| 253 | } |
| 254 | var buf bytes.Buffer |
| 255 | for i, stmt := range f.Stmts { |
| 256 | if i > 0 { |
| 257 | buf.WriteByte('\n') |
| 258 | } |
| 259 | writeTree(&buf, reflect.ValueOf(stmt)) |
| 260 | } |
| 261 | if got := buf.String(); test.want != got { |
| 262 | t.Errorf("parse `%s` = %s, want %s", test.input, got, test.want) |
| 263 | } |
| 264 | } |
| 265 | } |
| 266 | |
alandonovan | 30e71c6 | 2019-01-04 13:48:12 -0500 | [diff] [blame] | 267 | // TestCompoundStmt tests handling of REPL-style compound statements. |
| 268 | func TestCompoundStmt(t *testing.T) { |
| 269 | for _, test := range []struct { |
| 270 | input, want string |
| 271 | }{ |
| 272 | // blank lines |
| 273 | {"\n", |
| 274 | ``}, |
| 275 | {" \n", |
| 276 | ``}, |
| 277 | {"# comment\n", |
| 278 | ``}, |
| 279 | // simple statement |
| 280 | {"1\n", |
| 281 | `(ExprStmt X=1)`}, |
| 282 | {"print(1)\n", |
| 283 | `(ExprStmt X=(CallExpr Fn=print Args=(1)))`}, |
| 284 | {"1;2;3;\n", |
| 285 | `(ExprStmt X=1)(ExprStmt X=2)(ExprStmt X=3)`}, |
| 286 | {"f();g()\n", |
| 287 | `(ExprStmt X=(CallExpr Fn=f))(ExprStmt X=(CallExpr Fn=g))`}, |
| 288 | {"f();\n", |
| 289 | `(ExprStmt X=(CallExpr Fn=f))`}, |
| 290 | {"f(\n\n\n\n\n\n\n)\n", |
| 291 | `(ExprStmt X=(CallExpr Fn=f))`}, |
| 292 | // complex statements |
| 293 | {"def f():\n pass\n\n", |
alandonovan | 6ddc71c | 2019-06-04 09:08:55 -0400 | [diff] [blame] | 294 | `(DefStmt Name=f Body=((BranchStmt Token=pass)))`}, |
alandonovan | 30e71c6 | 2019-01-04 13:48:12 -0500 | [diff] [blame] | 295 | {"if cond:\n pass\n\n", |
| 296 | `(IfStmt Cond=cond True=((BranchStmt Token=pass)))`}, |
| 297 | // Even as a 1-liner, the following blank line is required. |
| 298 | {"if cond: pass\n\n", |
| 299 | `(IfStmt Cond=cond True=((BranchStmt Token=pass)))`}, |
alandonovan | 1258e4d | 2019-01-25 10:19:30 -0500 | [diff] [blame] | 300 | // github.com/google/starlark-go/issues/121 |
| 301 | {"a; b; c\n", |
| 302 | `(ExprStmt X=a)(ExprStmt X=b)(ExprStmt X=c)`}, |
| 303 | {"a; b c\n", |
| 304 | `invalid syntax`}, |
alandonovan | 30e71c6 | 2019-01-04 13:48:12 -0500 | [diff] [blame] | 305 | } { |
| 306 | |
| 307 | // Fake readline input from string. |
| 308 | // The ! suffix, which would cause a parse error, |
| 309 | // tests that the parser doesn't read more than necessary. |
| 310 | sc := bufio.NewScanner(strings.NewReader(test.input + "!")) |
| 311 | readline := func() ([]byte, error) { |
| 312 | if sc.Scan() { |
| 313 | return []byte(sc.Text() + "\n"), nil |
| 314 | } |
| 315 | return nil, sc.Err() |
| 316 | } |
| 317 | |
alandonovan | 1258e4d | 2019-01-25 10:19:30 -0500 | [diff] [blame] | 318 | var got string |
alandonovan | 30e71c6 | 2019-01-04 13:48:12 -0500 | [diff] [blame] | 319 | f, err := syntax.ParseCompoundStmt("foo.star", readline) |
| 320 | if err != nil { |
alandonovan | 1258e4d | 2019-01-25 10:19:30 -0500 | [diff] [blame] | 321 | got = stripPos(err) |
| 322 | } else { |
| 323 | for _, stmt := range f.Stmts { |
| 324 | got += treeString(stmt) |
| 325 | } |
alandonovan | 30e71c6 | 2019-01-04 13:48:12 -0500 | [diff] [blame] | 326 | } |
| 327 | if test.want != got { |
| 328 | t.Errorf("parse `%s` = %s, want %s", test.input, got, test.want) |
| 329 | } |
| 330 | } |
| 331 | } |
| 332 | |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 333 | func stripPos(err error) string { |
| 334 | s := err.Error() |
| 335 | if i := strings.Index(s, ": "); i >= 0 { |
| 336 | s = s[i+len(": "):] // strip file:line:col |
| 337 | } |
| 338 | return s |
| 339 | } |
| 340 | |
| 341 | // treeString prints a syntax node as a parenthesized tree. |
| 342 | // Idents are printed as foo and Literals as "foo" or 42. |
| 343 | // Structs are printed as (type name=value ...). |
| 344 | // Only non-empty fields are shown. |
| 345 | func treeString(n syntax.Node) string { |
| 346 | var buf bytes.Buffer |
| 347 | writeTree(&buf, reflect.ValueOf(n)) |
| 348 | return buf.String() |
| 349 | } |
| 350 | |
| 351 | func writeTree(out *bytes.Buffer, x reflect.Value) { |
| 352 | switch x.Kind() { |
| 353 | case reflect.String, reflect.Int, reflect.Bool: |
| 354 | fmt.Fprintf(out, "%v", x.Interface()) |
| 355 | case reflect.Ptr, reflect.Interface: |
| 356 | if elem := x.Elem(); elem.Kind() == 0 { |
| 357 | out.WriteString("nil") |
| 358 | } else { |
| 359 | writeTree(out, elem) |
| 360 | } |
| 361 | case reflect.Struct: |
| 362 | switch v := x.Interface().(type) { |
| 363 | case syntax.Literal: |
alandonovan | ebe61bd | 2021-02-12 16:57:32 -0500 | [diff] [blame] | 364 | switch v.Token { |
| 365 | case syntax.STRING: |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 366 | fmt.Fprintf(out, "%q", v.Value) |
alandonovan | ebe61bd | 2021-02-12 16:57:32 -0500 | [diff] [blame] | 367 | case syntax.BYTES: |
| 368 | fmt.Fprintf(out, "b%q", v.Value) |
| 369 | case syntax.INT: |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 370 | fmt.Fprintf(out, "%d", v.Value) |
| 371 | } |
| 372 | return |
| 373 | case syntax.Ident: |
| 374 | out.WriteString(v.Name) |
| 375 | return |
| 376 | } |
| 377 | fmt.Fprintf(out, "(%s", strings.TrimPrefix(x.Type().String(), "syntax.")) |
| 378 | for i, n := 0, x.NumField(); i < n; i++ { |
| 379 | f := x.Field(i) |
| 380 | if f.Type() == reflect.TypeOf(syntax.Position{}) { |
| 381 | continue // skip positions |
| 382 | } |
| 383 | name := x.Type().Field(i).Name |
Laurent Le Brun | 689fc22 | 2018-02-22 19:37:18 +0100 | [diff] [blame] | 384 | if name == "commentsRef" { |
| 385 | continue // skip comments fields |
| 386 | } |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 387 | if f.Type() == reflect.TypeOf(syntax.Token(0)) { |
| 388 | fmt.Fprintf(out, " %s=%s", name, f.Interface()) |
| 389 | continue |
| 390 | } |
| 391 | |
| 392 | switch f.Kind() { |
| 393 | case reflect.Slice: |
| 394 | if n := f.Len(); n > 0 { |
| 395 | fmt.Fprintf(out, " %s=(", name) |
| 396 | for i := 0; i < n; i++ { |
| 397 | if i > 0 { |
| 398 | out.WriteByte(' ') |
| 399 | } |
| 400 | writeTree(out, f.Index(i)) |
| 401 | } |
| 402 | out.WriteByte(')') |
| 403 | } |
| 404 | continue |
| 405 | case reflect.Ptr, reflect.Interface: |
| 406 | if f.IsNil() { |
| 407 | continue |
| 408 | } |
Alan Donovan | 5215385 | 2019-02-13 19:18:15 -0500 | [diff] [blame] | 409 | case reflect.Int: |
| 410 | if f.Int() != 0 { |
| 411 | fmt.Fprintf(out, " %s=%d", name, f.Int()) |
| 412 | } |
| 413 | continue |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 414 | case reflect.Bool: |
| 415 | if f.Bool() { |
| 416 | fmt.Fprintf(out, " %s", name) |
| 417 | } |
| 418 | continue |
| 419 | } |
| 420 | fmt.Fprintf(out, " %s=", name) |
| 421 | writeTree(out, f) |
| 422 | } |
| 423 | fmt.Fprintf(out, ")") |
| 424 | default: |
| 425 | fmt.Fprintf(out, "%T", x.Interface()) |
| 426 | } |
| 427 | } |
| 428 | |
| 429 | func TestParseErrors(t *testing.T) { |
Alan Donovan | 6beab7e | 2018-10-31 17:53:09 -0400 | [diff] [blame] | 430 | filename := starlarktest.DataFile("syntax", "testdata/errors.star") |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 431 | for _, chunk := range chunkedfile.Read(filename, t) { |
Laurent Le Brun | 689fc22 | 2018-02-22 19:37:18 +0100 | [diff] [blame] | 432 | _, err := syntax.Parse(filename, chunk.Source, 0) |
Alan Donovan | 312d1a5 | 2017-10-02 10:10:28 -0400 | [diff] [blame] | 433 | switch err := err.(type) { |
| 434 | case nil: |
| 435 | // ok |
| 436 | case syntax.Error: |
| 437 | chunk.GotError(int(err.Pos.Line), err.Msg) |
| 438 | default: |
| 439 | t.Error(err) |
| 440 | } |
| 441 | chunk.Done() |
| 442 | } |
| 443 | } |
| 444 | |
alandonovan | 0a10e4f | 2021-02-08 12:20:22 -0500 | [diff] [blame] | 445 | func TestFilePortion(t *testing.T) { |
| 446 | // Imagine that the Starlark file or expression print(x.f) is extracted |
| 447 | // from the middle of a file in some hypothetical template language; |
| 448 | // see https://github.com/google/starlark-go/issues/346. For example: |
| 449 | // -- |
| 450 | // {{loop x seq}} |
| 451 | // {{print(x.f)}} |
| 452 | // {{end}} |
| 453 | // -- |
| 454 | fp := syntax.FilePortion{Content: []byte("print(x.f)"), FirstLine: 2, FirstCol: 4} |
| 455 | file, err := syntax.Parse("foo.template", fp, 0) |
| 456 | if err != nil { |
| 457 | t.Fatal(err) |
| 458 | } |
| 459 | span := fmt.Sprint(file.Stmts[0].Span()) |
| 460 | want := "foo.template:2:4 foo.template:2:14" |
| 461 | if span != want { |
| 462 | t.Errorf("wrong span: got %q, want %q", span, want) |
| 463 | } |
| 464 | } |
| 465 | |
alandonovan | 30e71c6 | 2019-01-04 13:48:12 -0500 | [diff] [blame] | 466 | // dataFile is the same as starlarktest.DataFile. |
| 467 | // We make a copy to avoid a dependency cycle. |
| 468 | var dataFile = func(pkgdir, filename string) string { |
| 469 | return filepath.Join(build.Default.GOPATH, "src/go.starlark.net", pkgdir, filename) |
| 470 | } |
| 471 | |
| 472 | func BenchmarkParse(b *testing.B) { |
| 473 | filename := dataFile("syntax", "testdata/scan.star") |
| 474 | b.StopTimer() |
| 475 | data, err := ioutil.ReadFile(filename) |
| 476 | if err != nil { |
| 477 | b.Fatal(err) |
| 478 | } |
| 479 | b.StartTimer() |
| 480 | |
| 481 | for i := 0; i < b.N; i++ { |
| 482 | _, err := syntax.Parse(filename, data, 0) |
| 483 | if err != nil { |
| 484 | b.Fatal(err) |
| 485 | } |
| 486 | } |
| 487 | } |