Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 1 | // Copyright 2019 The Go Authors. All rights reserved. |
| 2 | // Use of this source code is governed by a BSD-style |
| 3 | // license that can be found in the LICENSE file. |
| 4 | |
Joe Tsai | f398784 | 2019-03-02 13:35:17 -0800 | [diff] [blame] | 5 | // +build ignore |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 6 | |
Joe Tsai | f398784 | 2019-03-02 13:35:17 -0800 | [diff] [blame] | 7 | package main |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 8 | |
| 9 | import ( |
| 10 | "archive/tar" |
| 11 | "bytes" |
| 12 | "compress/gzip" |
| 13 | "flag" |
| 14 | "fmt" |
| 15 | "io" |
| 16 | "io/ioutil" |
| 17 | "net/http" |
| 18 | "os" |
| 19 | "os/exec" |
Damien Neil | a80229e | 2019-06-20 12:53:48 -0700 | [diff] [blame] | 20 | "path" |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 21 | "path/filepath" |
| 22 | "regexp" |
| 23 | "runtime" |
| 24 | "strings" |
Joe Tsai | 62200db | 2019-07-11 17:34:10 -0700 | [diff] [blame] | 25 | "sync" |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 26 | "testing" |
Joe Tsai | f398784 | 2019-03-02 13:35:17 -0800 | [diff] [blame] | 27 | "time" |
Joe Tsai | 4f3de44 | 2019-08-07 19:33:51 -0700 | [diff] [blame] | 28 | |
| 29 | "google.golang.org/protobuf/runtime/protoimpl" |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 30 | ) |
| 31 | |
| 32 | var ( |
Joe Tsai | 4f3de44 | 2019-08-07 19:33:51 -0700 | [diff] [blame] | 33 | regenerate = flag.Bool("regenerate", false, "regenerate files") |
| 34 | buildRelease = flag.Bool("buildRelease", false, "build release binaries") |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 35 | |
Joe Tsai | abd06a8 | 2019-08-16 00:39:27 -0700 | [diff] [blame] | 36 | protobufVersion = "3.9.1" |
Joe Tsai | 08ff730 | 2019-08-21 13:54:27 -0700 | [diff] [blame] | 37 | golangVersions = []string{"1.9.7", "1.10.8", "1.11.13", "1.12.9", "1.13rc1"} |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 38 | golangLatest = golangVersions[len(golangVersions)-1] |
| 39 | |
Joe Tsai | 9e88bc0 | 2019-03-12 01:30:40 -0700 | [diff] [blame] | 40 | // purgeTimeout determines the maximum age of unused sub-directories. |
| 41 | purgeTimeout = 30 * 24 * time.Hour // 1 month |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 42 | |
| 43 | // Variables initialized by mustInitDeps. |
Herbie Ong | d64dceb | 2019-04-25 01:19:57 -0700 | [diff] [blame] | 44 | goPath string |
| 45 | modulePath string |
| 46 | protobufPath string |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 47 | ) |
| 48 | |
| 49 | func Test(t *testing.T) { |
| 50 | mustInitDeps(t) |
Joe Tsai | 4f3de44 | 2019-08-07 19:33:51 -0700 | [diff] [blame] | 51 | mustHandleFlags(t) |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 52 | |
Joe Tsai | 62200db | 2019-07-11 17:34:10 -0700 | [diff] [blame] | 53 | var wg sync.WaitGroup |
| 54 | sema := make(chan bool, (runtime.NumCPU()+1)/2) |
| 55 | for i := range golangVersions { |
| 56 | goVersion := golangVersions[i] |
| 57 | goLabel := "Go" + goVersion |
| 58 | runGo := func(label, workDir string, args ...string) { |
| 59 | wg.Add(1) |
| 60 | sema <- true |
| 61 | go func() { |
| 62 | defer wg.Done() |
| 63 | defer func() { <-sema }() |
| 64 | t.Run(goLabel+"/"+label, func(t *testing.T) { |
| 65 | args[0] += goVersion |
Joe Tsai | 7164af5 | 2019-08-07 19:27:43 -0700 | [diff] [blame] | 66 | command{Dir: workDir}.mustRun(t, args...) |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 67 | }) |
Joe Tsai | 62200db | 2019-07-11 17:34:10 -0700 | [diff] [blame] | 68 | }() |
| 69 | } |
| 70 | |
| 71 | workDir := filepath.Join(goPath, "src", modulePath) |
| 72 | runGo("Normal", workDir, "go", "test", "-race", "./...") |
| 73 | runGo("PureGo", workDir, "go", "test", "-race", "-tags", "purego", "./...") |
| 74 | runGo("Reflect", workDir, "go", "test", "-race", "-tags", "protoreflect", "./...") |
| 75 | if goVersion == golangLatest { |
Joe Tsai | 1799d11 | 2019-08-08 13:31:59 -0700 | [diff] [blame] | 76 | runGo("ProtoLegacy", workDir, "go", "test", "-race", "-tags", "protolegacy", "./...") |
Joe Tsai | 62200db | 2019-07-11 17:34:10 -0700 | [diff] [blame] | 77 | runGo("ProtocGenGo", "cmd/protoc-gen-go/testdata", "go", "test") |
| 78 | runGo("ProtocGenGoGRPC", "cmd/protoc-gen-go-grpc/testdata", "go", "test") |
Joe Tsai | dd271b6 | 2019-08-16 01:09:33 -0700 | [diff] [blame] | 79 | runGo("Conformance", "internal/conformance", "go", "test", "-execute") |
Joe Tsai | 62200db | 2019-07-11 17:34:10 -0700 | [diff] [blame] | 80 | } |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 81 | } |
Joe Tsai | 62200db | 2019-07-11 17:34:10 -0700 | [diff] [blame] | 82 | wg.Wait() |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 83 | |
| 84 | t.Run("GeneratedGoFiles", func(t *testing.T) { |
Joe Tsai | 1799d11 | 2019-08-08 13:31:59 -0700 | [diff] [blame] | 85 | diff := mustRunCommand(t, "go", "run", "-tags", "protolegacy", "./internal/cmd/generate-types") |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 86 | if strings.TrimSpace(diff) != "" { |
| 87 | t.Fatalf("stale generated files:\n%v", diff) |
| 88 | } |
Joe Tsai | 1799d11 | 2019-08-08 13:31:59 -0700 | [diff] [blame] | 89 | diff = mustRunCommand(t, "go", "run", "-tags", "protolegacy", "./internal/cmd/generate-protos") |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 90 | if strings.TrimSpace(diff) != "" { |
| 91 | t.Fatalf("stale generated files:\n%v", diff) |
| 92 | } |
| 93 | }) |
| 94 | t.Run("FormattedGoFiles", func(t *testing.T) { |
Joe Tsai | 7164af5 | 2019-08-07 19:27:43 -0700 | [diff] [blame] | 95 | files := strings.Split(strings.TrimSpace(mustRunCommand(t, "git", "ls-files", "*.go")), "\n") |
| 96 | diff := mustRunCommand(t, append([]string{"gofmt", "-d"}, files...)...) |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 97 | if strings.TrimSpace(diff) != "" { |
| 98 | t.Fatalf("unformatted source files:\n%v", diff) |
| 99 | } |
| 100 | }) |
| 101 | t.Run("CommittedGitChanges", func(t *testing.T) { |
Joe Tsai | 7164af5 | 2019-08-07 19:27:43 -0700 | [diff] [blame] | 102 | diff := mustRunCommand(t, "git", "diff", "--no-prefix", "HEAD") |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 103 | if strings.TrimSpace(diff) != "" { |
| 104 | t.Fatalf("uncommitted changes:\n%v", diff) |
| 105 | } |
| 106 | }) |
| 107 | t.Run("TrackedGitFiles", func(t *testing.T) { |
Joe Tsai | 7164af5 | 2019-08-07 19:27:43 -0700 | [diff] [blame] | 108 | diff := mustRunCommand(t, "git", "ls-files", "--others", "--exclude-standard") |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 109 | if strings.TrimSpace(diff) != "" { |
| 110 | t.Fatalf("untracked files:\n%v", diff) |
| 111 | } |
| 112 | }) |
| 113 | } |
| 114 | |
| 115 | func mustInitDeps(t *testing.T) { |
| 116 | check := func(err error) { |
| 117 | t.Helper() |
| 118 | if err != nil { |
| 119 | t.Fatal(err) |
| 120 | } |
| 121 | } |
| 122 | |
| 123 | // Determine the directory to place the test directory. |
| 124 | repoRoot, err := os.Getwd() |
| 125 | check(err) |
| 126 | testDir := filepath.Join(repoRoot, ".cache") |
| 127 | check(os.MkdirAll(testDir, 0775)) |
| 128 | |
Joe Tsai | f398784 | 2019-03-02 13:35:17 -0800 | [diff] [blame] | 129 | // Travis-CI has a hard-coded timeout where it kills the test after |
| 130 | // 10 minutes of a lack of activity on stdout. |
| 131 | // We work around this restriction by periodically printing the timestamp. |
| 132 | ticker := time.NewTicker(5 * time.Minute) |
| 133 | done := make(chan struct{}) |
| 134 | go func() { |
| 135 | now := time.Now() |
| 136 | for { |
| 137 | select { |
| 138 | case t := <-ticker.C: |
| 139 | fmt.Printf("\tt=%0.fmin\n", t.Sub(now).Minutes()) |
| 140 | case <-done: |
| 141 | return |
| 142 | } |
| 143 | } |
| 144 | }() |
| 145 | defer close(done) |
| 146 | defer ticker.Stop() |
| 147 | |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 148 | // Delete the current directory if non-empty, |
| 149 | // which only occurs if a dependency failed to initialize properly. |
| 150 | var workingDir string |
| 151 | defer func() { |
| 152 | if workingDir != "" { |
| 153 | os.RemoveAll(workingDir) // best-effort |
| 154 | } |
| 155 | }() |
| 156 | |
| 157 | // Delete other sub-directories that are no longer relevant. |
| 158 | defer func() { |
| 159 | subDirs := map[string]bool{"bin": true, "gocache": true, "gopath": true} |
| 160 | subDirs["protobuf-"+protobufVersion] = true |
| 161 | for _, v := range golangVersions { |
| 162 | subDirs["go"+v] = true |
| 163 | } |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 164 | |
Joe Tsai | 9e88bc0 | 2019-03-12 01:30:40 -0700 | [diff] [blame] | 165 | now := time.Now() |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 166 | fis, _ := ioutil.ReadDir(testDir) |
| 167 | for _, fi := range fis { |
Joe Tsai | 9e88bc0 | 2019-03-12 01:30:40 -0700 | [diff] [blame] | 168 | if subDirs[fi.Name()] { |
| 169 | os.Chtimes(filepath.Join(testDir, fi.Name()), now, now) // best-effort |
| 170 | continue |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 171 | } |
Joe Tsai | 9e88bc0 | 2019-03-12 01:30:40 -0700 | [diff] [blame] | 172 | if now.Sub(fi.ModTime()) < purgeTimeout { |
| 173 | continue |
| 174 | } |
| 175 | fmt.Printf("delete %v\n", fi.Name()) |
| 176 | os.RemoveAll(filepath.Join(testDir, fi.Name())) // best-effort |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 177 | } |
| 178 | }() |
| 179 | |
| 180 | // The bin directory contains symlinks to each tool by version. |
| 181 | // It is safe to delete this directory and run the test script from scratch. |
| 182 | binPath := filepath.Join(testDir, "bin") |
| 183 | check(os.RemoveAll(binPath)) |
| 184 | check(os.Mkdir(binPath, 0775)) |
| 185 | check(os.Setenv("PATH", binPath+":"+os.Getenv("PATH"))) |
| 186 | registerBinary := func(name, path string) { |
| 187 | check(os.Symlink(path, filepath.Join(binPath, name))) |
| 188 | } |
| 189 | |
| 190 | // Download and build the protobuf toolchain. |
| 191 | // We avoid downloading the pre-compiled binaries since they do not contain |
| 192 | // the conformance test runner. |
| 193 | workingDir = filepath.Join(testDir, "protobuf-"+protobufVersion) |
Herbie Ong | d64dceb | 2019-04-25 01:19:57 -0700 | [diff] [blame] | 194 | protobufPath = workingDir |
| 195 | if _, err := os.Stat(protobufPath); err != nil { |
| 196 | fmt.Printf("download %v\n", filepath.Base(protobufPath)) |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 197 | url := fmt.Sprintf("https://github.com/google/protobuf/releases/download/v%v/protobuf-all-%v.tar.gz", protobufVersion, protobufVersion) |
Herbie Ong | d64dceb | 2019-04-25 01:19:57 -0700 | [diff] [blame] | 198 | downloadArchive(check, protobufPath, url, "protobuf-"+protobufVersion) |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 199 | |
Herbie Ong | d64dceb | 2019-04-25 01:19:57 -0700 | [diff] [blame] | 200 | fmt.Printf("build %v\n", filepath.Base(protobufPath)) |
Joe Tsai | 7164af5 | 2019-08-07 19:27:43 -0700 | [diff] [blame] | 201 | command{Dir: protobufPath}.mustRun(t, "./autogen.sh") |
| 202 | command{Dir: protobufPath}.mustRun(t, "./configure") |
| 203 | command{Dir: protobufPath}.mustRun(t, "make") |
| 204 | command{Dir: filepath.Join(protobufPath, "conformance")}.mustRun(t, "make") |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 205 | } |
Damien Neil | a80229e | 2019-06-20 12:53:48 -0700 | [diff] [blame] | 206 | // The benchmark directory isn't present in the release download, |
| 207 | // so fetch needed files directly. |
| 208 | for _, path := range benchmarkProtos { |
| 209 | src := fmt.Sprintf("https://raw.githubusercontent.com/protocolbuffers/protobuf/v%v/%v", protobufVersion, path) |
| 210 | dst := filepath.Join(protobufPath, path) |
| 211 | if _, err := os.Stat(dst); err != nil { |
| 212 | downloadFile(check, dst, src) |
| 213 | } |
| 214 | } |
| 215 | benchdataPath := filepath.Join(testDir, "benchdata") |
| 216 | for _, path := range []string{ |
| 217 | "benchmarks/datasets/google_message1/proto2/dataset.google_message1_proto2.pb", |
| 218 | "benchmarks/datasets/google_message1/proto3/dataset.google_message1_proto3.pb", |
| 219 | "benchmarks/datasets/google_message2/dataset.google_message2.pb", |
| 220 | } { |
| 221 | src := fmt.Sprintf("https://raw.githubusercontent.com/protocolbuffers/protobuf/v%v/%v", protobufVersion, path) |
| 222 | dst := filepath.Join(benchdataPath, filepath.Base(path)) |
| 223 | if _, err := os.Stat(dst); err != nil { |
| 224 | downloadFile(check, dst, src) |
| 225 | } |
| 226 | } |
Herbie Ong | d64dceb | 2019-04-25 01:19:57 -0700 | [diff] [blame] | 227 | patchProtos(check, protobufPath) |
| 228 | check(os.Setenv("PROTOBUF_ROOT", protobufPath)) // for generate-protos |
| 229 | registerBinary("conform-test-runner", filepath.Join(protobufPath, "conformance", "conformance-test-runner")) |
| 230 | registerBinary("protoc", filepath.Join(protobufPath, "src", "protoc")) |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 231 | workingDir = "" |
| 232 | |
| 233 | // Download each Go toolchain version. |
| 234 | for _, v := range golangVersions { |
| 235 | workingDir = filepath.Join(testDir, "go"+v) |
| 236 | if _, err := os.Stat(workingDir); err != nil { |
| 237 | fmt.Printf("download %v\n", filepath.Base(workingDir)) |
| 238 | url := fmt.Sprintf("https://dl.google.com/go/go%v.%v-%v.tar.gz", v, runtime.GOOS, runtime.GOARCH) |
Joe Tsai | f398784 | 2019-03-02 13:35:17 -0800 | [diff] [blame] | 239 | downloadArchive(check, workingDir, url, "go") |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 240 | } |
| 241 | registerBinary("go"+v, filepath.Join(workingDir, "bin", "go")) |
| 242 | } |
| 243 | registerBinary("go", filepath.Join(testDir, "go"+golangLatest, "bin", "go")) |
| 244 | registerBinary("gofmt", filepath.Join(testDir, "go"+golangLatest, "bin", "gofmt")) |
| 245 | workingDir = "" |
| 246 | |
| 247 | // Travis-CI sets GOROOT, which confuses invocations of the Go toolchain. |
| 248 | // Explicitly clear GOROOT, so each toolchain uses their default GOROOT. |
| 249 | check(os.Unsetenv("GOROOT")) |
| 250 | |
Joe Tsai | 6a2180f | 2019-07-11 16:34:17 -0700 | [diff] [blame] | 251 | // Set a cache directory outside the test directory. |
| 252 | check(os.Setenv("GOCACHE", filepath.Join(repoRoot, ".gocache"))) |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 253 | |
| 254 | // Setup GOPATH for pre-module support (i.e., go1.10 and earlier). |
| 255 | goPath = filepath.Join(testDir, "gopath") |
Joe Tsai | 7164af5 | 2019-08-07 19:27:43 -0700 | [diff] [blame] | 256 | modulePath = strings.TrimSpace(command{Dir: testDir}.mustRun(t, "go", "list", "-m", "-f", "{{.Path}}")) |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 257 | check(os.RemoveAll(filepath.Join(goPath, "src"))) |
| 258 | check(os.MkdirAll(filepath.Join(goPath, "src", filepath.Dir(modulePath)), 0775)) |
| 259 | check(os.Symlink(repoRoot, filepath.Join(goPath, "src", modulePath))) |
Joe Tsai | 7164af5 | 2019-08-07 19:27:43 -0700 | [diff] [blame] | 260 | command{Dir: repoRoot}.mustRun(t, "go", "mod", "tidy") |
| 261 | command{Dir: repoRoot}.mustRun(t, "go", "mod", "vendor") |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 262 | check(os.Setenv("GOPATH", goPath)) |
| 263 | } |
| 264 | |
Damien Neil | a80229e | 2019-06-20 12:53:48 -0700 | [diff] [blame] | 265 | func downloadFile(check func(error), dstPath, srcURL string) { |
| 266 | resp, err := http.Get(srcURL) |
| 267 | check(err) |
| 268 | defer resp.Body.Close() |
| 269 | |
| 270 | check(os.MkdirAll(filepath.Dir(dstPath), 0775)) |
| 271 | f, err := os.Create(dstPath) |
| 272 | check(err) |
| 273 | |
| 274 | _, err = io.Copy(f, resp.Body) |
| 275 | check(err) |
| 276 | } |
| 277 | |
Joe Tsai | f398784 | 2019-03-02 13:35:17 -0800 | [diff] [blame] | 278 | func downloadArchive(check func(error), dstPath, srcURL, skipPrefix string) { |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 279 | check(os.RemoveAll(dstPath)) |
| 280 | |
| 281 | resp, err := http.Get(srcURL) |
| 282 | check(err) |
| 283 | defer resp.Body.Close() |
| 284 | |
| 285 | zr, err := gzip.NewReader(resp.Body) |
| 286 | check(err) |
| 287 | |
| 288 | tr := tar.NewReader(zr) |
| 289 | for { |
| 290 | h, err := tr.Next() |
| 291 | if err == io.EOF { |
| 292 | return |
| 293 | } |
| 294 | check(err) |
| 295 | |
Joe Tsai | f398784 | 2019-03-02 13:35:17 -0800 | [diff] [blame] | 296 | // Skip directories or files outside the prefix directory. |
| 297 | if len(skipPrefix) > 0 { |
| 298 | if !strings.HasPrefix(h.Name, skipPrefix) { |
| 299 | continue |
| 300 | } |
| 301 | if len(h.Name) > len(skipPrefix) && h.Name[len(skipPrefix)] != '/' { |
| 302 | continue |
| 303 | } |
| 304 | } |
| 305 | |
| 306 | path := strings.TrimPrefix(strings.TrimPrefix(h.Name, skipPrefix), "/") |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 307 | path = filepath.Join(dstPath, filepath.FromSlash(path)) |
| 308 | mode := os.FileMode(h.Mode & 0777) |
| 309 | switch h.Typeflag { |
| 310 | case tar.TypeReg: |
| 311 | b, err := ioutil.ReadAll(tr) |
| 312 | check(err) |
| 313 | check(ioutil.WriteFile(path, b, mode)) |
| 314 | case tar.TypeDir: |
| 315 | check(os.Mkdir(path, mode)) |
| 316 | } |
| 317 | } |
| 318 | } |
| 319 | |
| 320 | // patchProtos patches proto files with v2 locations of Go packages. |
| 321 | // TODO: Commit these changes upstream. |
| 322 | func patchProtos(check func(error), repoRoot string) { |
| 323 | javaPackageRx := regexp.MustCompile(`^option\s+java_package\s*=\s*".*"\s*;\s*$`) |
| 324 | goPackageRx := regexp.MustCompile(`^option\s+go_package\s*=\s*".*"\s*;\s*$`) |
| 325 | files := map[string]string{ |
Joe Tsai | a95b29f | 2019-05-16 12:47:20 -0700 | [diff] [blame] | 326 | "src/google/protobuf/any.proto": "google.golang.org/protobuf/types/known/anypb", |
| 327 | "src/google/protobuf/api.proto": "google.golang.org/protobuf/types/known/apipb", |
| 328 | "src/google/protobuf/duration.proto": "google.golang.org/protobuf/types/known/durationpb", |
| 329 | "src/google/protobuf/empty.proto": "google.golang.org/protobuf/types/known/emptypb", |
| 330 | "src/google/protobuf/field_mask.proto": "google.golang.org/protobuf/types/known/fieldmaskpb", |
| 331 | "src/google/protobuf/source_context.proto": "google.golang.org/protobuf/types/known/sourcecontextpb", |
| 332 | "src/google/protobuf/struct.proto": "google.golang.org/protobuf/types/known/structpb", |
| 333 | "src/google/protobuf/timestamp.proto": "google.golang.org/protobuf/types/known/timestamppb", |
| 334 | "src/google/protobuf/type.proto": "google.golang.org/protobuf/types/known/typepb", |
| 335 | "src/google/protobuf/wrappers.proto": "google.golang.org/protobuf/types/known/wrapperspb", |
| 336 | "src/google/protobuf/descriptor.proto": "google.golang.org/protobuf/types/descriptorpb", |
| 337 | "src/google/protobuf/compiler/plugin.proto": "google.golang.org/protobuf/types/pluginpb", |
| 338 | "conformance/conformance.proto": "google.golang.org/protobuf/internal/testprotos/conformance", |
| 339 | "src/google/protobuf/test_messages_proto2.proto": "google.golang.org/protobuf/internal/testprotos/conformance", |
| 340 | "src/google/protobuf/test_messages_proto3.proto": "google.golang.org/protobuf/internal/testprotos/conformance", |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 341 | } |
Damien Neil | a80229e | 2019-06-20 12:53:48 -0700 | [diff] [blame] | 342 | for _, p := range benchmarkProtos { |
| 343 | files[p] = path.Dir("google.golang.org/protobuf/internal/testprotos/" + p) |
| 344 | } |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 345 | for pbpath, gopath := range files { |
| 346 | b, err := ioutil.ReadFile(filepath.Join(repoRoot, pbpath)) |
| 347 | check(err) |
| 348 | ss := strings.Split(string(b), "\n") |
| 349 | |
| 350 | // Locate java_package and (possible) go_package options. |
| 351 | javaPackageIdx, goPackageIdx := -1, -1 |
| 352 | for i, s := range ss { |
| 353 | if javaPackageIdx < 0 && javaPackageRx.MatchString(s) { |
| 354 | javaPackageIdx = i |
| 355 | } |
| 356 | if goPackageIdx < 0 && goPackageRx.MatchString(s) { |
| 357 | goPackageIdx = i |
| 358 | } |
| 359 | } |
| 360 | |
| 361 | // Ensure the proto file has the correct go_package option. |
| 362 | opt := `option go_package = "` + gopath + `";` |
| 363 | if goPackageIdx >= 0 { |
| 364 | if ss[goPackageIdx] == opt { |
| 365 | continue // no changes needed |
| 366 | } |
| 367 | ss[goPackageIdx] = opt |
| 368 | } else { |
| 369 | // Insert go_package option before java_package option. |
| 370 | ss = append(ss[:javaPackageIdx], append([]string{opt}, ss[javaPackageIdx:]...)...) |
| 371 | } |
| 372 | |
| 373 | fmt.Println("patch " + pbpath) |
| 374 | b = []byte(strings.Join(ss, "\n")) |
| 375 | check(ioutil.WriteFile(filepath.Join(repoRoot, pbpath), b, 0664)) |
| 376 | } |
| 377 | } |
| 378 | |
Joe Tsai | 4f3de44 | 2019-08-07 19:33:51 -0700 | [diff] [blame] | 379 | func mustHandleFlags(t *testing.T) { |
| 380 | if *regenerate { |
| 381 | t.Run("Generate", func(t *testing.T) { |
| 382 | fmt.Print(mustRunCommand(t, "go", "run", "-tags", "protolegacy", "./internal/cmd/generate-types", "-execute")) |
| 383 | fmt.Print(mustRunCommand(t, "go", "run", "-tags", "protolegacy", "./internal/cmd/generate-protos", "-execute")) |
| 384 | files := strings.Split(strings.TrimSpace(mustRunCommand(t, "git", "ls-files", "*.go")), "\n") |
| 385 | mustRunCommand(t, append([]string{"gofmt", "-w"}, files...)...) |
| 386 | }) |
| 387 | } |
| 388 | if *buildRelease { |
| 389 | t.Run("BuildRelease", func(t *testing.T) { |
| 390 | v := protoimpl.VersionString() |
| 391 | for _, goos := range []string{"linux", "darwin", "windows"} { |
| 392 | for _, goarch := range []string{"386", "amd64"} { |
| 393 | binPath := filepath.Join("bin", fmt.Sprintf("protoc-gen-go.%v.%v.%v", v, goos, goarch)) |
| 394 | |
| 395 | // Build the binary. |
| 396 | cmd := command{Env: append(os.Environ(), "GOOS="+goos, "GOARCH="+goarch)} |
| 397 | cmd.mustRun(t, "go", "build", "-trimpath", "-ldflags", "-s -w", "-o", binPath, "./cmd/protoc-gen-go") |
| 398 | |
| 399 | // Archive and compress the binary. |
| 400 | in, err := ioutil.ReadFile(binPath) |
| 401 | if err != nil { |
| 402 | t.Fatal(err) |
| 403 | } |
| 404 | out := new(bytes.Buffer) |
| 405 | gz, _ := gzip.NewWriterLevel(out, gzip.BestCompression) |
| 406 | gz.Comment = fmt.Sprintf("protoc-gen-go VERSION=%v GOOS=%v GOARCH=%v", v, goos, goarch) |
| 407 | tw := tar.NewWriter(gz) |
| 408 | tw.WriteHeader(&tar.Header{ |
| 409 | Name: "protoc-gen-go", |
| 410 | Mode: int64(0775), |
| 411 | Size: int64(len(in)), |
| 412 | }) |
| 413 | tw.Write(in) |
| 414 | tw.Close() |
| 415 | gz.Close() |
| 416 | if err := ioutil.WriteFile(binPath+".tar.gz", out.Bytes(), 0664); err != nil { |
| 417 | t.Fatal(err) |
| 418 | } |
| 419 | } |
| 420 | } |
| 421 | }) |
| 422 | } |
| 423 | if *regenerate || *buildRelease { |
| 424 | t.SkipNow() |
| 425 | } |
| 426 | } |
| 427 | |
Joe Tsai | 7164af5 | 2019-08-07 19:27:43 -0700 | [diff] [blame] | 428 | type command struct { |
| 429 | Dir string |
| 430 | Env []string |
| 431 | } |
| 432 | |
| 433 | func (c command) mustRun(t *testing.T, args ...string) string { |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 434 | t.Helper() |
| 435 | stdout := new(bytes.Buffer) |
Herbie Ong | 4630b3d | 2019-03-19 16:42:01 -0700 | [diff] [blame] | 436 | stderr := new(bytes.Buffer) |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 437 | cmd := exec.Command(args[0], args[1:]...) |
Joe Tsai | 7164af5 | 2019-08-07 19:27:43 -0700 | [diff] [blame] | 438 | cmd.Dir = "." |
| 439 | if c.Dir != "" { |
| 440 | cmd.Dir = c.Dir |
| 441 | } |
| 442 | cmd.Env = os.Environ() |
| 443 | if c.Env != nil { |
| 444 | cmd.Env = c.Env |
| 445 | } |
| 446 | cmd.Env = append(cmd.Env, "PWD="+cmd.Dir) |
Herbie Ong | 4630b3d | 2019-03-19 16:42:01 -0700 | [diff] [blame] | 447 | cmd.Stdout = stdout |
| 448 | cmd.Stderr = stderr |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 449 | if err := cmd.Run(); err != nil { |
Herbie Ong | 4630b3d | 2019-03-19 16:42:01 -0700 | [diff] [blame] | 450 | t.Fatalf("executing (%v): %v\n%s%s", strings.Join(args, " "), err, stdout.String(), stderr.String()) |
Joe Tsai | 707894e | 2019-03-01 12:50:52 -0800 | [diff] [blame] | 451 | } |
| 452 | return stdout.String() |
| 453 | } |
Damien Neil | a80229e | 2019-06-20 12:53:48 -0700 | [diff] [blame] | 454 | |
Joe Tsai | 7164af5 | 2019-08-07 19:27:43 -0700 | [diff] [blame] | 455 | func mustRunCommand(t *testing.T, args ...string) string { |
| 456 | t.Helper() |
| 457 | return command{}.mustRun(t, args...) |
| 458 | } |
| 459 | |
Damien Neil | a80229e | 2019-06-20 12:53:48 -0700 | [diff] [blame] | 460 | var benchmarkProtos = []string{ |
| 461 | "benchmarks/benchmarks.proto", |
| 462 | "benchmarks/datasets/google_message1/proto2/benchmark_message1_proto2.proto", |
| 463 | "benchmarks/datasets/google_message1/proto3/benchmark_message1_proto3.proto", |
| 464 | "benchmarks/datasets/google_message2/benchmark_message2.proto", |
| 465 | "benchmarks/datasets/google_message3/benchmark_message3.proto", |
| 466 | "benchmarks/datasets/google_message3/benchmark_message3_1.proto", |
| 467 | "benchmarks/datasets/google_message3/benchmark_message3_2.proto", |
| 468 | "benchmarks/datasets/google_message3/benchmark_message3_3.proto", |
| 469 | "benchmarks/datasets/google_message3/benchmark_message3_4.proto", |
| 470 | "benchmarks/datasets/google_message3/benchmark_message3_5.proto", |
| 471 | "benchmarks/datasets/google_message3/benchmark_message3_6.proto", |
| 472 | "benchmarks/datasets/google_message3/benchmark_message3_7.proto", |
| 473 | "benchmarks/datasets/google_message3/benchmark_message3_8.proto", |
| 474 | "benchmarks/datasets/google_message4/benchmark_message4.proto", |
| 475 | "benchmarks/datasets/google_message4/benchmark_message4_1.proto", |
| 476 | "benchmarks/datasets/google_message4/benchmark_message4_2.proto", |
| 477 | "benchmarks/datasets/google_message4/benchmark_message4_3.proto", |
| 478 | } |