protogen, cmd/protoc-gen-go: initial commit
Package protogen provides support for writing protoc plugins.
A "plugin" in this case is a program run by protoc to generate output.
The protoc-gen-go command is a protoc plugin to generate Go code.
cmd/protoc-gen-go/golden_test.go is mostly a straight copy from
the golden test in github.com/golang/protobuf.
Change-Id: I332d0df1e4b60bb8cd926320b8721e16b99a4b71
Reviewed-on: https://go-review.googlesource.com/130175
Reviewed-by: Joe Tsai <thebrokentoaster@gmail.com>
diff --git a/.gitignore b/.gitignore
index 22d0d82..20e54cf 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
vendor
+cmd/protoc-gen-go/protoc-gen-go
diff --git a/cmd/protoc-gen-go/golden_test.go b/cmd/protoc-gen-go/golden_test.go
new file mode 100644
index 0000000..d0ed8ea
--- /dev/null
+++ b/cmd/protoc-gen-go/golden_test.go
@@ -0,0 +1,145 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+ "bytes"
+ "flag"
+ "go/build"
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "regexp"
+ "strings"
+ "testing"
+)
+
+// Set --regenerate to regenerate the golden files.
+var regenerate = flag.Bool("regenerate", false, "regenerate golden files")
+
+// When the environment variable RUN_AS_PROTOC_GEN_GO is set, we skip running
+// tests and instead act as protoc-gen-go. This allows the test binary to
+// pass itself to protoc.
+func init() {
+ if os.Getenv("RUN_AS_PROTOC_GEN_GO") != "" {
+ main()
+ os.Exit(0)
+ }
+}
+
+func TestGolden(t *testing.T) {
+ workdir, err := ioutil.TempDir("", "proto-test")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer os.RemoveAll(workdir)
+
+ // Find all the proto files we need to compile. We assume that each directory
+ // contains the files for a single package.
+ supportTypeAliases := hasReleaseTag("go1.9")
+ packages := map[string][]string{}
+ err = filepath.Walk("testdata", func(path string, info os.FileInfo, err error) error {
+ if filepath.Base(path) == "import_public" && !supportTypeAliases {
+ // Public imports require type alias support.
+ return filepath.SkipDir
+ }
+ if !strings.HasSuffix(path, ".proto") {
+ return nil
+ }
+ dir := filepath.Dir(path)
+ packages[dir] = append(packages[dir], path)
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Compile each package, using this binary as protoc-gen-go.
+ for _, sources := range packages {
+ args := []string{"-Itestdata", "--go_out=plugins=grpc,paths=source_relative:" + workdir}
+ args = append(args, sources...)
+ protoc(t, args)
+ }
+
+ // Compare each generated file to the golden version.
+ filepath.Walk(workdir, func(genPath string, info os.FileInfo, _ error) error {
+ if info.IsDir() {
+ return nil
+ }
+
+ // For each generated file, figure out the path to the corresponding
+ // golden file in the testdata directory.
+ relPath, err := filepath.Rel(workdir, genPath)
+ if err != nil {
+ t.Errorf("filepath.Rel(%q, %q): %v", workdir, genPath, err)
+ return nil
+ }
+ if filepath.SplitList(relPath)[0] == ".." {
+ t.Errorf("generated file %q is not relative to %q", genPath, workdir)
+ }
+ goldenPath := filepath.Join("testdata", relPath)
+
+ got, err := ioutil.ReadFile(genPath)
+ if err != nil {
+ t.Error(err)
+ return nil
+ }
+ if *regenerate {
+ // If --regenerate set, just rewrite the golden files.
+ err := ioutil.WriteFile(goldenPath, got, 0666)
+ if err != nil {
+ t.Error(err)
+ }
+ return nil
+ }
+
+ want, err := ioutil.ReadFile(goldenPath)
+ if err != nil {
+ t.Error(err)
+ return nil
+ }
+
+ want = fdescRE.ReplaceAll(want, nil)
+ got = fdescRE.ReplaceAll(got, nil)
+ if bytes.Equal(got, want) {
+ return nil
+ }
+
+ cmd := exec.Command("diff", "-u", goldenPath, genPath)
+ out, _ := cmd.CombinedOutput()
+ t.Errorf("golden file differs: %v\n%v", relPath, string(out))
+ return nil
+ })
+}
+
+var fdescRE = regexp.MustCompile(`(?ms)^var fileDescriptor.*}`)
+
+func protoc(t *testing.T, args []string) {
+ cmd := exec.Command("protoc", "--plugin=protoc-gen-go="+os.Args[0])
+ cmd.Args = append(cmd.Args, args...)
+ // We set the RUN_AS_PROTOC_GEN_GO environment variable to indicate that
+ // the subprocess should act as a proto compiler rather than a test.
+ cmd.Env = append(os.Environ(), "RUN_AS_PROTOC_GEN_GO=1")
+ out, err := cmd.CombinedOutput()
+ if len(out) > 0 || err != nil {
+ t.Log("RUNNING: ", strings.Join(cmd.Args, " "))
+ }
+ if len(out) > 0 {
+ t.Log(string(out))
+ }
+ if err != nil {
+ t.Fatalf("protoc: %v", err)
+ }
+}
+
+func hasReleaseTag(want string) bool {
+ for _, tag := range build.Default.ReleaseTags {
+ if tag == want {
+ return true
+ }
+ }
+ return false
+}
diff --git a/cmd/protoc-gen-go/main.go b/cmd/protoc-gen-go/main.go
new file mode 100644
index 0000000..9ea1420
--- /dev/null
+++ b/cmd/protoc-gen-go/main.go
@@ -0,0 +1,35 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// The protoc-gen-go binary is a protoc plugin to generate a Go protocol
+// buffer package.
+package main
+
+import (
+ "strings"
+
+ "google.golang.org/proto/protogen"
+)
+
+func main() {
+ protogen.Run(func(gen *protogen.Plugin) error {
+ for _, f := range gen.Files {
+ if !f.Generate {
+ continue
+ }
+ genFile(gen, f)
+ }
+ return nil
+ })
+}
+
+func genFile(gen *protogen.Plugin, f *protogen.File) {
+ g := gen.NewGeneratedFile(strings.TrimSuffix(f.Desc.GetName(), ".proto") + ".pb.go")
+ g.P("// Code generated by protoc-gen-go. DO NOT EDIT.")
+ g.P("// source: ", f.Desc.GetName())
+ g.P()
+ g.P("package TODO")
+
+ // TODO: Everything.
+}
diff --git a/cmd/protoc-gen-go/testdata/proto2/proto2.pb.go b/cmd/protoc-gen-go/testdata/proto2/proto2.pb.go
new file mode 100644
index 0000000..558363d
--- /dev/null
+++ b/cmd/protoc-gen-go/testdata/proto2/proto2.pb.go
@@ -0,0 +1,4 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// source: proto2/proto2.proto
+
+package TODO
diff --git a/cmd/protoc-gen-go/testdata/proto2/proto2.proto b/cmd/protoc-gen-go/testdata/proto2/proto2.proto
new file mode 100644
index 0000000..055bc71
--- /dev/null
+++ b/cmd/protoc-gen-go/testdata/proto2/proto2.proto
@@ -0,0 +1,12 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+syntax = "proto2";
+
+package goproto.protoc.proto2;
+
+option go_package = "google.golang.org/proto/cmd/protoc-gen-go/testdata/proto2";
+
+message Message {
+}
diff --git a/go.mod b/go.mod
index d94d903..6aa64ad 100644
--- a/go.mod
+++ b/go.mod
@@ -1,3 +1,8 @@
module google.golang.org/proto
-require github.com/google/go-cmp v0.2.0
+require (
+ github.com/golang/protobuf v1.2.0
+ github.com/google/go-cmp v0.2.0
+ golang.org/x/net v0.0.0-20180821023952-922f4815f713 // indirect
+ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f // indirect
+)
diff --git a/go.sum b/go.sum
index 5f4f636..1bfafa6 100644
--- a/go.sum
+++ b/go.sum
@@ -1,2 +1,8 @@
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+golang.org/x/net v0.0.0-20180821023952-922f4815f713 h1:rMJUcaDGbG+X967I4zGKCq5laYqcGKJmpB+3jhpOhPw=
+golang.org/x/net v0.0.0-20180821023952-922f4815f713/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
diff --git a/protogen/names.go b/protogen/names.go
new file mode 100644
index 0000000..1976cfd
--- /dev/null
+++ b/protogen/names.go
@@ -0,0 +1,54 @@
+package protogen
+
+import (
+ "go/token"
+ "strconv"
+ "strings"
+ "unicode"
+ "unicode/utf8"
+)
+
+// A GoImportPath is the import path of a Go package. e.g., "google.golang.org/genproto/protobuf".
+type GoImportPath string
+
+func (p GoImportPath) String() string { return strconv.Quote(string(p)) }
+
+// A GoPackageName is the name of a Go package. e.g., "protobuf".
+type GoPackageName string
+
+// cleanPacakgeName converts a string to a valid Go package name.
+func cleanPackageName(name string) GoPackageName {
+ name = strings.Map(badToUnderscore, name)
+ // Identifier must not be keyword: insert _.
+ if token.Lookup(name).IsKeyword() {
+ name = "_" + name
+ }
+ // Identifier must not begin with digit: insert _.
+ if r, _ := utf8.DecodeRuneInString(name); unicode.IsDigit(r) {
+ name = "_" + name
+ }
+ return GoPackageName(name)
+}
+
+// badToUnderscore is the mapping function used to generate Go names from package names,
+// which can be dotted in the input .proto file. It replaces non-identifier characters such as
+// dot or dash with underscore.
+func badToUnderscore(r rune) rune {
+ if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' {
+ return r
+ }
+ return '_'
+}
+
+// baseName returns the last path element of the name, with the last dotted suffix removed.
+func baseName(name string) string {
+ // First, find the last element
+ if i := strings.LastIndex(name, "/"); i >= 0 {
+ name = name[i+1:]
+ }
+ // Now drop the suffix
+ if i := strings.LastIndex(name, "."); i >= 0 {
+ name = name[:i]
+ }
+ return name
+}
diff --git a/protogen/protogen.go b/protogen/protogen.go
new file mode 100644
index 0000000..b10edad
--- /dev/null
+++ b/protogen/protogen.go
@@ -0,0 +1,224 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package protogen provides support for writing protoc plugins.
+//
+// Plugins for protoc, the Protocol Buffers Compiler, are programs which read
+// a CodeGeneratorRequest protocol buffer from standard input and write a
+// CodeGeneratorResponse protocol buffer to standard output. This package
+// provides support for writing plugins which generate Go code.
+package protogen
+
+import (
+ "bytes"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/golang/protobuf/proto"
+ descpb "github.com/golang/protobuf/protoc-gen-go/descriptor"
+ pluginpb "github.com/golang/protobuf/protoc-gen-go/plugin"
+)
+
+// Run executes a function as a protoc plugin.
+//
+// It reads a CodeGeneratorRequest message from os.Stdin, invokes the plugin
+// function, and writes a CodeGeneratorResponse message to os.Stdout.
+//
+// If a failure occurs while reading or writing, Run prints an error to
+// os.Stderr and calls os.Exit(1).
+func Run(f func(*Plugin) error) {
+ if err := run(f); err != nil {
+ fmt.Fprintf(os.Stderr, "%s: %v\n", filepath.Base(os.Args[0]), err)
+ os.Exit(1)
+ }
+}
+
+func run(f func(*Plugin) error) error {
+ in, err := ioutil.ReadAll(os.Stdin)
+ if err != nil {
+ return err
+ }
+ req := &pluginpb.CodeGeneratorRequest{}
+ if err := proto.Unmarshal(in, req); err != nil {
+ return err
+ }
+ gen, err := New(req)
+ if err != nil {
+ return err
+ }
+ if err := f(gen); err != nil {
+ // Errors from the plugin function are reported by setting the
+ // error field in the CodeGeneratorResponse.
+ //
+ // In contrast, errors that indicate a problem in protoc
+ // itself (unparsable input, I/O errors, etc.) are reported
+ // to stderr.
+ gen.Error(err)
+ }
+ resp := gen.Response()
+ out, err := proto.Marshal(resp)
+ if err != nil {
+ return err
+ }
+ if _, err := os.Stdout.Write(out); err != nil {
+ return err
+ }
+ return nil
+}
+
+// A Plugin is a protoc plugin invocation.
+type Plugin struct {
+ // Request is the CodeGeneratorRequest provided by protoc.
+ Request *pluginpb.CodeGeneratorRequest
+
+ // Files is the set of files to generate and everything they import.
+ // Files appear in topological order, so each file appears before any
+ // file that imports it.
+ Files []*File
+ filesByName map[string]*File
+
+ packageImportPath string // Go import path of the package we're generating code for.
+
+ genFiles []*GeneratedFile
+ err error
+}
+
+// New returns a new Plugin.
+func New(req *pluginpb.CodeGeneratorRequest) (*Plugin, error) {
+ gen := &Plugin{
+ Request: req,
+ filesByName: make(map[string]*File),
+ }
+
+ // TODO: Figure out how to pass parameters to the generator.
+ for _, param := range strings.Split(req.GetParameter(), ",") {
+ var value string
+ if i := strings.Index(param, "="); i >= 0 {
+ value = param[i+1:]
+ param = param[0:i]
+ }
+ switch param {
+ case "":
+ // Ignore.
+ case "import_prefix":
+ // TODO
+ case "import_path":
+ gen.packageImportPath = value
+ case "paths":
+ // TODO
+ case "plugins":
+ // TODO
+ case "annotate_code":
+ // TODO
+ default:
+ if param[0] != 'M' {
+ return nil, fmt.Errorf("unknown parameter %q", param)
+ }
+ // TODO
+ }
+ }
+
+ for _, fdesc := range gen.Request.ProtoFile {
+ f := newFile(gen, fdesc)
+ name := f.Desc.GetName()
+ if gen.filesByName[name] != nil {
+ return nil, fmt.Errorf("duplicate file name: %q", name)
+ }
+ gen.Files = append(gen.Files, f)
+ gen.filesByName[name] = f
+ }
+ for _, name := range gen.Request.FileToGenerate {
+ f, ok := gen.FileByName(name)
+ if !ok {
+ return nil, fmt.Errorf("no descriptor for generated file: %v", name)
+ }
+ f.Generate = true
+ }
+ return gen, nil
+}
+
+// Error records an error in code generation. The generator will report the
+// error back to protoc and will not produce output.
+func (gen *Plugin) Error(err error) {
+ if gen.err == nil {
+ gen.err = err
+ }
+}
+
+// Response returns the generator output.
+func (gen *Plugin) Response() *pluginpb.CodeGeneratorResponse {
+ resp := &pluginpb.CodeGeneratorResponse{}
+ if gen.err != nil {
+ resp.Error = proto.String(gen.err.Error())
+ return resp
+ }
+ for _, gf := range gen.genFiles {
+ resp.File = append(resp.File, &pluginpb.CodeGeneratorResponse_File{
+ Name: proto.String(gf.path),
+ Content: proto.String(string(gf.Content())),
+ })
+ }
+ return resp
+}
+
+// FileByName returns the file with the given name.
+func (gen *Plugin) FileByName(name string) (f *File, ok bool) {
+ f, ok = gen.filesByName[name]
+ return f, ok
+}
+
+// A File is a .proto source file.
+type File struct {
+ // TODO: Replace with protoreflect.FileDescriptor.
+ Desc *descpb.FileDescriptorProto
+
+ // Generate is true if the generator should generate code for this file.
+ Generate bool
+}
+
+func newFile(gen *Plugin, p *descpb.FileDescriptorProto) *File {
+ return &File{
+ Desc: p,
+ }
+}
+
+// A GeneratedFile is a generated file.
+type GeneratedFile struct {
+ path string
+ buf bytes.Buffer
+}
+
+// NewGeneratedFile creates a new generated file with the given path.
+func (gen *Plugin) NewGeneratedFile(path string) *GeneratedFile {
+ g := &GeneratedFile{
+ path: path,
+ }
+ gen.genFiles = append(gen.genFiles, g)
+ return g
+}
+
+// P prints a line to the generated output. It converts each parameter to a
+// string following the same rules as fmt.Print. It never inserts spaces
+// between parameters.
+//
+// TODO: .meta file annotations.
+func (g *GeneratedFile) P(v ...interface{}) {
+ for _, x := range v {
+ fmt.Fprint(&g.buf, x)
+ }
+ fmt.Fprintln(&g.buf)
+}
+
+// Write implements io.Writer.
+func (g *GeneratedFile) Write(p []byte) (n int, err error) {
+ return g.buf.Write(p)
+}
+
+// Content returns the contents of the generated file.
+func (g *GeneratedFile) Content() []byte {
+ return g.buf.Bytes()
+}
diff --git a/protogen/protogen_test.go b/protogen/protogen_test.go
new file mode 100644
index 0000000..1d23cc0
--- /dev/null
+++ b/protogen/protogen_test.go
@@ -0,0 +1,94 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package protogen
+
+import (
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/golang/protobuf/proto"
+ pluginpb "github.com/golang/protobuf/protoc-gen-go/plugin"
+)
+
+func TestFiles(t *testing.T) {
+ gen, err := New(makeRequest(t, "testdata/go_package/no_go_package_import.proto"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, test := range []struct {
+ path string
+ wantGenerate bool
+ }{
+ {
+ path: "go_package/no_go_package_import.proto",
+ wantGenerate: true,
+ },
+ {
+ path: "go_package/no_go_package.proto",
+ wantGenerate: false,
+ },
+ } {
+ f, ok := gen.FileByName(test.path)
+ if !ok {
+ t.Errorf("%q: not found by gen.FileByName", test.path)
+ continue
+ }
+ if f.Generate != test.wantGenerate {
+ t.Errorf("%q: Generate=%v, want %v", test.path, f.Generate, test.wantGenerate)
+ }
+ }
+}
+
+// makeRequest returns a CodeGeneratorRequest for the given protoc inputs.
+//
+// It does this by running protoc with the current binary as the protoc-gen-go
+// plugin. This "plugin" produces a single file, named 'request', which contains
+// the code generator request.
+func makeRequest(t *testing.T, args ...string) *pluginpb.CodeGeneratorRequest {
+ workdir, err := ioutil.TempDir("", "test")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer os.RemoveAll(workdir)
+
+ cmd := exec.Command("protoc", "--plugin=protoc-gen-go="+os.Args[0])
+ cmd.Args = append(cmd.Args, "--go_out="+workdir, "-Itestdata")
+ cmd.Args = append(cmd.Args, args...)
+ cmd.Env = append(os.Environ(), "RUN_AS_PROTOC_PLUGIN=1")
+ out, err := cmd.CombinedOutput()
+ if len(out) > 0 || err != nil {
+ t.Log("RUNNING: ", strings.Join(cmd.Args, " "))
+ }
+ if len(out) > 0 {
+ t.Log(string(out))
+ }
+ if err != nil {
+ t.Fatalf("protoc: %v", err)
+ }
+
+ b, err := ioutil.ReadFile(filepath.Join(workdir, "request"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ req := &pluginpb.CodeGeneratorRequest{}
+ if err := proto.UnmarshalText(string(b), req); err != nil {
+ t.Fatal(err)
+ }
+ return req
+}
+
+func init() {
+ if os.Getenv("RUN_AS_PROTOC_PLUGIN") != "" {
+ Run(func(p *Plugin) error {
+ g := p.NewGeneratedFile("request")
+ return proto.MarshalText(g, p.Request)
+ })
+ os.Exit(0)
+ }
+}
diff --git a/protogen/testdata/go_package/no_go_package.proto b/protogen/testdata/go_package/no_go_package.proto
new file mode 100644
index 0000000..d71884c
--- /dev/null
+++ b/protogen/testdata/go_package/no_go_package.proto
@@ -0,0 +1,9 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Proto source file with no go_package option.
+
+syntax = "proto3";
+package goproto.testdata;
+message M {}
diff --git a/protogen/testdata/go_package/no_go_package_import.proto b/protogen/testdata/go_package/no_go_package_import.proto
new file mode 100644
index 0000000..bb1a73f
--- /dev/null
+++ b/protogen/testdata/go_package/no_go_package_import.proto
@@ -0,0 +1,13 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Import of a proto source file with no go_package option.
+
+syntax = "proto3";
+package goproto.testdata;
+import "go_package/no_go_package.proto";
+message M1 {
+ M Field = 1;
+}
+