internal/filedesc, internal/filetype: initial commit
The internal/fileinit package is split apart into two packages:
* internal/filedesc constructs descriptors from the raw proto.
It is very similar to the previous internal/fileinit package.
* internal/filetype wraps descriptors with Go type information
Overview:
* The internal/fileinit package will be deleted in a future CL.
It is kept around since the v1 repo currently depends on it.
* The internal/prototype package is deleted. All former usages of it
are now using internal/filedesc instead. Most significantly,
the reflect/protodesc package was almost entirely re-written.
* The internal/impl package drops support for messages that do not
have a Descriptor method (pre-2016). This removes a significant amount
of technical debt.
filedesc.Builder to parse raw descriptors.
* The internal/encoding/defval package now handles enum values by name.
Change-Id: I3957bcc8588a70470fd6c7de1122216b80615ab7
Reviewed-on: https://go-review.googlesource.com/c/protobuf/+/182360
Reviewed-by: Damien Neil <dneil@google.com>
diff --git a/internal/cmd/generate-types/main.go b/internal/cmd/generate-types/main.go
index 66a852d..d6bb972 100644
--- a/internal/cmd/generate-types/main.go
+++ b/internal/cmd/generate-types/main.go
@@ -20,7 +20,6 @@
"strconv"
"strings"
"text/template"
- "unicode"
)
var (
@@ -38,8 +37,7 @@
repoRoot = strings.TrimSpace(string(out))
chdirRoot()
- writeSource("internal/fileinit/desc_list_gen.go", generateFileinitDescList())
- writeSource("internal/prototype/protofile_list_gen.go", generateListTypes())
+ writeSource("internal/filedesc/desc_list_gen.go", generateDescListTypes())
writeSource("internal/impl/encode_gen.go", generateImplEncode())
writeSource("proto/decode_gen.go", generateProtoDecode())
writeSource("proto/encode_gen.go", generateProtoEncode())
@@ -83,138 +81,19 @@
}
}
-func generateListTypes() string {
- // TODO: If Go2 has generics, replace this with a single container type.
- return mustExecute(listTypesTemplate, []DescriptorType{
- MessageDesc, FieldDesc, OneofDesc, ExtensionDesc, EnumDesc, EnumValueDesc, ServiceDesc, MethodDesc,
- })
-}
-
-var listTypesTemplate = template.Must(template.New("").Funcs(template.FuncMap{
- "unexport": func(t DescriptorType) Expr {
- return Expr(string(unicode.ToLower(rune(t[0]))) + string(t[1:]))
- },
-}).Parse(`
- {{- range .}}
- {{$nameList := (printf "%ss" (unexport .))}} {{/* e.g., "messages" */}}
- {{$nameListMeta := (printf "%ssMeta" (unexport .))}} {{/* e.g., "messagesMeta" */}}
- {{$nameMeta := (printf "%sMeta" (unexport .))}} {{/* e.g., "messageMeta" */}}
- {{$nameDesc := (printf "%sDesc" (unexport .))}} {{/* e.g., "messageDesc" */}}
-
- type {{$nameListMeta}} struct {
- once sync.Once
- typs []{{.}}
- nameOnce sync.Once
- byName map[protoreflect.Name]*{{.}}
- {{- if (eq . "Field")}}
- jsonOnce sync.Once
- byJSON map[string]*{{.}}
- {{- end}}
- {{- if .NumberExpr}}
- numOnce sync.Once
- byNum map[{{.NumberExpr}}]*{{.}}
- {{- end}}
- }
- type {{$nameList}} {{$nameListMeta}}
-
- func (p *{{$nameListMeta}}) lazyInit(parent protoreflect.Descriptor, ts []{{.}}) *{{$nameList}} {
- p.once.Do(func() {
- nb := getNameBuilder()
- defer putNameBuilder(nb)
- metas := make([]{{$nameMeta}}, len(ts))
- for i := range ts {
- t := &ts[i]
- if t.{{$nameMeta}} != nil {
- panic("already initialized")
- }
- t.{{$nameMeta}} = &metas[i]
- t.inheritedMeta.init(nb, parent, i, t.Name, {{printf "%v" (eq . "EnumValue")}})
- }
- p.typs = ts
- })
- return (*{{$nameList}})(p)
- }
- func (p *{{$nameList}}) Len() int { return len(p.typs) }
- func (p *{{$nameList}}) Get(i int) {{.Expr}} { return {{$nameDesc}}{&p.typs[i]} }
- func (p *{{$nameList}}) ByName(s protoreflect.Name) {{.Expr}} {
- p.nameOnce.Do(func() {
- if len(p.typs) > 0 {
- p.byName = make(map[protoreflect.Name]*{{.}}, len(p.typs))
- for i := range p.typs {
- t := &p.typs[i]
- p.byName[t.Name] = t
- }
- }
- })
- t := p.byName[s]
- if t == nil {
- return nil
- }
- return {{$nameDesc}}{t}
- }
- {{- if (eq . "Field")}}
- func (p *{{$nameList}}) ByJSONName(s string) {{.Expr}} {
- p.jsonOnce.Do(func() {
- if len(p.typs) > 0 {
- p.byJSON = make(map[string]*{{.}}, len(p.typs))
- for i := range p.typs {
- t := &p.typs[i]
- s := {{$nameDesc}}{t}.JSONName()
- if _, ok := p.byJSON[s]; !ok {
- p.byJSON[s] = t
- }
- }
- }
- })
- t := p.byJSON[s]
- if t == nil {
- return nil
- }
- return {{$nameDesc}}{t}
- }
- {{- end}}
- {{- if .NumberExpr}}
- func (p *{{$nameList}}) ByNumber(n {{.NumberExpr}}) {{.Expr}} {
- p.numOnce.Do(func() {
- if len(p.typs) > 0 {
- p.byNum = make(map[{{.NumberExpr}}]*{{.}}, len(p.typs))
- for i := range p.typs {
- t := &p.typs[i]
- if _, ok := p.byNum[t.Number]; !ok {
- p.byNum[t.Number] = t
- }
- }
- }
- })
- t := p.byNum[n]
- if t == nil {
- return nil
- }
- return {{$nameDesc}}{t}
- }
- {{- end}}
- func (p *{{$nameList}}) Format(s fmt.State, r rune) { descfmt.FormatList(s, r, p) }
- func (p *{{$nameList}}) ProtoInternal(pragma.DoNotImplement) {}
- {{- end}}
-`))
-
-func generateFileinitDescList() string {
- return mustExecute(fileinitDescListTemplate, []DescriptorType{
+func generateDescListTypes() string {
+ return mustExecute(descListTypesTemplate, []DescriptorType{
EnumDesc, EnumValueDesc, MessageDesc, FieldDesc, OneofDesc, ExtensionDesc, ServiceDesc, MethodDesc,
})
}
-var fileinitDescListTemplate = template.Must(template.New("").Funcs(template.FuncMap{
- "unexport": func(t DescriptorType) Expr {
- return Expr(string(unicode.ToLower(rune(t[0]))) + string(t[1:]))
- },
-}).Parse(`
+var descListTypesTemplate = template.Must(template.New("").Parse(`
{{- range .}}
- {{$nameList := (printf "%sDescs" (unexport .))}} {{/* e.g., "messageDescs" */}}
- {{$nameDesc := (printf "%sDesc" (unexport .))}} {{/* e.g., "messageDesc" */}}
+ {{$nameList := (printf "%ss" .)}} {{/* e.g., "Messages" */}}
+ {{$nameDesc := (printf "%s" .)}} {{/* e.g., "Message" */}}
type {{$nameList}} struct {
- list []{{$nameDesc}}
+ List []{{$nameDesc}}
once sync.Once
byName map[protoreflect.Name]*{{$nameDesc}} // protected by once
{{- if (eq . "Field")}}
@@ -226,22 +105,14 @@
}
func (p *{{$nameList}}) Len() int {
- return len(p.list)
+ return len(p.List)
}
func (p *{{$nameList}}) Get(i int) {{.Expr}} {
- {{- if (eq . "Message")}}
- return p.list[i].asDesc()
- {{- else}}
- return &p.list[i]
- {{- end}}
+ return &p.List[i]
}
func (p *{{$nameList}}) ByName(s protoreflect.Name) {{.Expr}} {
if d := p.lazyInit().byName[s]; d != nil {
- {{- if (eq . "Message")}}
- return d.asDesc()
- {{- else}}
return d
- {{- end}}
}
return nil
}
@@ -267,16 +138,16 @@
func (p *{{$nameList}}) ProtoInternal(pragma.DoNotImplement) {}
func (p *{{$nameList}}) lazyInit() *{{$nameList}} {
p.once.Do(func() {
- if len(p.list) > 0 {
- p.byName = make(map[protoreflect.Name]*{{$nameDesc}}, len(p.list))
+ if len(p.List) > 0 {
+ p.byName = make(map[protoreflect.Name]*{{$nameDesc}}, len(p.List))
{{- if (eq . "Field")}}
- p.byJSON = make(map[string]*{{$nameDesc}}, len(p.list))
+ p.byJSON = make(map[string]*{{$nameDesc}}, len(p.List))
{{- end}}
{{- if .NumberExpr}}
- p.byNum = make(map[{{.NumberExpr}}]*{{$nameDesc}}, len(p.list))
+ p.byNum = make(map[{{.NumberExpr}}]*{{$nameDesc}}, len(p.List))
{{- end}}
- for i := range p.list {
- d := &p.list[i]
+ for i := range p.List {
+ d := &p.List[i]
if _, ok := p.byName[d.Name()]; !ok {
p.byName[d.Name()] = d
}
diff --git a/internal/cmd/pbdump/pbdump.go b/internal/cmd/pbdump/pbdump.go
index 7a7c3ce..8bc1753 100644
--- a/internal/cmd/pbdump/pbdump.go
+++ b/internal/cmd/pbdump/pbdump.go
@@ -19,8 +19,8 @@
"google.golang.org/protobuf/internal/encoding/pack"
"google.golang.org/protobuf/internal/encoding/wire"
- "google.golang.org/protobuf/internal/prototype"
"google.golang.org/protobuf/internal/scalar"
+ "google.golang.org/protobuf/reflect/protodesc"
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/types/descriptorpb"
@@ -202,41 +202,43 @@
// Descriptor returns the field tree as a message descriptor.
func (fs fields) Descriptor() (protoreflect.MessageDescriptor, error) {
- ftyp, err := prototype.NewFile(&prototype.File{
- Syntax: protoreflect.Proto2,
- Messages: []prototype.Message{fs.messageDescriptor("M")},
- })
+ fd, err := protodesc.NewFile(&descriptorpb.FileDescriptorProto{
+ Name: scalar.String("dump.proto"),
+ Syntax: scalar.String("proto2"),
+ MessageType: []*descriptorpb.DescriptorProto{fs.messageDescriptor("M")},
+ }, nil)
if err != nil {
return nil, err
}
- return ftyp.Messages().Get(0), nil
+ return fd.Messages().Get(0), nil
}
-func (fs fields) messageDescriptor(name protoreflect.FullName) prototype.Message {
- m := prototype.Message{Name: name.Name()}
+func (fs fields) messageDescriptor(name protoreflect.FullName) *descriptorpb.DescriptorProto {
+ m := &descriptorpb.DescriptorProto{Name: scalar.String(string(name.Name()))}
for _, n := range fs.sortedNums() {
- f := prototype.Field{
- Name: protoreflect.Name(fmt.Sprintf("f%d", n)),
- Number: n,
- Cardinality: protoreflect.Optional,
- Kind: fs[n].kind,
+ k := fs[n].kind
+ if !k.IsValid() {
+ k = protoreflect.MessageKind
}
- if !f.Kind.IsValid() {
- f.Kind = protoreflect.MessageKind
+ f := &descriptorpb.FieldDescriptorProto{
+ Name: scalar.String(fmt.Sprintf("f%d", n)),
+ Number: scalar.Int32(int32(n)),
+ Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(),
+ Type: descriptorpb.FieldDescriptorProto_Type(k).Enum(),
}
- switch f.Kind {
+ switch k {
case protoreflect.BoolKind, protoreflect.EnumKind,
protoreflect.Int32Kind, protoreflect.Sint32Kind, protoreflect.Uint32Kind,
protoreflect.Int64Kind, protoreflect.Sint64Kind, protoreflect.Uint64Kind,
protoreflect.Sfixed32Kind, protoreflect.Fixed32Kind, protoreflect.FloatKind,
protoreflect.Sfixed64Kind, protoreflect.Fixed64Kind, protoreflect.DoubleKind:
- f.Cardinality = protoreflect.Repeated
+ f.Label = descriptorpb.FieldDescriptorProto_LABEL_REPEATED.Enum()
f.Options = &descriptorpb.FieldOptions{Packed: scalar.Bool(true)}
case protoreflect.MessageKind, protoreflect.GroupKind:
s := name.Append(protoreflect.Name(fmt.Sprintf("M%d", n)))
- f.MessageType = prototype.PlaceholderMessage(s)
- m.Messages = append(m.Messages, fs[n].sub.messageDescriptor(s))
+ f.TypeName = scalar.String(string("." + s))
+ m.NestedType = append(m.NestedType, fs[n].sub.messageDescriptor(s))
}
- m.Fields = append(m.Fields, f)
+ m.Field = append(m.Field, f)
}
return m
}
diff --git a/internal/cmd/pbdump/pbdump_test.go b/internal/cmd/pbdump/pbdump_test.go
index 57ec1b5..a3f6998 100644
--- a/internal/cmd/pbdump/pbdump_test.go
+++ b/internal/cmd/pbdump/pbdump_test.go
@@ -9,13 +9,22 @@
"strings"
"testing"
- "github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
-
- ptype "google.golang.org/protobuf/internal/prototype"
+ "google.golang.org/protobuf/encoding/prototext"
+ "google.golang.org/protobuf/proto"
pref "google.golang.org/protobuf/reflect/protoreflect"
+
+ "google.golang.org/protobuf/types/descriptorpb"
)
+func mustMakeMessage(s string) *descriptorpb.DescriptorProto {
+ s = fmt.Sprintf(`name:"test.proto" syntax:"proto2" message_type:[{%s}]`, s)
+ pb := new(descriptorpb.FileDescriptorProto)
+ if err := prototext.Unmarshal([]byte(s), pb); err != nil {
+ panic(err)
+ }
+ return pb.MessageType[0]
+}
+
func TestFields(t *testing.T) {
type fieldsKind struct {
kind pref.Kind
@@ -23,11 +32,11 @@
}
tests := []struct {
inFields []fieldsKind
- wantMsg ptype.Message
+ wantMsg *descriptorpb.DescriptorProto
wantErr string
}{{
inFields: []fieldsKind{{pref.MessageKind, ""}},
- wantMsg: ptype.Message{Name: "M"},
+ wantMsg: mustMakeMessage(`name:"M"`),
}, {
inFields: []fieldsKind{{pref.MessageKind, "987654321"}},
wantErr: "invalid field: 987654321",
@@ -52,44 +61,33 @@
{pref.MessageKind, " 10.20.30, 10.21 "},
{pref.GroupKind, "10"},
},
- wantMsg: ptype.Message{
- Name: "M",
- Fields: []ptype.Field{
- {Name: "f10", Number: 10, Cardinality: pref.Optional, Kind: pref.GroupKind, MessageType: ptype.PlaceholderMessage("M.M10")},
- },
- Messages: []ptype.Message{{
- Name: "M10",
- Fields: []ptype.Field{
- {Name: "f20", Number: 20, Cardinality: pref.Optional, Kind: pref.MessageKind, MessageType: ptype.PlaceholderMessage("M.M10.M20")},
- {Name: "f21", Number: 21, Cardinality: pref.Optional, Kind: pref.MessageKind, MessageType: ptype.PlaceholderMessage("M.M10.M21")},
- },
- Messages: []ptype.Message{{
- Name: "M20",
- Fields: []ptype.Field{
- {Name: "f30", Number: 30, Cardinality: pref.Optional, Kind: pref.MessageKind, MessageType: ptype.PlaceholderMessage("M.M10.M20.M30")},
- {Name: "f31", Number: 31, Cardinality: pref.Repeated, Kind: pref.Int32Kind},
- },
- Messages: []ptype.Message{{
- Name: "M30",
- }},
+ wantMsg: mustMakeMessage(`
+ name: "M"
+ field: [
+ {name:"f10" number:10 label:LABEL_OPTIONAL type:TYPE_GROUP type_name:".M.M10"}
+ ]
+ nested_type: [{
+ name: "M10"
+ field: [
+ {name:"f20" number:20 label:LABEL_OPTIONAL type:TYPE_MESSAGE type_name:".M.M10.M20"},
+ {name:"f21" number:21 label:LABEL_OPTIONAL type:TYPE_MESSAGE type_name:".M.M10.M21"}
+ ]
+ nested_type: [{
+ name: "M20"
+ field:[
+ {name:"f30" number:30 label:LABEL_OPTIONAL type:TYPE_MESSAGE, type_name:".M.M10.M20.M30"},
+ {name:"f31" number:31 label:LABEL_REPEATED type:TYPE_INT32 options:{packed:true}}
+ ]
+ nested_type: [{
+ name: "M30"
+ }]
}, {
- Name: "M21",
- }},
- }},
- },
+ name: "M21"
+ }]
+ }]
+ `),
}}
- opts := cmp.Options{
- cmp.Comparer(func(x, y pref.Descriptor) bool {
- if x == nil || y == nil {
- return x == nil && y == nil
- }
- return x.FullName() == y.FullName()
- }),
- cmpopts.IgnoreFields(ptype.Field{}, "Default"),
- cmpopts.IgnoreFields(ptype.Field{}, "Options"),
- cmpopts.IgnoreUnexported(ptype.Message{}, ptype.Field{}),
- }
for _, tt := range tests {
t.Run("", func(t *testing.T) {
var fields fields
@@ -106,8 +104,8 @@
t.Errorf("all Set calls succeeded, want %v error", tt.wantErr)
}
gotMsg := fields.messageDescriptor("M")
- if diff := cmp.Diff(tt.wantMsg, gotMsg, opts); diff != "" {
- t.Errorf("messageDescriptor() mismatch (-want +got):\n%v", diff)
+ if !proto.Equal(gotMsg, tt.wantMsg) {
+ t.Errorf("messageDescriptor() mismatch:\ngot %v\nwant %v", gotMsg, tt.wantMsg)
}
if _, err := fields.Descriptor(); err != nil {
t.Errorf("Descriptor() = %v, want nil error", err)