internal/filedesc, internal/filetype: initial commit
The internal/fileinit package is split apart into two packages:
* internal/filedesc constructs descriptors from the raw proto.
It is very similar to the previous internal/fileinit package.
* internal/filetype wraps descriptors with Go type information
Overview:
* The internal/fileinit package will be deleted in a future CL.
It is kept around since the v1 repo currently depends on it.
* The internal/prototype package is deleted. All former usages of it
are now using internal/filedesc instead. Most significantly,
the reflect/protodesc package was almost entirely re-written.
* The internal/impl package drops support for messages that do not
have a Descriptor method (pre-2016). This removes a significant amount
of technical debt.
filedesc.Builder to parse raw descriptors.
* The internal/encoding/defval package now handles enum values by name.
Change-Id: I3957bcc8588a70470fd6c7de1122216b80615ab7
Reviewed-on: https://go-review.googlesource.com/c/protobuf/+/182360
Reviewed-by: Damien Neil <dneil@google.com>
diff --git a/internal/fileinit/desc.go b/internal/fileinit/desc.go
index 938167b..21698b6 100644
--- a/internal/fileinit/desc.go
+++ b/internal/fileinit/desc.go
@@ -454,7 +454,7 @@
return fd.Message().Fields().ByNumber(2)
}
func (fd *fieldDesc) HasDefault() bool { return fd.defVal.has }
-func (fd *fieldDesc) Default() pref.Value { return fd.defVal.get() }
+func (fd *fieldDesc) Default() pref.Value { return fd.defVal.get(fd) }
func (fd *fieldDesc) DefaultEnumValue() pref.EnumValueDescriptor { return fd.defVal.enum }
func (fd *fieldDesc) ContainingOneof() pref.OneofDescriptor { return fd.oneofType }
func (fd *fieldDesc) ContainingMessage() pref.MessageDescriptor {
@@ -528,7 +528,7 @@
func (xd *extensionDesc) MapKey() pref.FieldDescriptor { return nil }
func (xd *extensionDesc) MapValue() pref.FieldDescriptor { return nil }
func (xd *extensionDesc) HasDefault() bool { return xd.lazyInit().defVal.has }
-func (xd *extensionDesc) Default() pref.Value { return xd.lazyInit().defVal.get() }
+func (xd *extensionDesc) Default() pref.Value { return xd.lazyInit().defVal.get(xd) }
func (xd *extensionDesc) DefaultEnumValue() pref.EnumValueDescriptor { return xd.lazyInit().defVal.enum }
func (xd *extensionDesc) ContainingOneof() pref.OneofDescriptor { return nil }
func (xd *extensionDesc) ContainingMessage() pref.MessageDescriptor { return xd.extendedType }
diff --git a/internal/fileinit/desc_lazy.go b/internal/fileinit/desc_lazy.go
index 0e26e17..673b100 100644
--- a/internal/fileinit/desc_lazy.go
+++ b/internal/fileinit/desc_lazy.go
@@ -12,8 +12,8 @@
defval "google.golang.org/protobuf/internal/encoding/defval"
wire "google.golang.org/protobuf/internal/encoding/wire"
fieldnum "google.golang.org/protobuf/internal/fieldnum"
+ fdesc "google.golang.org/protobuf/internal/filedesc"
pimpl "google.golang.org/protobuf/internal/impl"
- ptype "google.golang.org/protobuf/internal/prototype"
pvalue "google.golang.org/protobuf/internal/value"
pref "google.golang.org/protobuf/reflect/protoreflect"
preg "google.golang.org/protobuf/reflect/protoregistry"
@@ -95,7 +95,11 @@
fd.isPacked = true
}
}
- fd.defVal.lazyInit(fd.kind, file.enumValuesOf(fd.enumType))
+
+ // Default is resolved here since it depends on Enum being resolved.
+ if v := fd.defVal.val; v.IsValid() {
+ fd.defVal = unmarshalDefault(v.Bytes(), fd.kind, file, fd.enumType)
+ }
}
}
}
@@ -121,7 +125,7 @@
et := pimpl.Export{}.EnumTypeOf(reflect.Zero(typ).Interface())
xd.lazy.typ = typ
xd.lazy.new = func() pref.Value {
- return xd.lazy.defVal.get()
+ return xd.lazy.defVal.get(xd)
}
xd.lazy.valueOf = func(v interface{}) pref.Value {
ev := v.(pref.Enum)
@@ -146,7 +150,7 @@
default:
xd.lazy.typ = goTypeForPBKind[xd.lazy.kind]
xd.lazy.new = func() pref.Value {
- return xd.lazy.defVal.get()
+ return xd.lazy.defVal.get(xd)
}
xd.lazy.valueOf = func(v interface{}) pref.Value {
return pref.ValueOf(v)
@@ -179,7 +183,11 @@
case pref.MessageKind, pref.GroupKind:
xd.lazy.messageType = file.popMessageDependency()
}
- xd.lazy.defVal.lazyInit(xd.lazy.kind, file.enumValuesOf(xd.lazy.enumType))
+
+ // Default is resolved here since it depends on Enum being resolved.
+ if v := xd.lazy.defVal.val; v.IsValid() {
+ xd.lazy.defVal = unmarshalDefault(v.Bytes(), xd.lazy.kind, file, xd.lazy.enumType)
+ }
}
}
@@ -271,67 +279,84 @@
*fi = fileInit{} // clear fileInit for GC to reclaim resources
}
+func DefaultValue(v pref.Value, ev pref.EnumValueDescriptor) defaultValue {
+ dv := defaultValue{has: v.IsValid(), val: v, enum: ev}
+ if b, ok := v.Interface().([]byte); ok {
+ // Store a copy of the default bytes, so that we can detect
+ // accidental mutations of the original value.
+ dv.bytes = append([]byte(nil), b...)
+ }
+ return dv
+}
+
+func unmarshalDefault(b []byte, k pref.Kind, pf *fileDesc, ed pref.EnumDescriptor) defaultValue {
+ var evs pref.EnumValueDescriptors
+ if k == pref.EnumKind {
+ // If the enum is declared within the same file, be careful not to
+ // blindly call the Values method, lest we bind ourselves in a deadlock.
+ if ed, ok := ed.(*enumDesc); ok && ed.parentFile == pf {
+ evs = &ed.lazy.values
+ } else {
+ evs = ed.Values()
+ }
+ }
+
+ v, ev, err := defval.Unmarshal(string(b), k, evs, defval.Descriptor)
+ if err != nil {
+ panic(err)
+ }
+ dv := defaultValue{has: v.IsValid(), val: v, enum: ev}
+ if b, ok := v.Interface().([]byte); ok {
+ // Store a copy of the default bytes, so that we can detect
+ // accidental mutations of the original value.
+ dv.bytes = append([]byte(nil), b...)
+ }
+ return dv
+}
+
type defaultValue struct {
has bool
val pref.Value
enum pref.EnumValueDescriptor
- check func() // only set for non-empty bytes
+ bytes []byte
}
-func (dv *defaultValue) get() pref.Value {
- if dv.check != nil {
- dv.check()
+func (dv *defaultValue) get(fd pref.FieldDescriptor) pref.Value {
+ // Return the zero value as the default if unpopulated.
+ if !dv.has {
+ switch fd.Kind() {
+ case pref.BoolKind:
+ return pref.ValueOf(false)
+ case pref.Int32Kind, pref.Sint32Kind, pref.Sfixed32Kind:
+ return pref.ValueOf(int32(0))
+ case pref.Int64Kind, pref.Sint64Kind, pref.Sfixed64Kind:
+ return pref.ValueOf(int64(0))
+ case pref.Uint32Kind, pref.Fixed32Kind:
+ return pref.ValueOf(uint32(0))
+ case pref.Uint64Kind, pref.Fixed64Kind:
+ return pref.ValueOf(uint64(0))
+ case pref.FloatKind:
+ return pref.ValueOf(float32(0))
+ case pref.DoubleKind:
+ return pref.ValueOf(float64(0))
+ case pref.StringKind:
+ return pref.ValueOf(string(""))
+ case pref.BytesKind:
+ return pref.ValueOf([]byte(nil))
+ case pref.EnumKind:
+ return pref.ValueOf(fd.Enum().Values().Get(0).Number())
+ }
+ }
+
+ if len(dv.bytes) > 0 && !bytes.Equal(dv.bytes, dv.val.Bytes()) {
+ // TODO: Avoid panic if we're running with the race detector
+ // and instead spawn a goroutine that periodically resets
+ // this value back to the original to induce a race.
+ panic("detected mutation on the default bytes")
}
return dv.val
}
-func (dv *defaultValue) lazyInit(k pref.Kind, eds pref.EnumValueDescriptors) {
- if dv.has {
- switch k {
- case pref.EnumKind:
- // File descriptors always store default enums by name.
- dv.enum = eds.ByName(pref.Name(dv.val.String()))
- dv.val = pref.ValueOf(dv.enum.Number())
- case pref.BytesKind:
- // Store a copy of the default bytes, so that we can detect
- // accidental mutations of the original value.
- b := append([]byte(nil), dv.val.Bytes()...)
- dv.check = func() {
- if !bytes.Equal(b, dv.val.Bytes()) {
- // TODO: Avoid panic if we're running with the race detector
- // and instead spawn a goroutine that periodically resets
- // this value back to the original to induce a race.
- panic("detected mutation on the default bytes")
- }
- }
- }
- } else {
- switch k {
- case pref.BoolKind:
- dv.val = pref.ValueOf(false)
- case pref.Int32Kind, pref.Sint32Kind, pref.Sfixed32Kind:
- dv.val = pref.ValueOf(int32(0))
- case pref.Int64Kind, pref.Sint64Kind, pref.Sfixed64Kind:
- dv.val = pref.ValueOf(int64(0))
- case pref.Uint32Kind, pref.Fixed32Kind:
- dv.val = pref.ValueOf(uint32(0))
- case pref.Uint64Kind, pref.Fixed64Kind:
- dv.val = pref.ValueOf(uint64(0))
- case pref.FloatKind:
- dv.val = pref.ValueOf(float32(0))
- case pref.DoubleKind:
- dv.val = pref.ValueOf(float64(0))
- case pref.StringKind:
- dv.val = pref.ValueOf(string(""))
- case pref.BytesKind:
- dv.val = pref.ValueOf([]byte(nil))
- case pref.EnumKind:
- dv.enum = eds.Get(0)
- dv.val = pref.ValueOf(dv.enum.Number())
- }
- }
-}
-
func (fd *fileDesc) unmarshalFull(b []byte) {
nb := getNameBuilder()
defer putNameBuilder(nb)
@@ -368,7 +393,7 @@
}
case fieldnum.FileDescriptorProto_Dependency:
fd.lazy.imports = append(fd.lazy.imports, pref.FileImport{
- FileDescriptor: ptype.PlaceholderFile(nb.MakeString(v), ""),
+ FileDescriptor: fdesc.PlaceholderFile(nb.MakeString(v)),
})
case fieldnum.FileDescriptorProto_EnumType:
fd.enums.list[enumIdx].unmarshalFull(v, nb)
@@ -627,7 +652,6 @@
fd.parent = pd
fd.index = i
- var rawDefVal []byte
var rawTypeName []byte
for len(b) > 0 {
num, typ, n := wire.ConsumeTag(b)
@@ -664,8 +688,7 @@
fd.hasJSONName = true
fd.jsonName = nb.MakeString(v)
case fieldnum.FieldDescriptorProto_DefaultValue:
- fd.defVal.has = true
- rawDefVal = v
+ fd.defVal.val = pref.ValueOf(v) // temporarily store as bytes; later resolved in resolveMessages
case fieldnum.FieldDescriptorProto_TypeName:
rawTypeName = v
case fieldnum.FieldDescriptorProto_Options:
@@ -680,13 +703,6 @@
if !fd.hasJSONName {
fd.jsonName = nb.MakeJSONName(fd.Name())
}
- if rawDefVal != nil {
- var err error
- fd.defVal.val, err = defval.Unmarshal(string(rawDefVal), fd.kind, defval.Descriptor)
- if err != nil {
- panic(err)
- }
- }
if fd.isWeak {
if len(rawTypeName) == 0 || rawTypeName[0] != '.' {
panic("weak target name must be fully qualified")
@@ -695,7 +711,7 @@
name := pref.FullName(rawTypeName[1:])
fd.messageType, _ = preg.GlobalFiles.FindMessageByName(name)
if fd.messageType == nil {
- fd.messageType = ptype.PlaceholderMessage(pref.FullName(rawTypeName[1:]))
+ fd.messageType = fdesc.PlaceholderMessage(pref.FullName(rawTypeName[1:]))
}
}
}
@@ -749,7 +765,6 @@
}
func (xd *extensionDesc) unmarshalFull(b []byte, nb *nameBuilder) {
- var rawDefVal []byte
xd.lazy = new(extensionLazy)
for len(b) > 0 {
num, typ, n := wire.ConsumeTag(b)
@@ -772,8 +787,7 @@
xd.lazy.hasJSONName = true
xd.lazy.jsonName = nb.MakeString(v)
case fieldnum.FieldDescriptorProto_DefaultValue:
- xd.lazy.defVal.has = true
- rawDefVal = v
+ xd.lazy.defVal.val = pref.ValueOf(v) // temporarily store as bytes; later resolved in resolveExtensions
case fieldnum.FieldDescriptorProto_Options:
xd.unmarshalOptions(v)
}
@@ -782,14 +796,6 @@
b = b[m:]
}
}
-
- if rawDefVal != nil {
- var err error
- xd.lazy.defVal.val, err = defval.Unmarshal(string(rawDefVal), xd.lazy.kind, defval.Descriptor)
- if err != nil {
- panic(err)
- }
- }
}
func (xd *extensionDesc) unmarshalOptions(b []byte) {
diff --git a/internal/fileinit/fileinit_test.go b/internal/fileinit/fileinit_test.go
deleted file mode 100644
index aca3a20..0000000
--- a/internal/fileinit/fileinit_test.go
+++ /dev/null
@@ -1,113 +0,0 @@
-package fileinit_test
-
-import (
- "bytes"
- "compress/gzip"
- "io/ioutil"
- "testing"
-
- proto "github.com/golang/protobuf/proto"
- "google.golang.org/protobuf/reflect/protodesc"
- "google.golang.org/protobuf/reflect/protoreflect"
-
- testpb "google.golang.org/protobuf/internal/testprotos/test"
- "google.golang.org/protobuf/types/descriptorpb"
-)
-
-func TestInit(t *testing.T) {
- // Compare the FileDescriptorProto for the same test file from two different sources:
- //
- // 1. The result of passing the fileinit-produced FileDescriptor through protodesc.
- // 2. The protoc-generated wire-encoded message.
- //
- // This serves as a test of both fileinit and protodesc.
- got := protodesc.ToFileDescriptorProto(testpb.File_test_test_proto)
-
- want := &descriptorpb.FileDescriptorProto{}
- zb, _ := (&testpb.TestAllTypes{}).Descriptor()
- r, _ := gzip.NewReader(bytes.NewBuffer(zb))
- b, _ := ioutil.ReadAll(r)
- if err := proto.Unmarshal(b, want); err != nil {
- t.Fatal(err)
- }
-
- if !proto.Equal(got, want) {
- t.Errorf("protodesc.ToFileDescriptorProto(testpb.Test_protoFile) is not equal to the protoc-generated FileDescriptorProto for internal/testprotos/test/test.proto")
- }
-
- // Verify that the test proto file provides exhaustive coverage of all descriptor fields.
- seen := make(map[protoreflect.FullName]bool)
- visitFields(want.ProtoReflect(), func(field protoreflect.FieldDescriptor) {
- seen[field.FullName()] = true
- })
- ignore := map[protoreflect.FullName]bool{
- // The protoreflect descriptors don't include source info.
- "google.protobuf.FileDescriptorProto.source_code_info": true,
- "google.protobuf.FileDescriptorProto.syntax": true,
-
- // TODO: Test oneof and extension options. Testing these requires extending the
- // options messages (because they contain no user-settable fields), but importing
- // decriptor.proto from test.proto currently causes an import cycle. Add test
- // cases when that import cycle has been fixed.
- "google.protobuf.OneofDescriptorProto.options": true,
- }
- for _, messageName := range []protoreflect.Name{
- "FileDescriptorProto",
- "DescriptorProto",
- "FieldDescriptorProto",
- "OneofDescriptorProto",
- "EnumDescriptorProto",
- "EnumValueDescriptorProto",
- "ServiceDescriptorProto",
- "MethodDescriptorProto",
- } {
- message := descriptorpb.File_google_protobuf_descriptor_proto.Messages().ByName(messageName)
- for i, fields := 0, message.Fields(); i < fields.Len(); i++ {
- if name := fields.Get(i).FullName(); !seen[name] && !ignore[name] {
- t.Errorf("No test for descriptor field: %v", name)
- }
- }
- }
-
- // Verify that message descriptors for map entries have no Go type info.
- mapEntryName := protoreflect.FullName("goproto.proto.test.TestAllTypes.MapInt32Int32Entry")
- d := testpb.File_test_test_proto.Messages().ByName("TestAllTypes").Fields().ByName("map_int32_int32").Message()
- if gotName, wantName := d.FullName(), mapEntryName; gotName != wantName {
- t.Fatalf("looked up wrong descriptor: got %v, want %v", gotName, wantName)
- }
- if _, ok := d.(protoreflect.MessageType); ok {
- t.Errorf("message descriptor for %v must not implement protoreflect.MessageType", mapEntryName)
- }
-}
-
-// visitFields calls f for every field set in m and its children.
-func visitFields(m protoreflect.Message, f func(protoreflect.FieldDescriptor)) {
- m.Range(func(fd protoreflect.FieldDescriptor, value protoreflect.Value) bool {
- f(fd)
- switch fd.Kind() {
- case protoreflect.MessageKind, protoreflect.GroupKind:
- if fd.IsList() {
- for i, list := 0, value.List(); i < list.Len(); i++ {
- visitFields(list.Get(i).Message(), f)
- }
- } else {
- visitFields(value.Message(), f)
- }
- }
- return true
- })
-}
-
-func TestWeakInit(t *testing.T) {
- file := testpb.File_test_test_proto
- fd := file.Messages().ByName("TestWeak").Fields().ByName("weak_message")
- if want, got := fd.IsWeak(), true; got != want {
- t.Errorf("field %v: IsWeak() = %v, want %v", fd.FullName(), want, got)
- }
- if want, got := fd.Message().IsPlaceholder(), false; got != want {
- t.Errorf("field %v: Message.IsPlaceholder() = %v, want %v", fd.FullName(), want, got)
- }
- if fd.Message().Fields().Len() == 0 {
- t.Errorf("field %v: Message().Fields().Len() == 0, want >0", fd.FullName())
- }
-}