goprotobuf: Several optimisations:
- avoid allocation in dec_slice_struct.
- store backpointer to StructProperties in Properties to avoid calls to GetProperties.
- speed up XXX_unrecognized operations.
R=adg
CC=golang-dev
http://codereview.appspot.com/6450135
diff --git a/proto/all_test.go b/proto/all_test.go
index f83ae3e..5b4c93c 100644
--- a/proto/all_test.go
+++ b/proto/all_test.go
@@ -1511,3 +1511,30 @@
p2.Unmarshal(pbd)
}
}
+
+func BenchmarkUnmarshalUnrecognizedFields(b *testing.B) {
+ b.StopTimer()
+ pb := initGoTestField()
+ skip := &GoSkipTest{
+ SkipInt32: Int32(32),
+ SkipFixed32: Uint32(3232),
+ SkipFixed64: Uint64(6464),
+ SkipString: String("skipper"),
+ Skipgroup: &GoSkipTest_SkipGroup{
+ GroupInt32: Int32(75),
+ GroupString: String("wxyz"),
+ },
+ }
+
+ pbd := new(GoTestField)
+ p := NewBuffer(nil)
+ p.Marshal(pb)
+ p.Marshal(skip)
+ p2 := NewBuffer(nil)
+
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ p2.SetBuf(p.Bytes())
+ p2.Unmarshal(pbd)
+ }
+}
diff --git a/proto/decode.go b/proto/decode.go
index c5b1ad7..1929747 100644
--- a/proto/decode.go
+++ b/proto/decode.go
@@ -213,7 +213,7 @@
// Skip the next item in the buffer. Its wire type is decoded and presented as an argument.
// If the protocol buffer has extensions, and the field matches, add it as an extension.
// Otherwise, if the XXX_unrecognized field exists, append the skipped data there.
-func (o *Buffer) skipAndSave(t reflect.Type, tag, wire int, base uintptr) error {
+func (o *Buffer) skipAndSave(t reflect.Type, tag, wire int, base, unrecOffset uintptr) error {
oi := o.index
@@ -222,13 +222,11 @@
return err
}
- x := fieldIndex(t, "XXX_unrecognized")
- if x == nil {
+ if unrecOffset == 0 {
return nil
}
- p := propByIndex(t, x)
- ptr := (*[]byte)(unsafe.Pointer(base + p.offset))
+ ptr := (*[]byte)(unsafe.Pointer(base + unrecOffset))
if *ptr == nil {
// This is the first skipped element,
@@ -286,6 +284,8 @@
}
// Unmarshaler is the interface representing objects that can unmarshal themselves.
+// The argument points to data that may be overwritten, so implementations should
+// not keep references to the buffer.
type Unmarshaler interface {
Unmarshal([]byte) error
}
@@ -333,7 +333,7 @@
return err
}
- err = p.unmarshalType(typ, false, base)
+ err = p.unmarshalType(typ, GetProperties(typ.Elem()), false, base)
if collectStats {
stats.Decode++
@@ -343,9 +343,8 @@
}
// unmarshalType does the work of unmarshaling a structure.
-func (o *Buffer) unmarshalType(t reflect.Type, is_group bool, base uintptr) error {
+func (o *Buffer) unmarshalType(t reflect.Type, prop *StructProperties, is_group bool, base uintptr) error {
st := t.Elem()
- prop := GetProperties(st)
required, reqFields := prop.reqCount, uint64(0)
var err error
@@ -379,7 +378,7 @@
}
continue
}
- err = o.skipAndSave(st, tag, wire, base)
+ err = o.skipAndSave(st, tag, wire, base, prop.unrecOffset)
continue
}
p := prop.Prop[fieldnum]
@@ -656,7 +655,7 @@
structv := unsafe.Pointer(bas)
*ptr = (*struct{})(structv)
- err := o.unmarshalType(p.stype, true, bas)
+ err := o.unmarshalType(p.stype, p.sprop, true, bas)
return err
}
@@ -685,7 +684,7 @@
o.buf = raw
o.index = 0
- err = o.unmarshalType(p.stype, false, bas)
+ err = o.unmarshalType(p.stype, p.sprop, false, bas)
o.buf = obuf
o.index = oi
@@ -715,11 +714,11 @@
*v = y
if is_group {
- err := o.unmarshalType(p.stype, is_group, bas)
+ err := o.unmarshalType(p.stype, p.sprop, is_group, bas)
return err
}
- raw, err := o.DecodeRawBytes(true)
+ raw, err := o.DecodeRawBytes(false)
if err != nil {
return err
}
@@ -735,7 +734,7 @@
o.buf = raw
o.index = 0
- err = o.unmarshalType(p.stype, is_group, bas)
+ err = o.unmarshalType(p.stype, p.sprop, is_group, bas)
o.buf = obuf
o.index = oi
diff --git a/proto/encode.go b/proto/encode.go
index d2d63a8..105e6ce 100644
--- a/proto/encode.go
+++ b/proto/encode.go
@@ -197,7 +197,7 @@
t, b, err := getbase(pb)
if err == nil {
- err = p.enc_struct(t.Elem(), b)
+ err = p.enc_struct(t.Elem(), GetProperties(t.Elem()), b)
}
if collectStats {
@@ -295,7 +295,7 @@
obuf := o.buf
o.buf = o.bufalloc()
- err := o.enc_struct(typ, uintptr(structp))
+ err := o.enc_struct(typ, p.sprop, uintptr(structp))
nbuf := o.buf
o.buf = obuf
@@ -319,7 +319,7 @@
o.EncodeVarint(uint64((p.Tag << 3) | WireStartGroup))
b := uintptr(unsafe.Pointer(v))
typ := p.stype.Elem()
- err := o.enc_struct(typ, b)
+ err := o.enc_struct(typ, p.sprop, b)
if err != nil {
return err
}
@@ -495,7 +495,7 @@
obuf := o.buf
o.buf = o.bufalloc()
- err := o.enc_struct(typ, uintptr(structp))
+ err := o.enc_struct(typ, p.sprop, uintptr(structp))
nbuf := o.buf
o.buf = obuf
@@ -529,7 +529,7 @@
o.EncodeVarint(uint64((p.Tag << 3) | WireStartGroup))
b := uintptr(unsafe.Pointer(v))
- err := o.enc_struct(typ, b)
+ err := o.enc_struct(typ, p.sprop, b)
if err != nil {
if err == ErrNil {
@@ -556,8 +556,7 @@
}
// Encode a struct.
-func (o *Buffer) enc_struct(t reflect.Type, base uintptr) error {
- prop := GetProperties(t)
+func (o *Buffer) enc_struct(t reflect.Type, prop *StructProperties, base uintptr) error {
required := prop.reqCount
// Encode fields in tag order so that decoders may use optimizations
// that depend on the ordering.
diff --git a/proto/properties.go b/proto/properties.go
index 3da6997..81adb52 100644
--- a/proto/properties.go
+++ b/proto/properties.go
@@ -115,7 +115,8 @@
offset uintptr
tagcode []byte // encoding of EncodeVarint((Tag<<3)|WireType)
tagbuf [8]byte
- stype reflect.Type
+ stype reflect.Type // set for struct types only
+ sprop *StructProperties // set for struct types only
isMarshaler bool
isUnmarshaler bool
@@ -233,7 +234,7 @@
var protoMessageType = reflect.TypeOf((*Message)(nil)).Elem()
// Initialize the fields for encoding and decoding.
-func (p *Properties) setEncAndDec(typ reflect.Type) {
+func (p *Properties) setEncAndDec(typ reflect.Type, lockGetProp bool) {
p.enc = nil
p.dec = nil
@@ -385,6 +386,14 @@
}
p.tagbuf[i] = uint8(x)
p.tagcode = p.tagbuf[0 : i+1]
+
+ if p.stype != nil {
+ if lockGetProp {
+ p.sprop = GetProperties(p.stype.Elem())
+ } else {
+ p.sprop = getPropertiesLocked(p.stype.Elem())
+ }
+ }
}
var (
@@ -416,6 +425,10 @@
// Init populates the properties from a protocol buffer struct tag.
func (p *Properties) Init(typ reflect.Type, name, tag string, offset uintptr) {
+ p.init(typ, name, tag, offset, true)
+}
+
+func (p *Properties) init(typ reflect.Type, name, tag string, offset uintptr, lockGetProp bool) {
// "bytes,49,opt,def=hello!"
p.Name = name
p.OrigName = name
@@ -425,7 +438,7 @@
return
}
p.Parse(tag)
- p.setEncAndDec(typ)
+ p.setEncAndDec(typ, lockGetProp)
}
var (
@@ -436,8 +449,14 @@
// GetProperties returns the list of properties for the type represented by t.
func GetProperties(t reflect.Type) *StructProperties {
mutex.Lock()
+ sprop := getPropertiesLocked(t)
+ mutex.Unlock()
+ return sprop
+}
+
+// getPropertiesLocked requires that mutex is held.
+func getPropertiesLocked(t reflect.Type) *StructProperties {
if prop, ok := propertiesMap[t]; ok {
- mutex.Unlock()
if collectStats {
stats.Chit++
}
@@ -448,6 +467,8 @@
}
prop := new(StructProperties)
+ // in case of recursive protos, fill this in now.
+ propertiesMap[t] = prop
// build properties
prop.Prop = make([]*Properties, t.NumField())
@@ -455,7 +476,7 @@
for i := 0; i < t.NumField(); i++ {
f := t.Field(i)
p := new(Properties)
- p.Init(f.Type, f.Name, f.Tag.Get("protobuf"), f.Offset)
+ p.init(f.Type, f.Name, f.Tag.Get("protobuf"), f.Offset, false)
if f.Name == "XXX_extensions" { // special case
p.enc = (*Buffer).enc_map
p.dec = nil // not needed
@@ -499,20 +520,9 @@
}
prop.reqCount = reqCount
- propertiesMap[t] = prop
- mutex.Unlock()
return prop
}
-// Return the field index of the named field.
-// Returns nil if there is no such field.
-func fieldIndex(t reflect.Type, name string) []int {
- if field, ok := t.FieldByName(name); ok {
- return field.Index
- }
- return nil
-}
-
// Return the Properties object for the x[0]'th field of the structure.
func propByIndex(t reflect.Type, x []int) *Properties {
if len(x) != 1 {