internal/encoding/json: rewrite to a token-based encoder and decoder
Previous decoder decodes a JSON number into a float64, which lacks
64-bit integer precision.
I attempted to retrofit it with storing the raw bytes and parsed out
number parts, see golang.org/cl/164377. While that is possible, the
encoding logic for Value is not symmetrical with the decoding logic and
can be confusing since both utilizes the same Value struct.
Joe and I decided that it would be better to rewrite the JSON encoder
and decoder to be token-based instead, removing the need for sharing a
model type plus making it more efficient.
Change-Id: Ic0601428a824be4e20141623409ab4d92b6167c7
Reviewed-on: https://go-review.googlesource.com/c/protobuf/+/165677
Reviewed-by: Damien Neil <dneil@google.com>
diff --git a/internal/encoding/json/string.go b/internal/encoding/json/string.go
index 813d8fa..fbfb09f 100644
--- a/internal/encoding/json/string.go
+++ b/internal/encoding/json/string.go
@@ -15,20 +15,7 @@
"github.com/golang/protobuf/v2/internal/errors"
)
-func (p *encoder) marshalString(v Value) error {
- var err error
- p.out, err = appendString(p.out, v)
- return err
-}
-func appendString(out []byte, v Value) ([]byte, error) {
- if v.Type() != String {
- return nil, errors.New("invalid type %v, expected string", v.Type())
- }
- if len(v.raw) > 0 {
- return append(out, v.raw...), nil
- }
- in := v.String()
-
+func appendString(out []byte, in string) ([]byte, error) {
var nerr errors.NonFatal
out = append(out, '"')
i := indexNeedEscape(in)
@@ -68,19 +55,14 @@
return out, nerr.E
}
-func (p *decoder) unmarshalString() (Value, error) {
- v, n, err := consumeString(p.in)
- p.consume(n)
- return v, err
-}
-func consumeString(in []byte) (Value, int, error) {
+func (d *Decoder) parseString(in []byte) (string, int, error) {
var nerr errors.NonFatal
in0 := in
if len(in) == 0 {
- return Value{}, 0, io.ErrUnexpectedEOF
+ return "", 0, io.ErrUnexpectedEOF
}
if in[0] != '"' {
- return Value{}, 0, newSyntaxError("invalid character %q at start of string", in[0])
+ return "", 0, d.newSyntaxError("invalid character %q at start of string", in[0])
}
in = in[1:]
i := indexNeedEscape(string(in))
@@ -91,15 +73,14 @@
nerr.AppendInvalidUTF8("")
in, out = in[1:], append(out, in[0]) // preserve invalid byte
case r < ' ':
- return Value{}, 0, newSyntaxError("invalid character %q in string", r)
+ return "", 0, d.newSyntaxError("invalid character %q in string", r)
case r == '"':
in = in[1:]
n := len(in0) - len(in)
- v := rawValueOf(string(out), in0[:n:n])
- return v, n, nerr.E
+ return string(out), n, nerr.E
case r == '\\':
if len(in) < 2 {
- return Value{}, 0, io.ErrUnexpectedEOF
+ return "", 0, io.ErrUnexpectedEOF
}
switch r := in[1]; r {
case '"', '\\', '/':
@@ -116,36 +97,37 @@
in, out = in[2:], append(out, '\t')
case 'u':
if len(in) < 6 {
- return Value{}, 0, io.ErrUnexpectedEOF
+ return "", 0, io.ErrUnexpectedEOF
}
v, err := strconv.ParseUint(string(in[2:6]), 16, 16)
if err != nil {
- return Value{}, 0, newSyntaxError("invalid escape code %q in string", in[:6])
+ return "", 0, d.newSyntaxError("invalid escape code %q in string", in[:6])
}
in = in[6:]
r := rune(v)
if utf16.IsSurrogate(r) {
if len(in) < 6 {
- return Value{}, 0, io.ErrUnexpectedEOF
+ return "", 0, io.ErrUnexpectedEOF
}
v, err := strconv.ParseUint(string(in[2:6]), 16, 16)
r = utf16.DecodeRune(r, rune(v))
- if in[0] != '\\' || in[1] != 'u' || r == unicode.ReplacementChar || err != nil {
- return Value{}, 0, newSyntaxError("invalid escape code %q in string", in[:6])
+ if in[0] != '\\' || in[1] != 'u' ||
+ r == unicode.ReplacementChar || err != nil {
+ return "", 0, d.newSyntaxError("invalid escape code %q in string", in[:6])
}
in = in[6:]
}
out = append(out, string(r)...)
default:
- return Value{}, 0, newSyntaxError("invalid escape code %q in string", in[:2])
+ return "", 0, d.newSyntaxError("invalid escape code %q in string", in[:2])
}
default:
i := indexNeedEscape(string(in[n:]))
in, out = in[n+i:], append(out, in[:n+i]...)
}
}
- return Value{}, 0, io.ErrUnexpectedEOF
+ return "", 0, io.ErrUnexpectedEOF
}
// indexNeedEscape returns the index of the next character that needs escaping.