Skip to content

Commit 540f014

Browse files
dsnetdsnet
authored
Support MarshalOptions.Deterministic (#260)
The Deterministic flag specifies that marshaling the same input value should consistently return the same sequence of bytes when executed by the same version of the Go program.
1 parent 3fecd76 commit 540f014

10 files changed

+399
-106
lines changed

‎arshal.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,13 @@ type MarshalOptions struct {
3434
// unknown JSON object members.
3535
DiscardUnknownMembers bool
3636

37+
// Deterministic specifies that the same input value will be serialized
38+
// as the exact same output bytes. Different processes of
39+
// the same program will serialize equal values to the same bytes,
40+
// but different versions of the same program are not guaranteed
41+
// to produce the exact same sequence of bytes.
42+
Deterministic bool
43+
3744
// formatDepth is the depth at which we respect the format flag.
3845
formatDepth int
3946
// format is custom formatting for the value at the specified depth.

‎arshal_any.go

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -99,13 +99,32 @@ func marshalObjectAny(mo MarshalOptions, enc *Encoder, obj map[string]any) error
9999
if !enc.options.AllowInvalidUTF8 {
100100
enc.tokens.last.disableNamespace()
101101
}
102-
for name, val := range obj {
103-
if err := enc.WriteToken(String(name)); err != nil {
104-
return err
102+
if !mo.Deterministic || len(obj) <= 1 {
103+
for name, val := range obj {
104+
if err := enc.WriteToken(String(name)); err != nil {
105+
return err
106+
}
107+
if err := marshalValueAny(mo, enc, val); err != nil {
108+
return err
109+
}
105110
}
106-
if err := marshalValueAny(mo, enc, val); err != nil {
107-
return err
111+
} else {
112+
names := getStrings(len(obj))
113+
var i int
114+
for name := range obj {
115+
(*names)[i] = name
116+
i++
117+
}
118+
names.Sort()
119+
for _, name := range *names {
120+
if err := enc.WriteToken(String(name)); err != nil {
121+
return err
122+
}
123+
if err := marshalValueAny(mo, enc, obj[name]); err != nil {
124+
return err
125+
}
108126
}
127+
putStrings(names)
109128
}
110129
if err := enc.WriteToken(ObjectEnd); err != nil {
111130
return err

‎arshal_default.go

Lines changed: 70 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"fmt"
1414
"math"
1515
"reflect"
16+
"sort"
1617
"strconv"
1718
"sync"
1819
)
@@ -627,19 +628,76 @@ func makeMapArshaler(t reflect.Type) *arshaler {
627628
enc.tokens.last.disableNamespace()
628629
}
629630

630-
// NOTE: Map entries are serialized in a non-deterministic order.
631-
// Users that need stable output should call RawValue.Canonicalize.
632-
for iter := va.Value.MapRange(); iter.Next(); {
633-
k.SetIterKey(iter)
634-
if err := marshalKey(mko, enc, k); err != nil {
635-
// TODO: If err is errMissingName, then wrap it as a
636-
// SemanticError since this key type cannot be serialized
637-
// as a JSON string.
638-
return err
631+
switch {
632+
case !mo.Deterministic || n <= 1:
633+
for iter := va.Value.MapRange(); iter.Next(); {
634+
k.SetIterKey(iter)
635+
if err := marshalKey(mko, enc, k); err != nil {
636+
// TODO: If err is errMissingName, then wrap it as a
637+
// SemanticError since this key type cannot be serialized
638+
// as a JSON string.
639+
return err
640+
}
641+
v.SetIterValue(iter)
642+
if err := marshalVal(mo, enc, v); err != nil {
643+
return err
644+
}
639645
}
640-
v.SetIterValue(iter)
641-
if err := marshalVal(mo, enc, v); err != nil {
642-
return err
646+
case !nonDefaultKey && t.Key().Kind() == reflect.String:
647+
names := getStrings(n)
648+
for i, iter := 0, va.Value.MapRange(); i < n && iter.Next(); i++ {
649+
k.SetIterKey(iter)
650+
(*names)[i] = k.String()
651+
}
652+
names.Sort()
653+
for _, name := range *names {
654+
if err := enc.WriteToken(String(name)); err != nil {
655+
return err
656+
}
657+
// TODO(https://go.dev/issue/57061): Use v.SetMapIndexOf.
658+
k.SetString(name)
659+
v.Set(va.MapIndex(k.Value))
660+
if err := marshalVal(mo, enc, v); err != nil {
661+
return err
662+
}
663+
}
664+
putStrings(names)
665+
default:
666+
type member struct {
667+
name string // unquoted name
668+
key addressableValue
669+
}
670+
members := make([]member, n)
671+
keys := reflect.MakeSlice(reflect.SliceOf(t.Key()), n, n)
672+
for i, iter := 0, va.Value.MapRange(); i < n && iter.Next(); i++ {
673+
// Marshal the member name.
674+
k := addressableValue{keys.Index(i)} // indexed slice element is always addressable
675+
k.SetIterKey(iter)
676+
if err := marshalKey(mko, enc, k); err != nil {
677+
// TODO: If err is errMissingName, then wrap it as a
678+
// SemanticError since this key type cannot be serialized
679+
// as a JSON string.
680+
return err
681+
}
682+
name := enc.unwriteOnlyObjectMemberName()
683+
members[i] = member{name, k}
684+
}
685+
// TODO: If AllowDuplicateNames is enabled, then sort according
686+
// to reflect.Value as well if the names are equal.
687+
// See internal/fmtsort.
688+
// TODO(https://go.dev/issue/47619): Use slices.SortFunc instead.
689+
sort.Slice(members, func(i, j int) bool {
690+
return lessUTF16(members[i].name, members[j].name)
691+
})
692+
for _, member := range members {
693+
if err := enc.WriteToken(String(member.name)); err != nil {
694+
return err
695+
}
696+
// TODO(https://go.dev/issue/57061): Use v.SetMapIndexOf.
697+
v.Set(va.MapIndex(member.key.Value))
698+
if err := marshalVal(mo, enc, v); err != nil {
699+
return err
700+
}
643701
}
644702
}
645703
}

‎arshal_inlined.go

Lines changed: 42 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
package json
66

77
import (
8+
"bytes"
89
"errors"
910
"reflect"
1011
)
@@ -89,35 +90,61 @@ func marshalInlinedFallbackAll(mo MarshalOptions, enc *Encoder, va addressableVa
8990
}
9091
return nil
9192
} else {
92-
if v.Len() == 0 {
93+
m := v // must be a map[string]V
94+
n := m.Len()
95+
if n == 0 {
9396
return nil
9497
}
95-
m := v
98+
mk := newAddressableValue(stringType)
9699
mv := newAddressableValue(m.Type().Elem())
97-
for iter := m.MapRange(); iter.Next(); {
98-
b, err := appendString(enc.UnusedBuffer(), iter.Key().String(), !enc.options.AllowInvalidUTF8, nil)
100+
marshalKey := func(mk addressableValue) error {
101+
b, err := appendString(enc.UnusedBuffer(), mk.String(), !enc.options.AllowInvalidUTF8, nil)
99102
if err != nil {
100103
return err
101104
}
102105
if insertUnquotedName != nil {
103-
isVerbatim := consumeSimpleString(b) == len(b)
106+
isVerbatim := bytes.IndexByte(b, '\\') < 0
104107
name := unescapeStringMayCopy(b, isVerbatim)
105108
if !insertUnquotedName(name) {
106109
return &SyntacticError{str: "duplicate name " + string(b) + " in object"}
107110
}
108111
}
109-
if err := enc.WriteValue(b); err != nil {
110-
return err
112+
return enc.WriteValue(b)
113+
}
114+
marshalVal := f.fncs.marshal
115+
if mo.Marshalers != nil {
116+
marshalVal, _ = mo.Marshalers.lookup(marshalVal, mv.Type())
117+
}
118+
if !mo.Deterministic || n <= 1 {
119+
for iter := m.MapRange(); iter.Next(); {
120+
mk.SetIterKey(iter)
121+
if err := marshalKey(mk); err != nil {
122+
return err
123+
}
124+
mv.Set(iter.Value())
125+
if err := marshalVal(mo, enc, mv); err != nil {
126+
return err
127+
}
111128
}
112-
113-
mv.Set(iter.Value())
114-
marshal := f.fncs.marshal
115-
if mo.Marshalers != nil {
116-
marshal, _ = mo.Marshalers.lookup(marshal, mv.Type())
129+
} else {
130+
names := getStrings(n)
131+
for i, iter := 0, m.Value.MapRange(); i < n && iter.Next(); i++ {
132+
mk.SetIterKey(iter)
133+
(*names)[i] = mk.String()
117134
}
118-
if err := marshal(mo, enc, mv); err != nil {
119-
return err
135+
names.Sort()
136+
for _, name := range *names {
137+
mk.SetString(name)
138+
if err := marshalKey(mk); err != nil {
139+
return err
140+
}
141+
// TODO(https://go.dev/issue/57061): Use mv.SetMapIndexOf.
142+
mv.Set(m.MapIndex(mk.Value))
143+
if err := marshalVal(mo, enc, mv); err != nil {
144+
return err
145+
}
120146
}
147+
putStrings(names)
121148
}
122149
return nil
123150
}
@@ -162,7 +189,7 @@ func unmarshalInlinedFallbackNext(uo UnmarshalOptions, dec *Decoder, va addressa
162189
} else {
163190
name := string(unquotedName) // TODO: Intern this?
164191

165-
m := v
192+
m := v // must be a map[string]V
166193
if m.IsNil() {
167194
m.Set(reflect.MakeMap(m.Type()))
168195
}

0 commit comments

Comments
 (0)