bsoncodec/bsonrw: eliminate encoding allocations

This commit eliminates all allocations during marshalling (except for
the final []byte slice allocation, which is unavoidable). This matches
the behavior of encoding/json. It also prevents the vwPool from leaking
writers and includes a few small optimizations to the value_writer.

Other Changes:
   * bsoncodec: reduce use of reflect.Value.Interface()
   * bson: use a buffer pool instead of SliceWriter to eliminate 2 allocs
   * bsonrw: use references to slice indexes

This work builds off of #1313
and uses the BenchmarkCode* suite added by that PR.

The combined performance improvement is below:

```
goos: darwin
goarch: arm64
pkg: go.mongodb.org/mongo-driver/bson
                      │ base.20.txt  │             new.20.txt              │
                      │    sec/op    │   sec/op     vs base                │
CodeUnmarshal/BSON-10    3.177m ± 1%   2.202m ± 1%  -30.68% (p=0.000 n=20)
CodeMarshal/BSON-10     2922.7µ ± 0%   757.1µ ± 2%  -74.10% (p=0.000 n=20)
geomean                  3.047m        1.291m       -57.63%

                      │ base.20.txt  │               new.20.txt               │
                      │     B/s      │      B/s       vs base                 │
CodeUnmarshal/BSON-10   582.5Mi ± 1%    840.4Mi ± 1%   +44.26% (p=0.000 n=20)
CodeMarshal/BSON-10     633.2Mi ± 0%   2444.3Mi ± 2%  +286.03% (p=0.000 n=20)
geomean                 607.3Mi         1.400Gi       +135.99%

                      │ base.20.txt  │              new.20.txt              │
                      │     B/op     │     B/op      vs base                │
CodeUnmarshal/BSON-10   4.219Mi ± 0%   4.148Mi ± 0%   -1.69% (p=0.000 n=20)
CodeMarshal/BSON-10     2.818Mi ± 3%   1.630Mi ± 0%  -42.16% (p=0.000 n=20)
geomean                 3.448Mi        2.600Mi       -24.59%

                      │  base.20.txt   │              new.20.txt              │
                      │   allocs/op    │  allocs/op   vs base                 │
CodeUnmarshal/BSON-10      230.4k ± 0%   220.7k ± 0%    -4.21% (p=0.000 n=20)
CodeMarshal/BSON-10     94066.000 ± 0%    1.000 ± 0%  -100.00% (p=0.000 n=20)
geomean                    147.2k         469.7        -99.68%
```

Author: charlievieth

bson/bsoncodec/slice_codec.go

@@ -62,7 +62,7 @@ func (sc SliceCodec) EncodeValue(ec EncodeContext, vw bsonrw.ValueWriter, val re
 	}
 
 	// If we have a []primitive.E we want to treat it as a document instead of as an array.
-	if val.Type().ConvertibleTo(tD) {
+	if val.Type() == tD || val.Type().ConvertibleTo(tD) {
 		d := val.Convert(tD).Interface().(primitive.D)
 
 		dw, err := vw.WriteDocument()

bson/bsoncodec/struct_codec.go

@@ -192,15 +192,14 @@ func (sc *StructCodec) EncodeValue(ec EncodeContext, vw bsonrw.ValueWriter, val
 		encoder := desc.encoder
 
 		var zero bool
-		rvInterface := rv.Interface()
 		if cz, ok := encoder.(CodecZeroer); ok {
-			zero = cz.IsTypeZero(rvInterface)
+			zero = cz.IsTypeZero(rv.Interface())
 		} else if rv.Kind() == reflect.Interface {
 			// isZero will not treat an interface rv as an interface, so we need to check for the
 			// zero interface separately.
 			zero = rv.IsNil()
 		} else {
-			zero = isZero(rvInterface, sc.EncodeOmitDefaultStruct || ec.omitZeroStruct)
+			zero = isZero(rv, sc.EncodeOmitDefaultStruct || ec.omitZeroStruct)
 		}
 		if desc.omitEmpty && zero {
 			continue
@@ -394,56 +393,32 @@ func (sc *StructCodec) DecodeValue(dc DecodeContext, vr bsonrw.ValueReader, val
 	return nil
 }
 
-func isZero(i interface{}, omitZeroStruct bool) bool {
-	v := reflect.ValueOf(i)
-
-	// check the value validity
-	if !v.IsValid() {
-		return true
+func isZero(v reflect.Value, omitZeroStruct bool) bool {
+	kind := v.Kind()
+	if (kind != reflect.Pointer || !v.IsNil()) && v.Type().Implements(tZeroer) {
+		return v.Interface().(Zeroer).IsZero()
 	}
-
-	if z, ok := v.Interface().(Zeroer); ok && (v.Kind() != reflect.Ptr || !v.IsNil()) {
-		return z.IsZero()
-	}
-
-	switch v.Kind() {
-	case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
-		return v.Len() == 0
-	case reflect.Bool:
-		return !v.Bool()
-	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
-		return v.Int() == 0
-	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
-		return v.Uint() == 0
-	case reflect.Float32, reflect.Float64:
-		return v.Float() == 0
-	case reflect.Interface, reflect.Ptr:
-		return v.IsNil()
-	case reflect.Struct:
+	if kind == reflect.Struct {
 		if !omitZeroStruct {
 			return false
 		}
-
-		// TODO(GODRIVER-2820): Update the logic to be able to handle private struct fields.
-		// TODO Use condition "reflect.Zero(v.Type()).Equal(v)" instead.
-
 		vt := v.Type()
 		if vt == tTime {
 			return v.Interface().(time.Time).IsZero()
 		}
-		for i := 0; i < v.NumField(); i++ {
-			if vt.Field(i).PkgPath != "" && !vt.Field(i).Anonymous {
+		numField := vt.NumField()
+		for i := 0; i < numField; i++ {
+			ff := vt.Field(i)
+			if ff.PkgPath != "" && !ff.Anonymous {
 				continue // Private field
 			}
-			fld := v.Field(i)
-			if !isZero(fld.Interface(), omitZeroStruct) {
+			if !isZero(v.Field(i), omitZeroStruct) {
 				return false
 			}
 		}
 		return true
 	}
-
-	return false
+	return !v.IsValid() || v.IsZero()
 }
 
 type structDescription struct {
@@ -700,21 +675,21 @@ func getInlineField(val reflect.Value, index []int) (reflect.Value, error) {
 
 // DeepZero returns recursive zero object
 func deepZero(st reflect.Type) (result reflect.Value) {
-	result = reflect.Indirect(reflect.New(st))
-
-	if result.Kind() == reflect.Struct {
-		for i := 0; i < result.NumField(); i++ {
-			if f := result.Field(i); f.Kind() == reflect.Ptr {
-				if f.CanInterface() {
-					if ft := reflect.TypeOf(f.Interface()); ft.Elem().Kind() == reflect.Struct {
-						result.Field(i).Set(recursivePointerTo(deepZero(ft.Elem())))
-					}
+	if st.Kind() == reflect.Struct {
+		numField := st.NumField()
+		for i := 0; i < numField; i++ {
+			if result == emptyValue {
+				result = reflect.Indirect(reflect.New(st))
+			}
+			f := result.Field(i)
+			if f.CanInterface() {
+				if f.Type().Kind() == reflect.Struct {
+					result.Field(i).Set(recursivePointerTo(deepZero(f.Type().Elem())))
 				}
 			}
 		}
 	}
-
-	return
+	return result
 }
 
 // recursivePointerTo calls reflect.New(v.Type) but recursively for its fields inside

bson/bsoncodec/struct_codec_test.go

@@ -7,6 +7,7 @@
 package bsoncodec
 
 import (
+	"reflect"
 	"testing"
 	"time"
 
@@ -147,7 +148,7 @@ func TestIsZero(t *testing.T) {
 		t.Run(tc.description, func(t *testing.T) {
 			t.Parallel()
 
-			got := isZero(tc.value, tc.omitZeroStruct)
+			got := isZero(reflect.ValueOf(tc.value), tc.omitZeroStruct)
 			assert.Equal(t, tc.want, got, "expected and actual isZero return are different")
 		})
 	}

bson/bsoncodec/types.go

@@ -34,6 +34,7 @@ var tValueUnmarshaler = reflect.TypeOf((*ValueUnmarshaler)(nil)).Elem()
 var tMarshaler = reflect.TypeOf((*Marshaler)(nil)).Elem()
 var tUnmarshaler = reflect.TypeOf((*Unmarshaler)(nil)).Elem()
 var tProxy = reflect.TypeOf((*Proxy)(nil)).Elem()
+var tZeroer = reflect.TypeOf((*Zeroer)(nil)).Elem()
 
 var tBinary = reflect.TypeOf(primitive.Binary{})
 var tUndefined = reflect.TypeOf(primitive.Undefined{})

bson/bsonrw/copier.go

@@ -193,7 +193,7 @@ func (c Copier) AppendDocumentBytes(dst []byte, src ValueReader) ([]byte, error)
 	}
 
 	vw := vwPool.Get().(*valueWriter)
-	defer vwPool.Put(vw)
+	defer putValueWriter(vw)
 
 	vw.reset(dst)
 
@@ -213,7 +213,7 @@ func (c Copier) AppendArrayBytes(dst []byte, src ValueReader) ([]byte, error) {
 	}
 
 	vw := vwPool.Get().(*valueWriter)
-	defer vwPool.Put(vw)
+	defer putValueWriter(vw)
 
 	vw.reset(dst)
 
@@ -258,7 +258,7 @@ func (c Copier) AppendValueBytes(dst []byte, src ValueReader) (bsontype.Type, []
 	}
 
 	vw := vwPool.Get().(*valueWriter)
-	defer vwPool.Put(vw)
+	defer putValueWriter(vw)
 
 	start := len(dst)
 

bson/bsonrw/value_reader.go

@@ -739,8 +739,7 @@ func (vr *valueReader) ReadValue() (ValueReader, error) {
 		return nil, ErrEOA
 	}
 
-	_, err = vr.readCString()
-	if err != nil {
+	if err := vr.consumeCString(); err != nil {
 		return nil, err
 	}
 
@@ -794,6 +793,15 @@ func (vr *valueReader) readByte() (byte, error) {
 	return vr.d[vr.offset-1], nil
 }
 
+func (vr *valueReader) consumeCString() error {
+	idx := bytes.IndexByte(vr.d[vr.offset:], 0x00)
+	if idx < 0 {
+		return io.EOF
+	}
+	vr.offset += int64(idx) + 1
+	return nil
+}
+
 func (vr *valueReader) readCString() (string, error) {
 	idx := bytes.IndexByte(vr.d[vr.offset:], 0x00)
 	if idx < 0 {

bson/bsonrw/value_writer.go

@@ -28,6 +28,13 @@ var vwPool = sync.Pool{
 	},
 }
 
+func putValueWriter(vw *valueWriter) {
+	if vw != nil {
+		vw.w = nil // don't leak the writer
+		vwPool.Put(vw)
+	}
+}
+
 // BSONValueWriterPool is a pool for BSON ValueWriters.
 //
 // Deprecated: BSONValueWriterPool will not be supported in Go Driver 2.0.
@@ -149,32 +156,21 @@ type valueWriter struct {
 }
 
 func (vw *valueWriter) advanceFrame() {
-	if vw.frame+1 >= int64(len(vw.stack)) { // We need to grow the stack
-		length := len(vw.stack)
-		if length+1 >= cap(vw.stack) {
-			// double it
-			buf := make([]vwState, 2*cap(vw.stack)+1)
-			copy(buf, vw.stack)
-			vw.stack = buf
-		}
-		vw.stack = vw.stack[:length+1]
-	}
 	vw.frame++
+	if vw.frame >= int64(len(vw.stack)) {
+		vw.stack = append(vw.stack, vwState{})
+	}
 }
 
 func (vw *valueWriter) push(m mode) {
 	vw.advanceFrame()
 
 	// Clean the stack
-	vw.stack[vw.frame].mode = m
-	vw.stack[vw.frame].key = ""
-	vw.stack[vw.frame].arrkey = 0
-	vw.stack[vw.frame].start = 0
+	vw.stack[vw.frame] = vwState{mode: m}
 
-	vw.stack[vw.frame].mode = m
 	switch m {
 	case mDocument, mArray, mCodeWithScope:
-		vw.reserveLength()
+		vw.reserveLength() // WARN: this is not needed
 	}
 }
 
@@ -213,6 +209,7 @@ func newValueWriter(w io.Writer) *valueWriter {
 	return vw
 }
 
+// TODO: only used in tests
 func newValueWriterFromSlice(buf []byte) *valueWriter {
 	vw := new(valueWriter)
 	stack := make([]vwState, 1, 5)
@@ -249,17 +246,17 @@ func (vw *valueWriter) invalidTransitionError(destination mode, name string, mod
 }
 
 func (vw *valueWriter) writeElementHeader(t bsontype.Type, destination mode, callerName string, addmodes ...mode) error {
-	switch vw.stack[vw.frame].mode {
+	frame := &vw.stack[vw.frame]
+	switch frame.mode {
 	case mElement:
-		key := vw.stack[vw.frame].key
+		key := frame.key
 		if !isValidCString(key) {
 			return errors.New("BSON element key cannot contain null bytes")
 		}
-
-		vw.buf = bsoncore.AppendHeader(vw.buf, t, key)
+		vw.appendHeader(t, key)
 	case mValue:
 		// TODO: Do this with a cache of the first 1000 or so array keys.
-		vw.buf = bsoncore.AppendHeader(vw.buf, t, strconv.Itoa(vw.stack[vw.frame].arrkey))
+		vw.appendIntHeader(t, frame.arrkey)
 	default:
 		modes := []mode{mElement, mValue}
 		if addmodes != nil {
@@ -601,9 +598,11 @@ func (vw *valueWriter) writeLength() error {
 	if length > maxSize {
 		return errMaxDocumentSizeExceeded{size: int64(len(vw.buf))}
 	}
-	length = length - int(vw.stack[vw.frame].start)
-	start := vw.stack[vw.frame].start
+	frame := &vw.stack[vw.frame]
+	length = length - int(frame.start)
+	start := frame.start
 
+	_ = vw.buf[start+3] // BCE
 	vw.buf[start+0] = byte(length)
 	vw.buf[start+1] = byte(length >> 8)
 	vw.buf[start+2] = byte(length >> 16)
@@ -612,5 +611,21 @@ func (vw *valueWriter) writeLength() error {
 }
 
 func isValidCString(cs string) bool {
-	return !strings.ContainsRune(cs, '\x00')
+	return strings.IndexByte(cs, 0) == -1
+}
+
+// appendHeader is the same as bsoncore.AppendHeader but does not check if the
+// key is a valid C string since the caller has already checked for that.
+//
+// The caller of this function must check if key is a valid C string.
+func (vw *valueWriter) appendHeader(t bsontype.Type, key string) {
+	vw.buf = bsoncore.AppendType(vw.buf, t)
+	vw.buf = append(vw.buf, key...)
+	vw.buf = append(vw.buf, 0x00)
+}
+
+func (vw *valueWriter) appendIntHeader(t bsontype.Type, key int) {
+	vw.buf = bsoncore.AppendType(vw.buf, t)
+	vw.buf = strconv.AppendInt(vw.buf, int64(key), 10)
+	vw.buf = append(vw.buf, 0x00)
 }

bson/marshal.go

@@ -9,6 +9,7 @@ package bson
 import (
 	"bytes"
 	"encoding/json"
+	"sync"
 
 	"go.mongodb.org/mongo-driver/bson/bsoncodec"
 	"go.mongodb.org/mongo-driver/bson/bsonrw"
@@ -141,6 +142,13 @@ func MarshalAppendWithRegistry(r *bsoncodec.Registry, dst []byte, val interface{
 	return MarshalAppendWithContext(bsoncodec.EncodeContext{Registry: r}, dst, val)
 }
 
+// Pool of buffers for marshalling BSON.
+var bufPool = sync.Pool{
+	New: func() interface{} {
+		return new(bytes.Buffer)
+	},
+}
+
 // MarshalAppendWithContext will encode val as a BSON document using Registry r and EncodeContext ec and append the
 // bytes to dst. If dst is not large enough to hold the bytes, it will be grown. If val is not a type that can be
 // transformed into a document, MarshalValueAppendWithContext should be used instead.
@@ -162,8 +170,9 @@ func MarshalAppendWithRegistry(r *bsoncodec.Registry, dst []byte, val interface{
 //
 // See [Encoder] for more examples.
 func MarshalAppendWithContext(ec bsoncodec.EncodeContext, dst []byte, val interface{}) ([]byte, error) {
-	sw := new(bsonrw.SliceWriter)
-	*sw = dst
+	sw := bufPool.Get().(*bytes.Buffer)
+	defer bufPool.Put(sw)
+	sw.Reset()
 	vw := bvwPool.Get(sw)
 	defer bvwPool.Put(vw)
 
@@ -184,7 +193,7 @@ func MarshalAppendWithContext(ec bsoncodec.EncodeContext, dst []byte, val interf
 		return nil, err
 	}
 
-	return *sw, nil
+	return append(dst, sw.Bytes()...), nil
 }
 
 // MarshalValue returns the BSON encoding of val.