strict erroring for custom marshalers trying to use a deterministic buffer as discussed in #648

Author: jmarais

proto/all_test.go

@@ -2303,6 +2303,17 @@ func TestInvalidUTF8(t *testing.T) {
 	}
 }
 
+func TestDeterministicErrorOnCustomMarshaler(t *testing.T) {
+	u := uint64(0)
+	in := &CustomDeterministicMarshaler{Field1: &u}
+	var b1 Buffer
+	b1.SetDeterministic(true)
+	err := b1.Marshal(in)
+	if !strings.Contains(err.Error(), "deterministic") {
+		t.Fatalf("Expected: %s but got %s", "proto: deterministic not supported by the Marshal method of CustomDeterministicMarshaler", err.Error())
+	}
+}
+
 // Benchmarks
 
 func testMsg() *GoTest {

proto/table_marshal.go

@@ -225,6 +225,9 @@ func (u *marshalInfo) marshal(b []byte, ptr pointer, deterministic bool) ([]byte
 	// If the message can marshal itself, let it do it, for compatibility.
 	// NOTE: This is not efficient.
 	if u.hasmarshaler {
+		if deterministic {
+			return nil, errors.New("proto: deterministic not supported by the Marshal method of " + u.typ.Name())
+		}
 		m := ptr.asPointerTo(u.typ).Interface().(Marshaler)
 		b1, err := m.Marshal()
 		b = append(b, b1...)

proto/test_proto/deterministic.go

@@ -0,0 +1,5 @@
+package test_proto
+
+func (m *CustomDeterministicMarshaler) Marshal() ([]byte, error) {
+	return []byte{}, nil
+}