proto: return errors when using deterministic marshaling with custom Marshalers

Custom marshalers cannot report that they are definitely deterministic. It is better to report that this is not supported than to give undefined behavior.

Author: jmarais

proto/all_test.go

@@ -2303,6 +2303,17 @@ func TestInvalidUTF8(t *testing.T) {
 	}
 }
 
+func TestDeterministicErrorOnCustomMarshaler(t *testing.T) {
+	u := uint64(0)
+	in := &CustomDeterministicMarshaler{Field1: &u}
+	var b1 Buffer
+	b1.SetDeterministic(true)
+	err := b1.Marshal(in)
+	if !strings.Contains(err.Error(), "deterministic") {
+		t.Fatalf("Expected: %s but got %s", "proto: deterministic not supported by the Marshal method of test_proto.CustomDeterministicMarshaler", err.Error())
+	}
+}
+
 // Benchmarks
 
 func testMsg() *GoTest {

proto/table_marshal.go

@@ -225,6 +225,9 @@ func (u *marshalInfo) marshal(b []byte, ptr pointer, deterministic bool) ([]byte
 	// If the message can marshal itself, let it do it, for compatibility.
 	// NOTE: This is not efficient.
 	if u.hasmarshaler {
+		if deterministic {
+			return nil, errors.New("proto: deterministic not supported by the Marshal method of " + u.typ.String())
+		}
 		m := ptr.asPointerTo(u.typ).Interface().(Marshaler)
 		b1, err := m.Marshal()
 		b = append(b, b1...)

proto/test_proto/deterministic.go

@@ -0,0 +1,5 @@
+package test_proto
+
+func (m *CustomDeterministicMarshaler) Marshal() ([]byte, error) {
+	return []byte{}, nil
+}