proto: fix and cleanup test for deterministic marshal with custom marshalers

proto/all_test.go

@@ -2303,14 +2303,22 @@ func TestInvalidUTF8(t *testing.T) {
 	}
 }
 
+type CustomRawMessage []byte
+
+func (m *CustomRawMessage) Marshal() ([]byte, error) {
+	return []byte(*m), nil
+}
+func (m *CustomRawMessage) Reset()         { *m = nil }
+func (m *CustomRawMessage) String() string { return fmt.Sprintf("%x", *m) }
+func (m *CustomRawMessage) ProtoMessage()  {}
+
 func TestDeterministicErrorOnCustomMarshaler(t *testing.T) {
-	u := uint64(0)
-	in := &CustomDeterministicMarshaler{Field1: &u}
+	in := CustomRawMessage{1, 2, 3}
 	var b1 Buffer
 	b1.SetDeterministic(true)
-	err := b1.Marshal(in)
-	if !strings.Contains(err.Error(), "deterministic") {
-		t.Fatalf("Expected: %s but got %s", "proto: deterministic not supported by the Marshal method of test_proto.CustomDeterministicMarshaler", err.Error())
+	err := b1.Marshal(&in)
+	if err == nil || !strings.Contains(err.Error(), "deterministic") {
+		t.Fatalf("Marshal error:\ngot  %v\nwant deterministic not supported error", err)
 	}
 }
 

proto/table_marshal.go

@@ -225,9 +225,6 @@ func (u *marshalInfo) marshal(b []byte, ptr pointer, deterministic bool) ([]byte
 	// If the message can marshal itself, let it do it, for compatibility.
 	// NOTE: This is not efficient.
 	if u.hasmarshaler {
-		if deterministic {
-			return nil, errors.New("proto: deterministic not supported by the Marshal method of " + u.typ.String())
-		}
 		m := ptr.asPointerTo(u.typ).Interface().(Marshaler)
 		b1, err := m.Marshal()
 		b = append(b, b1...)
@@ -2718,6 +2715,11 @@ func Marshal(pb Message) ([]byte, error) {
 // a Buffer for most applications.
 func (p *Buffer) Marshal(pb Message) error {
 	var err error
+	if p.deterministic {
+		if _, ok := pb.(Marshaler); ok {
+			return fmt.Errorf("proto: deterministic not supported by the Marshal method of %T", pb)
+		}
+	}
 	if m, ok := pb.(newMarshaler); ok {
 		siz := m.XXX_Size()
 		p.grow(siz) // make sure buf has enough capacity