proto: fix and cleanup test for deterministic marshal with custom marshalers

PR #650 added a check to error out when a custom marshaler was called and
deterministic marshaling was also specified.

That change performed the check in a relatively obscure location that did not
catch all code paths. Since determinism can only be enabled on the Buffer type,
we check it up front in Buffer.Marshal.

Also, change the test to avoid code injection into generated sources.

Author: dsnet

proto/all_test.go

@@ -2303,14 +2303,22 @@ func TestInvalidUTF8(t *testing.T) {
 	}
 }
 
+type CustomRawMessage []byte
+
+func (m *CustomRawMessage) Marshal() ([]byte, error) {
+	return []byte(*m), nil
+}
+func (m *CustomRawMessage) Reset()         { *m = nil }
+func (m *CustomRawMessage) String() string { return fmt.Sprintf("%x", *m) }
+func (m *CustomRawMessage) ProtoMessage()  {}
+
 func TestDeterministicErrorOnCustomMarshaler(t *testing.T) {
-	u := uint64(0)
-	in := &CustomDeterministicMarshaler{Field1: &u}
+	in := CustomRawMessage{1, 2, 3}
 	var b1 Buffer
 	b1.SetDeterministic(true)
-	err := b1.Marshal(in)
-	if !strings.Contains(err.Error(), "deterministic") {
-		t.Fatalf("Expected: %s but got %s", "proto: deterministic not supported by the Marshal method of test_proto.CustomDeterministicMarshaler", err.Error())
+	err := b1.Marshal(&in)
+	if err == nil || !strings.Contains(err.Error(), "deterministic") {
+		t.Fatalf("Marshal error:\ngot  %v\nwant deterministic not supported error", err)
 	}
 }
 

proto/table_marshal.go

@@ -225,9 +225,6 @@ func (u *marshalInfo) marshal(b []byte, ptr pointer, deterministic bool) ([]byte
 	// If the message can marshal itself, let it do it, for compatibility.
 	// NOTE: This is not efficient.
 	if u.hasmarshaler {
-		if deterministic {
-			return nil, errors.New("proto: deterministic not supported by the Marshal method of " + u.typ.String())
-		}
 		m := ptr.asPointerTo(u.typ).Interface().(Marshaler)
 		b1, err := m.Marshal()
 		b = append(b, b1...)
@@ -2718,6 +2715,11 @@ func Marshal(pb Message) ([]byte, error) {
 // a Buffer for most applications.
 func (p *Buffer) Marshal(pb Message) error {
 	var err error
+	if p.deterministic {
+		if _, ok := pb.(Marshaler); ok {
+			return fmt.Errorf("proto: deterministic not supported by the Marshal method of %T", pb)
+		}
+	}
 	if m, ok := pb.(newMarshaler); ok {
 		siz := m.XXX_Size()
 		p.grow(siz) // make sure buf has enough capacity