crypto/internal/fips/sha3: reduce s390x divergence

It's a little annoying, but we can fit the IBM instructions on top of
the regular state, avoiding more intrusive interventions.

Going forward we should not accept assembly that replaces the whole
implementation, because it doubles the work to do any refactoring like
the one in this chain.

Also, it took me a while to find the specification of these
instructions, which should have been linked from the source for the next
person who'd have to touch this.

Finally, it's really painful to test this without a LUCI TryBot, per #67307.

For #69536

Change-Id: I90632a90f06b2aa2e863967de972b12dbaa5b2ae
Reviewed-on: https://go-review.googlesource.com/c/go/+/617359
LUCI-TryBot-Result: Go LUCI <[email protected]>
Auto-Submit: Filippo Valsorda <[email protected]>
Reviewed-by: Carlos Amedee <[email protected]>
Reviewed-by: Daniel McCarney <[email protected]>
Reviewed-by: Roland Shoemaker <[email protected]>

Author: FiloSottile

src/crypto/internal/fips/sha3/_asm/keccakf_amd64_asm.go

@@ -101,7 +101,7 @@ const (
 
 func main() {
 	Package("golang.org/x/crypto/sha3")
-	ConstraintExpr("amd64,!purego,gc")
+	ConstraintExpr("!purego")
 	keccakF1600()
 	Generate()
 }

src/crypto/internal/fips/sha3/hashes.go

@@ -6,22 +6,22 @@ package sha3
 
 // New224 returns a new Digest computing the SHA3-224 hash.
 func New224() *Digest {
-	return new224()
+	return &Digest{rate: rateK448, outputLen: 28, dsbyte: dsbyteSHA3}
 }
 
 // New256 returns a new Digest computing the SHA3-256 hash.
 func New256() *Digest {
-	return new256()
+	return &Digest{rate: rateK512, outputLen: 32, dsbyte: dsbyteSHA3}
 }
 
 // New384 returns a new Digest computing the SHA3-384 hash.
 func New384() *Digest {
-	return new384()
+	return &Digest{rate: rateK768, outputLen: 48, dsbyte: dsbyteSHA3}
 }
 
 // New512 returns a new Digest computing the SHA3-512 hash.
 func New512() *Digest {
-	return new512()
+	return &Digest{rate: rateK1024, outputLen: 64, dsbyte: dsbyteSHA3}
 }
 
 // TODO(fips): do this in the stdlib crypto/sha3 package.
@@ -46,22 +46,6 @@ const (
 	rateK1024 = (1600 - 1024) / 8
 )
 
-func new224Generic() *Digest {
-	return &Digest{rate: rateK448, outputLen: 28, dsbyte: dsbyteSHA3}
-}
-
-func new256Generic() *Digest {
-	return &Digest{rate: rateK512, outputLen: 32, dsbyte: dsbyteSHA3}
-}
-
-func new384Generic() *Digest {
-	return &Digest{rate: rateK768, outputLen: 48, dsbyte: dsbyteSHA3}
-}
-
-func new512Generic() *Digest {
-	return &Digest{rate: rateK1024, outputLen: 64, dsbyte: dsbyteSHA3}
-}
-
 // NewLegacyKeccak256 returns a new Digest computing the legacy, non-standard
 // Keccak-256 hash.
 func NewLegacyKeccak256() *Digest {

src/crypto/internal/fips/sha3/keccakf.go

@@ -2,11 +2,14 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build !amd64 || purego || !gc
-
 package sha3
 
-import "math/bits"
+import (
+	"internal/byteorder"
+	"internal/goarch"
+	"math/bits"
+	"unsafe"
+)
 
 // rc stores the round constants for use in the ι step.
 var rc = [24]uint64{
@@ -36,9 +39,23 @@ var rc = [24]uint64{
 	0x8000000080008008,
 }
 
-// keccakF1600 applies the Keccak permutation to a 1600b-wide
-// state represented as a slice of 25 uint64s.
-func keccakF1600(a *[25]uint64) {
+// keccakF1600Generic applies the Keccak permutation.
+func keccakF1600Generic(da *[200]byte) {
+	var a *[25]uint64
+	if goarch.BigEndian {
+		a = new([25]uint64)
+		for i := range a {
+			a[i] = byteorder.LeUint64(da[i*8:])
+		}
+		defer func() {
+			for i := range a {
+				byteorder.LePutUint64(da[i*8:], a[i])
+			}
+		}()
+	} else {
+		a = (*[25]uint64)(unsafe.Pointer(da))
+	}
+
 	// Implementation translated from Keccak-inplace.c
 	// in the keccak reference code.
 	var t, bc0, bc1, bc2, bc3, bc4, d0, d1, d2, d3, d4 uint64

src/crypto/internal/fips/sha3/keccakf_amd64.go

@@ -13,9 +13,6 @@ package sha3
 import (
 	"crypto/internal/fips/subtle"
 	"errors"
-	"internal/byteorder"
-	"internal/goarch"
-	"unsafe"
 )
 
 // spongeDirection indicates the direction bytes are flowing through the sponge.
@@ -77,24 +74,8 @@ func (d *Digest) Clone() *Digest {
 
 // permute applies the KeccakF-1600 permutation.
 func (d *Digest) permute() {
-	var a *[25]uint64
-	if goarch.BigEndian {
-		a = new([25]uint64)
-		for i := range a {
-			a[i] = byteorder.LeUint64(d.a[i*8:])
-		}
-	} else {
-		a = (*[25]uint64)(unsafe.Pointer(&d.a))
-	}
-
-	keccakF1600(a)
+	keccakF1600(&d.a)
 	d.n = 0
-
-	if goarch.BigEndian {
-		for i := range a {
-			byteorder.LePutUint64(d.a[i*8:], a[i])
-		}
-	}
 }
 
 // padAndPermute appends the domain separation bits in dsbyte, applies
@@ -115,7 +96,8 @@ func (d *Digest) padAndPermute() {
 }
 
 // Write absorbs more data into the hash's state.
-func (d *Digest) Write(p []byte) (n int, err error) {
+func (d *Digest) Write(p []byte) (n int, err error) { return d.write(p) }
+func (d *Digest) writeGeneric(p []byte) (n int, err error) {
 	if d.state != spongeAbsorbing {
 		panic("sha3: Write after Read")
 	}
@@ -137,7 +119,7 @@ func (d *Digest) Write(p []byte) (n int, err error) {
 }
 
 // read squeezes an arbitrary number of bytes from the sponge.
-func (d *Digest) read(out []byte) (n int, err error) {
+func (d *Digest) readGeneric(out []byte) (n int, err error) {
 	// If we're still absorbing, pad and apply the permutation.
 	if d.state == spongeAbsorbing {
 		d.padAndPermute()
@@ -162,7 +144,8 @@ func (d *Digest) read(out []byte) (n int, err error) {
 
 // Sum appends the current hash to b and returns the resulting slice.
 // It does not change the underlying hash state.
-func (d *Digest) Sum(b []byte) []byte {
+func (d *Digest) Sum(b []byte) []byte { return d.sum(b) }
+func (d *Digest) sumGeneric(b []byte) []byte {
 	if d.state != spongeAbsorbing {
 		panic("sha3: Sum after Read")
 	}

src/crypto/internal/fips/sha3/sha3.go

@@ -0,0 +1,20 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !purego
+
+package sha3
+
+//go:noescape
+func keccakF1600(a *[200]byte)
+
+func (d *Digest) write(p []byte) (n int, err error) {
+	return d.writeGeneric(p)
+}
+func (d *Digest) read(out []byte) (n int, err error) {
+	return d.readGeneric(out)
+}
+func (d *Digest) sum(b []byte) []byte {
+	return d.sumGeneric(b)
+}

src/crypto/internal/fips/sha3/sha3_amd64.go

@@ -1,8 +1,8 @@
 // Code generated by command: go run keccakf_amd64_asm.go -out ../keccakf_amd64.s -pkg sha3. DO NOT EDIT.
 
-//go:build amd64 && !purego && gc
+//go:build !purego
 
-// func keccakF1600(a *[25]uint64)
+// func keccakF1600(a *[200]byte)
 TEXT ·keccakF1600(SB), $200-8
 	MOVQ a+0(FP), DI
 

src/crypto/internal/fips/sha3/keccakf_amd64.s renamed to src/crypto/internal/fips/sha3/sha3_amd64.s

@@ -2,30 +2,20 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build !gc || purego || !s390x || !ignore
+//go:build (!amd64 && !s390x) || purego
 
 package sha3
 
-func new224() *Digest {
-	return new224Generic()
+func keccakF1600(a *[200]byte) {
+	keccakF1600Generic(a)
 }
 
-func new256() *Digest {
-	return new256Generic()
+func (d *Digest) write(p []byte) (n int, err error) {
+	return d.writeGeneric(p)
 }
-
-func new384() *Digest {
-	return new384Generic()
-}
-
-func new512() *Digest {
-	return new512Generic()
+func (d *Digest) read(out []byte) (n int, err error) {
+	return d.readGeneric(out)
 }
-
-func newShake128() *SHAKE {
-	return newShake128Generic()
-}
-
-func newShake256() *SHAKE {
-	return newShake256Generic()
+func (d *Digest) sum(b []byte) []byte {
+	return d.sumGeneric(b)
 }