Add optimized assembly xorBytes for ARM (NEON and non-NEON)

Author: adriancable

src/crypto/cipher/xor_arm.go

@@ -0,0 +1,58 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cipher
+
+import (
+	"internal/cpu"
+	"unsafe"
+)
+
+const wordSize = int(unsafe.Sizeof(uintptr(0)))
+
+var hasNEON = cpu.HWCap&(1<<12) != 0
+
+func isAligned(a *byte) bool {
+	return uintptr(unsafe.Pointer(a))%uintptr(wordSize) == 0
+}
+
+// xorBytes xors the bytes in a and b. The destination should have enough
+// space, otherwise xorBytes will panic. Returns the number of bytes xor'd.
+func xorBytes(dst, a, b []byte) int {
+	n := len(a)
+	if len(b) < n {
+		n = len(b)
+	}
+	if n == 0 {
+		return 0
+	}
+	// make sure dst has enough space
+	_ = dst[n-1]
+
+	if hasNEON {
+		xorBytesNEON32(&dst[0], &a[0], &b[0], n)
+	} else if isAligned(&dst[0]) && isAligned(&a[0]) && isAligned(&b[0]) {
+		xorBytesARM32(&dst[0], &a[0], &b[0], n)
+	} else {
+		safeXORBytes(dst, a, b, n)
+	}
+	return n
+}
+
+// n needs to be smaller or equal than the length of a and b.
+func safeXORBytes(dst, a, b []byte, n int) {
+	for i := 0; i < n; i++ {
+		dst[i] = a[i] ^ b[i]
+	}
+}
+
+func xorWords(dst, a, b []byte) {
+	xorBytes(dst, a, b)
+}
+
+//go:noescape
+func xorBytesARM32(dst, a, b *byte, n int)
+
+//go:noescape
+func xorBytesNEON32(dst, a, b *byte, n int)

src/crypto/cipher/xor_arm.s

@@ -0,0 +1,114 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#include "textflag.h"
+
+// func xorBytesARM32(dst, a, b *byte, n int)
+TEXT ·xorBytesARM32(SB), NOSPLIT|NOFRAME, $0
+	MOVW	dst+0(FP), R0
+	MOVW	a+4(FP), R1
+	MOVW	b+8(FP), R2
+	MOVW	n+12(FP), R3
+	CMP	$4, R3
+	BLT	less_than4
+
+loop_4:
+	MOVW.P	4(R1), R4
+	MOVW.P	4(R2), R5
+	EOR	R4, R5, R5
+	MOVW.P	R5, 4(R0)
+
+	SUB	$4, R3
+	CMP	$4, R3
+	BGE	loop_4
+
+less_than4:
+	CMP	$2, R3
+	BLT	less_than2
+	MOVH.P	2(R1), R4
+	MOVH.P	2(R2), R5
+	EOR	R4, R5, R5
+	MOVH.P	R5, 2(R0)
+
+	SUB	$2, R3
+
+less_than2:
+	CMP	$0, R3
+	BEQ	end
+	MOVB	(R1), R4
+	MOVB	(R2), R5
+	EOR	R4, R5, R5
+	MOVB	R5, (R0)
+end:
+	RET
+
+// func xorBytesNEON32(dst, a, b *byte, n int)
+TEXT ·xorBytesNEON32(SB), NOSPLIT|NOFRAME, $0
+	MOVW	dst+0(FP), R0
+	MOVW	a+4(FP), R1
+	MOVW	b+8(FP), R2
+	MOVW	n+12(FP), R3
+	CMP	$32, R3
+	BLT	less_than32
+
+loop_32:
+	WORD	$0xF421020D // vld1.u8 {q0, q1}, [r1]!
+	WORD	$0xF422420D // vld1.u8 {q2, q3}, [r2]!
+	WORD	$0xF3004154 // veor q2, q0, q2
+	WORD	$0xF3026156 // veor q3, q1, q3
+	WORD	$0xF400420D // vst1.u8 {q2, q3}, [r0]!
+
+	SUB	$32, R3
+	CMP	$32, R3
+	BGE	loop_32
+
+less_than32:
+	CMP	$16, R3
+	BLT	less_than16
+	WORD	$0xF4210A0D // vld1.u8 q0, [r1]!
+	WORD	$0xF4222A0D // vld1.u8 q1, [r2]!
+	WORD	$0xF3002152 // veor q1, q0, q1
+	WORD	$0xF4002A0D // vst1.u8 {q1}, [r0]!
+
+	SUB	$16, R3
+
+less_than16:
+	CMP	$8, R3
+	BLT	less_than8
+	WORD	$0xF421070D // vld1.u8 d0, [r1]!
+	WORD	$0xF422170D // vld1.u8 d1, [r2]!
+	WORD	$0xF3001111 // veor d1, d0, d1
+	WORD	$0xF400170D // vst1.u8 {d1}, [r0]!
+
+	SUB	$8, R3
+
+less_than8:
+	CMP	$4, R3
+	BLT	less_than4
+	MOVW.P	4(R1), R4
+	MOVW.P	4(R2), R5
+	EOR	R4, R5, R5
+	MOVW.P	R5, 4(R0)
+
+	SUB	$4, R3
+
+less_than4:
+	CMP	$2, R3
+	BLT	less_than2
+	MOVH.P	2(R1), R4
+	MOVH.P	2(R2), R5
+	EOR	R4, R5, R5
+	MOVH.P	R5, 2(R0)
+
+	SUB	$2, R3
+
+less_than2:
+	CMP	$0, R3
+	BEQ	end
+	MOVB	(R1), R4
+	MOVB	(R2), R5
+	EOR	R4, R5, R5
+	MOVB	R5, (R0)
+end:
+	RET

src/crypto/cipher/xor_generic.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build !amd64 && !ppc64 && !ppc64le && !arm64
+//go:build !amd64 && !ppc64 && !ppc64le && !arm64 && !arm
 
 package cipher