Module Name: src
Committed By: riastradh
Date: Mon Jul 27 20:51:29 UTC 2020
Modified Files:
src/sys/arch/arm/conf: files.arm
src/sys/arch/arm/vfp: vfp_init.c
src/sys/crypto/chacha/arch/arm: chacha_neon.c chacha_neon.h
files.chacha_arm
src/tests/sys/crypto/chacha: Makefile t_chacha.c
Log Message:
Enable ChaCha NEON code on armv7 too.
The 4-blocks-at-a-time assembly helper is disabled for now; adapting
it to armv7 is going to be a little annoying with only 16 128-bit
vector registers.
(Should also do a fifth block in the integer registers for 320 bytes
at a time.)
To generate a diff of this commit:
cvs rdiff -u -r1.159 -r1.160 src/sys/arch/arm/conf/files.arm
cvs rdiff -u -r1.69 -r1.70 src/sys/arch/arm/vfp/vfp_init.c
cvs rdiff -u -r1.2 -r1.3 src/sys/crypto/chacha/arch/arm/chacha_neon.c
cvs rdiff -u -r1.1 -r1.2 src/sys/crypto/chacha/arch/arm/chacha_neon.h \
src/sys/crypto/chacha/arch/arm/files.chacha_arm
cvs rdiff -u -r1.1 -r1.2 src/tests/sys/crypto/chacha/Makefile \
src/tests/sys/crypto/chacha/t_chacha.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/arm/conf/files.arm
diff -u src/sys/arch/arm/conf/files.arm:1.159 src/sys/arch/arm/conf/files.arm:1.160
--- src/sys/arch/arm/conf/files.arm:1.159 Mon Jul 27 20:49:45 2020
+++ src/sys/arch/arm/conf/files.arm Mon Jul 27 20:51:29 2020
@@ -1,4 +1,4 @@
-# $NetBSD: files.arm,v 1.159 2020/07/27 20:49:45 riastradh Exp $
+# $NetBSD: files.arm,v 1.160 2020/07/27 20:51:29 riastradh Exp $
# temporary define to allow easy moving to ../arch/arm/arm32
defflag ARM32
@@ -265,3 +265,6 @@ file dev/tprof/tprof_armv7.c tprof
# vpaes with ARM NEON
include "crypto/aes/arch/arm/files.aesneon"
+
+# ChaCha with ARM NEON
+include "crypto/chacha/arch/arm/files.chacha_arm"
Index: src/sys/arch/arm/vfp/vfp_init.c
diff -u src/sys/arch/arm/vfp/vfp_init.c:1.69 src/sys/arch/arm/vfp/vfp_init.c:1.70
--- src/sys/arch/arm/vfp/vfp_init.c:1.69 Sat Jul 25 22:12:56 2020
+++ src/sys/arch/arm/vfp/vfp_init.c Mon Jul 27 20:51:29 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vfp_init.c,v 1.69 2020/07/25 22:12:56 riastradh Exp $ */
+/* $NetBSD: vfp_init.c,v 1.70 2020/07/27 20:51:29 riastradh Exp $ */
/*
* Copyright (c) 2008 ARM Ltd
@@ -32,7 +32,7 @@
#include "opt_cputypes.h"
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: vfp_init.c,v 1.69 2020/07/25 22:12:56 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfp_init.c,v 1.70 2020/07/27 20:51:29 riastradh Exp $");
#include <sys/param.h>
#include <sys/types.h>
@@ -52,6 +52,8 @@ __KERNEL_RCSID(0, "$NetBSD: vfp_init.c,v
#include <crypto/aes/aes_impl.h>
#include <crypto/aes/arch/arm/aes_neon.h>
+#include <crypto/chacha/arch/arm/chacha_neon.h>
+#include <crypto/chacha/chacha_impl.h>
#ifdef FPU_VFP
@@ -409,6 +411,7 @@ vfp_attach(struct cpu_info *ci)
install_coproc_handler(CORE_UNKNOWN_HANDLER,
neon_handler);
aes_md_init(&aes_neon_impl);
+ chacha_md_init(&chacha_neon_impl);
}
#endif
}
Index: src/sys/crypto/chacha/arch/arm/chacha_neon.c
diff -u src/sys/crypto/chacha/arch/arm/chacha_neon.c:1.2 src/sys/crypto/chacha/arch/arm/chacha_neon.c:1.3
--- src/sys/crypto/chacha/arch/arm/chacha_neon.c:1.2 Mon Jul 27 20:48:18 2020
+++ src/sys/crypto/chacha/arch/arm/chacha_neon.c Mon Jul 27 20:51:29 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: chacha_neon.c,v 1.2 2020/07/27 20:48:18 riastradh Exp $ */
+/* $NetBSD: chacha_neon.c,v 1.3 2020/07/27 20:51:29 riastradh Exp $ */
/*-
* Copyright (c) 2020 The NetBSD Foundation, Inc.
@@ -150,8 +150,10 @@ chacha_stream_neon(uint8_t *restrict s,
unsigned nr)
{
+#ifdef __aarch64__
for (; n >= 256; s += 256, n -= 256, blkno += 4)
chacha_stream256_neon(s, blkno, nonce, k, chacha_const32, nr);
+#endif
if (n) {
const uint32x4_t blkno_inc = {1,0,0,0};
@@ -208,9 +210,11 @@ chacha_stream_xor_neon(uint8_t *s, const
unsigned nr)
{
+#ifdef __aarch64__
for (; n >= 256; s += 256, p += 256, n -= 256, blkno += 4)
chacha_stream_xor256_neon(s, p, blkno, nonce, k,
chacha_const32, nr);
+#endif
if (n) {
const uint32x4_t blkno_inc = {1,0,0,0};
Index: src/sys/crypto/chacha/arch/arm/chacha_neon.h
diff -u src/sys/crypto/chacha/arch/arm/chacha_neon.h:1.1 src/sys/crypto/chacha/arch/arm/chacha_neon.h:1.2
--- src/sys/crypto/chacha/arch/arm/chacha_neon.h:1.1 Sat Jul 25 22:51:57 2020
+++ src/sys/crypto/chacha/arch/arm/chacha_neon.h Mon Jul 27 20:51:29 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: chacha_neon.h,v 1.1 2020/07/25 22:51:57 riastradh Exp $ */
+/* $NetBSD: chacha_neon.h,v 1.2 2020/07/27 20:51:29 riastradh Exp $ */
/*-
* Copyright (c) 2020 The NetBSD Foundation, Inc.
@@ -64,7 +64,8 @@ void xchacha_stream_xor_neon(uint8_t *,
const uint8_t[static 32],
unsigned);
-/* Assembly helpers */
+#ifdef __aarch64__
+/* Assembly helpers -- aarch64 only for now */
void chacha_stream256_neon(uint8_t[restrict static 256], uint32_t,
const uint8_t[static 12],
const uint8_t[static 32],
@@ -77,6 +78,7 @@ void chacha_stream_xor256_neon(uint8_t[r
const uint8_t[static 32],
const uint8_t[static 16],
unsigned);
+#endif /* __aarch64__ */
extern const struct chacha_impl chacha_neon_impl;
Index: src/sys/crypto/chacha/arch/arm/files.chacha_arm
diff -u src/sys/crypto/chacha/arch/arm/files.chacha_arm:1.1 src/sys/crypto/chacha/arch/arm/files.chacha_arm:1.2
--- src/sys/crypto/chacha/arch/arm/files.chacha_arm:1.1 Sat Jul 25 22:51:57 2020
+++ src/sys/crypto/chacha/arch/arm/files.chacha_arm Mon Jul 27 20:51:29 2020
@@ -1,9 +1,11 @@
-# $NetBSD: files.chacha_arm,v 1.1 2020/07/25 22:51:57 riastradh Exp $
+# $NetBSD: files.chacha_arm,v 1.2 2020/07/27 20:51:29 riastradh Exp $
ifdef aarch64
makeoptions chacha "COPTS.chacha_neon.c"+="-march=armv8-a"
+else
+makeoptions aes "COPTS.chacha_neon.c"+="-mfloat-abi=softfp -mfpu=neon"
endif
-file crypto/chacha/arch/arm/chacha_neon.c chacha
-file crypto/chacha/arch/arm/chacha_neon_64.S chacha & aarch64
-file crypto/chacha/arch/arm/chacha_neon_impl.c chacha
+file crypto/chacha/arch/arm/chacha_neon.c chacha & (cpu_cortex | aarch64)
+file crypto/chacha/arch/arm/chacha_neon_64.S chacha & aarch64
+file crypto/chacha/arch/arm/chacha_neon_impl.c chacha & (cpu_cortex | aarch64)
Index: src/tests/sys/crypto/chacha/Makefile
diff -u src/tests/sys/crypto/chacha/Makefile:1.1 src/tests/sys/crypto/chacha/Makefile:1.2
--- src/tests/sys/crypto/chacha/Makefile:1.1 Sat Jul 25 22:53:38 2020
+++ src/tests/sys/crypto/chacha/Makefile Mon Jul 27 20:51:29 2020
@@ -1,4 +1,4 @@
-# $NetBSD: Makefile,v 1.1 2020/07/25 22:53:38 riastradh Exp $
+# $NetBSD: Makefile,v 1.2 2020/07/27 20:51:29 riastradh Exp $
.include <bsd.own.mk>
@@ -16,16 +16,18 @@ SRCS.t_chacha+= t_chacha.c
SRCS.t_chacha+= chacha_ref.c
SRCS.t_chacha+= chacha_selftest.c
-.if !empty(MACHINE_ARCH:Maarch64*)
+.if !empty(MACHINE_ARCH:Mearmv7*) || !empty(MACHINE_ARCH:Maarch64*)
.PATH: ${NETBSDSRCDIR}/sys/crypto/chacha/arch/arm
CPPFLAGS+= -I${NETBSDSRCDIR}/sys/crypto/chacha/arch/arm
SRCS.t_chacha+= chacha_neon.c
+.if !empty(MACHINE_ARCH:Maarch64*)
SRCS.t_chacha+= chacha_neon_64.S
+.endif
SRCS.t_chacha+= chacha_neon_impl.c
-.endif # aarch64
+.endif # earmv7 or aarch64
.if ${MACHINE_ARCH} == "i386" || ${MACHINE_ARCH} == "x86_64"
Index: src/tests/sys/crypto/chacha/t_chacha.c
diff -u src/tests/sys/crypto/chacha/t_chacha.c:1.1 src/tests/sys/crypto/chacha/t_chacha.c:1.2
--- src/tests/sys/crypto/chacha/t_chacha.c:1.1 Sat Jul 25 22:53:38 2020
+++ src/tests/sys/crypto/chacha/t_chacha.c Mon Jul 27 20:51:29 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: t_chacha.c,v 1.1 2020/07/25 22:53:38 riastradh Exp $ */
+/* $NetBSD: t_chacha.c,v 1.2 2020/07/27 20:51:29 riastradh Exp $ */
/*-
* Copyright (c) 2020 The NetBSD Foundation, Inc.
@@ -35,7 +35,7 @@
#include <crypto/chacha/arch/x86/chacha_sse2.h>
#endif
-#ifdef __aarch64__
+#ifdef __ARM_NEON__
#include <crypto/chacha/arch/arm/chacha_neon.h>
#endif
@@ -81,7 +81,7 @@ ATF_TC_BODY(name, tc) \
atf_tc_fail("%s self-test failed", (impl)->ci_name); \
}
-#ifdef __aarch64__
+#ifdef __ARM_NEON__
CHACHA_SELFTEST(chacha_neon_selftest, &chacha_neon_impl,
"ARM NEON ChaCha self-test")
#endif
@@ -96,7 +96,7 @@ ATF_TP_ADD_TCS(tp)
ATF_TP_ADD_TC(tp, chacha_ref_selftest);
-#ifdef __aarch64__
+#ifdef __ARM_NEON__
ATF_TP_ADD_TC(tp, chacha_neon_selftest);
#endif
