On Mon, 28 Apr 2025 10:00:32 PDT (-0700), ebigg...@kernel.org wrote:
> From: Eric Biggers <ebigg...@google.com>
>
> Instead of providing crypto_shash algorithms for the arch-optimized
> SHA-256 code, instead implement the SHA-256 library. This is much
> simpler, it makes the SHA-256 library functions be arch-optimized, and
> it fixes the longstanding issue where the arch-optimized SHA-256 was
> disabled by default. SHA-256 still remains available through
> crypto_shash, but individual architectures no longer need to handle it.
>
> To match sha256_blocks_arch(), change the type of the nblocks parameter
> of the assembly function from int to size_t. The assembly function
> actually already treated it as size_t.
>
> Signed-off-by: Eric Biggers <ebigg...@google.com>
> ---
> arch/riscv/crypto/Kconfig | 11 --
> arch/riscv/crypto/Makefile | 3 -
> arch/riscv/crypto/sha256-riscv64-glue.c | 125 ------------------
> arch/riscv/lib/crypto/Kconfig | 7 +
> arch/riscv/lib/crypto/Makefile | 3 +
> .../sha256-riscv64-zvknha_or_zvknhb-zvkb.S | 4 +-
> arch/riscv/lib/crypto/sha256.c | 62 +++++++++
> 7 files changed, 74 insertions(+), 141 deletions(-)
> delete mode 100644 arch/riscv/crypto/sha256-riscv64-glue.c
> rename arch/riscv/{ => lib}/crypto/sha256-riscv64-zvknha_or_zvknhb-zvkb.S
> (98%)
> create mode 100644 arch/riscv/lib/crypto/sha256.c
>
> diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig
> index 4863be2a4ec2f..cd9b776602f89 100644
> --- a/arch/riscv/crypto/Kconfig
> +++ b/arch/riscv/crypto/Kconfig
> @@ -26,21 +26,10 @@ config CRYPTO_GHASH_RISCV64
> GCM GHASH function (NIST SP 800-38D)
>
> Architecture: riscv64 using:
> - Zvkg vector crypto extension
>
> -config CRYPTO_SHA256_RISCV64
> - tristate "Hash functions: SHA-224 and SHA-256"
> - depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
> - select CRYPTO_SHA256
> - help
> - SHA-224 and SHA-256 secure hash algorithm (FIPS 180)
> -
> - Architecture: riscv64 using:
> - - Zvknha or Zvknhb vector crypto extensions
> - - Zvkb vector crypto extension
> -
> config CRYPTO_SHA512_RISCV64
> tristate "Hash functions: SHA-384 and SHA-512"
> depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
> select CRYPTO_SHA512
> help
> diff --git a/arch/riscv/crypto/Makefile b/arch/riscv/crypto/Makefile
> index 4ae9bf762e907..e10e8257734e3 100644
> --- a/arch/riscv/crypto/Makefile
> +++ b/arch/riscv/crypto/Makefile
> @@ -5,13 +5,10 @@ aes-riscv64-y := aes-riscv64-glue.o aes-riscv64-zvkned.o \
> aes-riscv64-zvkned-zvbb-zvkg.o aes-riscv64-zvkned-zvkb.o
>
> obj-$(CONFIG_CRYPTO_GHASH_RISCV64) += ghash-riscv64.o
> ghash-riscv64-y := ghash-riscv64-glue.o ghash-riscv64-zvkg.o
>
> -obj-$(CONFIG_CRYPTO_SHA256_RISCV64) += sha256-riscv64.o
> -sha256-riscv64-y := sha256-riscv64-glue.o
> sha256-riscv64-zvknha_or_zvknhb-zvkb.o
> -
> obj-$(CONFIG_CRYPTO_SHA512_RISCV64) += sha512-riscv64.o
> sha512-riscv64-y := sha512-riscv64-glue.o sha512-riscv64-zvknhb-zvkb.o
>
> obj-$(CONFIG_CRYPTO_SM3_RISCV64) += sm3-riscv64.o
> sm3-riscv64-y := sm3-riscv64-glue.o sm3-riscv64-zvksh-zvkb.o
> diff --git a/arch/riscv/crypto/sha256-riscv64-glue.c
> b/arch/riscv/crypto/sha256-riscv64-glue.c
> deleted file mode 100644
> index c998300ab8435..0000000000000
> --- a/arch/riscv/crypto/sha256-riscv64-glue.c
> +++ /dev/null
> @@ -1,125 +0,0 @@
> -// SPDX-License-Identifier: GPL-2.0-or-later
> -/*
> - * SHA-256 and SHA-224 using the RISC-V vector crypto extensions
> - *
> - * Copyright (C) 2022 VRULL GmbH
> - * Author: Heiko Stuebner <heiko.stueb...@vrull.eu>
> - *
> - * Copyright (C) 2023 SiFive, Inc.
> - * Author: Jerry Shih <jerry.s...@sifive.com>
> - */
> -
> -#include <asm/simd.h>
> -#include <asm/vector.h>
> -#include <crypto/internal/hash.h>
> -#include <crypto/internal/simd.h>
> -#include <crypto/sha256_base.h>
> -#include <linux/kernel.h>
> -#include <linux/module.h>
> -
> -/*
> - * Note: the asm function only uses the 'state' field of struct sha256_state.
> - * It is assumed to be the first field.
> - */
> -asmlinkage void sha256_transform_zvknha_or_zvknhb_zvkb(
> - struct crypto_sha256_state *state, const u8 *data, int num_blocks);
> -
> -static void sha256_block(struct crypto_sha256_state *state, const u8 *data,
> - int num_blocks)
> -{
> - /*
> - * Ensure struct crypto_sha256_state begins directly with the SHA-256
> - * 256-bit internal state, as this is what the asm function expects.
> - */
> - BUILD_BUG_ON(offsetof(struct crypto_sha256_state, state) != 0);
> -
> - if (crypto_simd_usable()) {
> - kernel_vector_begin();
> - sha256_transform_zvknha_or_zvknhb_zvkb(state, data, num_blocks);
> - kernel_vector_end();
> - } else
> - sha256_transform_blocks(state, data, num_blocks);
> -}
> -
> -static int riscv64_sha256_update(struct shash_desc *desc, const u8 *data,
> - unsigned int len)
> -{
> - return sha256_base_do_update_blocks(desc, data, len, sha256_block);
> -}
> -
> -static int riscv64_sha256_finup(struct shash_desc *desc, const u8 *data,
> - unsigned int len, u8 *out)
> -{
> - sha256_base_do_finup(desc, data, len, sha256_block);
> - return sha256_base_finish(desc, out);
> -}
> -
> -static int riscv64_sha256_digest(struct shash_desc *desc, const u8 *data,
> - unsigned int len, u8 *out)
> -{
> - return sha256_base_init(desc) ?:
> - riscv64_sha256_finup(desc, data, len, out);
> -}
> -
> -static struct shash_alg riscv64_sha256_algs[] = {
> - {
> - .init = sha256_base_init,
> - .update = riscv64_sha256_update,
> - .finup = riscv64_sha256_finup,
> - .digest = riscv64_sha256_digest,
> - .descsize = sizeof(struct crypto_sha256_state),
> - .digestsize = SHA256_DIGEST_SIZE,
> - .base = {
> - .cra_blocksize = SHA256_BLOCK_SIZE,
> - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
> - CRYPTO_AHASH_ALG_FINUP_MAX,
> - .cra_priority = 300,
> - .cra_name = "sha256",
> - .cra_driver_name =
> "sha256-riscv64-zvknha_or_zvknhb-zvkb",
> - .cra_module = THIS_MODULE,
> - },
> - }, {
> - .init = sha224_base_init,
> - .update = riscv64_sha256_update,
> - .finup = riscv64_sha256_finup,
> - .descsize = sizeof(struct crypto_sha256_state),
> - .digestsize = SHA224_DIGEST_SIZE,
> - .base = {
> - .cra_blocksize = SHA224_BLOCK_SIZE,
> - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
> - CRYPTO_AHASH_ALG_FINUP_MAX,
> - .cra_priority = 300,
> - .cra_name = "sha224",
> - .cra_driver_name =
> "sha224-riscv64-zvknha_or_zvknhb-zvkb",
> - .cra_module = THIS_MODULE,
> - },
> - },
> -};
> -
> -static int __init riscv64_sha256_mod_init(void)
> -{
> - /* Both zvknha and zvknhb provide the SHA-256 instructions. */
> - if ((riscv_isa_extension_available(NULL, ZVKNHA) ||
> - riscv_isa_extension_available(NULL, ZVKNHB)) &&
> - riscv_isa_extension_available(NULL, ZVKB) &&
> - riscv_vector_vlen() >= 128)
> - return crypto_register_shashes(riscv64_sha256_algs,
> - ARRAY_SIZE(riscv64_sha256_algs));
> -
> - return -ENODEV;
> -}
> -
> -static void __exit riscv64_sha256_mod_exit(void)
> -{
> - crypto_unregister_shashes(riscv64_sha256_algs,
> - ARRAY_SIZE(riscv64_sha256_algs));
> -}
> -
> -module_init(riscv64_sha256_mod_init);
> -module_exit(riscv64_sha256_mod_exit);
> -
> -MODULE_DESCRIPTION("SHA-256 (RISC-V accelerated)");
> -MODULE_AUTHOR("Heiko Stuebner <heiko.stueb...@vrull.eu>");
> -MODULE_LICENSE("GPL");
> -MODULE_ALIAS_CRYPTO("sha256");
> -MODULE_ALIAS_CRYPTO("sha224");
> diff --git a/arch/riscv/lib/crypto/Kconfig b/arch/riscv/lib/crypto/Kconfig
> index bc7a43f33eb3a..c100571feb7e8 100644
> --- a/arch/riscv/lib/crypto/Kconfig
> +++ b/arch/riscv/lib/crypto/Kconfig
> @@ -4,5 +4,12 @@ config CRYPTO_CHACHA_RISCV64
> tristate
> depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
> default CRYPTO_LIB_CHACHA
> select CRYPTO_ARCH_HAVE_LIB_CHACHA
> select CRYPTO_LIB_CHACHA_GENERIC
> +
> +config CRYPTO_SHA256_RISCV64
> + tristate
> + depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
> + default CRYPTO_LIB_SHA256
> + select CRYPTO_ARCH_HAVE_LIB_SHA256
> + select CRYPTO_LIB_SHA256_GENERIC
> diff --git a/arch/riscv/lib/crypto/Makefile b/arch/riscv/lib/crypto/Makefile
> index e27b78f317fc8..b7cb877a2c07e 100644
> --- a/arch/riscv/lib/crypto/Makefile
> +++ b/arch/riscv/lib/crypto/Makefile
> @@ -1,4 +1,7 @@
> # SPDX-License-Identifier: GPL-2.0-only
>
> obj-$(CONFIG_CRYPTO_CHACHA_RISCV64) += chacha-riscv64.o
> chacha-riscv64-y := chacha-riscv64-glue.o chacha-riscv64-zvkb.o
> +
> +obj-$(CONFIG_CRYPTO_SHA256_RISCV64) += sha256-riscv64.o
> +sha256-riscv64-y := sha256.o sha256-riscv64-zvknha_or_zvknhb-zvkb.o
> diff --git a/arch/riscv/crypto/sha256-riscv64-zvknha_or_zvknhb-zvkb.S
> b/arch/riscv/lib/crypto/sha256-riscv64-zvknha_or_zvknhb-zvkb.S
> similarity index 98%
> rename from arch/riscv/crypto/sha256-riscv64-zvknha_or_zvknhb-zvkb.S
> rename to arch/riscv/lib/crypto/sha256-riscv64-zvknha_or_zvknhb-zvkb.S
> index f1f5779e47323..fad501ad06171 100644
> --- a/arch/riscv/crypto/sha256-riscv64-zvknha_or_zvknhb-zvkb.S
> +++ b/arch/riscv/lib/crypto/sha256-riscv64-zvknha_or_zvknhb-zvkb.S
> @@ -104,12 +104,12 @@
> sha256_4rounds \last, \k1, W1, W2, W3, W0
> sha256_4rounds \last, \k2, W2, W3, W0, W1
> sha256_4rounds \last, \k3, W3, W0, W1, W2
> .endm
>
> -// void sha256_transform_zvknha_or_zvknhb_zvkb(u32 state[8], const u8 *data,
> -// int num_blocks);
> +// void sha256_transform_zvknha_or_zvknhb_zvkb(u32 state[SHA256_STATE_WORDS],
> +// const u8 *data, size_t nblocks);
> SYM_FUNC_START(sha256_transform_zvknha_or_zvknhb_zvkb)
>
> // Load the round constants into K0-K15.
> vsetivli zero, 4, e32, m1, ta, ma
> la t0, K256
> diff --git a/arch/riscv/lib/crypto/sha256.c b/arch/riscv/lib/crypto/sha256.c
> new file mode 100644
> index 0000000000000..18b84030f0b39
> --- /dev/null
> +++ b/arch/riscv/lib/crypto/sha256.c
> @@ -0,0 +1,62 @@
> +// SPDX-License-Identifier: GPL-2.0-or-later
> +/*
> + * SHA-256 (RISC-V accelerated)
> + *
> + * Copyright (C) 2022 VRULL GmbH
> + * Author: Heiko Stuebner <heiko.stueb...@vrull.eu>
> + *
> + * Copyright (C) 2023 SiFive, Inc.
> + * Author: Jerry Shih <jerry.s...@sifive.com>
> + */
> +
> +#include <asm/simd.h>
> +#include <asm/vector.h>
> +#include <crypto/internal/sha2.h>
> +#include <crypto/internal/simd.h>
> +#include <linux/kernel.h>
> +#include <linux/module.h>
> +
> +asmlinkage void sha256_transform_zvknha_or_zvknhb_zvkb(
> + u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks);
> +
> +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_extensions);
> +
> +void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS],
> + const u8 *data, size_t nblocks)
> +{
> + if (static_branch_likely(&have_extensions) && crypto_simd_usable()) {
> + kernel_vector_begin();
> + sha256_transform_zvknha_or_zvknhb_zvkb(state, data, nblocks);
> + kernel_vector_end();
> + } else {
> + sha256_blocks_generic(state, data, nblocks);
> + }
> +}
> +EXPORT_SYMBOL(sha256_blocks_arch);
> +
> +bool sha256_is_arch_optimized(void)
> +{
> + return static_key_enabled(&have_extensions);
> +}
> +EXPORT_SYMBOL(sha256_is_arch_optimized);
> +
> +static int __init riscv64_sha256_mod_init(void)
> +{
> + /* Both zvknha and zvknhb provide the SHA-256 instructions. */
> + if ((riscv_isa_extension_available(NULL, ZVKNHA) ||
> + riscv_isa_extension_available(NULL, ZVKNHB)) &&
> + riscv_isa_extension_available(NULL, ZVKB) &&
> + riscv_vector_vlen() >= 128)
> + static_branch_enable(&have_extensions);
> + return 0;
> +}
> +arch_initcall(riscv64_sha256_mod_init);
> +
> +static void __exit riscv64_sha256_mod_exit(void)
> +{
> +}
> +module_exit(riscv64_sha256_mod_exit);
> +
> +MODULE_DESCRIPTION("SHA-256 (RISC-V accelerated)");
> +MODULE_AUTHOR("Heiko Stuebner <heiko.stueb...@vrull.eu>");
> +MODULE_LICENSE("GPL");
Acked-by: Palmer Dabbelt <pal...@rivosinc.com>
I assume you want to keep these all together somewhere, so I'm going to
drop it from the RISC-V patchwork.
