A Clang Full LTO bug[1] was fixed in Clang 23 where multiple .o files performing an address-take of the same asm-originating typeid weak symbols would have those symbols get completely removed instead of just keeping one, seen at link time:
ld.lld: error: undefined symbol: __kcfi_typeid_aria_aesni_avx_encrypt_16way >>> referenced by aria-aesni-avx-asm_64.S:962 >>> vmlinux.o:(__cfi_aria_aesni_avx_encrypt_16way) As a work-around for earlier Clang, add aria_avx_setup_16way_ops() and aria_avx2_setup_32way_ops(), exported by the glue that owns each tier's asm functions, and fill all three tables with them, so that each asm function has its address taken in only one file. Build tested ARCH=x86_64 with the kernel test robot's randconfig (CONFIG_LTO_CLANG_FULL=y, CONFIG_CFI=y, and all three ARIA drivers built in) and W=1, with Clang 22.1.8 and Clang 24.0.0git: vmlinux.o defines all fifteen __kcfi_typeid_aria_* symbols, and vmlinux links. Without this change, Clang 22.1.8 fails to link it as above, and Clang 24.0.0git, which has the LLVM fix, links it. Reported-by: kernel test robot <[email protected]> Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/ Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/ Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/ Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/ Link: https://github.com/llvm/llvm-project/pull/204025 [1] Assisted-by: LLM Signed-off-by: Kees Cook <[email protected]> --- arch/x86/crypto/aria-avx.h | 3 +++ arch/x86/crypto/aria_aesni_avx2_glue.c | 35 ++++++++++++++----------- arch/x86/crypto/aria_aesni_avx_glue.c | 29 +++++++++++++------- arch/x86/crypto/aria_gfni_avx512_glue.c | 8 ++---- 4 files changed, 45 insertions(+), 30 deletions(-) diff --git a/arch/x86/crypto/aria-avx.h b/arch/x86/crypto/aria-avx.h index 6e1b2d8a31ed..0fff24861088 100644 --- a/arch/x86/crypto/aria-avx.h +++ b/arch/x86/crypto/aria-avx.h @@ -59,4 +59,7 @@ struct aria_avx_ops { }; + +void aria_avx_setup_16way_ops(struct aria_avx_ops *ops, bool use_gfni); +void aria_avx2_setup_32way_ops(struct aria_avx_ops *ops, bool use_gfni); #endif diff --git a/arch/x86/crypto/aria_aesni_avx2_glue.c b/arch/x86/crypto/aria_aesni_avx2_glue.c index 1487a49bfbac..65c850601f49 100644 --- a/arch/x86/crypto/aria_aesni_avx2_glue.c +++ b/arch/x86/crypto/aria_aesni_avx2_glue.c @@ -37,6 +37,24 @@ asmlinkage void aria_aesni_avx2_gfni_ctr_crypt_32way(const void *ctx, u8 *dst, u8 *keystream, u8 *iv); EXPORT_SYMBOL_GPL(aria_aesni_avx2_gfni_ctr_crypt_32way); +/* + * Set up function pointers for the AVX 32-way asm functions for both this + * module and the the higher-tier modules. + */ +void aria_avx2_setup_32way_ops(struct aria_avx_ops *ops, bool use_gfni) +{ + if (use_gfni) { + ops->aria_encrypt_32way = aria_aesni_avx2_gfni_encrypt_32way; + ops->aria_decrypt_32way = aria_aesni_avx2_gfni_decrypt_32way; + ops->aria_ctr_crypt_32way = aria_aesni_avx2_gfni_ctr_crypt_32way; + } else { + ops->aria_encrypt_32way = aria_aesni_avx2_encrypt_32way; + ops->aria_decrypt_32way = aria_aesni_avx2_decrypt_32way; + ops->aria_ctr_crypt_32way = aria_aesni_avx2_ctr_crypt_32way; + } +} +EXPORT_SYMBOL_GPL(aria_avx2_setup_32way_ops); + static struct aria_avx_ops aria_ops; struct aria_avx2_request_ctx { @@ -211,21 +229,8 @@ static int __init aria_avx2_init(void) return -ENODEV; } - if (boot_cpu_has(X86_FEATURE_GFNI)) { - aria_ops.aria_encrypt_16way = aria_aesni_avx_gfni_encrypt_16way; - aria_ops.aria_decrypt_16way = aria_aesni_avx_gfni_decrypt_16way; - aria_ops.aria_ctr_crypt_16way = aria_aesni_avx_gfni_ctr_crypt_16way; - aria_ops.aria_encrypt_32way = aria_aesni_avx2_gfni_encrypt_32way; - aria_ops.aria_decrypt_32way = aria_aesni_avx2_gfni_decrypt_32way; - aria_ops.aria_ctr_crypt_32way = aria_aesni_avx2_gfni_ctr_crypt_32way; - } else { - aria_ops.aria_encrypt_16way = aria_aesni_avx_encrypt_16way; - aria_ops.aria_decrypt_16way = aria_aesni_avx_decrypt_16way; - aria_ops.aria_ctr_crypt_16way = aria_aesni_avx_ctr_crypt_16way; - aria_ops.aria_encrypt_32way = aria_aesni_avx2_encrypt_32way; - aria_ops.aria_decrypt_32way = aria_aesni_avx2_decrypt_32way; - aria_ops.aria_ctr_crypt_32way = aria_aesni_avx2_ctr_crypt_32way; - } + aria_avx_setup_16way_ops(&aria_ops, boot_cpu_has(X86_FEATURE_GFNI)); + aria_avx2_setup_32way_ops(&aria_ops, boot_cpu_has(X86_FEATURE_GFNI)); return crypto_register_skciphers(aria_algs, ARRAY_SIZE(aria_algs)); } diff --git a/arch/x86/crypto/aria_aesni_avx_glue.c b/arch/x86/crypto/aria_aesni_avx_glue.c index e4e3d78915a5..07baf5a27143 100644 --- a/arch/x86/crypto/aria_aesni_avx_glue.c +++ b/arch/x86/crypto/aria_aesni_avx_glue.c @@ -39,6 +39,25 @@ EXPORT_SYMBOL_GPL(aria_aesni_avx_gfni_ctr_crypt_16way); static struct aria_avx_ops aria_ops; +/* + * Set up function pointers for the AVX 16-way asm functions for both this + * module's own dispatch table and the higher-tier (AVX2, GFNI/AVX-512) + * modules. + */ +void aria_avx_setup_16way_ops(struct aria_avx_ops *ops, bool use_gfni) +{ + if (use_gfni) { + ops->aria_encrypt_16way = aria_aesni_avx_gfni_encrypt_16way; + ops->aria_decrypt_16way = aria_aesni_avx_gfni_decrypt_16way; + ops->aria_ctr_crypt_16way = aria_aesni_avx_gfni_ctr_crypt_16way; + } else { + ops->aria_encrypt_16way = aria_aesni_avx_encrypt_16way; + ops->aria_decrypt_16way = aria_aesni_avx_decrypt_16way; + ops->aria_ctr_crypt_16way = aria_aesni_avx_ctr_crypt_16way; + } +} +EXPORT_SYMBOL_GPL(aria_avx_setup_16way_ops); + struct aria_avx_request_ctx { u8 keystream[ARIA_AESNI_PARALLEL_BLOCK_SIZE]; }; @@ -197,15 +216,7 @@ static int __init aria_avx_init(void) return -ENODEV; } - if (boot_cpu_has(X86_FEATURE_GFNI)) { - aria_ops.aria_encrypt_16way = aria_aesni_avx_gfni_encrypt_16way; - aria_ops.aria_decrypt_16way = aria_aesni_avx_gfni_decrypt_16way; - aria_ops.aria_ctr_crypt_16way = aria_aesni_avx_gfni_ctr_crypt_16way; - } else { - aria_ops.aria_encrypt_16way = aria_aesni_avx_encrypt_16way; - aria_ops.aria_decrypt_16way = aria_aesni_avx_decrypt_16way; - aria_ops.aria_ctr_crypt_16way = aria_aesni_avx_ctr_crypt_16way; - } + aria_avx_setup_16way_ops(&aria_ops, boot_cpu_has(X86_FEATURE_GFNI)); return crypto_register_skciphers(aria_algs, ARRAY_SIZE(aria_algs)); } diff --git a/arch/x86/crypto/aria_gfni_avx512_glue.c b/arch/x86/crypto/aria_gfni_avx512_glue.c index 363cbf4399cc..1eccfceae314 100644 --- a/arch/x86/crypto/aria_gfni_avx512_glue.c +++ b/arch/x86/crypto/aria_gfni_avx512_glue.c @@ -214,12 +214,8 @@ static int __init aria_avx512_init(void) return -ENODEV; } - aria_ops.aria_encrypt_16way = aria_aesni_avx_gfni_encrypt_16way; - aria_ops.aria_decrypt_16way = aria_aesni_avx_gfni_decrypt_16way; - aria_ops.aria_ctr_crypt_16way = aria_aesni_avx_gfni_ctr_crypt_16way; - aria_ops.aria_encrypt_32way = aria_aesni_avx2_gfni_encrypt_32way; - aria_ops.aria_decrypt_32way = aria_aesni_avx2_gfni_decrypt_32way; - aria_ops.aria_ctr_crypt_32way = aria_aesni_avx2_gfni_ctr_crypt_32way; + aria_avx_setup_16way_ops(&aria_ops, true); + aria_avx2_setup_32way_ops(&aria_ops, true); aria_ops.aria_encrypt_64way = aria_gfni_avx512_encrypt_64way; aria_ops.aria_decrypt_64way = aria_gfni_avx512_decrypt_64way; aria_ops.aria_ctr_crypt_64way = aria_gfni_avx512_ctr_crypt_64way; -- 2.55.0
