https://gcc.gnu.org/g:ee2f4aeb3c55f2603e41cf4772275660f987f046
commit r17-6103-gee2f4aeb3c55f2603e41cf4772275660f987f046 Author: Kyrylo Tkachov <[email protected]> Date: Mon Nov 9 10:53:43 2026 +0100 aarch64: Implement clmul and clmul_widen optabs [PR128111] __builtin_clmulg expands through the clmul and clmul_widen optabs, and without them it falls back to a multiply-based software sequence. AArch64 has carry-less multiplication in PMULL: the 64x64->128 form with +aes (FEAT_PMULL) and the 8x8->16 form in base AdvSIMD. Add clmuldi3 and clmul_widendi3 for TARGET_AES on top of aarch64_crypto_pmulldi. The middle-end zero-extends narrower operands to DImode, so these also cover 16-bit and 32-bit clmul and the 32-bit widening form. The high half is extracted as a lane of the V2DI view of the product. A highpart subreg of the TImode product is spilled to the stack when it is the only half used. These are not available in streaming mode, so the expanders require TARGET_NON_STREAMING. Add clmul_widenqi3 for TARGET_SIMD using the 8x8->16 PMULL, so 8-bit clmul does not need +aes. For example, with +aes the low half of a 64x64->128 carry-less product (clmul) and both halves (clmul_widen): unsigned long long clmul (unsigned long long a, unsigned long long b) { return __builtin_clmulg (a, b); } unsigned long long clmul_widen (unsigned long long a, unsigned long long b, unsigned long long *hi) { return __builtin_clmulg (a, b, hi); } generate: clmul: fmov d31, x0 fmov d30, x1 pmull v30.1q, v31.1d, v30.1d fmov x0, d30 ret clmul_widen: fmov d31, x0 fmov d30, x1 pmull v30.1q, v31.1d, v30.1d fmov x0, d30 st1 {v30.d}[1], [x2] ret and without +aes the 8-bit clmul_widen generates: clmul8_widen: dup v31.8b, w0 dup v30.8b, w1 pmull v30.8h, v31.8b, v30.8b ushr v31.2s, v30.2s, 8 umov w0, v30.b[0] str b31, [x2] ret The runtime test clmul_4.c needs FEAT_PMULL, so it requires the new effective target aarch64_pmull_hw. Bootstrapped and tested on aarch64-linux-gnu. gcc/ChangeLog: PR middle-end/128111 * config/aarch64/aarch64-simd.md (clmuldi3, clmul_widendi3): New expanders, enabled for TARGET_AES in non-streaming mode. (clmul_widenqi3): New expander. gcc/testsuite/ChangeLog: PR middle-end/128111 * lib/target-supports.exp (check_effective_target_aarch64_pmull_hw): New. * gcc.target/aarch64/clmul_1.c: New test. * gcc.target/aarch64/clmul_2.c: Likewise. * gcc.target/aarch64/clmul_3.c: Likewise. * gcc.target/aarch64/clmul_4.c: Likewise. * gcc.target/aarch64/clmul_5.c: Likewise. * gcc.target/aarch64/clmul_6.c: Likewise. * gcc.target/aarch64/sme/clmul_1.c: Likewise. Signed-off-by: Kyrylo Tkachov <[email protected]> Diff: --- gcc/config/aarch64/aarch64-simd.md | 53 +++++++++++++++++++++++ gcc/testsuite/gcc.target/aarch64/clmul_1.c | 52 +++++++++++++++++++++++ gcc/testsuite/gcc.target/aarch64/clmul_2.c | 59 ++++++++++++++++++++++++++ gcc/testsuite/gcc.target/aarch64/clmul_3.c | 38 +++++++++++++++++ gcc/testsuite/gcc.target/aarch64/clmul_4.c | 15 +++++++ gcc/testsuite/gcc.target/aarch64/clmul_5.c | 15 +++++++ gcc/testsuite/gcc.target/aarch64/clmul_6.c | 25 +++++++++++ gcc/testsuite/gcc.target/aarch64/sme/clmul_1.c | 28 ++++++++++++ gcc/testsuite/lib/target-supports.exp | 18 ++++++++ 9 files changed, 303 insertions(+) diff --git a/gcc/config/aarch64/aarch64-simd.md b/gcc/config/aarch64/aarch64-simd.md index 7ea023633139..20958df82a48 100644 --- a/gcc/config/aarch64/aarch64-simd.md +++ b/gcc/config/aarch64/aarch64-simd.md @@ -10833,6 +10833,59 @@ [(set_attr "type" "crypto_pmull")] ) +;; Carry-less multiplication. clmul<mode>3 returns the low half of the +;; product. clmul_widen<mode>3 returns the low half in operand 0 and the +;; high half in operand 1. The DImode pattern also serves narrower modes, +;; which the middle-end zero-extends to DImode. + +(define_expand "clmuldi3" + [(match_operand:DI 0 "register_operand") + (match_operand:DI 1 "register_operand") + (match_operand:DI 2 "register_operand")] + "TARGET_AES && TARGET_NON_STREAMING" +{ + rtx prod = gen_reg_rtx (TImode); + emit_insn (gen_aarch64_crypto_pmulldi (prod, operands[1], operands[2])); + emit_move_insn (operands[0], gen_lowpart (DImode, prod)); + DONE; +}) + +(define_expand "clmul_widendi3" + [(match_operand:DI 0 "register_operand") + (match_operand:DI 1 "register_operand") + (match_operand:DI 2 "register_operand") + (match_operand:DI 3 "register_operand")] + "TARGET_AES && TARGET_NON_STREAMING" +{ + rtx prod = gen_reg_rtx (TImode); + emit_insn (gen_aarch64_crypto_pmulldi (prod, operands[2], operands[3])); + emit_move_insn (operands[0], gen_lowpart (DImode, prod)); + emit_insn (gen_vec_extractv2didi (operands[1], + gen_lowpart (V2DImode, prod), + aarch64_endian_lane_rtx (V2DImode, 1))); + DONE; +}) + +(define_expand "clmul_widenqi3" + [(match_operand:QI 0 "register_operand") + (match_operand:QI 1 "register_operand") + (match_operand:QI 2 "register_operand") + (match_operand:QI 3 "register_operand")] + "TARGET_SIMD" +{ + rtx x = gen_reg_rtx (V8QImode); + rtx y = gen_reg_rtx (V8QImode); + rtx prod = gen_reg_rtx (V8HImode); + rtx half = gen_lowpart (HImode, prod); + emit_insn (gen_aarch64_simd_dupv8qi (x, operands[2])); + emit_insn (gen_aarch64_simd_dupv8qi (y, operands[3])); + emit_insn (gen_aarch64_pmullv8qi (prod, x, y)); + emit_move_insn (operands[0], gen_lowpart (QImode, half)); + rtx hi = expand_shift (RSHIFT_EXPR, HImode, half, 8, NULL_RTX, 1); + emit_move_insn (operands[1], gen_lowpart (QImode, hi)); + DONE; +}) + ;; Sign- or zero-extend a 64-bit integer vector to a 128-bit vector. (define_insn_and_split "<optab><Vnarrowq><mode>2" [(set (match_operand:VQN 0 "register_operand" "=w") diff --git a/gcc/testsuite/gcc.target/aarch64/clmul_1.c b/gcc/testsuite/gcc.target/aarch64/clmul_1.c new file mode 100644 index 000000000000..a8c930d4eaa9 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/clmul_1.c @@ -0,0 +1,52 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=armv8-a+aes" } */ + +typedef unsigned long long u64; +typedef unsigned int u32; +typedef unsigned short u16; + +/* +** clmul64: +** fmov d([0-9]+), x0 +** fmov d([0-9]+), x1 +** pmull v\2\.1q, v\1\.1d, v\2\.1d +** fmov x0, d\2 +** ret +*/ +u64 +clmul64 (u64 a, u64 b) +{ + return __builtin_clmulg (a, b); +} + +/* +** clmul32: +** fmov s([0-9]+), w0 +** fmov s([0-9]+), w1 +** pmull v\1\.1q, v\1\.1d, v\2\.1d +** fmov w0, s\1 +** ret +*/ +u32 +clmul32 (u32 a, u32 b) +{ + return __builtin_clmulg (a, b); +} + +/* +** clmul16: +** and x0, x0, 65535 +** and x1, x1, 65535 +** fmov d([0-9]+), x0 +** fmov d([0-9]+), x1 +** pmull v\2\.1q, v\1\.1d, v\2\.1d +** umov w0, v\2\.h\[0\] +** ret +*/ +u16 +clmul16 (u16 a, u16 b) +{ + return __builtin_clmulg (a, b); +} + +/* { dg-final { check-function-bodies "**" "" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/clmul_2.c b/gcc/testsuite/gcc.target/aarch64/clmul_2.c new file mode 100644 index 000000000000..bbfc5916ce40 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/clmul_2.c @@ -0,0 +1,59 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=armv8-a+aes" } */ + +typedef unsigned long long u64; +typedef unsigned int u32; + +/* Both halves of a 64x64->128 product. */ + +/* +** clmul64_widen: +** fmov d([0-9]+), x0 +** fmov d([0-9]+), x1 +** pmull v\2\.1q, v\1\.1d, v\2\.1d +** fmov x0, d\2 +** st1 \{v\2\.d\}\[1\], \[x2\] +** ret +*/ +u64 +clmul64_widen (u64 a, u64 b, u64 *hi) +{ + return __builtin_clmulg (a, b, hi); +} + +/* Only the high half is used. Must not go through the stack. */ + +/* +** clmul64_high: +** fmov d([0-9]+), x0 +** fmov d([0-9]+), x1 +** pmull v\2\.1q, v\1\.1d, v\2\.1d +** umov x0, v\2\.d\[1\] +** ret +*/ +u64 +clmul64_high (u64 a, u64 b) +{ + u64 h; + __builtin_clmulg (a, b, &h); + return h; +} + +/* +** clmul32_widen: +** fmov s([0-9]+), w0 +** fmov s([0-9]+), w1 +** pmull v\1\.1q, v\1\.1d, v\2\.1d +** fmov x0, d\1 +** lsr x0, x0, 32 +** str w0, \[x2\] +** fmov w0, s\1 +** ret +*/ +u32 +clmul32_widen (u32 a, u32 b, u32 *hi) +{ + return __builtin_clmulg (a, b, hi); +} + +/* { dg-final { check-function-bodies "**" "" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/clmul_3.c b/gcc/testsuite/gcc.target/aarch64/clmul_3.c new file mode 100644 index 000000000000..353ce780dcde --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/clmul_3.c @@ -0,0 +1,38 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=armv8-a" } */ + +typedef unsigned char u8; + +/* 8x8->16 carry-less multiplication needs only base AdvSIMD. */ + +/* +** clmul8: +** dup v([0-9]+)\.8b, w0 +** dup v([0-9]+)\.8b, w1 +** pmull v\2\.8h, v\1\.8b, v\2\.8b +** umov w0, v\2\.b\[0\] +** ret +*/ +u8 +clmul8 (u8 a, u8 b) +{ + return __builtin_clmulg (a, b); +} + +/* +** clmul8_widen: +** dup v([0-9]+)\.8b, w0 +** dup v([0-9]+)\.8b, w1 +** pmull v\2\.8h, v\1\.8b, v\2\.8b +** ushr v\1\.2s, v\2\.2s, 8 +** umov w0, v\2\.b\[0\] +** str b\1, \[x2\] +** ret +*/ +u8 +clmul8_widen (u8 a, u8 b, u8 *hi) +{ + return __builtin_clmulg (a, b, hi); +} + +/* { dg-final { check-function-bodies "**" "" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/clmul_4.c b/gcc/testsuite/gcc.target/aarch64/clmul_4.c new file mode 100644 index 000000000000..3144c1aa12c1 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/clmul_4.c @@ -0,0 +1,15 @@ +/* { dg-do run { target aarch64_pmull_hw } } */ +/* { dg-options "-O2 -march=armv8-a+aes" } */ + +#define main do_main + +#include "../../c-c++-common/builtin-clmulg-1.c" + +#undef main + +int +main () +{ + do_main (); + return 0; +} diff --git a/gcc/testsuite/gcc.target/aarch64/clmul_5.c b/gcc/testsuite/gcc.target/aarch64/clmul_5.c new file mode 100644 index 000000000000..8472ec54325e --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/clmul_5.c @@ -0,0 +1,15 @@ +/* { dg-do run } */ +/* { dg-options "-O2 -march=armv8-a" } */ + +#define main do_main + +#include "../../c-c++-common/builtin-clmulg-1.c" + +#undef main + +int +main () +{ + do_main (); + return 0; +} diff --git a/gcc/testsuite/gcc.target/aarch64/clmul_6.c b/gcc/testsuite/gcc.target/aarch64/clmul_6.c new file mode 100644 index 000000000000..6a41799c088f --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/clmul_6.c @@ -0,0 +1,25 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=armv8-a+aes -mbig-endian" } */ + +typedef unsigned long long u64; + +/* The high half of the product is the second doubleword of the PMULL + result regardless of endianness. */ + +/* +** clmul64_high: +** fmov d([0-9]+), x0 +** fmov d([0-9]+), x1 +** pmull v\2\.1q, v\1\.1d, v\2\.1d +** umov x0, v\2\.d\[1\] +** ret +*/ +u64 +clmul64_high (u64 a, u64 b) +{ + u64 h; + __builtin_clmulg (a, b, &h); + return h; +} + +/* { dg-final { check-function-bodies "**" "" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sme/clmul_1.c b/gcc/testsuite/gcc.target/aarch64/sme/clmul_1.c new file mode 100644 index 000000000000..69f3bfea45a6 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sme/clmul_1.c @@ -0,0 +1,28 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=armv9-a+sme+aes" } */ + +typedef unsigned long long u64; +typedef unsigned char u8; + +/* PMULL is an Advanced SIMD instruction and is not available in streaming + mode, so the carry-less multiplication must be open-coded. */ + +u64 +clmul64 (u64 a, u64 b) [[arm::streaming]] +{ + return __builtin_clmulg (a, b); +} + +u64 +clmul64_widen (u64 a, u64 b, u64 *hi) [[arm::streaming_compatible]] +{ + return __builtin_clmulg (a, b, hi); +} + +u8 +clmul8 (u8 a, u8 b) [[arm::streaming]] +{ + return __builtin_clmulg (a, b); +} + +/* { dg-final { scan-assembler-not {\tpmull\t} } } */ diff --git a/gcc/testsuite/lib/target-supports.exp b/gcc/testsuite/lib/target-supports.exp index c5ef4cbb333d..4dcb7f41b459 100644 --- a/gcc/testsuite/lib/target-supports.exp +++ b/gcc/testsuite/lib/target-supports.exp @@ -6974,6 +6974,24 @@ proc check_effective_target_arm_neon_hw { } { } [add_options_for_arm_neon ""]] } +# Return true if this is an AArch64 target that can run the 64-bit +# polynomial multiply (PMULL) instructions of FEAT_PMULL. + +proc check_effective_target_aarch64_pmull_hw { } { + if { ![istarget aarch64*-*-*] } { + return 0 + } + return [check_runtime aarch64_pmull_hw_available { + #pragma GCC target "+aes" + int + main (void) + { + asm volatile ("pmull v0.1q, v1.1d, v2.1d"); + return 0; + } + }] +} + # Return true if this is an AArch64 target that can run SVE code. proc check_effective_target_aarch64_sve_hw { } {
