https://gcc.gnu.org/g:ee2f4aeb3c55f2603e41cf4772275660f987f046

commit r17-6103-gee2f4aeb3c55f2603e41cf4772275660f987f046
Author: Kyrylo Tkachov <[email protected]>
Date:   Mon Nov 9 10:53:43 2026 +0100

    aarch64: Implement clmul and clmul_widen optabs [PR128111]
    
    __builtin_clmulg expands through the clmul and clmul_widen optabs, and
    without them it falls back to a multiply-based software sequence.
    AArch64 has carry-less multiplication in PMULL: the 64x64->128 form
    with +aes (FEAT_PMULL) and the 8x8->16 form in base AdvSIMD.
    
    Add clmuldi3 and clmul_widendi3 for TARGET_AES on top of
    aarch64_crypto_pmulldi.  The middle-end zero-extends narrower operands
    to DImode, so these also cover 16-bit and 32-bit clmul and the 32-bit
    widening form.  The high half is extracted as a lane of the V2DI view
    of the product.  A highpart subreg of the TImode product is spilled
    to the stack when it is the only half used.
    
    These are not available in streaming mode, so the expanders require
    TARGET_NON_STREAMING.
    
    Add clmul_widenqi3 for TARGET_SIMD using the 8x8->16 PMULL, so 8-bit
    clmul does not need +aes.
    
    For example, with +aes the low half of a 64x64->128 carry-less product
    (clmul) and both halves (clmul_widen):
    
      unsigned long long clmul (unsigned long long a, unsigned long long b)
      {
        return __builtin_clmulg (a, b);
      }
    
      unsigned long long clmul_widen (unsigned long long a, unsigned long long 
b,
                                      unsigned long long *hi)
      {
        return __builtin_clmulg (a, b, hi);
      }
    
    generate:
    
      clmul:
            fmov    d31, x0
            fmov    d30, x1
            pmull   v30.1q, v31.1d, v30.1d
            fmov    x0, d30
            ret
    
      clmul_widen:
            fmov    d31, x0
            fmov    d30, x1
            pmull   v30.1q, v31.1d, v30.1d
            fmov    x0, d30
            st1     {v30.d}[1], [x2]
            ret
    
    and without +aes the 8-bit clmul_widen generates:
    
      clmul8_widen:
            dup     v31.8b, w0
            dup     v30.8b, w1
            pmull   v30.8h, v31.8b, v30.8b
            ushr    v31.2s, v30.2s, 8
            umov    w0, v30.b[0]
            str     b31, [x2]
            ret
    
    The runtime test clmul_4.c needs FEAT_PMULL, so it requires the new
    effective target aarch64_pmull_hw.
    
    Bootstrapped and tested on aarch64-linux-gnu.
    
    gcc/ChangeLog:
    
            PR middle-end/128111
            * config/aarch64/aarch64-simd.md (clmuldi3, clmul_widendi3): New
            expanders, enabled for TARGET_AES in non-streaming mode.
            (clmul_widenqi3): New expander.
    
    gcc/testsuite/ChangeLog:
    
            PR middle-end/128111
            * lib/target-supports.exp (check_effective_target_aarch64_pmull_hw):
            New.
            * gcc.target/aarch64/clmul_1.c: New test.
            * gcc.target/aarch64/clmul_2.c: Likewise.
            * gcc.target/aarch64/clmul_3.c: Likewise.
            * gcc.target/aarch64/clmul_4.c: Likewise.
            * gcc.target/aarch64/clmul_5.c: Likewise.
            * gcc.target/aarch64/clmul_6.c: Likewise.
            * gcc.target/aarch64/sme/clmul_1.c: Likewise.
    
    Signed-off-by: Kyrylo Tkachov <[email protected]>

Diff:
---
 gcc/config/aarch64/aarch64-simd.md             | 53 +++++++++++++++++++++++
 gcc/testsuite/gcc.target/aarch64/clmul_1.c     | 52 +++++++++++++++++++++++
 gcc/testsuite/gcc.target/aarch64/clmul_2.c     | 59 ++++++++++++++++++++++++++
 gcc/testsuite/gcc.target/aarch64/clmul_3.c     | 38 +++++++++++++++++
 gcc/testsuite/gcc.target/aarch64/clmul_4.c     | 15 +++++++
 gcc/testsuite/gcc.target/aarch64/clmul_5.c     | 15 +++++++
 gcc/testsuite/gcc.target/aarch64/clmul_6.c     | 25 +++++++++++
 gcc/testsuite/gcc.target/aarch64/sme/clmul_1.c | 28 ++++++++++++
 gcc/testsuite/lib/target-supports.exp          | 18 ++++++++
 9 files changed, 303 insertions(+)

diff --git a/gcc/config/aarch64/aarch64-simd.md 
b/gcc/config/aarch64/aarch64-simd.md
index 7ea023633139..20958df82a48 100644
--- a/gcc/config/aarch64/aarch64-simd.md
+++ b/gcc/config/aarch64/aarch64-simd.md
@@ -10833,6 +10833,59 @@
   [(set_attr "type" "crypto_pmull")]
 )
 
+;; Carry-less multiplication.  clmul<mode>3 returns the low half of the
+;; product.  clmul_widen<mode>3 returns the low half in operand 0 and the
+;; high half in operand 1.  The DImode pattern also serves narrower modes,
+;; which the middle-end zero-extends to DImode.
+
+(define_expand "clmuldi3"
+  [(match_operand:DI 0 "register_operand")
+   (match_operand:DI 1 "register_operand")
+   (match_operand:DI 2 "register_operand")]
+  "TARGET_AES && TARGET_NON_STREAMING"
+{
+  rtx prod = gen_reg_rtx (TImode);
+  emit_insn (gen_aarch64_crypto_pmulldi (prod, operands[1], operands[2]));
+  emit_move_insn (operands[0], gen_lowpart (DImode, prod));
+  DONE;
+})
+
+(define_expand "clmul_widendi3"
+  [(match_operand:DI 0 "register_operand")
+   (match_operand:DI 1 "register_operand")
+   (match_operand:DI 2 "register_operand")
+   (match_operand:DI 3 "register_operand")]
+  "TARGET_AES && TARGET_NON_STREAMING"
+{
+  rtx prod = gen_reg_rtx (TImode);
+  emit_insn (gen_aarch64_crypto_pmulldi (prod, operands[2], operands[3]));
+  emit_move_insn (operands[0], gen_lowpart (DImode, prod));
+  emit_insn (gen_vec_extractv2didi (operands[1],
+                                   gen_lowpart (V2DImode, prod),
+                                   aarch64_endian_lane_rtx (V2DImode, 1)));
+  DONE;
+})
+
+(define_expand "clmul_widenqi3"
+  [(match_operand:QI 0 "register_operand")
+   (match_operand:QI 1 "register_operand")
+   (match_operand:QI 2 "register_operand")
+   (match_operand:QI 3 "register_operand")]
+  "TARGET_SIMD"
+{
+  rtx x = gen_reg_rtx (V8QImode);
+  rtx y = gen_reg_rtx (V8QImode);
+  rtx prod = gen_reg_rtx (V8HImode);
+  rtx half = gen_lowpart (HImode, prod);
+  emit_insn (gen_aarch64_simd_dupv8qi (x, operands[2]));
+  emit_insn (gen_aarch64_simd_dupv8qi (y, operands[3]));
+  emit_insn (gen_aarch64_pmullv8qi (prod, x, y));
+  emit_move_insn (operands[0], gen_lowpart (QImode, half));
+  rtx hi = expand_shift (RSHIFT_EXPR, HImode, half, 8, NULL_RTX, 1);
+  emit_move_insn (operands[1], gen_lowpart (QImode, hi));
+  DONE;
+})
+
 ;; Sign- or zero-extend a 64-bit integer vector to a 128-bit vector.
 (define_insn_and_split "<optab><Vnarrowq><mode>2"
   [(set (match_operand:VQN 0 "register_operand" "=w")
diff --git a/gcc/testsuite/gcc.target/aarch64/clmul_1.c 
b/gcc/testsuite/gcc.target/aarch64/clmul_1.c
new file mode 100644
index 000000000000..a8c930d4eaa9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/clmul_1.c
@@ -0,0 +1,52 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=armv8-a+aes" } */
+
+typedef unsigned long long u64;
+typedef unsigned int u32;
+typedef unsigned short u16;
+
+/*
+** clmul64:
+**     fmov    d([0-9]+), x0
+**     fmov    d([0-9]+), x1
+**     pmull   v\2\.1q, v\1\.1d, v\2\.1d
+**     fmov    x0, d\2
+**     ret
+*/
+u64
+clmul64 (u64 a, u64 b)
+{
+  return __builtin_clmulg (a, b);
+}
+
+/*
+** clmul32:
+**     fmov    s([0-9]+), w0
+**     fmov    s([0-9]+), w1
+**     pmull   v\1\.1q, v\1\.1d, v\2\.1d
+**     fmov    w0, s\1
+**     ret
+*/
+u32
+clmul32 (u32 a, u32 b)
+{
+  return __builtin_clmulg (a, b);
+}
+
+/*
+** clmul16:
+**     and     x0, x0, 65535
+**     and     x1, x1, 65535
+**     fmov    d([0-9]+), x0
+**     fmov    d([0-9]+), x1
+**     pmull   v\2\.1q, v\1\.1d, v\2\.1d
+**     umov    w0, v\2\.h\[0\]
+**     ret
+*/
+u16
+clmul16 (u16 a, u16 b)
+{
+  return __builtin_clmulg (a, b);
+}
+
+/* { dg-final { check-function-bodies "**" "" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/clmul_2.c 
b/gcc/testsuite/gcc.target/aarch64/clmul_2.c
new file mode 100644
index 000000000000..bbfc5916ce40
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/clmul_2.c
@@ -0,0 +1,59 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=armv8-a+aes" } */
+
+typedef unsigned long long u64;
+typedef unsigned int u32;
+
+/* Both halves of a 64x64->128 product.  */
+
+/*
+** clmul64_widen:
+**     fmov    d([0-9]+), x0
+**     fmov    d([0-9]+), x1
+**     pmull   v\2\.1q, v\1\.1d, v\2\.1d
+**     fmov    x0, d\2
+**     st1     \{v\2\.d\}\[1\], \[x2\]
+**     ret
+*/
+u64
+clmul64_widen (u64 a, u64 b, u64 *hi)
+{
+  return __builtin_clmulg (a, b, hi);
+}
+
+/* Only the high half is used.  Must not go through the stack.  */
+
+/*
+** clmul64_high:
+**     fmov    d([0-9]+), x0
+**     fmov    d([0-9]+), x1
+**     pmull   v\2\.1q, v\1\.1d, v\2\.1d
+**     umov    x0, v\2\.d\[1\]
+**     ret
+*/
+u64
+clmul64_high (u64 a, u64 b)
+{
+  u64 h;
+  __builtin_clmulg (a, b, &h);
+  return h;
+}
+
+/*
+** clmul32_widen:
+**     fmov    s([0-9]+), w0
+**     fmov    s([0-9]+), w1
+**     pmull   v\1\.1q, v\1\.1d, v\2\.1d
+**     fmov    x0, d\1
+**     lsr     x0, x0, 32
+**     str     w0, \[x2\]
+**     fmov    w0, s\1
+**     ret
+*/
+u32
+clmul32_widen (u32 a, u32 b, u32 *hi)
+{
+  return __builtin_clmulg (a, b, hi);
+}
+
+/* { dg-final { check-function-bodies "**" "" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/clmul_3.c 
b/gcc/testsuite/gcc.target/aarch64/clmul_3.c
new file mode 100644
index 000000000000..353ce780dcde
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/clmul_3.c
@@ -0,0 +1,38 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=armv8-a" } */
+
+typedef unsigned char u8;
+
+/* 8x8->16 carry-less multiplication needs only base AdvSIMD.  */
+
+/*
+** clmul8:
+**     dup     v([0-9]+)\.8b, w0
+**     dup     v([0-9]+)\.8b, w1
+**     pmull   v\2\.8h, v\1\.8b, v\2\.8b
+**     umov    w0, v\2\.b\[0\]
+**     ret
+*/
+u8
+clmul8 (u8 a, u8 b)
+{
+  return __builtin_clmulg (a, b);
+}
+
+/*
+** clmul8_widen:
+**     dup     v([0-9]+)\.8b, w0
+**     dup     v([0-9]+)\.8b, w1
+**     pmull   v\2\.8h, v\1\.8b, v\2\.8b
+**     ushr    v\1\.2s, v\2\.2s, 8
+**     umov    w0, v\2\.b\[0\]
+**     str     b\1, \[x2\]
+**     ret
+*/
+u8
+clmul8_widen (u8 a, u8 b, u8 *hi)
+{
+  return __builtin_clmulg (a, b, hi);
+}
+
+/* { dg-final { check-function-bodies "**" "" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/clmul_4.c 
b/gcc/testsuite/gcc.target/aarch64/clmul_4.c
new file mode 100644
index 000000000000..3144c1aa12c1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/clmul_4.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target aarch64_pmull_hw } } */
+/* { dg-options "-O2 -march=armv8-a+aes" } */
+
+#define main do_main
+
+#include "../../c-c++-common/builtin-clmulg-1.c"
+
+#undef main
+
+int
+main ()
+{
+  do_main ();
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/clmul_5.c 
b/gcc/testsuite/gcc.target/aarch64/clmul_5.c
new file mode 100644
index 000000000000..8472ec54325e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/clmul_5.c
@@ -0,0 +1,15 @@
+/* { dg-do run } */
+/* { dg-options "-O2 -march=armv8-a" } */
+
+#define main do_main
+
+#include "../../c-c++-common/builtin-clmulg-1.c"
+
+#undef main
+
+int
+main ()
+{
+  do_main ();
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/clmul_6.c 
b/gcc/testsuite/gcc.target/aarch64/clmul_6.c
new file mode 100644
index 000000000000..6a41799c088f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/clmul_6.c
@@ -0,0 +1,25 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=armv8-a+aes -mbig-endian" } */
+
+typedef unsigned long long u64;
+
+/* The high half of the product is the second doubleword of the PMULL
+   result regardless of endianness.  */
+
+/*
+** clmul64_high:
+**     fmov    d([0-9]+), x0
+**     fmov    d([0-9]+), x1
+**     pmull   v\2\.1q, v\1\.1d, v\2\.1d
+**     umov    x0, v\2\.d\[1\]
+**     ret
+*/
+u64
+clmul64_high (u64 a, u64 b)
+{
+  u64 h;
+  __builtin_clmulg (a, b, &h);
+  return h;
+}
+
+/* { dg-final { check-function-bodies "**" "" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sme/clmul_1.c 
b/gcc/testsuite/gcc.target/aarch64/sme/clmul_1.c
new file mode 100644
index 000000000000..69f3bfea45a6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme/clmul_1.c
@@ -0,0 +1,28 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=armv9-a+sme+aes" } */
+
+typedef unsigned long long u64;
+typedef unsigned char u8;
+
+/* PMULL is an Advanced SIMD instruction and is not available in streaming
+   mode, so the carry-less multiplication must be open-coded.  */
+
+u64
+clmul64 (u64 a, u64 b) [[arm::streaming]]
+{
+  return __builtin_clmulg (a, b);
+}
+
+u64
+clmul64_widen (u64 a, u64 b, u64 *hi) [[arm::streaming_compatible]]
+{
+  return __builtin_clmulg (a, b, hi);
+}
+
+u8
+clmul8 (u8 a, u8 b) [[arm::streaming]]
+{
+  return __builtin_clmulg (a, b);
+}
+
+/* { dg-final { scan-assembler-not {\tpmull\t} } } */
diff --git a/gcc/testsuite/lib/target-supports.exp 
b/gcc/testsuite/lib/target-supports.exp
index c5ef4cbb333d..4dcb7f41b459 100644
--- a/gcc/testsuite/lib/target-supports.exp
+++ b/gcc/testsuite/lib/target-supports.exp
@@ -6974,6 +6974,24 @@ proc check_effective_target_arm_neon_hw { } {
     } [add_options_for_arm_neon ""]]
 }
 
+# Return true if this is an AArch64 target that can run the 64-bit
+# polynomial multiply (PMULL) instructions of FEAT_PMULL.
+
+proc check_effective_target_aarch64_pmull_hw { } {
+    if { ![istarget aarch64*-*-*] } {
+       return 0
+    }
+    return [check_runtime aarch64_pmull_hw_available {
+       #pragma GCC target "+aes"
+       int
+       main (void)
+       {
+         asm volatile ("pmull v0.1q, v1.1d, v2.1d");
+         return 0;
+       }
+    }]
+}
+
 # Return true if this is an AArch64 target that can run SVE code.
 
 proc check_effective_target_aarch64_sve_hw { } {

Reply via email to