https://gcc.gnu.org/g:f819469be78f6e4b408e04ff35e8c5781ec416f4

commit r17-6104-gf819469be78f6e4b408e04ff35e8c5781ec416f4
Author: Kyrylo Tkachov <[email protected]>
Date:   Mon Nov 9 11:14:12 2026 +0100

    aarch64: Use SVE2 PMULLB for clmul in streaming mode [PR128111]
    
    Advanced SIMD PMULL is not available in streaming mode, so the clmul
    expanders are limited to non-streaming mode and __builtin_clmulg falls
    back to a multiply-based sequence there.  SVE2 has PMULLB, which is
    available in streaming mode.  Its pair form writes the least and
    most significant halves of each product to adjacent elements, which is
    the shape of clmul_widen:
    
      PMULLB Zd.H, Zn.B, Zm.B   8x8->16
      PMULLB Zd.D, Zn.S, Zm.S   32x32->64
      PMULLB Zd.Q, Zn.D, Zm.D   64x64->128 (+sve-aes, or +ssve-aes in
                                streaming mode)
    
    Add aarch64_expand_sve_clmul.  For 8-bit and 32-bit inputs it uses a
    scalar pattern for PMULLB that names the overlapping scalar registers
    with the %Z modifier.  The product is one element of twice the width,
    which is split with a shift.  For 64-bit inputs it loads the operands into
    element 0 of zeroed vectors with FMOV, uses the pair form and moves
    element 1 to element 0 with UZP2.
    
    Use it for clmuldi3 and clmul_widendi3 when the Advanced SIMD PMULL is
    not usable but TARGET_SVE_AES is, for clmul_widenqi3 when TARGET_SIMD is
    not available, and for a new clmul_widensi3.  The 32-bit pattern is also
    used in non-streaming mode with +sve2 and without +aes, where it replaces
    the multiply-based fallback.  Advanced SIMD PMULL stays preferred when
    available.
    
    For example, in a streaming function:
    
      unsigned int f (unsigned int a, unsigned int b)
      {
        return __builtin_clmulg (a, b);
      }
    
    now generates:
    
      f:
            fmov    s31, w0
            fmov    s30, w1
            pmullb  z30.d, z31.s, z30.s
            fmov    w0, s30
            ret
    
    instead of a 44 instruction sequence with 16 multiplies.
    
    Bootstrapped and tested on aarch64-linux-gnu.
    
    gcc/ChangeLog:
    
            PR middle-end/128111
            * config/aarch64/aarch64-protos.h (aarch64_expand_sve_clmul): 
Declare.
            * config/aarch64/aarch64.cc (aarch64_expand_sve_clmul): New 
function.
            * config/aarch64/aarch64-sve2.md (aarch64_sve_pmullb_scalar<mode>): 
New
            pattern.
            * config/aarch64/iterators.md (SVE_PMULL_SCALAR, Vpmull_etype)
            (Vpmull_wetype): New iterator and attributes.
            * config/aarch64/aarch64-simd.md (clmuldi3, clmul_widendi3): Also
            enable for TARGET_SVE_AES and use aarch64_expand_sve_clmul when
            Advanced SIMD PMULL is not usable.
            (clmul_widensi3): New expander.
            (clmul_widenqi3): Also enable for TARGET_SVE2.
    
    gcc/testsuite/ChangeLog:
    
            PR middle-end/128111
            * gcc.target/aarch64/sme/clmul_1.c: Replace.
            * gcc.target/aarch64/sme/clmul_2.c: New test.
            * gcc.target/aarch64/sve2/clmul_1.c: Likewise.
    
    Signed-off-by: Kyrylo Tkachov <[email protected]>

Diff:
---
 gcc/config/aarch64/aarch64-protos.h             |  1 +
 gcc/config/aarch64/aarch64-simd.md              | 80 +++++++++++++++++--------
 gcc/config/aarch64/aarch64-sve2.md              | 14 +++++
 gcc/config/aarch64/aarch64.cc                   | 75 +++++++++++++++++++++++
 gcc/config/aarch64/iterators.md                 |  6 ++
 gcc/testsuite/gcc.target/aarch64/sme/clmul_1.c  | 65 ++++++++++++++++----
 gcc/testsuite/gcc.target/aarch64/sme/clmul_2.c  | 55 +++++++++++++++++
 gcc/testsuite/gcc.target/aarch64/sve2/clmul_1.c | 18 ++++++
 8 files changed, 280 insertions(+), 34 deletions(-)

diff --git a/gcc/config/aarch64/aarch64-protos.h 
b/gcc/config/aarch64/aarch64-protos.h
index d8a282d769fb..500886568f04 100644
--- a/gcc/config/aarch64/aarch64-protos.h
+++ b/gcc/config/aarch64/aarch64-protos.h
@@ -1305,6 +1305,7 @@ extern void aarch64_adjust_reg_alloc_order ();
 bool aarch64_optimize_mode_switching (aarch64_mode_entity);
 void aarch64_restore_za (rtx);
 void aarch64_expand_crc_using_pmull (scalar_mode, scalar_mode, rtx *);
+void aarch64_expand_sve_clmul (rtx, rtx, rtx, rtx);
 void aarch64_expand_reversed_crc_using_pmull (scalar_mode, scalar_mode, rtx *);
 
 void aarch64_expand_fp_spaceship (rtx, rtx, rtx, rtx);
diff --git a/gcc/config/aarch64/aarch64-simd.md 
b/gcc/config/aarch64/aarch64-simd.md
index 20958df82a48..09ddca1a5e83 100644
--- a/gcc/config/aarch64/aarch64-simd.md
+++ b/gcc/config/aarch64/aarch64-simd.md
@@ -10836,17 +10836,24 @@
 ;; Carry-less multiplication.  clmul<mode>3 returns the low half of the
 ;; product.  clmul_widen<mode>3 returns the low half in operand 0 and the
 ;; high half in operand 1.  The DImode pattern also serves narrower modes,
-;; which the middle-end zero-extends to DImode.
+;; which the middle-end zero-extends to DImode.  Advanced SIMD PMULL is
+;; used when available.  Otherwise SVE2 PMULLB is used, which is also
+;; available in streaming mode.
 
 (define_expand "clmuldi3"
   [(match_operand:DI 0 "register_operand")
    (match_operand:DI 1 "register_operand")
    (match_operand:DI 2 "register_operand")]
-  "TARGET_AES && TARGET_NON_STREAMING"
+  "(TARGET_AES && TARGET_NON_STREAMING) || TARGET_SVE_AES"
 {
-  rtx prod = gen_reg_rtx (TImode);
-  emit_insn (gen_aarch64_crypto_pmulldi (prod, operands[1], operands[2]));
-  emit_move_insn (operands[0], gen_lowpart (DImode, prod));
+  if (TARGET_AES && TARGET_NON_STREAMING)
+    {
+      rtx prod = gen_reg_rtx (TImode);
+      emit_insn (gen_aarch64_crypto_pmulldi (prod, operands[1], operands[2]));
+      emit_move_insn (operands[0], gen_lowpart (DImode, prod));
+    }
+  else
+    aarch64_expand_sve_clmul (operands[0], NULL_RTX, operands[1], operands[2]);
   DONE;
 })
 
@@ -10855,14 +10862,33 @@
    (match_operand:DI 1 "register_operand")
    (match_operand:DI 2 "register_operand")
    (match_operand:DI 3 "register_operand")]
-  "TARGET_AES && TARGET_NON_STREAMING"
-{
-  rtx prod = gen_reg_rtx (TImode);
-  emit_insn (gen_aarch64_crypto_pmulldi (prod, operands[2], operands[3]));
-  emit_move_insn (operands[0], gen_lowpart (DImode, prod));
-  emit_insn (gen_vec_extractv2didi (operands[1],
-                                   gen_lowpart (V2DImode, prod),
-                                   aarch64_endian_lane_rtx (V2DImode, 1)));
+  "(TARGET_AES && TARGET_NON_STREAMING) || TARGET_SVE_AES"
+{
+  if (TARGET_AES && TARGET_NON_STREAMING)
+    {
+      rtx prod = gen_reg_rtx (TImode);
+      emit_insn (gen_aarch64_crypto_pmulldi (prod, operands[2], operands[3]));
+      emit_move_insn (operands[0], gen_lowpart (DImode, prod));
+      emit_insn (gen_vec_extractv2didi (operands[1],
+                                       gen_lowpart (V2DImode, prod),
+                                       aarch64_endian_lane_rtx (V2DImode, 1)));
+    }
+  else
+    aarch64_expand_sve_clmul (operands[0], operands[1], operands[2],
+                             operands[3]);
+  DONE;
+})
+
+;; Without the 64-bit PMULL, 32-bit operands use SVE2 PMULLB.
+(define_expand "clmul_widensi3"
+  [(match_operand:SI 0 "register_operand")
+   (match_operand:SI 1 "register_operand")
+   (match_operand:SI 2 "register_operand")
+   (match_operand:SI 3 "register_operand")]
+  "TARGET_SVE2 && !(TARGET_AES && TARGET_NON_STREAMING)"
+{
+  aarch64_expand_sve_clmul (operands[0], operands[1], operands[2],
+                           operands[3]);
   DONE;
 })
 
@@ -10871,18 +10897,24 @@
    (match_operand:QI 1 "register_operand")
    (match_operand:QI 2 "register_operand")
    (match_operand:QI 3 "register_operand")]
-  "TARGET_SIMD"
+  "TARGET_SIMD || TARGET_SVE2"
 {
-  rtx x = gen_reg_rtx (V8QImode);
-  rtx y = gen_reg_rtx (V8QImode);
-  rtx prod = gen_reg_rtx (V8HImode);
-  rtx half = gen_lowpart (HImode, prod);
-  emit_insn (gen_aarch64_simd_dupv8qi (x, operands[2]));
-  emit_insn (gen_aarch64_simd_dupv8qi (y, operands[3]));
-  emit_insn (gen_aarch64_pmullv8qi (prod, x, y));
-  emit_move_insn (operands[0], gen_lowpart (QImode, half));
-  rtx hi = expand_shift (RSHIFT_EXPR, HImode, half, 8, NULL_RTX, 1);
-  emit_move_insn (operands[1], gen_lowpart (QImode, hi));
+  if (TARGET_SIMD)
+    {
+      rtx x = gen_reg_rtx (V8QImode);
+      rtx y = gen_reg_rtx (V8QImode);
+      rtx prod = gen_reg_rtx (V8HImode);
+      rtx half = gen_lowpart (HImode, prod);
+      emit_insn (gen_aarch64_simd_dupv8qi (x, operands[2]));
+      emit_insn (gen_aarch64_simd_dupv8qi (y, operands[3]));
+      emit_insn (gen_aarch64_pmullv8qi (prod, x, y));
+      emit_move_insn (operands[0], gen_lowpart (QImode, half));
+      rtx hi = expand_shift (RSHIFT_EXPR, HImode, half, 8, NULL_RTX, 1);
+      emit_move_insn (operands[1], gen_lowpart (QImode, hi));
+    }
+  else
+    aarch64_expand_sve_clmul (operands[0], operands[1], operands[2],
+                             operands[3]);
   DONE;
 })
 
diff --git a/gcc/config/aarch64/aarch64-sve2.md 
b/gcc/config/aarch64/aarch64-sve2.md
index a60ff46d2bf6..56fab8eecb1b 100644
--- a/gcc/config/aarch64/aarch64-sve2.md
+++ b/gcc/config/aarch64/aarch64-sve2.md
@@ -4250,6 +4250,20 @@
   [(set_attr "sve_type" "sve_int_pmul")]
 )
 
+;; PMULLB on scalar operands that overlap element 0 of SVE registers.  The
+;; result is the product of the 8-bit or 32-bit sources in element 0 of the
+;; destination, which has twice the element width.
+(define_insn "@aarch64_sve_pmullb_scalar<mode>"
+  [(set (match_operand:<DWI> 0 "register_operand" "=w")
+       (unspec:<DWI>
+         [(match_operand:SVE_PMULL_SCALAR 1 "register_operand" "w")
+          (match_operand:SVE_PMULL_SCALAR 2 "register_operand" "w")]
+         UNSPEC_PMULLB))]
+  "TARGET_SVE2"
+  "pmullb\t%Z0.<Vpmull_wetype>, %Z1.<Vpmull_etype>, %Z2.<Vpmull_etype>"
+  [(set_attr "sve_type" "sve_int_pmul")]
+)
+
 ;; Extending PMUL, with the results modeled as pairs of values.
 ;; This representation works for .H, .D and .Q, with .Q requiring
 ;; the AES extension.  (This is enforced by the mode iterator.)
diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index 1ef3afc9a80a..bd33334b4bdb 100644
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -34098,6 +34098,81 @@ aarch64::report_not_enum (location_t location, tree 
fndecl, unsigned int argno,
            " a valid %qT value", actual, argno + 1, fndecl, enumtype);
 }
 
+/* Return the scalar X, which has 32 or 64 bits, in a new register of the SVE
+   vector mode VMODE with the elements of VMODE other than element 0 set to
+   zero.  Element 0 is loaded with a scalar FMOV.  */
+
+static rtx
+aarch64_sve_load_elt0 (machine_mode vmode, rtx x)
+{
+  rtx res = gen_reg_rtx (vmode);
+  emit_set_insn (res, gen_rtx_UNSPEC (vmode,
+                                     gen_rtvec (2, CONST0_RTX (vmode), x),
+                                     UNSPEC_INSR));
+  return res;
+}
+
+/* Store element 0 of the SVE vector VEC, which has mode VMODE, in the
+   scalar register DEST.  */
+
+static void
+aarch64_sve_store_elt0 (rtx dest, rtx vec, machine_mode vmode)
+{
+  insn_code icode = convert_optab_handler (vec_extract_optab, vmode,
+                                          GET_MODE (dest));
+  class expand_operand ops[3];
+  create_output_operand (&ops[0], dest, GET_MODE (dest));
+  create_input_operand (&ops[1], vec, vmode);
+  create_integer_operand (&ops[2], 0);
+  expand_insn (icode, 3, ops);
+  if (!rtx_equal_p (ops[0].value, dest))
+    emit_move_insn (dest, ops[0].value);
+}
+
+/* Expand the carry-less multiplication of A and B, which are scalars of the
+   same integer mode, using SVE2 PMULLB.  Store the least significant half of
+   the product in LO and, if HI is nonnull, the most significant half in HI.
+
+   For 8-bit and 32-bit inputs PMULLB produces the whole product as one
+   element of twice the width, in the scalar register that overlaps element 0
+   of the result.  Split it into two halves with a shift.  For 64-bit inputs
+   the product has 128 bits, so use the pair form of PMULLB, which writes the
+   two halves to elements 0 and 1.  Move element 1 to element 0 with UZP2.  */
+
+void
+aarch64_expand_sve_clmul (rtx lo, rtx hi, rtx a, rtx b)
+{
+  scalar_mode mode = as_a <scalar_mode> (GET_MODE (a));
+
+  if (mode == DImode)
+    {
+      machine_mode vmode = aarch64_full_sve_mode (mode).require ();
+      rtx x = aarch64_sve_load_elt0 (vmode, a);
+      rtx y = rtx_equal_p (a, b) ? x : aarch64_sve_load_elt0 (vmode, b);
+      rtx prod = gen_reg_rtx (vmode);
+      emit_insn (gen_aarch64_sve (UNSPEC_PMULLB_PAIR, vmode, prod, x, y));
+      aarch64_sve_store_elt0 (lo, prod, vmode);
+      if (hi)
+       {
+         rtx high = gen_reg_rtx (vmode);
+         emit_insn (gen_aarch64_sve (UNSPEC_UZP2, vmode, high, prod, prod));
+         aarch64_sve_store_elt0 (hi, high, vmode);
+       }
+      return;
+    }
+
+  scalar_mode wmode = GET_MODE_WIDER_MODE (mode).require ();
+  rtx wide = gen_reg_rtx (wmode);
+  emit_insn (gen_aarch64_sve_pmullb_scalar (mode, wide, a, b));
+  emit_move_insn (lo, gen_lowpart (mode, wide));
+  if (hi)
+    {
+      rtx high = expand_shift (RSHIFT_EXPR, wmode, wide,
+                              GET_MODE_BITSIZE (mode), NULL_RTX, 1);
+      emit_move_insn (hi, gen_lowpart (mode, high));
+    }
+}
+
 /* Generate assembly to calculate CRC
    using carry-less multiplication instruction.
    OPERANDS[1] is input CRC,
diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md
index d3ca1c09e396..92b61159cc33 100644
--- a/gcc/config/aarch64/iterators.md
+++ b/gcc/config/aarch64/iterators.md
@@ -1550,6 +1550,12 @@
 ;; For doubling width of an integer mode
 (define_mode_attr DWI [(QI "HI") (HI "SI") (SI "DI") (DI "TI")])
 
+;; Scalar integer modes for which SVE2 PMULLB has a non-pair form, and the
+;; element sizes of its sources and result.
+(define_mode_iterator SVE_PMULL_SCALAR [QI SI])
+(define_mode_attr Vpmull_etype [(QI "b") (SI "s")])
+(define_mode_attr Vpmull_wetype [(QI "h") (SI "d")])
+
 (define_mode_attr fcvt_change_mode [(SI "df") (DI "sf")])
 
 (define_mode_attr FCVT_CHANGE_MODE [(SI "DF") (DI "SF")])
diff --git a/gcc/testsuite/gcc.target/aarch64/sme/clmul_1.c 
b/gcc/testsuite/gcc.target/aarch64/sme/clmul_1.c
index 69f3bfea45a6..30173ef0a6ad 100644
--- a/gcc/testsuite/gcc.target/aarch64/sme/clmul_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sme/clmul_1.c
@@ -1,28 +1,73 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -march=armv9-a+sme+aes" } */
+/* { dg-options "-O2 -march=armv9-a+sme" } */
 
-typedef unsigned long long u64;
+typedef unsigned int u32;
+typedef unsigned short u16;
 typedef unsigned char u8;
 
 /* PMULL is an Advanced SIMD instruction and is not available in streaming
-   mode, so the carry-less multiplication must be open-coded.  */
+   mode.  8-bit and 32-bit carry-less multiplications use SVE2 PMULLB.  */
 
-u64
-clmul64 (u64 a, u64 b) [[arm::streaming]]
+/*
+** clmul8:
+**     fmov    s([0-9]+), w0
+**     fmov    s([0-9]+), w1
+**     pmullb  z\1\.h, z\1\.b, z\2\.b
+**     umov    w0, v\1\.h\[0\]
+**     ret
+*/
+u8
+clmul8 (u8 a, u8 b) [[arm::streaming]]
 {
   return __builtin_clmulg (a, b);
 }
 
-u64
-clmul64_widen (u64 a, u64 b, u64 *hi) [[arm::streaming_compatible]]
+/*
+** clmul8_widen:
+**     fmov    s([0-9]+), w1
+**     fmov    s([0-9]+), w0
+**     pmullb  z\2\.h, z\2\.b, z\1\.b
+**     umov    w0, v\2\.h\[0\]
+**     lsr     w1, w0, 8
+**     strb    w1, \[x2\]
+**     ret
+*/
+u8
+clmul8_widen (u8 a, u8 b, u8 *hi) [[arm::streaming]]
 {
   return __builtin_clmulg (a, b, hi);
 }
 
-u8
-clmul8 (u8 a, u8 b) [[arm::streaming]]
+/*
+** clmul16:
+**     and     w0, w0, 65535
+**     and     w1, w1, 65535
+**     fmov    s([0-9]+), w0
+**     fmov    s([0-9]+), w1
+**     pmullb  z\1\.d, z\1\.s, z\2\.s
+**     fmov    x0, d\1
+**     ret
+*/
+u16
+clmul16 (u16 a, u16 b) [[arm::streaming]]
 {
   return __builtin_clmulg (a, b);
 }
 
-/* { dg-final { scan-assembler-not {\tpmull\t} } } */
+/*
+** clmul32_widen:
+**     fmov    s([0-9]+), w1
+**     fmov    s([0-9]+), w0
+**     pmullb  z\2\.d, z\2\.s, z\1\.s
+**     fmov    x0, d\2
+**     lsr     x1, x0, 32
+**     str     w1, \[x2\]
+**     ret
+*/
+u32
+clmul32_widen (u32 a, u32 b, u32 *hi) [[arm::streaming]]
+{
+  return __builtin_clmulg (a, b, hi);
+}
+
+/* { dg-final { check-function-bodies "**" "" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sme/clmul_2.c 
b/gcc/testsuite/gcc.target/aarch64/sme/clmul_2.c
new file mode 100644
index 000000000000..68767d1c64eb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme/clmul_2.c
@@ -0,0 +1,55 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=armv9-a+sme+ssve-aes" } */
+
+typedef unsigned long long u64;
+
+/* With +ssve-aes the 64x64->128 product uses SVE2 PMULLB.  */
+
+/*
+** clmul64:
+**     fmov    d([0-9]+), x0
+**     fmov    d([0-9]+), x1
+**     pmullb  z\2\.q, z\1\.d, z\2\.d
+**     fmov    x0, d\2
+**     ret
+*/
+u64
+clmul64 (u64 a, u64 b) [[arm::streaming]]
+{
+  return __builtin_clmulg (a, b);
+}
+
+/*
+** clmul64_widen:
+**     fmov    d([0-9]+), x0
+**     fmov    d([0-9]+), x1
+**     pmullb  z\2\.q, z\1\.d, z\2\.d
+**     uzp2    z([0-9]+)\.d, z\2\.d, z\2\.d
+**     fmov    x0, d\2
+**     str     d\3, \[x2\]
+**     ret
+*/
+u64
+clmul64_widen (u64 a, u64 b, u64 *hi) [[arm::streaming]]
+{
+  return __builtin_clmulg (a, b, hi);
+}
+
+/*
+** clmul64_high:
+**     fmov    d([0-9]+), x0
+**     fmov    d([0-9]+), x1
+**     pmullb  z\2\.q, z\1\.d, z\2\.d
+**     uzp2    z\2\.d, z\2\.d, z\2\.d
+**     fmov    x0, d\2
+**     ret
+*/
+u64
+clmul64_high (u64 a, u64 b) [[arm::streaming]]
+{
+  u64 h;
+  __builtin_clmulg (a, b, &h);
+  return h;
+}
+
+/* { dg-final { check-function-bodies "**" "" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/clmul_1.c 
b/gcc/testsuite/gcc.target/aarch64/sve2/clmul_1.c
new file mode 100644
index 000000000000..b40df8601731
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/clmul_1.c
@@ -0,0 +1,18 @@
+/* { dg-do run { target aarch64_sve2_hw } } */
+/* { dg-options "-O2 -march=armv8-a+sve2" } */
+
+/* Without +aes, 32-bit and 16-bit carry-less multiplications use SVE2
+   PMULLB.  */
+
+#define main do_main
+
+#include "../../../c-c++-common/builtin-clmulg-1.c"
+
+#undef main
+
+int
+main ()
+{
+  do_main ();
+  return 0;
+}

Reply via email to