https://gcc.gnu.org/g:b48cd3589a15f0ffee1bc2210234d7609c9c233f

commit r17-6111-gb48cd3589a15f0ffee1bc2210234d7609c9c233f
Author: Tamar Christina <[email protected]>
Date:   Tue Nov 10 12:44:56 2026 +0000

    AArch64: implement floating point dot-product optabs.
    
    With the middle-end support now pushed, we can now recognize loops
    such as
    
    float
    dot (const _Float16 *restrict a,
         const _Float16 *restrict b, int n)
    {
      float sum = 0.0f;
      for (int i = 0; i < n; ++i)
        sum += (float) a[i] * (float) b[i];
      return sum;
    }
    
    as floating-point dot products.
    
    AArch64 already has patterns for the AdvSIMD and SVE FDOT instructions, but 
they
    aren't both exposed under the optab names that the vectorizer needs. This 
patch
    connects those existing patterns to sdot_prod so that the loop can use FDOT
    directly.
    
    For AdvSIMD the pattern name now includes both the result and input modes,
    matching the optab.  The existing Neon intrinsic names are kept through 
aliases
    for the renamed instruction codes.
    
    For SVE the floating-point pattern is added to the same generated 
dot-product
    lookup used by the integer patterns.  The ordinary SVE dot intrinsics then 
use
    that lookup too, replacing the hard-coded choice for the half-to-float
    instruction.  The FP8 intrinsics that set FPMR keep their existing expansion
    path.
    
    Concretely, with -O3 -ffast-math and the appropriate architecture 
extensions,
    the loop above can use
    
      fdot v0.4s, v1.8h, v2.8h
    
    for AdvSIMD, or
    
      fdot z0.s, z1.h, z2.h
    
    for SVE.  Each instruction multiplies pairs of half-precision inputs and
    accumulates them into single-precision elements.  This avoids having to 
widen
    the inputs and perform the multiply and accumulate as separate instructions.
    
    The existing feature checks are retained: AdvSIMD requires +f16f32dot, 
while the
    SVE form requires SVE2.1 or SME2.  The tests check both AdvSIMD widths, 
scalable
    SVE and streaming SME2 code generation, and that we don't emit FDOT without 
the
    extension. There is also coverage for the FP8 intrinsics in streaming and
    non-streaming functions.
    
    This also fixes two bugs with the fdot's RTL introduced in GCC17.
    
    1. the accumulator of the generic pattern was in the wrong place, it should 
be
       last not first.
    2. The endian lane swapping was wrong. the two way fdots work in pairs of 
input
       so the lanes should be flipped in groups of 2.
    
    gcc/ChangeLog:
    
            * config/aarch64/aarch64-builtins.cc (CODE_FOR_sdot_prodv4hf): Add
            alias for CODE_FOR_sdot_prodv2sfv4hf.
            (CODE_FOR_sdot_prodv8hf): Add alias for CODE_FOR_sdot_prodv4sfv8hf.
            * config/aarch64/aarch64-simd-builtins.def (sdot_prod): Update the
            comment to include both result and input modes in the pattern name.
            * config/aarch64/aarch64-simd.md
            (sdot_prod<VDQSF:vsi2qi><vczle><vczbe>): Rename to...
            (sdot_prod<VDQSF:mode><vsi2qi><vczle><vczbe>): ...this, including 
the
            result mode to match the sdot_prod optab.  Put the multiplicands in
            operands 1 and 2 and the accumulator in operand 3.  Update the
            assembly template to match.
            (sdot_lane<VF:isquadop><VDQSF:vsi2qi><vczle><vczbe>): Use the number
            of halfword pairs when mapping lane indices on big-endian targets.
            * config/aarch64/aarch64-sve-builtins-base.cc (svdot_impl::expand):
            Use the result mode as the input mode for intrinsics with one type
            suffix.  Use code_for_dot_prod for all dot intrinsics that do not
            set FPMR, including floating-point dot products.
            * config/aarch64/aarch64-sve.md (<sur>dot_prod<mode><vsi2qi>):
            Rename to...
            (@<sur>dot_prod<mode><vsi2qi>): ...this, generating an instruction
            lookup.
            * config/aarch64/aarch64-sve2.md (aarch64_sve_fdotvnx4sfvnx8hf):
            Rename to...
            (@<sur>dot_prod<VNx4SF_ONLY:mode><VNx8HF_ONLY:mode>): ...this.  Use
            the VNx4SF_ONLY and VNx8HF_ONLY mode iterators and DOTPROD_FP 
integer
            iterator to expose FDOT through the sdot_prod optab and generated
            dot-product lookup.
            * config/aarch64/arm_neon.h (vdot_f32_f16, vdotq_f32_f16): Update
            builtin argument order to match the sdot_prod optab.
            * config/aarch64/iterators.md (DOTPROD_FP): New integer iterator.
            (sur): Add UNSPEC_FDOT mapping.
    
    gcc/testsuite/ChangeLog:
    
            * gcc.target/aarch64/advsimd-intrinsics/fdot-1.c: Use fat LTO 
objects.
            * gcc.target/aarch64/advsimd-intrinsics/fdot-2.c: Likewise.
            * gcc.target/aarch64/advsimd-intrinsics/fdot-3.c: Likewise.
            * gcc.target/aarch64/fdot-autovec-advsimd-run.c: New AdvSIMD runtime
            test.
            * gcc.target/aarch64/fdot-autovec-advsimd.c: New test.
            * gcc.target/aarch64/fdot-autovec-noext.c: New test.
            * gcc.target/aarch64/fdot-autovec-run.h: New shared runtime test
            helper.  Check normal and subnormal inputs and lengths from 0 to 
257.
            * gcc.target/aarch64/sve2/fdot-autovec-run.c: New SVE2.1 runtime 
test.
            * gcc.target/aarch64/sve2/fdot-autovec-scalable.c: New test.
            * gcc.target/aarch64/sve2/fdot-autovec-sme2.c: New test.
            * gcc.target/aarch64/sve2/fdot-fp8-vla.c: New test.
            * lib/target-supports.exp
            (check_effective_target_aarch64_f16f32dot_hw): New procedure to 
check
            whether the target can execute AdvSIMD FP16-to-FP32 dot products.

Diff:
---
 gcc/config/aarch64/aarch64-builtins.cc             |  4 +++
 gcc/config/aarch64/aarch64-simd-builtins.def       |  2 +-
 gcc/config/aarch64/aarch64-simd.md                 | 14 +++++----
 gcc/config/aarch64/aarch64-sve-builtins-base.cc    | 17 ++++------
 gcc/config/aarch64/aarch64-sve.md                  |  2 +-
 gcc/config/aarch64/aarch64-sve2.md                 | 17 +++++-----
 gcc/config/aarch64/arm_neon.h                      |  4 +--
 gcc/config/aarch64/iterators.md                    |  2 ++
 .../gcc.target/aarch64/advsimd-intrinsics/fdot-1.c |  2 +-
 .../gcc.target/aarch64/advsimd-intrinsics/fdot-2.c |  2 +-
 .../gcc.target/aarch64/advsimd-intrinsics/fdot-3.c |  2 +-
 .../gcc.target/aarch64/fdot-autovec-advsimd-run.c  |  7 +++++
 .../gcc.target/aarch64/fdot-autovec-advsimd.c      | 16 ++++++++++
 .../gcc.target/aarch64/fdot-autovec-noext.c        | 14 +++++++++
 .../gcc.target/aarch64/fdot-autovec-run.h          | 36 ++++++++++++++++++++++
 .../gcc.target/aarch64/sve2/fdot-autovec-run.c     |  6 ++++
 .../aarch64/sve2/fdot-autovec-scalable.c           | 14 +++++++++
 .../gcc.target/aarch64/sve2/fdot-autovec-sme2.c    | 14 +++++++++
 .../gcc.target/aarch64/sve2/fdot-fp8-vla.c         | 33 ++++++++++++++++++++
 gcc/testsuite/lib/target-supports.exp              | 17 ++++++++++
 20 files changed, 192 insertions(+), 33 deletions(-)

diff --git a/gcc/config/aarch64/aarch64-builtins.cc 
b/gcc/config/aarch64/aarch64-builtins.cc
index d94d75b9afc1..1d8baaef5c41 100644
--- a/gcc/config/aarch64/aarch64-builtins.cc
+++ b/gcc/config/aarch64/aarch64-builtins.cc
@@ -459,6 +459,10 @@ constexpr insn_code CODE_FOR_aarch64_udot_prodv16qi
   = CODE_FOR_udot_prodv4siv16qi;
 constexpr insn_code CODE_FOR_aarch64_usdot_prodv16qi
   = CODE_FOR_usdot_prodv4siv16qi;
+constexpr insn_code CODE_FOR_sdot_prodv4hf
+  = CODE_FOR_sdot_prodv2sfv4hf;
+constexpr insn_code CODE_FOR_sdot_prodv8hf
+  = CODE_FOR_sdot_prodv4sfv8hf;
 
 #define CF0(N, X) CODE_FOR_aarch64_##N##X
 #define CF1(N, X) CODE_FOR_##N##X##1
diff --git a/gcc/config/aarch64/aarch64-simd-builtins.def 
b/gcc/config/aarch64/aarch64-simd-builtins.def
index 925d7a0052b1..0a84dfd459d1 100644
--- a/gcc/config/aarch64/aarch64-simd-builtins.def
+++ b/gcc/config/aarch64/aarch64-simd-builtins.def
@@ -860,7 +860,7 @@
   VAR2 (QUADOP_LANE_PAIR, bfdot_lane, 0, QUIET, v2sf, v4sf)
   VAR2 (QUADOP_LANE_PAIR, bfdot_laneq, 0, QUIET, v2sf, v4sf)
 
-  /* Implemented by sdot_prod<mode>.  */
+  /* Implemented by sdot_prod<result_mode><input_mode>.  */
   BUILTIN_VF (TERNOP, sdot_prod, 10, DEFAULT)
   /* Implemented by sdot_lane{q}<dot_mode>.  */
   BUILTIN_VF (QUADOP_LANE_PAIR, sdot_lane, 10, DEFAULT)
diff --git a/gcc/config/aarch64/aarch64-simd.md 
b/gcc/config/aarch64/aarch64-simd.md
index 09ddca1a5e83..11705bc90738 100644
--- a/gcc/config/aarch64/aarch64-simd.md
+++ b/gcc/config/aarch64/aarch64-simd.md
@@ -799,16 +799,16 @@
   [(set_attr "type" "neon_dot<VS:q>")]
 )
 
-(define_insn "sdot_prod<VDQSF:vsi2qi><vczle><vczbe>"
+(define_insn "sdot_prod<VDQSF:mode><vsi2qi><vczle><vczbe>"
   [(set (match_operand:VDQSF 0 "register_operand" "=w")
        (plus:VDQSF
          (unspec:VDQSF
-          [(match_operand:<VSI2QI> 2 "register_operand" "w")
-           (match_operand:<VSI2QI> 3 "register_operand" "w")]
+          [(match_operand:<VSI2QI> 1 "register_operand" "w")
+           (match_operand:<VSI2QI> 2 "register_operand" "w")]
            UNSPEC_FDOT)
-         (match_operand:VDQSF 1 "register_operand" "0")))]
+         (match_operand:VDQSF 3 "register_operand" "0")))]
   "TARGET_F16F32DOT"
-  "fdot\t%0.<Vtype>, %2.<Vbfdottype>, %3.<Vbfdottype>"
+  "fdot\t%0.<Vtype>, %1.<Vbfdottype>, %2.<Vbfdottype>"
   [(set_attr "type" "neon_dot<q>")]
 )
 
@@ -823,7 +823,9 @@
          (match_operand:VDQSF 1 "register_operand" "0")))]
   "TARGET_F16F32DOT"
 {
-  operands[4] = aarch64_endian_lane_rtx (<VF:MODE>mode, INTVAL (operands[4]));
+  int nunits = GET_MODE_NUNITS (<VF:MODE>mode).to_constant ();
+  int lane = INTVAL (operands[4]);
+  operands[4] = gen_int_mode (ENDIAN_LANE_N (nunits / 2, lane), SImode);
   return "fdot\t%0.<VDQSF:Vtype>, %2.<VDQSF:Vbfdottype>, %3.2h[%4]";
 }
   [(set_attr "type" "neon_dot<VDQSF:q>")]
diff --git a/gcc/config/aarch64/aarch64-sve-builtins-base.cc 
b/gcc/config/aarch64/aarch64-sve-builtins-base.cc
index d6d53b3d41f5..93443751b19f 100644
--- a/gcc/config/aarch64/aarch64-sve-builtins-base.cc
+++ b/gcc/config/aarch64/aarch64-sve-builtins-base.cc
@@ -979,23 +979,18 @@ public:
     insn_code icode;
     machine_mode mode0 = GET_MODE (e.args[0]);
     machine_mode mode1 = GET_MODE (e.args[1]);
+    /* Without a second type suffix, both multiplicands have the same mode.  */
+    if (e.type_suffix_ids[1] == NUM_TYPE_SUFFIXES)
+      mode1 = mode0;
     if (e.fpm_mode == aarch64_acle::FPM_set)
       icode = code_for_aarch64_sve_dot (e.result_mode ());
     else
       {
-       /* In the optab, the multiplication operands come before the accumulator
-          operand.  The optab is keyed off the multiplication mode.  */
+       /* The dot patterns place the multiplication operands before the
+          accumulator.  */
        e.rotate_inputs_left (0, 3);
        int unspec = unspec_for (e);
-       if (e.type_suffix_ids[1] == NUM_TYPE_SUFFIXES)
-         icode = e.convert_optab_handler_for_sign (sdot_prod_optab,
-                                                   udot_prod_optab,
-                                                   0, e.result_mode (),
-                                                   GET_MODE (e.args[0]));
-       else
-         icode = (e.type_suffix (0).float_p
-                  ? CODE_FOR_aarch64_sve_fdotvnx4sfvnx8hf
-                  : code_for_dot_prod (unspec, mode0, mode1));
+       icode = code_for_dot_prod (unspec, mode0, mode1);
       }
     return e.use_unpred_insn (icode);
   }
diff --git a/gcc/config/aarch64/aarch64-sve.md 
b/gcc/config/aarch64/aarch64-sve.md
index 8ba24d4d4e10..d0b3612f340a 100644
--- a/gcc/config/aarch64/aarch64-sve.md
+++ b/gcc/config/aarch64/aarch64-sve.md
@@ -7818,7 +7818,7 @@
 ;; -------------------------------------------------------------------------
 
 ;; Four-element integer dot-product with accumulation.
-(define_insn "<sur>dot_prod<mode><vsi2qi>"
+(define_insn "@<sur>dot_prod<mode><vsi2qi>"
   [(set (match_operand:SVE_FULL_SDI 0 "register_operand")
        (plus:SVE_FULL_SDI
          (unspec:SVE_FULL_SDI
diff --git a/gcc/config/aarch64/aarch64-sve2.md 
b/gcc/config/aarch64/aarch64-sve2.md
index 56fab8eecb1b..3a8b81b3a695 100644
--- a/gcc/config/aarch64/aarch64-sve2.md
+++ b/gcc/config/aarch64/aarch64-sve2.md
@@ -2998,15 +2998,14 @@
   [(set_attr "sve_type" "sve_<sve_type_unspec>")]
 )
 
-;; Two-way dot-product.
-(define_insn "aarch64_sve_fdotvnx4sfvnx8hf"
-  [(set (match_operand:VNx4SF 0 "register_operand")
-       (plus:VNx4SF
-         (unspec:VNx4SF
-           [(match_operand:VNx8HF 1 "register_operand")
-            (match_operand:VNx8HF 2 "register_operand")]
-           UNSPEC_FDOT)
-         (match_operand:VNx4SF 3 "register_operand")))]
+(define_insn "@<sur>dot_prod<VNx4SF_ONLY:mode><VNx8HF_ONLY:mode>"
+  [(set (match_operand:VNx4SF_ONLY 0 "register_operand")
+       (plus:VNx4SF_ONLY
+         (unspec:VNx4SF_ONLY
+           [(match_operand:VNx8HF_ONLY 1 "register_operand")
+            (match_operand:VNx8HF_ONLY 2 "register_operand")]
+           DOTPROD_FP)
+         (match_operand:VNx4SF_ONLY 3 "register_operand")))]
   "TARGET_SVE2p1_OR_SME2"
   {@ [ cons: =0 , 1 , 2 , 3 ; attrs: movprfx ]
      [ w        , w , w , 0 ; *              ] fdot\t%0.s, %1.h, %2.h
diff --git a/gcc/config/aarch64/arm_neon.h b/gcc/config/aarch64/arm_neon.h
index cd37b6669639..9caa0a2088d8 100644
--- a/gcc/config/aarch64/arm_neon.h
+++ b/gcc/config/aarch64/arm_neon.h
@@ -19559,14 +19559,14 @@ __extension__ extern __inline float32x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vdot_f32_f16 (float32x2_t __r, float16x4_t __a, float16x4_t __b)
 {
-  return __builtin_aarch64_sdot_prodv4hf (__r, __a, __b);
+  return __builtin_aarch64_sdot_prodv4hf (__a, __b, __r);
 }
 
 __extension__ extern __inline float32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vdotq_f32_f16 (float32x4_t __r, float16x8_t __a, float16x8_t __b)
 {
-  return __builtin_aarch64_sdot_prodv8hf (__r, __a, __b);
+  return __builtin_aarch64_sdot_prodv8hf (__a, __b, __r);
 }
 
 __extension__ extern __inline float32x2_t
diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md
index 92b61159cc33..dcaa5a3b708a 100644
--- a/gcc/config/aarch64/iterators.md
+++ b/gcc/config/aarch64/iterators.md
@@ -3761,6 +3761,7 @@
 (define_int_iterator BSL_DUP [1 2])
 
 (define_int_iterator DOTPROD [UNSPEC_SDOT UNSPEC_UDOT])
+(define_int_iterator DOTPROD_FP [UNSPEC_FDOT])
 
 (define_int_iterator DOTPROD_I8MM [UNSPEC_USDOT UNSPEC_SUDOT])
 (define_int_iterator DOTPROD_US_ONLY [UNSPEC_USDOT])
@@ -4841,6 +4842,7 @@
                      (UNSPEC_URSHL  "ur") (UNSPEC_SRSHL  "sr")
                      (UNSPEC_UQRSHL  "u") (UNSPEC_SQRSHL  "s")
                      (UNSPEC_SDOT "s") (UNSPEC_UDOT "u")
+                     (UNSPEC_FDOT "s")
                      (UNSPEC_USDOT "us") (UNSPEC_SUDOT "su")
                      (UNSPEC_SMATMUL "s") (UNSPEC_UMATMUL "u")
                      (UNSPEC_USMATMUL "us")
diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-1.c 
b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-1.c
index 4abbdf57ef69..d85bccbac768 100644
--- a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-1.c
+++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-1.c
@@ -1,7 +1,7 @@
 /* { dg-do assemble { target { aarch64*-*-* } } } */
 /* { dg-require-effective-target aarch64_f16f32dot_neon_ok } */
 /* { dg-add-options aarch64_f16f32dot_neon }  */
-/* { dg-additional-options "-save-temps" } */
+/* { dg-additional-options "-save-temps -ffat-lto-objects" } */
 /* { dg-final { check-function-bodies "**" "" {-O[^0]} } } */
 /* { dg-skip-if "" { *-*-* } { "-fno-fat-lto-objects" } } */
 
diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-2.c 
b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-2.c
index b42bfa09c77c..7f8684049a8b 100644
--- a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-2.c
+++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-2.c
@@ -2,7 +2,7 @@
 /* { dg-require-effective-target stdint_types_mbig_endian } */
 /* { dg-require-effective-target aarch64_f16f32dot_neon_ok } */
 /* { dg-add-options aarch64_f16f32dot_neon }  */
-/* { dg-additional-options "-mbig-endian --save-temps" } */
+/* { dg-additional-options "-mbig-endian --save-temps -ffat-lto-objects" } */
 /* { dg-final { check-function-bodies "**" "" {-O[^0]} } } */
 /* { dg-skip-if "" { *-*-* } { "-fno-fat-lto-objects" } } */
 
diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-3.c 
b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-3.c
index 314322a08345..72c5f628c9cd 100644
--- a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-3.c
+++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-3.c
@@ -2,7 +2,7 @@
 /* { dg-skip-if "" { *-*-* } { "-fno-fat-lto-objects" } } */
 /* { dg-require-effective-target aarch64_f16f32dot_neon_ok } */
 /* { dg-add-options aarch64_f16f32dot_neon }  */
-/* { dg-additional-options "--save-temps" } */
+/* { dg-additional-options "--save-temps -ffat-lto-objects" } */
 
 #include <arm_neon.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/fdot-autovec-advsimd-run.c 
b/gcc/testsuite/gcc.target/aarch64/fdot-autovec-advsimd-run.c
new file mode 100644
index 000000000000..deb4f59e0abf
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/fdot-autovec-advsimd-run.c
@@ -0,0 +1,7 @@
+/* { dg-do run { target aarch64_f16f32dot_hw } } */
+/* { dg-options "-O3 -ffast-math -march=armv8.2-a+fp16+f16f32dot 
-mautovec-preference=asimd-only -save-temps" } */
+
+#include "fdot-autovec-run.h"
+
+/* { dg-final { scan-assembler {\tfdot\tv[0-9]+\.4s, v[0-9]+\.8h, v[0-9]+\.8h} 
} } */
+/* { dg-final { scan-assembler {\tfdot\tv[0-9]+\.2s, v[0-9]+\.4h, v[0-9]+\.4h} 
} } */
diff --git a/gcc/testsuite/gcc.target/aarch64/fdot-autovec-advsimd.c 
b/gcc/testsuite/gcc.target/aarch64/fdot-autovec-advsimd.c
new file mode 100644
index 000000000000..6326f1c3eb1e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/fdot-autovec-advsimd.c
@@ -0,0 +1,16 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -ffast-math -march=armv8.2-a+fp16+f16f32dot 
-mautovec-preference=asimd-only -fdump-tree-vect-details" } */
+
+float
+dot_half_advsimd (const _Float16 *restrict a,
+                 const _Float16 *restrict b, int n)
+{
+  float sum = 0.0f;
+  for (int i = 0; i < n; ++i)
+    sum += (float) a[i] * (float) b[i];
+  return sum;
+}
+
+/* { dg-final { scan-assembler {\tfdot\tv[0-9]+\.4s, v[0-9]+\.8h, v[0-9]+\.8h} 
} } */
+/* { dg-final { scan-assembler {\tfdot\tv[0-9]+\.2s, v[0-9]+\.4h, v[0-9]+\.4h} 
} } */
+/* { dg-final { scan-tree-dump "dot_prod pattern recognized" "vect" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/fdot-autovec-noext.c 
b/gcc/testsuite/gcc.target/aarch64/fdot-autovec-noext.c
new file mode 100644
index 000000000000..c452637d5b56
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/fdot-autovec-noext.c
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -ffast-math -march=armv8.2-a+fp16 
-mautovec-preference=asimd-only" } */
+
+float
+dot_half_noext (const _Float16 *restrict a,
+               const _Float16 *restrict b, int n)
+{
+  float sum = 0.0f;
+  for (int i = 0; i < n; ++i)
+    sum += (float) a[i] * (float) b[i];
+  return sum;
+}
+
+/* { dg-final { scan-assembler-not {\tfdot\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/fdot-autovec-run.h 
b/gcc/testsuite/gcc.target/aarch64/fdot-autovec-run.h
new file mode 100644
index 000000000000..8fce986db0fc
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/fdot-autovec-run.h
@@ -0,0 +1,36 @@
+__attribute__ ((noipa)) float
+dot_half (const _Float16 *restrict a, const _Float16 *restrict b, int n)
+{
+  float sum = 0.0f;
+  for (int i = 0; i < n; ++i)
+    sum += (float) a[i] * (float) b[i];
+  return sum;
+}
+
+int
+main (void)
+{
+  _Float16 a[257], b[257];
+  for (int small = 0; small < 2; ++small)
+    {
+      /* Both normal and subnormal inputs, with exact FP32 products and
+        sums.  The small products would underflow if evaluated in FP16.  */
+      float scale = small ? 0x1p-24f : 0.0625f;
+      for (int i = 0; i < 257; ++i)
+       {
+         a[i] = (i % 31 - 15) * scale;
+         b[i] = (i % 13 - 6) * scale;
+       }
+
+      double expected = 0.0;
+      for (int n = 0; n <= 257; ++n)
+       {
+         if (dot_half (a, b, n) != (float) expected)
+           __builtin_abort ();
+         if (n < 257)
+           expected += (double) a[n] * (double) b[n];
+       }
+    }
+  return 0;
+}
+
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/fdot-autovec-run.c 
b/gcc/testsuite/gcc.target/aarch64/sve2/fdot-autovec-run.c
new file mode 100644
index 000000000000..7743c6ca6023
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/fdot-autovec-run.c
@@ -0,0 +1,6 @@
+/* { dg-do run { target aarch64_sve2p1_hw } } */
+/* { dg-options "-O3 -ffast-math -march=armv9.4-a+sve2p1 
-mautovec-preference=sve-only -save-temps" } */
+
+#include "../fdot-autovec-run.h"
+
+/* { dg-final { scan-assembler {\tfdot\tz[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h} } 
} */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/fdot-autovec-scalable.c 
b/gcc/testsuite/gcc.target/aarch64/sve2/fdot-autovec-scalable.c
new file mode 100644
index 000000000000..ecb833ac3846
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/fdot-autovec-scalable.c
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -ffast-math -march=armv9.4-a+sve2p1 
-mautovec-preference=sve-only" } */
+
+float
+dot_half_scalable (const _Float16 *restrict a,
+                  const _Float16 *restrict b, int n)
+{
+  float sum = 0.0f;
+  for (int i = 0; i < n; ++i)
+    sum += (float) a[i] * (float) b[i];
+  return sum;
+}
+
+/* { dg-final { scan-assembler {\tfdot\tz[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h} } 
} */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/fdot-autovec-sme2.c 
b/gcc/testsuite/gcc.target/aarch64/sve2/fdot-autovec-sme2.c
new file mode 100644
index 000000000000..e3b018853483
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/fdot-autovec-sme2.c
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -ffast-math -march=armv9.2-a+sme2 
-mautovec-preference=sve-only" } */
+
+__arm_streaming float
+dot_half_sme2 (const _Float16 *restrict a,
+              const _Float16 *restrict b, int n)
+{
+  float sum = 0.0f;
+  for (int i = 0; i < n; ++i)
+    sum += (float) a[i] * (float) b[i];
+  return sum;
+}
+
+/* { dg-final { scan-assembler {\tfdot\tz[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h} } 
} */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/fdot-fp8-vla.c 
b/gcc/testsuite/gcc.target/aarch64/sve2/fdot-fp8-vla.c
new file mode 100644
index 000000000000..7d000811dbfb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/fdot-fp8-vla.c
@@ -0,0 +1,33 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 
-march=armv9.4-a+sve2+fp8dot2+fp8dot4+ssve-fp8dot2+ssve-fp8dot4" } */
+
+#include <arm_sve.h>
+
+svfloat16_t
+dot_fp8_to_f16 (svfloat16_t acc, svmfloat8_t a, svmfloat8_t b, fpm_t fpm)
+{
+  return svdot_f16_mf8_fpm (acc, a, b, fpm);
+}
+
+svfloat32_t
+dot_fp8_to_f32 (svfloat32_t acc, svmfloat8_t a, svmfloat8_t b, fpm_t fpm)
+{
+  return svdot_f32_mf8_fpm (acc, a, b, fpm);
+}
+
+__arm_streaming svfloat16_t
+dot_fp8_to_f16_streaming (svfloat16_t acc, svmfloat8_t a, svmfloat8_t b,
+                         fpm_t fpm)
+{
+  return svdot_f16_mf8_fpm (acc, a, b, fpm);
+}
+
+__arm_streaming svfloat32_t
+dot_fp8_to_f32_streaming (svfloat32_t acc, svmfloat8_t a, svmfloat8_t b,
+                         fpm_t fpm)
+{
+  return svdot_f32_mf8_fpm (acc, a, b, fpm);
+}
+
+/* { dg-final { scan-assembler-times {\tfdot\tz[0-9]+\.h, z[0-9]+\.b, 
z[0-9]+\.b} 2 } } */
+/* { dg-final { scan-assembler-times {\tfdot\tz[0-9]+\.s, z[0-9]+\.b, 
z[0-9]+\.b} 2 } } */
diff --git a/gcc/testsuite/lib/target-supports.exp 
b/gcc/testsuite/lib/target-supports.exp
index 4dcb7f41b459..3ce06189d2e6 100644
--- a/gcc/testsuite/lib/target-supports.exp
+++ b/gcc/testsuite/lib/target-supports.exp
@@ -7716,6 +7716,23 @@ proc add_options_for_aarch64_f16f32dot_neon { flags } {
     return "$flags $et_aarch64_f16f32dot_neon_flags"
 }
 
+# Return true if the target can execute AdvSIMD FP16-to-FP32 dot products.
+
+proc check_effective_target_aarch64_f16f32dot_hw { } {
+    if { ![check_effective_target_aarch64_f16f32dot_neon_ok] } {
+       return 0
+    }
+    return [check_runtime aarch64_f16f32dot_hw_available {
+       int
+       main (void)
+       {
+         asm volatile ("movi v0.4s, 0; fdot v0.4s, v0.8h, v0.8h"
+                       : : : "v0");
+         return 0;
+       }
+    } [add_options_for_aarch64_f16f32dot_neon ""]]
+}
+
 # Return 1 if the target supports BFloat16 SIMD instructions, 0 otherwise.
 # The test is valid for ARM and for AArch64.

Reply via email to