https://gcc.gnu.org/g:b48cd3589a15f0ffee1bc2210234d7609c9c233f
commit r17-6111-gb48cd3589a15f0ffee1bc2210234d7609c9c233f Author: Tamar Christina <[email protected]> Date: Tue Nov 10 12:44:56 2026 +0000 AArch64: implement floating point dot-product optabs. With the middle-end support now pushed, we can now recognize loops such as float dot (const _Float16 *restrict a, const _Float16 *restrict b, int n) { float sum = 0.0f; for (int i = 0; i < n; ++i) sum += (float) a[i] * (float) b[i]; return sum; } as floating-point dot products. AArch64 already has patterns for the AdvSIMD and SVE FDOT instructions, but they aren't both exposed under the optab names that the vectorizer needs. This patch connects those existing patterns to sdot_prod so that the loop can use FDOT directly. For AdvSIMD the pattern name now includes both the result and input modes, matching the optab. The existing Neon intrinsic names are kept through aliases for the renamed instruction codes. For SVE the floating-point pattern is added to the same generated dot-product lookup used by the integer patterns. The ordinary SVE dot intrinsics then use that lookup too, replacing the hard-coded choice for the half-to-float instruction. The FP8 intrinsics that set FPMR keep their existing expansion path. Concretely, with -O3 -ffast-math and the appropriate architecture extensions, the loop above can use fdot v0.4s, v1.8h, v2.8h for AdvSIMD, or fdot z0.s, z1.h, z2.h for SVE. Each instruction multiplies pairs of half-precision inputs and accumulates them into single-precision elements. This avoids having to widen the inputs and perform the multiply and accumulate as separate instructions. The existing feature checks are retained: AdvSIMD requires +f16f32dot, while the SVE form requires SVE2.1 or SME2. The tests check both AdvSIMD widths, scalable SVE and streaming SME2 code generation, and that we don't emit FDOT without the extension. There is also coverage for the FP8 intrinsics in streaming and non-streaming functions. This also fixes two bugs with the fdot's RTL introduced in GCC17. 1. the accumulator of the generic pattern was in the wrong place, it should be last not first. 2. The endian lane swapping was wrong. the two way fdots work in pairs of input so the lanes should be flipped in groups of 2. gcc/ChangeLog: * config/aarch64/aarch64-builtins.cc (CODE_FOR_sdot_prodv4hf): Add alias for CODE_FOR_sdot_prodv2sfv4hf. (CODE_FOR_sdot_prodv8hf): Add alias for CODE_FOR_sdot_prodv4sfv8hf. * config/aarch64/aarch64-simd-builtins.def (sdot_prod): Update the comment to include both result and input modes in the pattern name. * config/aarch64/aarch64-simd.md (sdot_prod<VDQSF:vsi2qi><vczle><vczbe>): Rename to... (sdot_prod<VDQSF:mode><vsi2qi><vczle><vczbe>): ...this, including the result mode to match the sdot_prod optab. Put the multiplicands in operands 1 and 2 and the accumulator in operand 3. Update the assembly template to match. (sdot_lane<VF:isquadop><VDQSF:vsi2qi><vczle><vczbe>): Use the number of halfword pairs when mapping lane indices on big-endian targets. * config/aarch64/aarch64-sve-builtins-base.cc (svdot_impl::expand): Use the result mode as the input mode for intrinsics with one type suffix. Use code_for_dot_prod for all dot intrinsics that do not set FPMR, including floating-point dot products. * config/aarch64/aarch64-sve.md (<sur>dot_prod<mode><vsi2qi>): Rename to... (@<sur>dot_prod<mode><vsi2qi>): ...this, generating an instruction lookup. * config/aarch64/aarch64-sve2.md (aarch64_sve_fdotvnx4sfvnx8hf): Rename to... (@<sur>dot_prod<VNx4SF_ONLY:mode><VNx8HF_ONLY:mode>): ...this. Use the VNx4SF_ONLY and VNx8HF_ONLY mode iterators and DOTPROD_FP integer iterator to expose FDOT through the sdot_prod optab and generated dot-product lookup. * config/aarch64/arm_neon.h (vdot_f32_f16, vdotq_f32_f16): Update builtin argument order to match the sdot_prod optab. * config/aarch64/iterators.md (DOTPROD_FP): New integer iterator. (sur): Add UNSPEC_FDOT mapping. gcc/testsuite/ChangeLog: * gcc.target/aarch64/advsimd-intrinsics/fdot-1.c: Use fat LTO objects. * gcc.target/aarch64/advsimd-intrinsics/fdot-2.c: Likewise. * gcc.target/aarch64/advsimd-intrinsics/fdot-3.c: Likewise. * gcc.target/aarch64/fdot-autovec-advsimd-run.c: New AdvSIMD runtime test. * gcc.target/aarch64/fdot-autovec-advsimd.c: New test. * gcc.target/aarch64/fdot-autovec-noext.c: New test. * gcc.target/aarch64/fdot-autovec-run.h: New shared runtime test helper. Check normal and subnormal inputs and lengths from 0 to 257. * gcc.target/aarch64/sve2/fdot-autovec-run.c: New SVE2.1 runtime test. * gcc.target/aarch64/sve2/fdot-autovec-scalable.c: New test. * gcc.target/aarch64/sve2/fdot-autovec-sme2.c: New test. * gcc.target/aarch64/sve2/fdot-fp8-vla.c: New test. * lib/target-supports.exp (check_effective_target_aarch64_f16f32dot_hw): New procedure to check whether the target can execute AdvSIMD FP16-to-FP32 dot products. Diff: --- gcc/config/aarch64/aarch64-builtins.cc | 4 +++ gcc/config/aarch64/aarch64-simd-builtins.def | 2 +- gcc/config/aarch64/aarch64-simd.md | 14 +++++---- gcc/config/aarch64/aarch64-sve-builtins-base.cc | 17 ++++------ gcc/config/aarch64/aarch64-sve.md | 2 +- gcc/config/aarch64/aarch64-sve2.md | 17 +++++----- gcc/config/aarch64/arm_neon.h | 4 +-- gcc/config/aarch64/iterators.md | 2 ++ .../gcc.target/aarch64/advsimd-intrinsics/fdot-1.c | 2 +- .../gcc.target/aarch64/advsimd-intrinsics/fdot-2.c | 2 +- .../gcc.target/aarch64/advsimd-intrinsics/fdot-3.c | 2 +- .../gcc.target/aarch64/fdot-autovec-advsimd-run.c | 7 +++++ .../gcc.target/aarch64/fdot-autovec-advsimd.c | 16 ++++++++++ .../gcc.target/aarch64/fdot-autovec-noext.c | 14 +++++++++ .../gcc.target/aarch64/fdot-autovec-run.h | 36 ++++++++++++++++++++++ .../gcc.target/aarch64/sve2/fdot-autovec-run.c | 6 ++++ .../aarch64/sve2/fdot-autovec-scalable.c | 14 +++++++++ .../gcc.target/aarch64/sve2/fdot-autovec-sme2.c | 14 +++++++++ .../gcc.target/aarch64/sve2/fdot-fp8-vla.c | 33 ++++++++++++++++++++ gcc/testsuite/lib/target-supports.exp | 17 ++++++++++ 20 files changed, 192 insertions(+), 33 deletions(-) diff --git a/gcc/config/aarch64/aarch64-builtins.cc b/gcc/config/aarch64/aarch64-builtins.cc index d94d75b9afc1..1d8baaef5c41 100644 --- a/gcc/config/aarch64/aarch64-builtins.cc +++ b/gcc/config/aarch64/aarch64-builtins.cc @@ -459,6 +459,10 @@ constexpr insn_code CODE_FOR_aarch64_udot_prodv16qi = CODE_FOR_udot_prodv4siv16qi; constexpr insn_code CODE_FOR_aarch64_usdot_prodv16qi = CODE_FOR_usdot_prodv4siv16qi; +constexpr insn_code CODE_FOR_sdot_prodv4hf + = CODE_FOR_sdot_prodv2sfv4hf; +constexpr insn_code CODE_FOR_sdot_prodv8hf + = CODE_FOR_sdot_prodv4sfv8hf; #define CF0(N, X) CODE_FOR_aarch64_##N##X #define CF1(N, X) CODE_FOR_##N##X##1 diff --git a/gcc/config/aarch64/aarch64-simd-builtins.def b/gcc/config/aarch64/aarch64-simd-builtins.def index 925d7a0052b1..0a84dfd459d1 100644 --- a/gcc/config/aarch64/aarch64-simd-builtins.def +++ b/gcc/config/aarch64/aarch64-simd-builtins.def @@ -860,7 +860,7 @@ VAR2 (QUADOP_LANE_PAIR, bfdot_lane, 0, QUIET, v2sf, v4sf) VAR2 (QUADOP_LANE_PAIR, bfdot_laneq, 0, QUIET, v2sf, v4sf) - /* Implemented by sdot_prod<mode>. */ + /* Implemented by sdot_prod<result_mode><input_mode>. */ BUILTIN_VF (TERNOP, sdot_prod, 10, DEFAULT) /* Implemented by sdot_lane{q}<dot_mode>. */ BUILTIN_VF (QUADOP_LANE_PAIR, sdot_lane, 10, DEFAULT) diff --git a/gcc/config/aarch64/aarch64-simd.md b/gcc/config/aarch64/aarch64-simd.md index 09ddca1a5e83..11705bc90738 100644 --- a/gcc/config/aarch64/aarch64-simd.md +++ b/gcc/config/aarch64/aarch64-simd.md @@ -799,16 +799,16 @@ [(set_attr "type" "neon_dot<VS:q>")] ) -(define_insn "sdot_prod<VDQSF:vsi2qi><vczle><vczbe>" +(define_insn "sdot_prod<VDQSF:mode><vsi2qi><vczle><vczbe>" [(set (match_operand:VDQSF 0 "register_operand" "=w") (plus:VDQSF (unspec:VDQSF - [(match_operand:<VSI2QI> 2 "register_operand" "w") - (match_operand:<VSI2QI> 3 "register_operand" "w")] + [(match_operand:<VSI2QI> 1 "register_operand" "w") + (match_operand:<VSI2QI> 2 "register_operand" "w")] UNSPEC_FDOT) - (match_operand:VDQSF 1 "register_operand" "0")))] + (match_operand:VDQSF 3 "register_operand" "0")))] "TARGET_F16F32DOT" - "fdot\t%0.<Vtype>, %2.<Vbfdottype>, %3.<Vbfdottype>" + "fdot\t%0.<Vtype>, %1.<Vbfdottype>, %2.<Vbfdottype>" [(set_attr "type" "neon_dot<q>")] ) @@ -823,7 +823,9 @@ (match_operand:VDQSF 1 "register_operand" "0")))] "TARGET_F16F32DOT" { - operands[4] = aarch64_endian_lane_rtx (<VF:MODE>mode, INTVAL (operands[4])); + int nunits = GET_MODE_NUNITS (<VF:MODE>mode).to_constant (); + int lane = INTVAL (operands[4]); + operands[4] = gen_int_mode (ENDIAN_LANE_N (nunits / 2, lane), SImode); return "fdot\t%0.<VDQSF:Vtype>, %2.<VDQSF:Vbfdottype>, %3.2h[%4]"; } [(set_attr "type" "neon_dot<VDQSF:q>")] diff --git a/gcc/config/aarch64/aarch64-sve-builtins-base.cc b/gcc/config/aarch64/aarch64-sve-builtins-base.cc index d6d53b3d41f5..93443751b19f 100644 --- a/gcc/config/aarch64/aarch64-sve-builtins-base.cc +++ b/gcc/config/aarch64/aarch64-sve-builtins-base.cc @@ -979,23 +979,18 @@ public: insn_code icode; machine_mode mode0 = GET_MODE (e.args[0]); machine_mode mode1 = GET_MODE (e.args[1]); + /* Without a second type suffix, both multiplicands have the same mode. */ + if (e.type_suffix_ids[1] == NUM_TYPE_SUFFIXES) + mode1 = mode0; if (e.fpm_mode == aarch64_acle::FPM_set) icode = code_for_aarch64_sve_dot (e.result_mode ()); else { - /* In the optab, the multiplication operands come before the accumulator - operand. The optab is keyed off the multiplication mode. */ + /* The dot patterns place the multiplication operands before the + accumulator. */ e.rotate_inputs_left (0, 3); int unspec = unspec_for (e); - if (e.type_suffix_ids[1] == NUM_TYPE_SUFFIXES) - icode = e.convert_optab_handler_for_sign (sdot_prod_optab, - udot_prod_optab, - 0, e.result_mode (), - GET_MODE (e.args[0])); - else - icode = (e.type_suffix (0).float_p - ? CODE_FOR_aarch64_sve_fdotvnx4sfvnx8hf - : code_for_dot_prod (unspec, mode0, mode1)); + icode = code_for_dot_prod (unspec, mode0, mode1); } return e.use_unpred_insn (icode); } diff --git a/gcc/config/aarch64/aarch64-sve.md b/gcc/config/aarch64/aarch64-sve.md index 8ba24d4d4e10..d0b3612f340a 100644 --- a/gcc/config/aarch64/aarch64-sve.md +++ b/gcc/config/aarch64/aarch64-sve.md @@ -7818,7 +7818,7 @@ ;; ------------------------------------------------------------------------- ;; Four-element integer dot-product with accumulation. -(define_insn "<sur>dot_prod<mode><vsi2qi>" +(define_insn "@<sur>dot_prod<mode><vsi2qi>" [(set (match_operand:SVE_FULL_SDI 0 "register_operand") (plus:SVE_FULL_SDI (unspec:SVE_FULL_SDI diff --git a/gcc/config/aarch64/aarch64-sve2.md b/gcc/config/aarch64/aarch64-sve2.md index 56fab8eecb1b..3a8b81b3a695 100644 --- a/gcc/config/aarch64/aarch64-sve2.md +++ b/gcc/config/aarch64/aarch64-sve2.md @@ -2998,15 +2998,14 @@ [(set_attr "sve_type" "sve_<sve_type_unspec>")] ) -;; Two-way dot-product. -(define_insn "aarch64_sve_fdotvnx4sfvnx8hf" - [(set (match_operand:VNx4SF 0 "register_operand") - (plus:VNx4SF - (unspec:VNx4SF - [(match_operand:VNx8HF 1 "register_operand") - (match_operand:VNx8HF 2 "register_operand")] - UNSPEC_FDOT) - (match_operand:VNx4SF 3 "register_operand")))] +(define_insn "@<sur>dot_prod<VNx4SF_ONLY:mode><VNx8HF_ONLY:mode>" + [(set (match_operand:VNx4SF_ONLY 0 "register_operand") + (plus:VNx4SF_ONLY + (unspec:VNx4SF_ONLY + [(match_operand:VNx8HF_ONLY 1 "register_operand") + (match_operand:VNx8HF_ONLY 2 "register_operand")] + DOTPROD_FP) + (match_operand:VNx4SF_ONLY 3 "register_operand")))] "TARGET_SVE2p1_OR_SME2" {@ [ cons: =0 , 1 , 2 , 3 ; attrs: movprfx ] [ w , w , w , 0 ; * ] fdot\t%0.s, %1.h, %2.h diff --git a/gcc/config/aarch64/arm_neon.h b/gcc/config/aarch64/arm_neon.h index cd37b6669639..9caa0a2088d8 100644 --- a/gcc/config/aarch64/arm_neon.h +++ b/gcc/config/aarch64/arm_neon.h @@ -19559,14 +19559,14 @@ __extension__ extern __inline float32x2_t __attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) vdot_f32_f16 (float32x2_t __r, float16x4_t __a, float16x4_t __b) { - return __builtin_aarch64_sdot_prodv4hf (__r, __a, __b); + return __builtin_aarch64_sdot_prodv4hf (__a, __b, __r); } __extension__ extern __inline float32x4_t __attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) vdotq_f32_f16 (float32x4_t __r, float16x8_t __a, float16x8_t __b) { - return __builtin_aarch64_sdot_prodv8hf (__r, __a, __b); + return __builtin_aarch64_sdot_prodv8hf (__a, __b, __r); } __extension__ extern __inline float32x2_t diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md index 92b61159cc33..dcaa5a3b708a 100644 --- a/gcc/config/aarch64/iterators.md +++ b/gcc/config/aarch64/iterators.md @@ -3761,6 +3761,7 @@ (define_int_iterator BSL_DUP [1 2]) (define_int_iterator DOTPROD [UNSPEC_SDOT UNSPEC_UDOT]) +(define_int_iterator DOTPROD_FP [UNSPEC_FDOT]) (define_int_iterator DOTPROD_I8MM [UNSPEC_USDOT UNSPEC_SUDOT]) (define_int_iterator DOTPROD_US_ONLY [UNSPEC_USDOT]) @@ -4841,6 +4842,7 @@ (UNSPEC_URSHL "ur") (UNSPEC_SRSHL "sr") (UNSPEC_UQRSHL "u") (UNSPEC_SQRSHL "s") (UNSPEC_SDOT "s") (UNSPEC_UDOT "u") + (UNSPEC_FDOT "s") (UNSPEC_USDOT "us") (UNSPEC_SUDOT "su") (UNSPEC_SMATMUL "s") (UNSPEC_UMATMUL "u") (UNSPEC_USMATMUL "us") diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-1.c b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-1.c index 4abbdf57ef69..d85bccbac768 100644 --- a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-1.c +++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-1.c @@ -1,7 +1,7 @@ /* { dg-do assemble { target { aarch64*-*-* } } } */ /* { dg-require-effective-target aarch64_f16f32dot_neon_ok } */ /* { dg-add-options aarch64_f16f32dot_neon } */ -/* { dg-additional-options "-save-temps" } */ +/* { dg-additional-options "-save-temps -ffat-lto-objects" } */ /* { dg-final { check-function-bodies "**" "" {-O[^0]} } } */ /* { dg-skip-if "" { *-*-* } { "-fno-fat-lto-objects" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-2.c b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-2.c index b42bfa09c77c..7f8684049a8b 100644 --- a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-2.c +++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-2.c @@ -2,7 +2,7 @@ /* { dg-require-effective-target stdint_types_mbig_endian } */ /* { dg-require-effective-target aarch64_f16f32dot_neon_ok } */ /* { dg-add-options aarch64_f16f32dot_neon } */ -/* { dg-additional-options "-mbig-endian --save-temps" } */ +/* { dg-additional-options "-mbig-endian --save-temps -ffat-lto-objects" } */ /* { dg-final { check-function-bodies "**" "" {-O[^0]} } } */ /* { dg-skip-if "" { *-*-* } { "-fno-fat-lto-objects" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-3.c b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-3.c index 314322a08345..72c5f628c9cd 100644 --- a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-3.c +++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-3.c @@ -2,7 +2,7 @@ /* { dg-skip-if "" { *-*-* } { "-fno-fat-lto-objects" } } */ /* { dg-require-effective-target aarch64_f16f32dot_neon_ok } */ /* { dg-add-options aarch64_f16f32dot_neon } */ -/* { dg-additional-options "--save-temps" } */ +/* { dg-additional-options "--save-temps -ffat-lto-objects" } */ #include <arm_neon.h> diff --git a/gcc/testsuite/gcc.target/aarch64/fdot-autovec-advsimd-run.c b/gcc/testsuite/gcc.target/aarch64/fdot-autovec-advsimd-run.c new file mode 100644 index 000000000000..deb4f59e0abf --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/fdot-autovec-advsimd-run.c @@ -0,0 +1,7 @@ +/* { dg-do run { target aarch64_f16f32dot_hw } } */ +/* { dg-options "-O3 -ffast-math -march=armv8.2-a+fp16+f16f32dot -mautovec-preference=asimd-only -save-temps" } */ + +#include "fdot-autovec-run.h" + +/* { dg-final { scan-assembler {\tfdot\tv[0-9]+\.4s, v[0-9]+\.8h, v[0-9]+\.8h} } } */ +/* { dg-final { scan-assembler {\tfdot\tv[0-9]+\.2s, v[0-9]+\.4h, v[0-9]+\.4h} } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/fdot-autovec-advsimd.c b/gcc/testsuite/gcc.target/aarch64/fdot-autovec-advsimd.c new file mode 100644 index 000000000000..6326f1c3eb1e --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/fdot-autovec-advsimd.c @@ -0,0 +1,16 @@ +/* { dg-do compile } */ +/* { dg-options "-O3 -ffast-math -march=armv8.2-a+fp16+f16f32dot -mautovec-preference=asimd-only -fdump-tree-vect-details" } */ + +float +dot_half_advsimd (const _Float16 *restrict a, + const _Float16 *restrict b, int n) +{ + float sum = 0.0f; + for (int i = 0; i < n; ++i) + sum += (float) a[i] * (float) b[i]; + return sum; +} + +/* { dg-final { scan-assembler {\tfdot\tv[0-9]+\.4s, v[0-9]+\.8h, v[0-9]+\.8h} } } */ +/* { dg-final { scan-assembler {\tfdot\tv[0-9]+\.2s, v[0-9]+\.4h, v[0-9]+\.4h} } } */ +/* { dg-final { scan-tree-dump "dot_prod pattern recognized" "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/fdot-autovec-noext.c b/gcc/testsuite/gcc.target/aarch64/fdot-autovec-noext.c new file mode 100644 index 000000000000..c452637d5b56 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/fdot-autovec-noext.c @@ -0,0 +1,14 @@ +/* { dg-do compile } */ +/* { dg-options "-O3 -ffast-math -march=armv8.2-a+fp16 -mautovec-preference=asimd-only" } */ + +float +dot_half_noext (const _Float16 *restrict a, + const _Float16 *restrict b, int n) +{ + float sum = 0.0f; + for (int i = 0; i < n; ++i) + sum += (float) a[i] * (float) b[i]; + return sum; +} + +/* { dg-final { scan-assembler-not {\tfdot\t} } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/fdot-autovec-run.h b/gcc/testsuite/gcc.target/aarch64/fdot-autovec-run.h new file mode 100644 index 000000000000..8fce986db0fc --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/fdot-autovec-run.h @@ -0,0 +1,36 @@ +__attribute__ ((noipa)) float +dot_half (const _Float16 *restrict a, const _Float16 *restrict b, int n) +{ + float sum = 0.0f; + for (int i = 0; i < n; ++i) + sum += (float) a[i] * (float) b[i]; + return sum; +} + +int +main (void) +{ + _Float16 a[257], b[257]; + for (int small = 0; small < 2; ++small) + { + /* Both normal and subnormal inputs, with exact FP32 products and + sums. The small products would underflow if evaluated in FP16. */ + float scale = small ? 0x1p-24f : 0.0625f; + for (int i = 0; i < 257; ++i) + { + a[i] = (i % 31 - 15) * scale; + b[i] = (i % 13 - 6) * scale; + } + + double expected = 0.0; + for (int n = 0; n <= 257; ++n) + { + if (dot_half (a, b, n) != (float) expected) + __builtin_abort (); + if (n < 257) + expected += (double) a[n] * (double) b[n]; + } + } + return 0; +} + diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/fdot-autovec-run.c b/gcc/testsuite/gcc.target/aarch64/sve2/fdot-autovec-run.c new file mode 100644 index 000000000000..7743c6ca6023 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve2/fdot-autovec-run.c @@ -0,0 +1,6 @@ +/* { dg-do run { target aarch64_sve2p1_hw } } */ +/* { dg-options "-O3 -ffast-math -march=armv9.4-a+sve2p1 -mautovec-preference=sve-only -save-temps" } */ + +#include "../fdot-autovec-run.h" + +/* { dg-final { scan-assembler {\tfdot\tz[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h} } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/fdot-autovec-scalable.c b/gcc/testsuite/gcc.target/aarch64/sve2/fdot-autovec-scalable.c new file mode 100644 index 000000000000..ecb833ac3846 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve2/fdot-autovec-scalable.c @@ -0,0 +1,14 @@ +/* { dg-do compile } */ +/* { dg-options "-O3 -ffast-math -march=armv9.4-a+sve2p1 -mautovec-preference=sve-only" } */ + +float +dot_half_scalable (const _Float16 *restrict a, + const _Float16 *restrict b, int n) +{ + float sum = 0.0f; + for (int i = 0; i < n; ++i) + sum += (float) a[i] * (float) b[i]; + return sum; +} + +/* { dg-final { scan-assembler {\tfdot\tz[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h} } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/fdot-autovec-sme2.c b/gcc/testsuite/gcc.target/aarch64/sve2/fdot-autovec-sme2.c new file mode 100644 index 000000000000..e3b018853483 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve2/fdot-autovec-sme2.c @@ -0,0 +1,14 @@ +/* { dg-do compile } */ +/* { dg-options "-O3 -ffast-math -march=armv9.2-a+sme2 -mautovec-preference=sve-only" } */ + +__arm_streaming float +dot_half_sme2 (const _Float16 *restrict a, + const _Float16 *restrict b, int n) +{ + float sum = 0.0f; + for (int i = 0; i < n; ++i) + sum += (float) a[i] * (float) b[i]; + return sum; +} + +/* { dg-final { scan-assembler {\tfdot\tz[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h} } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/fdot-fp8-vla.c b/gcc/testsuite/gcc.target/aarch64/sve2/fdot-fp8-vla.c new file mode 100644 index 000000000000..7d000811dbfb --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve2/fdot-fp8-vla.c @@ -0,0 +1,33 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=armv9.4-a+sve2+fp8dot2+fp8dot4+ssve-fp8dot2+ssve-fp8dot4" } */ + +#include <arm_sve.h> + +svfloat16_t +dot_fp8_to_f16 (svfloat16_t acc, svmfloat8_t a, svmfloat8_t b, fpm_t fpm) +{ + return svdot_f16_mf8_fpm (acc, a, b, fpm); +} + +svfloat32_t +dot_fp8_to_f32 (svfloat32_t acc, svmfloat8_t a, svmfloat8_t b, fpm_t fpm) +{ + return svdot_f32_mf8_fpm (acc, a, b, fpm); +} + +__arm_streaming svfloat16_t +dot_fp8_to_f16_streaming (svfloat16_t acc, svmfloat8_t a, svmfloat8_t b, + fpm_t fpm) +{ + return svdot_f16_mf8_fpm (acc, a, b, fpm); +} + +__arm_streaming svfloat32_t +dot_fp8_to_f32_streaming (svfloat32_t acc, svmfloat8_t a, svmfloat8_t b, + fpm_t fpm) +{ + return svdot_f32_mf8_fpm (acc, a, b, fpm); +} + +/* { dg-final { scan-assembler-times {\tfdot\tz[0-9]+\.h, z[0-9]+\.b, z[0-9]+\.b} 2 } } */ +/* { dg-final { scan-assembler-times {\tfdot\tz[0-9]+\.s, z[0-9]+\.b, z[0-9]+\.b} 2 } } */ diff --git a/gcc/testsuite/lib/target-supports.exp b/gcc/testsuite/lib/target-supports.exp index 4dcb7f41b459..3ce06189d2e6 100644 --- a/gcc/testsuite/lib/target-supports.exp +++ b/gcc/testsuite/lib/target-supports.exp @@ -7716,6 +7716,23 @@ proc add_options_for_aarch64_f16f32dot_neon { flags } { return "$flags $et_aarch64_f16f32dot_neon_flags" } +# Return true if the target can execute AdvSIMD FP16-to-FP32 dot products. + +proc check_effective_target_aarch64_f16f32dot_hw { } { + if { ![check_effective_target_aarch64_f16f32dot_neon_ok] } { + return 0 + } + return [check_runtime aarch64_f16f32dot_hw_available { + int + main (void) + { + asm volatile ("movi v0.4s, 0; fdot v0.4s, v0.8h, v0.8h" + : : : "v0"); + return 0; + } + } [add_options_for_aarch64_f16f32dot_neon ""]] +} + # Return 1 if the target supports BFloat16 SIMD instructions, 0 otherwise. # The test is valid for ARM and for AArch64.
