From: Richard Ball <[email protected]>
This patch adds the following advsimd intrinsics:
*vdot_f32_f16
*vdotq_f32_f16
*vdot_lane_f32_f16
*vdot_laneq_f32_f16
*vdotq_lane_f32_f16
*vdotq_laneq_f32_f16
gcc/ChangeLog:
* config/aarch64/aarch64-c.cc
(aarch64_update_cpp_builtins): New target flag.
* config/aarch64/aarch64-simd-builtins.def: New functions.
* config/aarch64/aarch64-simd.md
(aarch64_simdfdot<mode>): New pattern.
(aarch64_simdfdot_lane<VF:isquadop><VDQSF:mode>): Likewise.
* config/aarch64/aarch64.h
(TARGET_F16F32DOT): New target flag.
* config/aarch64/arm_neon.h
(target): Add f16f32dot section
(__attribute__): New builtin function calls.
(vdot_f32_f16): Likewise.
(vdotq_f32_f16): Likewise.
(vdot_lane_f32_f16): Likewise.
(vdot_laneq_f32_f16): Likewise.
(vdotq_lane_f32_f16): Likewise.
(vdotq_laneq_f32_f16): Likewise.
* config/aarch64/iterators.md (q): New options for isquadop.
* doc/invoke.texi: Add f16f32dot.
gcc/testsuite/ChangeLog:
* lib/target-supports.exp: New check_effective_target.
* gcc.target/aarch64/advsimd-intrinsics/fdot-1.c: New test.
* gcc.target/aarch64/advsimd-intrinsics/fdot-2.c: New test.
* gcc.target/aarch64/advsimd-intrinsics/fdot-3.c: New test.
---
gcc/config/aarch64/aarch64-c.cc | 1 +
gcc/config/aarch64/aarch64-simd-builtins.def | 5 ++
gcc/config/aarch64/aarch64-simd.md | 32 +++++++++
gcc/config/aarch64/aarch64.h | 2 +
gcc/config/aarch64/arm_neon.h | 50 +++++++++++++
gcc/config/aarch64/iterators.md | 6 +-
gcc/doc/invoke.texi | 2 +
.../aarch64/advsimd-intrinsics/fdot-1.c | 68 ++++++++++++++++++
.../aarch64/advsimd-intrinsics/fdot-2.c | 70 +++++++++++++++++++
.../aarch64/advsimd-intrinsics/fdot-3.c | 32 +++++++++
gcc/testsuite/lib/target-supports.exp | 41 +++++++++++
11 files changed, 308 insertions(+), 1 deletion(-)
create mode 100644 gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-1.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-2.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-3.c
diff --git a/gcc/config/aarch64/aarch64-c.cc b/gcc/config/aarch64/aarch64-c.cc
index eb08882af55f..41318e4ee0bd 100644
--- a/gcc/config/aarch64/aarch64-c.cc
+++ b/gcc/config/aarch64/aarch64-c.cc
@@ -314,6 +314,7 @@ aarch64_update_cpp_builtins (cpp_reader *pfile)
aarch64_def_or_undef (TARGET_SME2p2, "__ARM_FEATURE_SME2p2", pfile);
aarch64_def_or_undef (TARGET_FAMINMAX, "__ARM_FEATURE_FAMINMAX", pfile);
aarch64_def_or_undef (TARGET_PCDPHINT, "__ARM_FEATURE_PCDPHINT", pfile);
+ aarch64_def_or_undef (TARGET_F16F32DOT, "__ARM_FEATURE_F16F32DOT", pfile);
// Function multi-versioning defines
aarch64_def_or_undef (targetm.has_ifunc_p (),
diff --git a/gcc/config/aarch64/aarch64-simd-builtins.def
b/gcc/config/aarch64/aarch64-simd-builtins.def
index 8677df3f488f..764d3ac411bd 100644
--- a/gcc/config/aarch64/aarch64-simd-builtins.def
+++ b/gcc/config/aarch64/aarch64-simd-builtins.def
@@ -941,6 +941,11 @@
VAR2 (QUADOP_LANE_PAIR, bfdot_lane, 0, QUIET, v2sf, v4sf)
VAR2 (QUADOP_LANE_PAIR, bfdot_laneq, 0, QUIET, v2sf, v4sf)
+ /* Implemented by aarch64_simdfdot{_lane}{q}<mode>. */
+ VAR2 (TERNOP, simdfdot, 0, QUIET, v2sf, v4sf)
+ VAR2 (QUADOP_LANE_PAIR, simdfdot_lane, 0, QUIET, v2sf, v4sf)
+ VAR2 (QUADOP_LANE_PAIR, simdfdot_laneq, 0, QUIET, v2sf, v4sf)
+
/* Implemented by aarch64_bfmmlaqv4sf */
VAR1 (TERNOP, bfmmlaq, 0, QUIET, v4sf)
diff --git a/gcc/config/aarch64/aarch64-simd.md
b/gcc/config/aarch64/aarch64-simd.md
index ec14474fe520..6d2959cc0d0b 100644
--- a/gcc/config/aarch64/aarch64-simd.md
+++ b/gcc/config/aarch64/aarch64-simd.md
@@ -10623,6 +10623,38 @@
[(set_attr "type" "neon_dot<VDQSF:q>")]
)
+(define_insn "aarch64_simdfdot<mode>"
+ [(set (match_operand:VDQSF 0 "register_operand" "=w")
+ (plus:VDQSF
+ (unspec:VDQSF
+ [(match_operand:<VFMLA_W> 2 "register_operand" "w")
+ (match_operand:<VFMLA_W> 3 "register_operand" "w")]
+ UNSPEC_FDOT)
+ (match_operand:VDQSF 1 "register_operand" "0")))]
+ "TARGET_F16F32DOT"
+ "fdot\t%0.<Vtype>, %2.<Vbfdottype>, %3.<Vbfdottype>"
+ [(set_attr "type" "neon_dot<q>")]
+)
+
+(define_insn "aarch64_simdfdot_lane<VF:isquadop><VDQSF:mode>"
+ [(set (match_operand:VDQSF 0 "register_operand" "=w")
+ (plus:VDQSF
+ (unspec:VDQSF
+ [(match_operand:<VDQSF:VFMLA_W> 2 "register_operand" "w")
+ (match_operand:VF 3 "register_operand" "w")
+ (match_operand:SI 4 "const_int_operand" "n")]
+ UNSPEC_FDOT)
+ (match_operand:VDQSF 1 "register_operand" "0")))]
+ "TARGET_F16F32DOT"
+{
+ int nunits = GET_MODE_NUNITS (<VF:MODE>mode).to_constant ();
+ int lane = INTVAL (operands[4]);
+ operands[4] = gen_int_mode (ENDIAN_LANE_N (nunits / 2, lane), SImode);
+ return "fdot\t%0.<VDQSF:Vtype>, %2.<VDQSF:Vbfdottype>, %3.2h[%4]";
+}
+ [(set_attr "type" "neon_dot<VDQSF:q>")]
+)
+
;; bfmmla
(define_insn "aarch64_bfmmlaqv4sf"
[(set (match_operand:V4SF 0 "register_operand" "=w")
diff --git a/gcc/config/aarch64/aarch64.h b/gcc/config/aarch64/aarch64.h
index 5bd3379cb0db..97b33221e366 100644
--- a/gcc/config/aarch64/aarch64.h
+++ b/gcc/config/aarch64/aarch64.h
@@ -425,6 +425,8 @@ constexpr auto AARCH64_FL_DEFAULT_ISA_MODE ATTRIBUTE_UNUSED
#define TARGET_F8F16MM (AARCH64_HAVE_ISA (F8F16MM))
/* SVE_F16F32MM instructions, enabled through +sve-f16f32mm. */
#define TARGET_SVE_F16F32MM (AARCH64_HAVE_ISA (SVE_F16F32MM))
+/* F16F32DOT instructions enabled through +f16f32dot. */
+#define TARGET_F16F32DOT (AARCH64_HAVE_ISA (F16F32DOT))
/* Make sure this is always defined so we don't have to check for ifdefs
but rather use normal ifs. */
diff --git a/gcc/config/aarch64/arm_neon.h b/gcc/config/aarch64/arm_neon.h
index 82cf94b51739..2e0a84e2e963 100644
--- a/gcc/config/aarch64/arm_neon.h
+++ b/gcc/config/aarch64/arm_neon.h
@@ -28420,6 +28420,56 @@ vst4q_lane_bf16 (bfloat16_t *__ptr, bfloat16x8x4_t
__val, const int __lane)
#pragma GCC pop_options
+#pragma GCC push_options
+#pragma GCC target ("+nothing+f16f32dot")
+
+__extension__ extern __inline float32x2_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vdot_f32_f16 (float32x2_t __r, float16x4_t __a, float16x4_t __b)
+{
+ return __builtin_aarch64_simdfdotv2sf (__r, __a, __b);
+}
+
+__extension__ extern __inline float32x4_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vdotq_f32_f16 (float32x4_t __r, float16x8_t __a, float16x8_t __b)
+{
+ return __builtin_aarch64_simdfdotv4sf (__r, __a, __b);
+}
+
+__extension__ extern __inline float32x2_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vdot_lane_f32_f16 (float32x2_t __r, float16x4_t __a, float16x4_t __b,
+ const int __index)
+{
+ return __builtin_aarch64_simdfdot_lanev2sf (__r, __a, __b, __index);
+}
+
+__extension__ extern __inline float32x2_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vdot_laneq_f32_f16 (float32x2_t __r, float16x4_t __a, float16x8_t __b,
+ const int __index)
+{
+ return __builtin_aarch64_simdfdot_laneqv2sf (__r, __a, __b, __index);
+}
+
+__extension__ extern __inline float32x4_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vdotq_lane_f32_f16 (float32x4_t __r, float16x8_t __a, float16x4_t __b,
+ const int __index)
+{
+ return __builtin_aarch64_simdfdot_lanev4sf (__r, __a, __b, __index);
+}
+
+__extension__ extern __inline float32x4_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vdotq_laneq_f32_f16 (float32x4_t __r, float16x8_t __a, float16x8_t __b,
+ const int __index)
+{
+ return __builtin_aarch64_simdfdot_laneqv4sf (__r, __a, __b, __index);
+}
+
+#pragma GCC pop_options
/* AdvSIMD 8-bit Integer Matrix Multiply (I8MM) intrinsics. */
#pragma GCC push_options
diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md
index 37f819ce6493..ccb10bdb2741 100644
--- a/gcc/config/aarch64/iterators.md
+++ b/gcc/config/aarch64/iterators.md
@@ -169,6 +169,9 @@
;; BFmode vector modes.
(define_mode_iterator VBF [V4BF V8BF])
+;; Fmode vector modes.
+(define_mode_iterator VF [V4HF V8HF])
+
;; This mode iterator allows :P to be used for patterns that operate on
;; addresses in different modes. In LP64, only DI will match, while in
;; ILP32, either can match.
@@ -2707,7 +2710,8 @@
(define_mode_attr f16quad [(V2SF "") (V4SF "q")])
-(define_mode_attr isquadop [(V8QI "") (V16QI "q") (V4BF "") (V8BF "q")])
+(define_mode_attr isquadop [(V8QI "") (V16QI "q") (V4BF "") (V8BF "q")
+ (V4HF "") (V8HF "q")])
(define_code_attr f16mac [(plus "a") (minus "s")])
diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
index 09321ca5da95..6312177b893d 100644
--- a/gcc/doc/invoke.texi
+++ b/gcc/doc/invoke.texi
@@ -22076,6 +22076,8 @@ extension.
@item ssve-fp8dot2
Enable the fp8 (8-bit floating point) to half-precision 2-way dot product
extension in streaming mode.
+@item f16f32dot
+Enable the Half-precision to single-precision dot product extension.
@item faminmax
Enable the Floating Point Absolute Maximum/Minimum extension.
@item lut
diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-1.c
b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-1.c
new file mode 100644
index 000000000000..821f14c9a32e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-1.c
@@ -0,0 +1,68 @@
+/* { dg-do assemble { target { aarch64*-*-* } } } */
+/* { dg-require-effective-target aarch64_v8_2a_f16f32dot_neon_ok } */
+/* { dg-add-options aarch64_v8_2a_f16f32dot_neon } */
+/* { dg-additional-options "-save-temps" } */
+/* { dg-final { check-function-bodies "**" "" {-O[^0]} } } */
+/* { dg-skip-if "" { *-*-* } { "-fno-fat-lto-objects" } } */
+
+#include <arm_neon.h>
+
+/*
+**ufoo:
+** fdot v0.2s, (v1.4h, v2.4h|v2.4h, v1.4h)
+** ret
+*/
+float32x2_t ufoo(float32x2_t r, float16x4_t x, float16x4_t y)
+{
+ return vdot_f32_f16 (r, x, y);
+}
+
+/*
+**ufooq:
+** fdot v0.4s, (v1.8h, v2.8h|v2.8h, v1.8h)
+** ret
+*/
+float32x4_t ufooq(float32x4_t r, float16x8_t x, float16x8_t y)
+{
+ return vdotq_f32_f16 (r, x, y);
+}
+
+/*
+**ufoo_lane:
+** fdot v0.2s, v1.4h, v2.2h\[0\]
+** ret
+*/
+float32x2_t ufoo_lane(float32x2_t r, float16x4_t x, float16x4_t y)
+{
+ return vdot_lane_f32_f16 (r, x, y, 0);
+}
+
+/*
+**ufoo_laneq:
+** fdot v0.2s, v1.4h, v2.2h\[2\]
+** ret
+*/
+float32x2_t ufoo_laneq(float32x2_t r, float16x4_t x, float16x8_t y)
+{
+ return vdot_laneq_f32_f16 (r, x, y, 2);
+}
+
+/*
+**ufooq_lane:
+** fdot v0.4s, v1.8h, v2.2h\[1\]
+** ret
+*/
+float32x4_t ufooq_lane(float32x4_t r, float16x8_t x, float16x4_t y)
+{
+ return vdotq_lane_f32_f16 (r, x, y, 1);
+}
+
+/*
+**ufooq_laneq:
+** fdot v0.4s, v1.8h, v2.2h\[2\]
+** ret
+*/
+float32x4_t ufooq_laneq(float32x4_t r, float16x8_t x, float16x8_t y)
+{
+ return vdotq_laneq_f32_f16 (r, x, y, 2);
+}
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-2.c
b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-2.c
new file mode 100644
index 000000000000..4c0ca215ebf1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-2.c
@@ -0,0 +1,70 @@
+/* { dg-do assemble { target { aarch64*-*-* } } } */
+/* { dg-require-effective-target stdint_types_mbig_endian } */
+/* { dg-require-effective-target aarch64_v8_2a_f16f32dot_neon_ok } */
+/* { dg-add-options aarch64_v8_2a_f16f32dot_neon } */
+/* { dg-additional-options "-mbig-endian --save-temps" } */
+/* { dg-final { check-function-bodies "**" "" {-O[^0]} } } */
+/* { dg-skip-if "" { *-*-* } { "-fno-fat-lto-objects" } } */
+
+
+#include <arm_neon.h>
+
+/*
+**ufoo:
+** fdot v0.2s, (v1.4h, v2.4h|v2.4h, v1.4h)
+** ret
+*/
+float32x2_t ufoo(float32x2_t r, float16x4_t x, float16x4_t y)
+{
+ return vdot_f32_f16 (r, x, y);
+}
+
+/*
+**ufooq:
+** fdot v0.4s, (v1.8h, v2.8h|v2.8h, v1.8h)
+** ret
+*/
+float32x4_t ufooq(float32x4_t r, float16x8_t x, float16x8_t y)
+{
+ return vdotq_f32_f16 (r, x, y);
+}
+
+/*
+**ufoo_lane:
+** fdot v0.2s, v1.4h, v2.2h\[0\]
+** ret
+*/
+float32x2_t ufoo_lane(float32x2_t r, float16x4_t x, float16x4_t y)
+{
+ return vdot_lane_f32_f16 (r, x, y, 0);
+}
+
+/*
+**ufoo_laneq:
+** fdot v0.2s, v1.4h, v2.2h\[2\]
+** ret
+*/
+float32x2_t ufoo_laneq(float32x2_t r, float16x4_t x, float16x8_t y)
+{
+ return vdot_laneq_f32_f16 (r, x, y, 2);
+}
+
+/*
+**ufooq_lane:
+** fdot v0.4s, v1.8h, v2.2h\[1\]
+** ret
+*/
+float32x4_t ufooq_lane(float32x4_t r, float16x8_t x, float16x4_t y)
+{
+ return vdotq_lane_f32_f16 (r, x, y, 1);
+}
+
+/*
+**ufooq_laneq:
+** fdot v0.4s, v1.8h, v2.2h\[2\]
+** ret
+*/
+float32x4_t ufooq_laneq(float32x4_t r, float16x8_t x, float16x8_t y)
+{
+ return vdotq_laneq_f32_f16 (r, x, y, 2);
+}
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-3.c
b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-3.c
new file mode 100644
index 000000000000..a0eba1256534
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-3.c
@@ -0,0 +1,32 @@
+/* { dg-do assemble { target { aarch64*-*-* } } } */
+/* { dg-skip-if "" { *-*-* } { "-fno-fat-lto-objects" } } */
+/* { dg-require-effective-target aarch64_v8_2a_f16f32dot_neon_ok } */
+/* { dg-add-options aarch64_v8_2a_f16f32dot_neon } */
+/* { dg-additional-options "--save-temps" } */
+
+#include <arm_neon.h>
+
+float32x2_t ufoo_lane(float32x2_t r, float16x4_t x, float16x4_t y)
+{
+ return vdot_lane_f32_f16 (r, x, y, 2);
+}
+
+float32x2_t ufoo_laneq(float32x2_t r, float16x4_t x, float16x8_t y)
+{
+ return vdot_laneq_f32_f16 (r, x, y, 4);
+}
+
+float32x4_t ufooq_lane(float32x4_t r, float16x8_t x, float16x4_t y)
+{
+ return vdotq_lane_f32_f16 (r, x, y, 3);
+}
+
+float32x4_t ufooq_laneq(float32x4_t r, float16x8_t x, float16x8_t y)
+{
+ return vdotq_laneq_f32_f16 (r, x, y, 5);
+}
+
+/* { dg-error {lane 2 out of range 0 - 1} "" { target *-*-* } 0 } */
+/* { dg-error {lane 4 out of range 0 - 3} "" { target *-*-* } 0 } */
+/* { dg-error {lane 3 out of range 0 - 1} "" { target *-*-* } 0 } */
+/* { dg-error {lane 5 out of range 0 - 3} "" { target *-*-* } 0 } */
\ No newline at end of file
diff --git a/gcc/testsuite/lib/target-supports.exp
b/gcc/testsuite/lib/target-supports.exp
index ca79080629f5..55d8619977dd 100644
--- a/gcc/testsuite/lib/target-supports.exp
+++ b/gcc/testsuite/lib/target-supports.exp
@@ -7291,6 +7291,47 @@ proc add_options_for_arm_fp16fml_neon { flags } {
return "$flags $et_arm_fp16fml_neon_flags"
}
+# Return 1 if the target supports F16F32DOT
+# instructions, 0 otherwise. This test is valid
+# for AARCH64.
+# Record the command line options needed.
+
+proc check_effective_target_aarch64_v8_2a_f16f32dot_neon_ok_nocache { } {
+ global et_aarch64_v8_2a_f16f32dot_neon_flags
+ set et_aarch64_v8_2a_f16f32dot_neon_flags ""
+
+ if { ![istarget aarch64*-*-*] } {
+ return 0;
+ }
+
+ foreach flags {"" "-mfloat-abi=softfp -mfpu=neon-fp-armv8"
"-mfloat-abi=hard -mfpu=neon-fp-armv8" } {
+ if { [check_no_compiler_messages_nocache
aarch64_v8_2a_f16f32dot_neon_ok object {
+ #include <arm_neon.h>
+ #if !defined (__ARM_FEATURE_F16F32DOT)
+ #error "__ARM_FEATURE_F16F32DOT not defined"
+ #endif
+ } "$flags -march=armv8.2-a+f16f32dot"] } {
+ set et_aarch64_v8_2a_f16f32dot_neon_flags "$flags
-march=armv8.2-a+f16f32dot"
+ return 1
+ }
+ }
+
+ return 0;
+}
+
+proc check_effective_target_aarch64_v8_2a_f16f32dot_neon_ok { } {
+ return [check_cached_effective_target aarch64_v8_2a_f16f32dot_neon_ok \
+ check_effective_target_aarch64_v8_2a_f16f32dot_neon_ok_nocache]
+}
+
+proc add_options_for_aarch64_v8_2a_f16f32dot_neon { flags } {
+ if { ! [check_effective_target_aarch64_v8_2a_f16f32dot_neon_ok] } {
+ return "$flags"
+ }
+ global et_aarch64_v8_2a_f16f32dot_neon_flags
+ return "$flags $et_aarch64_v8_2a_f16f32dot_neon_flags"
+}
+
# Return 1 if the target supports BFloat16 SIMD instructions, 0 otherwise.
# The test is valid for ARM and for AArch64.
--
2.54.0
