From: Richard Ball <[email protected]>
This patch adds the following advsimd intrinsics:
*vdot_f32_f16
*vdotq_f32_f16
*vdot_lane_f32_f16
*vdot_laneq_f32_f16
*vdotq_lane_f32_f16
*vdotq_laneq_f32_f16
gcc/ChangeLog:
* config/aarch64/aarch64-c.cc
(aarch64_update_cpp_builtins): New target flag.
* config/aarch64/aarch64-simd-builtins.def: New functions.
* config/aarch64/aarch64-simd.md
(aarch64_simdfdot<mode>): New pattern.
(aarch64_simdfdot_lane<VF:isquadop><VDQSF:mode>): Likewise.
* config/aarch64/aarch64.h
(TARGET_F16F32DOT): New target flag.
* config/aarch64/arm_neon.h
(target): Add f16f32dot section
(__attribute__): New builtin function calls.
(vdot_f32_f16): Likewise.
(vdotq_f32_f16): Likewise.
(vdot_lane_f32_f16): Likewise.
(vdot_laneq_f32_f16): Likewise.
(vdotq_lane_f32_f16): Likewise.
(vdotq_laneq_f32_f16): Likewise.
* config/aarch64/iterators.md (q): New options for isquadop.
* doc/invoke.texi: Add f16f32dot.
gcc/testsuite/ChangeLog:
* lib/target-supports.exp: New check_effective_target.
* gcc.target/aarch64/advsimd-intrinsics/fdot-1.c: New test.
* gcc.target/aarch64/advsimd-intrinsics/fdot-2.c: New test.
* gcc.target/aarch64/advsimd-intrinsics/fdot-3.c: New test.
---
gcc/config/aarch64/aarch64-c.cc | 1 +
gcc/config/aarch64/aarch64-simd-builtins.def | 6 ++
gcc/config/aarch64/aarch64-simd.md | 30 ++++++++
gcc/config/aarch64/aarch64.h | 2 +
gcc/config/aarch64/arm_neon.h | 50 +++++++++++++
gcc/config/aarch64/iterators.md | 12 +++-
gcc/doc/invoke.texi | 2 +
.../aarch64/advsimd-intrinsics/fdot-1.c | 68 ++++++++++++++++++
.../aarch64/advsimd-intrinsics/fdot-2.c | 70 +++++++++++++++++++
.../aarch64/advsimd-intrinsics/fdot-3.c | 32 +++++++++
gcc/testsuite/lib/target-supports.exp | 41 +++++++++++
11 files changed, 311 insertions(+), 3 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-1.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-2.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-3.c
diff --git a/gcc/config/aarch64/aarch64-c.cc b/gcc/config/aarch64/aarch64-c.cc
index c4ddcc6d717d..eb47a57ede44 100644
--- a/gcc/config/aarch64/aarch64-c.cc
+++ b/gcc/config/aarch64/aarch64-c.cc
@@ -320,6 +320,7 @@ aarch64_update_cpp_builtins (cpp_reader *pfile)
aarch64_def_or_undef (AARCH64_HAVE_ISA (SSVE_FEXPA),
"__ARM_FEATURE_SSVE_FEXPA", pfile);
builtin_define ("__ARM_PREFETCH_RANGE");
+ aarch64_def_or_undef (TARGET_F16F32DOT, "__ARM_FEATURE_F16F32DOT", pfile);
// Function multi-versioning defines
aarch64_def_or_undef (targetm.has_ifunc_p (),
diff --git a/gcc/config/aarch64/aarch64-simd-builtins.def
b/gcc/config/aarch64/aarch64-simd-builtins.def
index 02938d48b4eb..9c61cc05a050 100644
--- a/gcc/config/aarch64/aarch64-simd-builtins.def
+++ b/gcc/config/aarch64/aarch64-simd-builtins.def
@@ -921,6 +921,12 @@
VAR2 (QUADOP_LANE_PAIR, bfdot_lane, 0, QUIET, v2sf, v4sf)
VAR2 (QUADOP_LANE_PAIR, bfdot_laneq, 0, QUIET, v2sf, v4sf)
+ /* Implemented by sdot_prod<mode>. */
+ BUILTIN_VF (TERNOP, sdot_prod, 10, DEFAULT)
+ /* Implemented by sdot_lane{q}<dot_mode>. */
+ BUILTIN_VF (QUADOP_LANE_PAIR, sdot_lane, 10, DEFAULT)
+ BUILTIN_VF (QUADOP_LANE_PAIR, sdot_laneq, 10, DEFAULT)
+
/* Implemented by aarch64_bfmmlaqv4sf */
VAR1 (TERNOP, bfmmlaq, 0, QUIET, v4sf)
diff --git a/gcc/config/aarch64/aarch64-simd.md
b/gcc/config/aarch64/aarch64-simd.md
index ce51e24da36a..433f16052bf2 100644
--- a/gcc/config/aarch64/aarch64-simd.md
+++ b/gcc/config/aarch64/aarch64-simd.md
@@ -782,6 +782,36 @@
[(set_attr "type" "neon_dot<VS:q>")]
)
+(define_insn "sdot_prod<VDQSF:vsi2qi><vczle><vczbe>"
+ [(set (match_operand:VDQSF 0 "register_operand" "=w")
+ (plus:VDQSF
+ (unspec:VDQSF
+ [(match_operand:<VSI2QI> 2 "register_operand" "w")
+ (match_operand:<VSI2QI> 3 "register_operand" "w")]
+ UNSPEC_FDOT)
+ (match_operand:VDQSF 1 "register_operand" "0")))]
+ "TARGET_F16F32DOT"
+ "fdot\t%0.<Vtype>, %2.<Vbfdottype>, %3.<Vbfdottype>"
+ [(set_attr "type" "neon_dot<q>")]
+)
+
+(define_insn "sdot_lane<VF:isquadop><VDQSF:vsi2qi><vczle><vczbe>"
+ [(set (match_operand:VDQSF 0 "register_operand" "=w")
+ (plus:VDQSF
+ (unspec:VDQSF
+ [(match_operand:<VDQSF:VSI2QI> 2 "register_operand" "w")
+ (match_operand:VF 3 "register_operand" "w")
+ (match_operand:SI 4 "immediate_operand" "i")]
+ UNSPEC_FDOT)
+ (match_operand:VDQSF 1 "register_operand" "0")))]
+ "TARGET_F16F32DOT"
+{
+ operands[4] = aarch64_endian_lane_rtx (<VF:MODE>mode, INTVAL (operands[4]));
+ return "fdot\t%0.<VDQSF:Vtype>, %2.<VDQSF:Vbfdottype>, %3.2h[%4]";
+}
+ [(set_attr "type" "neon_dot<VDQSF:q>")]
+)
+
(define_expand "copysign<mode>3"
[(match_operand:VHSDF 0 "register_operand")
(match_operand:VHSDF 1 "register_operand")
diff --git a/gcc/config/aarch64/aarch64.h b/gcc/config/aarch64/aarch64.h
index 26db4958a80f..a544a1877ef3 100644
--- a/gcc/config/aarch64/aarch64.h
+++ b/gcc/config/aarch64/aarch64.h
@@ -427,6 +427,8 @@ constexpr auto AARCH64_FL_DEFAULT_ISA_MODE ATTRIBUTE_UNUSED
#define TARGET_F8F16MM (AARCH64_HAVE_ISA (F8F16MM))
/* SVE_F16F32MM instructions, enabled through +sve-f16f32mm. */
#define TARGET_SVE_F16F32MM (AARCH64_HAVE_ISA (SVE_F16F32MM))
+/* F16F32DOT instructions enabled through +f16f32dot. */
+#define TARGET_F16F32DOT (AARCH64_HAVE_ISA (F16F32DOT))
/* Make sure this is always defined so we don't have to check for ifdefs
but rather use normal ifs. */
diff --git a/gcc/config/aarch64/arm_neon.h b/gcc/config/aarch64/arm_neon.h
index 3e401dbe6a0f..101a6e521068 100644
--- a/gcc/config/aarch64/arm_neon.h
+++ b/gcc/config/aarch64/arm_neon.h
@@ -22404,6 +22404,56 @@ vst4q_lane_bf16 (bfloat16_t *__ptr, bfloat16x8x4_t
__val, const int __lane)
#pragma GCC pop_options
+#pragma GCC push_options
+#pragma GCC target ("+nothing+f16f32dot")
+
+__extension__ extern __inline float32x2_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vdot_f32_f16 (float32x2_t __r, float16x4_t __a, float16x4_t __b)
+{
+ return __builtin_aarch64_sdot_prodv4hf (__r, __a, __b);
+}
+
+__extension__ extern __inline float32x4_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vdotq_f32_f16 (float32x4_t __r, float16x8_t __a, float16x8_t __b)
+{
+ return __builtin_aarch64_sdot_prodv8hf (__r, __a, __b);
+}
+
+__extension__ extern __inline float32x2_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vdot_lane_f32_f16 (float32x2_t __r, float16x4_t __a, float16x4_t __b,
+ const int __index)
+{
+ return __builtin_aarch64_sdot_lanev4hf (__r, __a, __b, __index);
+}
+
+__extension__ extern __inline float32x2_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vdot_laneq_f32_f16 (float32x2_t __r, float16x4_t __a, float16x8_t __b,
+ const int __index)
+{
+ return __builtin_aarch64_sdot_laneqv4hf (__r, __a, __b, __index);
+}
+
+__extension__ extern __inline float32x4_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vdotq_lane_f32_f16 (float32x4_t __r, float16x8_t __a, float16x4_t __b,
+ const int __index)
+{
+ return __builtin_aarch64_sdot_lanev8hf (__r, __a, __b, __index);
+}
+
+__extension__ extern __inline float32x4_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vdotq_laneq_f32_f16 (float32x4_t __r, float16x8_t __a, float16x8_t __b,
+ const int __index)
+{
+ return __builtin_aarch64_sdot_laneqv8hf (__r, __a, __b, __index);
+}
+
+#pragma GCC pop_options
/* AdvSIMD 8-bit Integer Matrix Multiply (I8MM) intrinsics. */
#pragma GCC push_options
diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md
index a8b976e4b71e..5dfb27449703 100644
--- a/gcc/config/aarch64/iterators.md
+++ b/gcc/config/aarch64/iterators.md
@@ -172,6 +172,9 @@
;; BFmode vector modes.
(define_mode_iterator VBF [V4BF V8BF])
+;; Fmode vector modes.
+(define_mode_iterator VF [V4HF V8HF])
+
;; This mode iterator allows :P to be used for patterns that operate on
;; addresses in different modes. In LP64, only DI will match, while in
;; ILP32, either can match.
@@ -2668,9 +2671,11 @@
(V4HF "v") (V8HF "v")])
(define_mode_attr vsi2qi [(V2SI "v8qi") (V4SI "v16qi")
- (VNx4SI "vnx16qi") (VNx2DI "vnx8hi")])
+ (VNx4SI "vnx16qi") (VNx2DI "vnx8hi")
+ (V2SF "v4hf") (V4SF "v8hf")])
(define_mode_attr VSI2QI [(V2SI "V8QI") (V4SI "V16QI")
- (VNx4SI "VNx16QI") (VNx2DI "VNx8HI")])
+ (VNx4SI "VNx16QI") (VNx2DI "VNx8HI")
+ (V2SF "V4HF") (V4SF "V8HF")])
;; Register suffix for DOTPROD input types from the return type.
@@ -2720,7 +2725,8 @@
(define_mode_attr f16quad [(V2SF "") (V4SF "q")])
-(define_mode_attr isquadop [(V8QI "") (V16QI "q") (V4BF "") (V8BF "q")])
+(define_mode_attr isquadop [(V8QI "") (V16QI "q") (V4BF "") (V8BF "q")
+ (V4HF "") (V8HF "q")])
(define_code_attr f16mac [(plus "a") (minus "s")])
diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
index 78052b229a58..498cd795c5c0 100644
--- a/gcc/doc/invoke.texi
+++ b/gcc/doc/invoke.texi
@@ -22099,6 +22099,8 @@ extension.
@item ssve-fp8dot2
Enable the fp8 (8-bit floating point) to half-precision 2-way dot product
extension in streaming mode.
+@item f16f32dot
+Enable the Half-precision to single-precision dot product extension.
@item faminmax
Enable the Floating Point Absolute Maximum/Minimum extension.
@item lut
diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-1.c
b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-1.c
new file mode 100644
index 000000000000..4abbdf57ef69
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-1.c
@@ -0,0 +1,68 @@
+/* { dg-do assemble { target { aarch64*-*-* } } } */
+/* { dg-require-effective-target aarch64_f16f32dot_neon_ok } */
+/* { dg-add-options aarch64_f16f32dot_neon } */
+/* { dg-additional-options "-save-temps" } */
+/* { dg-final { check-function-bodies "**" "" {-O[^0]} } } */
+/* { dg-skip-if "" { *-*-* } { "-fno-fat-lto-objects" } } */
+
+#include <arm_neon.h>
+
+/*
+**ufoo:
+** fdot v0.2s, (v1.4h, v2.4h|v2.4h, v1.4h)
+** ret
+*/
+float32x2_t ufoo(float32x2_t r, float16x4_t x, float16x4_t y)
+{
+ return vdot_f32_f16 (r, x, y);
+}
+
+/*
+**ufooq:
+** fdot v0.4s, (v1.8h, v2.8h|v2.8h, v1.8h)
+** ret
+*/
+float32x4_t ufooq(float32x4_t r, float16x8_t x, float16x8_t y)
+{
+ return vdotq_f32_f16 (r, x, y);
+}
+
+/*
+**ufoo_lane:
+** fdot v0.2s, v1.4h, v2.2h\[0\]
+** ret
+*/
+float32x2_t ufoo_lane(float32x2_t r, float16x4_t x, float16x4_t y)
+{
+ return vdot_lane_f32_f16 (r, x, y, 0);
+}
+
+/*
+**ufoo_laneq:
+** fdot v0.2s, v1.4h, v2.2h\[2\]
+** ret
+*/
+float32x2_t ufoo_laneq(float32x2_t r, float16x4_t x, float16x8_t y)
+{
+ return vdot_laneq_f32_f16 (r, x, y, 2);
+}
+
+/*
+**ufooq_lane:
+** fdot v0.4s, v1.8h, v2.2h\[1\]
+** ret
+*/
+float32x4_t ufooq_lane(float32x4_t r, float16x8_t x, float16x4_t y)
+{
+ return vdotq_lane_f32_f16 (r, x, y, 1);
+}
+
+/*
+**ufooq_laneq:
+** fdot v0.4s, v1.8h, v2.2h\[2\]
+** ret
+*/
+float32x4_t ufooq_laneq(float32x4_t r, float16x8_t x, float16x8_t y)
+{
+ return vdotq_laneq_f32_f16 (r, x, y, 2);
+}
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-2.c
b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-2.c
new file mode 100644
index 000000000000..b42bfa09c77c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-2.c
@@ -0,0 +1,70 @@
+/* { dg-do assemble { target { aarch64*-*-* } } } */
+/* { dg-require-effective-target stdint_types_mbig_endian } */
+/* { dg-require-effective-target aarch64_f16f32dot_neon_ok } */
+/* { dg-add-options aarch64_f16f32dot_neon } */
+/* { dg-additional-options "-mbig-endian --save-temps" } */
+/* { dg-final { check-function-bodies "**" "" {-O[^0]} } } */
+/* { dg-skip-if "" { *-*-* } { "-fno-fat-lto-objects" } } */
+
+
+#include <arm_neon.h>
+
+/*
+**ufoo:
+** fdot v0.2s, (v1.4h, v2.4h|v2.4h, v1.4h)
+** ret
+*/
+float32x2_t ufoo(float32x2_t r, float16x4_t x, float16x4_t y)
+{
+ return vdot_f32_f16 (r, x, y);
+}
+
+/*
+**ufooq:
+** fdot v0.4s, (v1.8h, v2.8h|v2.8h, v1.8h)
+** ret
+*/
+float32x4_t ufooq(float32x4_t r, float16x8_t x, float16x8_t y)
+{
+ return vdotq_f32_f16 (r, x, y);
+}
+
+/*
+**ufoo_lane:
+** fdot v0.2s, v1.4h, v2.2h\[0\]
+** ret
+*/
+float32x2_t ufoo_lane(float32x2_t r, float16x4_t x, float16x4_t y)
+{
+ return vdot_lane_f32_f16 (r, x, y, 0);
+}
+
+/*
+**ufoo_laneq:
+** fdot v0.2s, v1.4h, v2.2h\[2\]
+** ret
+*/
+float32x2_t ufoo_laneq(float32x2_t r, float16x4_t x, float16x8_t y)
+{
+ return vdot_laneq_f32_f16 (r, x, y, 2);
+}
+
+/*
+**ufooq_lane:
+** fdot v0.4s, v1.8h, v2.2h\[1\]
+** ret
+*/
+float32x4_t ufooq_lane(float32x4_t r, float16x8_t x, float16x4_t y)
+{
+ return vdotq_lane_f32_f16 (r, x, y, 1);
+}
+
+/*
+**ufooq_laneq:
+** fdot v0.4s, v1.8h, v2.2h\[2\]
+** ret
+*/
+float32x4_t ufooq_laneq(float32x4_t r, float16x8_t x, float16x8_t y)
+{
+ return vdotq_laneq_f32_f16 (r, x, y, 2);
+}
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-3.c
b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-3.c
new file mode 100644
index 000000000000..314322a08345
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-3.c
@@ -0,0 +1,32 @@
+/* { dg-do assemble { target { aarch64*-*-* } } } */
+/* { dg-skip-if "" { *-*-* } { "-fno-fat-lto-objects" } } */
+/* { dg-require-effective-target aarch64_f16f32dot_neon_ok } */
+/* { dg-add-options aarch64_f16f32dot_neon } */
+/* { dg-additional-options "--save-temps" } */
+
+#include <arm_neon.h>
+
+float32x2_t ufoo_lane(float32x2_t r, float16x4_t x, float16x4_t y)
+{
+ return vdot_lane_f32_f16 (r, x, y, 2);
+}
+
+float32x2_t ufoo_laneq(float32x2_t r, float16x4_t x, float16x8_t y)
+{
+ return vdot_laneq_f32_f16 (r, x, y, 4);
+}
+
+float32x4_t ufooq_lane(float32x4_t r, float16x8_t x, float16x4_t y)
+{
+ return vdotq_lane_f32_f16 (r, x, y, 3);
+}
+
+float32x4_t ufooq_laneq(float32x4_t r, float16x8_t x, float16x8_t y)
+{
+ return vdotq_laneq_f32_f16 (r, x, y, 5);
+}
+
+/* { dg-error {lane 2 out of range 0 - 1} "" { target *-*-* } 0 } */
+/* { dg-error {lane 4 out of range 0 - 3} "" { target *-*-* } 0 } */
+/* { dg-error {lane 3 out of range 0 - 1} "" { target *-*-* } 0 } */
+/* { dg-error {lane 5 out of range 0 - 3} "" { target *-*-* } 0 } */
\ No newline at end of file
diff --git a/gcc/testsuite/lib/target-supports.exp
b/gcc/testsuite/lib/target-supports.exp
index edd1290bd9e3..b494e0157136 100644
--- a/gcc/testsuite/lib/target-supports.exp
+++ b/gcc/testsuite/lib/target-supports.exp
@@ -7335,6 +7335,47 @@ proc add_options_for_arm_fp16fml_neon { flags } {
return "$flags $et_arm_fp16fml_neon_flags"
}
+# Return 1 if the target supports F16F32DOT
+# instructions, 0 otherwise. This test is valid
+# for AARCH64.
+# Record the command line options needed.
+
+proc check_effective_target_aarch64_f16f32dot_neon_ok_nocache { } {
+ global et_aarch64_f16f32dot_neon_flags
+ set et_aarch64_f16f32dot_neon_flags ""
+
+ if { ![istarget aarch64*-*-*] } {
+ return 0;
+ }
+
+ foreach flags {"" "" } {
+ if { [check_no_compiler_messages_nocache aarch64_f16f32dot_neon_ok
object {
+ #include <arm_neon.h>
+ #if !defined (__ARM_FEATURE_F16F32DOT)
+ #error "__ARM_FEATURE_F16F32DOT not defined"
+ #endif
+ } "$flags -march=armv8.2-a+fp16+f16f32dot"] } {
+ set et_aarch64_f16f32dot_neon_flags "$flags
-march=armv8.2-a+fp16+f16f32dot"
+ return 1
+ }
+ }
+
+ return 0;
+}
+
+proc check_effective_target_aarch64_f16f32dot_neon_ok { } {
+ return [check_cached_effective_target aarch64_f16f32dot_neon_ok \
+ check_effective_target_aarch64_f16f32dot_neon_ok_nocache]
+}
+
+proc add_options_for_aarch64_f16f32dot_neon { flags } {
+ if { ! [check_effective_target_aarch64_f16f32dot_neon_ok] } {
+ return "$flags"
+ }
+ global et_aarch64_f16f32dot_neon_flags
+ return "$flags $et_aarch64_f16f32dot_neon_flags"
+}
+
# Return 1 if the target supports BFloat16 SIMD instructions, 0 otherwise.
# The test is valid for ARM and for AArch64.
--
2.54.0
