From: Richard Ball <[email protected]>
---
.../aarch64/aarch64-sve-builtins-base.cc | 29 +++++-
.../aarch64/aarch64-sve-builtins-sve2.def | 5 +
gcc/config/aarch64/aarch64-sve-builtins.cc | 6 ++
gcc/config/aarch64/aarch64-sve2.md | 35 +++++++
.../aarch64/sme2/acle-asm/dot_lane_s16.c | 94 +++++++++++++++++++
.../aarch64/sme2/acle-asm/dot_lane_u16.c | 94 +++++++++++++++++++
.../aarch64/sme2/acle-asm/dot_s16.c | 45 +++++++++
.../aarch64/sme2/acle-asm/dot_u16.c | 45 +++++++++
8 files changed, 349 insertions(+), 4 deletions(-)
create mode 100644
gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/dot_lane_s16.c
create mode 100644
gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/dot_lane_u16.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/dot_s16.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/dot_u16.c
diff --git a/gcc/config/aarch64/aarch64-sve-builtins-base.cc
b/gcc/config/aarch64/aarch64-sve-builtins-base.cc
index 1fa7473283d..212002574e8 100644
--- a/gcc/config/aarch64/aarch64-sve-builtins-base.cc
+++ b/gcc/config/aarch64/aarch64-sve-builtins-base.cc
@@ -958,11 +958,23 @@ public:
0, e.result_mode (),
GET_MODE (e.args[0]));
else
- icode = (e.type_suffix (0).float_p
+ {
+ if (e.type_suffix (0).element_bits > 16)
+ {
+ icode = (e.type_suffix (0).float_p
? CODE_FOR_aarch64_sve_fdotvnx4sfvnx8hf
: e.type_suffix (0).unsigned_p
? CODE_FOR_udot_prodvnx4sivnx8hi
: CODE_FOR_sdot_prodvnx4sivnx8hi);
+ }
+ else
+ {
+ icode = (e.type_suffix (0).unsigned_p
+ ? CODE_FOR_udot_prodvnx8hivnx16qi
+ : CODE_FOR_sdot_prodvnx8hivnx16qi);
+ }
+ }
+
}
return e.use_unpred_insn (icode);
}
@@ -989,10 +1001,19 @@ public:
accumulator last. */
e.rotate_inputs_left (0, 4);
int unspec = unspec_for (e);
- if (unspec == UNSPEC_FDOT)
- icode = CODE_FOR_aarch64_fdot_prod_lanevnx4sfvnx8hf;
+ if (e.type_suffix (0).element_bits > 16)
+ {
+ if (unspec == UNSPEC_FDOT)
+ icode = CODE_FOR_aarch64_fdot_prod_lanevnx4sfvnx8hf;
+ else
+ icode = code_for_aarch64_dot_prod_lane (unspec, mode0, mode1);
+ }
else
- icode = code_for_aarch64_dot_prod_lane (unspec, mode0, mode1);
+ {
+ icode = (e.type_suffix (0).unsigned_p
+ ? CODE_FOR_udot_prod_lanevnx8hivnx16qi
+ : CODE_FOR_sdot_prod_lanevnx8hivnx16qi);
+ }
}
return e.use_exact_insn (icode);
}
diff --git a/gcc/config/aarch64/aarch64-sve-builtins-sve2.def
b/gcc/config/aarch64/aarch64-sve-builtins-sve2.def
index 1a55de890cf..2c9a89becd6 100644
--- a/gcc/config/aarch64/aarch64-sve-builtins-sve2.def
+++ b/gcc/config/aarch64/aarch64-sve-builtins-sve2.def
@@ -464,6 +464,11 @@ DEF_SVE_FUNCTION_GS_FPM (svmmla, mmla, h_float_mf8, none,
none, set)
DEF_SVE_FUNCTION_GS_FPM (svmmla, mmla, s_float_mf8, none, none, set)
#undef REQUIRED_EXTENSIONS
+#define REQUIRED_EXTENSIONS sve_and_sme (AARCH64_FL_SVE2p3, AARCH64_FL_SME2p3)
+DEF_SVE_FUNCTION (svdot, ternary_qq_opt_n_or_011, s_narrow_fsu_sve2p3, none)
+DEF_SVE_FUNCTION (svdot_lane, ternary_qq_or_011_lane, s_narrow_fsu_sve2p3,
none)
+#undef REQUIRED_EXTENSIONS
+
#define REQUIRED_EXTENSIONS nonstreaming_sve (AARCH64_FL_SVE_F16F32MM)
DEF_SVE_FUNCTION (svmmla, mmla, cvt_f32_f16, none)
#undef REQUIRED_EXTENSIONS
diff --git a/gcc/config/aarch64/aarch64-sve-builtins.cc
b/gcc/config/aarch64/aarch64-sve-builtins.cc
index 5617582315d..6edfd9a32e5 100644
--- a/gcc/config/aarch64/aarch64-sve-builtins.cc
+++ b/gcc/config/aarch64/aarch64-sve-builtins.cc
@@ -636,6 +636,11 @@ CONSTEXPR const group_suffix_info group_suffixes[] = {
#define TYPES_s_narrow_fsu(S, D, T) \
D (f32, f16), D (s32, s16), D (u32, u16)
+/* _s16_s8
+ _u16_u8. */
+#define TYPES_s_narrow_fsu_sve2p3(S, D, T) \
+ D (s16, s8), D (u16, u8)
+
/* _za8 _za16 _za32 _za64 _za128. */
#define TYPES_all_za(S, D, T) \
S (za8), S (za16), S (za32), S (za64), S (za128)
@@ -938,6 +943,7 @@ DEF_SVE_TYPES_ARRAY (while);
DEF_SVE_TYPES_ARRAY (while_x);
DEF_SVE_TYPES_ARRAY (while_x_c);
DEF_SVE_TYPES_ARRAY (s_narrow_fsu);
+DEF_SVE_TYPES_ARRAY (s_narrow_fsu_sve2p3);
DEF_SVE_TYPES_ARRAY (all_za);
DEF_SVE_TYPES_ARRAY (d_za);
DEF_SVE_TYPES_ARRAY (za_bhsd_data);
diff --git a/gcc/config/aarch64/aarch64-sve2.md
b/gcc/config/aarch64/aarch64-sve2.md
index 1d428619c07..2f8c61e115c 100644
--- a/gcc/config/aarch64/aarch64-sve2.md
+++ b/gcc/config/aarch64/aarch64-sve2.md
@@ -2861,6 +2861,41 @@ (define_insn "<sur>dot_prodvnx4sivnx8hi"
[(set_attr "sve_type" "sve_int_dot")]
)
+;; Two-way dot-product SME2p3 || SVE2p3.
+(define_insn "<sur>dot_prodvnx8hivnx16qi"
+ [(set (match_operand:VNx8HI 0 "register_operand")
+ (plus:VNx8HI
+ (unspec:VNx8HI
+ [(match_operand:VNx16QI 1 "register_operand")
+ (match_operand:VNx16QI 2 "register_operand")]
+ DOTPROD)
+ (match_operand:VNx8HI 3 "register_operand")))]
+ "TARGET_SVE2p3_OR_SME2p3"
+ {@ [ cons: =0 , 1 , 2 , 3 ; attrs: movprfx ]
+ [ w , w , w , 0 ; * ] <sur>dot\t%0.h, %1.b, %2.b
+ [ ?&w , w , w , w ; yes ] movprfx\t%0,
%3\;<sur>dot\t%0.h, %1.b, %2.b
+ }
+ [(set_attr "sve_type" "sve_int_dot")]
+)
+
+(define_insn "<sur>dot_prod_lanevnx8hivnx16qi"
+ [(set (match_operand:VNx8HI 0 "register_operand")
+ (plus:VNx8HI
+ (unspec:VNx8HI
+ [(match_operand:VNx16QI 1 "register_operand")
+ (unspec:VNx16QI
+ [(match_operand:VNx16QI 2 "register_operand")
+ (match_operand:SI 3 "const_int_operand")]
+ UNSPEC_SVE_LANE_SELECT)]
+ DOTPROD)
+ (match_operand:VNx8HI 4 "register_operand")))]
+ "TARGET_SVE2p3_OR_SME2p3"
+ {@ [ cons: =0 , 1 , 2 , 4 ; attrs: movprfx ]
+ [ w , w , y , 0 ; * ] <sur>dot\t%0.h, %1.b, %2.b[%3]
+ [ ?&w , w , y , w ; yes ] movprfx\t%0,
%4\;<sur>dot\t%0.h, %1.b, %2.b[%3]
+ }
+ [(set_attr "sve_type" "sve_fp_mul")]
+)
;; -------------------------------------------------------------------------
;; ---- [FP] Multi-register operations
;; -------------------------------------------------------------------------
diff --git a/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/dot_lane_s16.c
b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/dot_lane_s16.c
new file mode 100644
index 00000000000..f51cba33e69
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/dot_lane_s16.c
@@ -0,0 +1,94 @@
+/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */
+
+#include "test_sme2_acle.h"
+#pragma GCC target "+sme2p3"
+
+/*
+** dot_lane_0_s16_tied1:
+** sdot z0\.h, z4\.b, z5\.b\[0\]
+** ret
+*/
+TEST_DUAL_Z (dot_lane_0_s16_tied1, svint16_t, svint8_t,
+ z0 = svdot_lane_s16_s8 (z0, z4, z5, 0),
+ z0 = svdot_lane (z0, z4, z5, 0))
+
+/*
+** dot_lane_0_s16_tied2:
+** mov (z[0-9]+)\.d, z0\.d
+** movprfx z0, z4
+** sdot z0\.h, \1\.b, z1\.b\[0\]
+** ret
+*/
+TEST_DUAL_Z_REV (dot_lane_0_s16_tied2, svint16_t, svint8_t,
+ z0_res = svdot_lane_s16_s8 (z4, z0, z1, 0),
+ z0_res = svdot_lane (z4, z0, z1, 0))
+
+/*
+** dot_lane_0_s16_tied3:
+** mov (z[0-9]+)\.d, z0\.d
+** movprfx z0, z4
+** sdot z0\.h, z1\.b, \1\.b\[0\]
+** ret
+*/
+TEST_DUAL_Z_REV (dot_lane_0_s16_tied3, svint16_t, svint8_t,
+ z0_res = svdot_lane_s16_s8 (z4, z1, z0, 0),
+ z0_res = svdot_lane (z4, z1, z0, 0))
+
+/*
+** dot_lane_0_s16_untied:
+** movprfx z0, z1
+** sdot z0\.h, z4\.b, z5\.b\[0\]
+** ret
+*/
+TEST_DUAL_Z (dot_lane_0_s16_untied, svint16_t, svint8_t,
+ z0 = svdot_lane_s16_s8 (z1, z4, z5, 0),
+ z0 = svdot_lane (z1, z4, z5, 0))
+
+/*
+** dot_lane_1_s16:
+** sdot z0\.h, z4\.b, z5\.b\[1\]
+** ret
+*/
+TEST_DUAL_Z (dot_lane_1_s16, svint16_t, svint8_t,
+ z0 = svdot_lane_s16_s8 (z0, z4, z5, 1),
+ z0 = svdot_lane (z0, z4, z5, 1))
+
+/*
+** dot_lane_2_s16:
+** sdot z0\.h, z4\.b, z5\.b\[2\]
+** ret
+*/
+TEST_DUAL_Z (dot_lane_2_s16, svint16_t, svint8_t,
+ z0 = svdot_lane_s16_s8 (z0, z4, z5, 2),
+ z0 = svdot_lane (z0, z4, z5, 2))
+
+/*
+** dot_lane_3_s16:
+** sdot z0\.h, z4\.b, z5\.b\[3\]
+** ret
+*/
+TEST_DUAL_Z (dot_lane_3_s16, svint16_t, svint8_t,
+ z0 = svdot_lane_s16_s8 (z0, z4, z5, 3),
+ z0 = svdot_lane (z0, z4, z5, 3))
+
+/*
+** dot_lane_z8_s16:
+** str d8, \[sp, -16\]!
+** mov (z[0-7])\.d, z8\.d
+** sdot z0\.h, z1\.b, \1\.b\[1\]
+** ldr d8, \[sp\], 16
+** ret
+*/
+TEST_DUAL_LANE_REG (dot_lane_z8_s16, svint16_t, svint8_t, z8,
+ z0 = svdot_lane_s16_s8 (z0, z1, z8, 1),
+ z0 = svdot_lane (z0, z1, z8, 1))
+
+/*
+** dot_lane_z16_s16:
+** mov (z[0-7])\.d, z16\.d
+** sdot z0\.h, z1\.b, \1\.b\[1\]
+** ret
+*/
+TEST_DUAL_LANE_REG (dot_lane_z16_s16, svint16_t, svint8_t, z16,
+ z0 = svdot_lane_s16_s8 (z0, z1, z16, 1),
+ z0 = svdot_lane (z0, z1, z16, 1))
diff --git a/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/dot_lane_u16.c
b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/dot_lane_u16.c
new file mode 100644
index 00000000000..f6f5b81dec3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/dot_lane_u16.c
@@ -0,0 +1,94 @@
+/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */
+
+#include "test_sme2_acle.h"
+#pragma GCC target "+sme2p3"
+
+/*
+** dot_lane_0_u16_tied1:
+** udot z0\.h, z4\.b, z5\.b\[0\]
+** ret
+*/
+TEST_DUAL_Z (dot_lane_0_u16_tied1, svuint16_t, svuint8_t,
+ z0 = svdot_lane_u16_u8 (z0, z4, z5, 0),
+ z0 = svdot_lane (z0, z4, z5, 0))
+
+/*
+** dot_lane_0_u16_tied2:
+** mov (z[0-9]+)\.d, z0\.d
+** movprfx z0, z4
+** udot z0\.h, \1\.b, z1\.b\[0\]
+** ret
+*/
+TEST_DUAL_Z_REV (dot_lane_0_u16_tied2, svuint16_t, svuint8_t,
+ z0_res = svdot_lane_u16_u8 (z4, z0, z1, 0),
+ z0_res = svdot_lane (z4, z0, z1, 0))
+
+/*
+** dot_lane_0_u16_tied3:
+** mov (z[0-9]+)\.d, z0\.d
+** movprfx z0, z4
+** udot z0\.h, z1\.b, \1\.b\[0\]
+** ret
+*/
+TEST_DUAL_Z_REV (dot_lane_0_u16_tied3, svuint16_t, svuint8_t,
+ z0_res = svdot_lane_u16_u8 (z4, z1, z0, 0),
+ z0_res = svdot_lane (z4, z1, z0, 0))
+
+/*
+** dot_lane_0_u16_untied:
+** movprfx z0, z1
+** udot z0\.h, z4\.b, z5\.b\[0\]
+** ret
+*/
+TEST_DUAL_Z (dot_lane_0_u16_untied, svuint16_t, svuint8_t,
+ z0 = svdot_lane_u16_u8 (z1, z4, z5, 0),
+ z0 = svdot_lane (z1, z4, z5, 0))
+
+/*
+** dot_lane_1_u16:
+** udot z0\.h, z4\.b, z5\.b\[1\]
+** ret
+*/
+TEST_DUAL_Z (dot_lane_1_u16, svuint16_t, svuint8_t,
+ z0 = svdot_lane_u16_u8 (z0, z4, z5, 1),
+ z0 = svdot_lane (z0, z4, z5, 1))
+
+/*
+** dot_lane_2_u16:
+** udot z0\.h, z4\.b, z5\.b\[2\]
+** ret
+*/
+TEST_DUAL_Z (dot_lane_2_u16, svuint16_t, svuint8_t,
+ z0 = svdot_lane_u16_u8 (z0, z4, z5, 2),
+ z0 = svdot_lane (z0, z4, z5, 2))
+
+/*
+** dot_lane_3_u16:
+** udot z0\.h, z4\.b, z5\.b\[3\]
+** ret
+*/
+TEST_DUAL_Z (dot_lane_3_u16, svuint16_t, svuint8_t,
+ z0 = svdot_lane_u16_u8 (z0, z4, z5, 3),
+ z0 = svdot_lane (z0, z4, z5, 3))
+
+/*
+** dot_lane_z8_u16:
+** str d8, \[sp, -16\]!
+** mov (z[0-7])\.d, z8\.d
+** udot z0\.h, z1\.b, \1\.b\[1\]
+** ldr d8, \[sp\], 16
+** ret
+*/
+TEST_DUAL_LANE_REG (dot_lane_z8_u16, svuint16_t, svuint8_t, z8,
+ z0 = svdot_lane_u16_u8 (z0, z1, z8, 1),
+ z0 = svdot_lane (z0, z1, z8, 1))
+
+/*
+** dot_lane_z16_u16:
+** mov (z[0-7])\.d, z16\.d
+** udot z0\.h, z1\.b, \1\.b\[1\]
+** ret
+*/
+TEST_DUAL_LANE_REG (dot_lane_z16_u16, svuint16_t, svuint8_t, z16,
+ z0 = svdot_lane_u16_u8 (z0, z1, z16, 1),
+ z0 = svdot_lane (z0, z1, z16, 1))
diff --git a/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/dot_s16.c
b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/dot_s16.c
new file mode 100644
index 00000000000..dd1b827cbba
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/dot_s16.c
@@ -0,0 +1,45 @@
+/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */
+
+#include "test_sme2_acle.h"
+#pragma GCC target "+sme2p3"
+
+/*
+** dot_f32_tied1:
+** sdot z0\.h, z4\.b, z5\.b
+** ret
+*/
+TEST_DUAL_Z (dot_f32_tied1, svint16_t, svint8_t,
+ z0 = svdot_s16_s8 (z0, z4, z5),
+ z0 = svdot (z0, z4, z5))
+
+/*
+** dot_f32_tied2:
+** mov (z[0-9]+)\.d, z0\.d
+** movprfx z0, z4
+** sdot z0\.h, \1\.b, z1\.b
+** ret
+*/
+TEST_DUAL_Z_REV (dot_f32_tied2, svint16_t, svint8_t,
+ z0_res = svdot_s16_s8 (z4, z0, z1),
+ z0_res = svdot (z4, z0, z1))
+
+/*
+** dot_f32_tied3:
+** mov (z[0-9]+)\.d, z0\.d
+** movprfx z0, z4
+** sdot z0\.h, z1\.b, \1\.b
+** ret
+*/
+TEST_DUAL_Z_REV (dot_f32_tied3, svint16_t, svint8_t,
+ z0_res = svdot_s16_s8 (z4, z1, z0),
+ z0_res = svdot (z4, z1, z0))
+
+/*
+** dot_f32_untied:
+** movprfx z0, z1
+** sdot z0\.h, z4\.b, z5\.b
+** ret
+*/
+TEST_DUAL_Z (dot_f32_untied, svint16_t, svint8_t,
+ z0 = svdot_s16_s8 (z1, z4, z5),
+ z0 = svdot (z1, z4, z5))
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/dot_u16.c
b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/dot_u16.c
new file mode 100644
index 00000000000..3ea9fbfbcc3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/dot_u16.c
@@ -0,0 +1,45 @@
+/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */
+
+#include "test_sme2_acle.h"
+#pragma GCC target "+sme2p3"
+
+/*
+** dot_f32_tied1:
+** udot z0\.h, z4\.b, z5\.b
+** ret
+*/
+TEST_DUAL_Z (dot_f32_tied1, svuint16_t, svuint8_t,
+ z0 = svdot_u16_u8 (z0, z4, z5),
+ z0 = svdot (z0, z4, z5))
+
+/*
+** dot_f32_tied2:
+** mov (z[0-9]+)\.d, z0\.d
+** movprfx z0, z4
+** udot z0\.h, \1\.b, z1\.b
+** ret
+*/
+TEST_DUAL_Z_REV (dot_f32_tied2, svuint16_t, svuint8_t,
+ z0_res = svdot_u16_u8 (z4, z0, z1),
+ z0_res = svdot (z4, z0, z1))
+
+/*
+** dot_f32_tied3:
+** mov (z[0-9]+)\.d, z0\.d
+** movprfx z0, z4
+** udot z0\.h, z1\.b, \1\.b
+** ret
+*/
+TEST_DUAL_Z_REV (dot_f32_tied3, svuint16_t, svuint8_t,
+ z0_res = svdot_u16_u8 (z4, z1, z0),
+ z0_res = svdot (z4, z1, z0))
+
+/*
+** dot_f32_untied:
+** movprfx z0, z1
+** udot z0\.h, z4\.b, z5\.b
+** ret
+*/
+TEST_DUAL_Z (dot_f32_untied, svuint16_t, svuint8_t,
+ z0 = svdot_u16_u8 (z1, z4, z5),
+ z0 = svdot (z1, z4, z5))
\ No newline at end of file
--
2.43.0
