From: Ezra Sitorus <[email protected]>
Add new internal function COMPLEX_DOT_PROD_ROT0 and optab cdot_prod_rot0_optab
to represent complex integer dot product.
COMPLEX_DOT_PROD_ROT0 (A, B, ACC) is a lane-reducing signed integer complex
dot product. The 2 input vectors contain interleaved W-bit real and imaginary
components in even/odd numbered lanes respectively. Each 4W-bit result element
accumulates the 0-rotation products of the 2 complex input pairs in the
corresponding group of 4 input lanes:
result[j] = acc[j]
+ a[k].re * b[k].re
- a[k].im * b[k].im
+ a[k + 1].re * b[k + 1].re
- a[k + 1].im * b[k + 1].im
where 2*j = k.
0-rotation means that the imaginary components in the second source vector are
negated, and so the overall operation is accumulating the real part of the
complex dot product. Other rotations can be added in future patches.
In internal-fn.cc: complex_dot_prod_direct is defined to { -1, 1, false }
because it uses the return type as the accumulator/result type and the input
type is either of the narrower source type.
In AArch64, this corresponds to the CDOT instruction. This patch also teaches
GCC to convert the optab into the correct define_insn pattern.
gcc/ChangeLog:
* config/aarch64/aarch64-sve2.md (cdot_prod_rot0<mode><vsi2qi>): New
define_expand.
* internal-fn.cc (complex_dot_prod_direct): New static initializers.
* internal-fn.def (COMPLEX_DOT_PROD_ROT0) New internal function.
* optabs.def (cdot_prod_rot0_optab): New optab.
---
gcc/config/aarch64/aarch64-sve2.md | 10 ++++++++++
gcc/internal-fn.cc | 5 +++++
gcc/internal-fn.def | 1 +
gcc/optabs.def | 1 +
4 files changed, 17 insertions(+)
diff --git a/gcc/config/aarch64/aarch64-sve2.md
b/gcc/config/aarch64/aarch64-sve2.md
index c33df283cc2a..ad4febb60bf8 100644
--- a/gcc/config/aarch64/aarch64-sve2.md
+++ b/gcc/config/aarch64/aarch64-sve2.md
@@ -3578,6 +3578,16 @@
;; - CDOT
;; -------------------------------------------------------------------------
+(define_expand "cdot_prod_rot0<mode><vsi2qi>"
+ [(set (match_operand:SVE_FULL_SDI 0 "register_operand")
+ (unspec:SVE_FULL_SDI
+ [(match_operand:SVE_FULL_SDI 3 "register_operand")
+ (match_operand:<VSI2QI> 1 "register_operand")
+ (match_operand:<VSI2QI> 2 "register_operand")]
+ UNSPEC_CDOT))]
+ "TARGET_SVE2"
+)
+
(define_insn "@aarch64_sve_<optab><mode>"
[(set (match_operand:SVE_FULL_SDI 0 "register_operand")
(unspec:SVE_FULL_SDI
diff --git a/gcc/internal-fn.cc b/gcc/internal-fn.cc
index c57b594ac591..cc968109261b 100644
--- a/gcc/internal-fn.cc
+++ b/gcc/internal-fn.cc
@@ -180,6 +180,7 @@ init_internal_fns ()
#define unary_convert_direct { -1, 0, true }
#define binary_direct { 0, 0, true }
#define ternary_direct { 0, 0, true }
+#define complex_dot_prod_direct { -1, 1, false }
#define cond_unary_direct { 1, 1, true }
#define cond_binary_direct { 1, 1, true }
#define cond_ternary_direct { 1, 1, true }
@@ -4315,6 +4316,9 @@ expand_reduc_sbool_optab_fn (internal_fn fn, gcall *stmt,
direct_optab optab)
#define expand_select_vl_optab_fn(FN, STMT, OPTAB) \
expand_convert_optab_fn (FN, STMT, OPTAB, 3)
+#define expand_complex_dot_prod_optab_fn(FN, STMT, OPTAB) \
+ expand_convert_optab_fn (FN, STMT, OPTAB, 3)
+
/* Expanders for optabs that can use expand_convert_optab_fn. */
#define expand_unary_convert_optab_fn(FN, STMT, OPTAB) \
@@ -4398,6 +4402,7 @@ multi_vector_optab_supported_p (convert_optab optab,
tree_pair types,
#define direct_unary_convert_optab_supported_p convert_optab_supported_p
#define direct_binary_optab_supported_p direct_optab_supported_p
#define direct_ternary_optab_supported_p direct_optab_supported_p
+#define direct_complex_dot_prod_optab_supported_p convert_optab_supported_p
#define direct_cond_unary_optab_supported_p direct_optab_supported_p
#define direct_cond_binary_optab_supported_p direct_optab_supported_p
#define direct_cond_ternary_optab_supported_p direct_optab_supported_p
diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def
index af9f92950c73..08bdedc30045 100644
--- a/gcc/internal-fn.def
+++ b/gcc/internal-fn.def
@@ -453,6 +453,7 @@ DEF_INTERNAL_OPTAB_FN (COMPLEX_ADD_ROT90, ECF_CONST,
cadd90, binary)
DEF_INTERNAL_OPTAB_FN (COMPLEX_ADD_ROT270, ECF_CONST, cadd270, binary)
DEF_INTERNAL_OPTAB_FN (COMPLEX_MUL, ECF_CONST, cmul, binary)
DEF_INTERNAL_OPTAB_FN (COMPLEX_MUL_CONJ, ECF_CONST, cmul_conj, binary)
+DEF_INTERNAL_OPTAB_FN (COMPLEX_DOT_PROD_ROT0, ECF_CONST, cdot_prod_rot0,
complex_dot_prod)
DEF_INTERNAL_OPTAB_FN (VEC_ADDSUB, ECF_CONST, vec_addsub, binary)
DEF_INTERNAL_OPTAB_FN (VEC_TRUNC_ADD_HIGH, ECF_CONST | ECF_NOTHROW,
vec_trunc_add_high, binary)
diff --git a/gcc/optabs.def b/gcc/optabs.def
index 9d95f047cb50..b30e2bec6b2b 100644
--- a/gcc/optabs.def
+++ b/gcc/optabs.def
@@ -114,6 +114,7 @@ OPTAB_CD(vec_init_optab, "vec_init$a$b")
OPTAB_CD (sdot_prod_optab, "sdot_prod$I$a$b")
OPTAB_CD (udot_prod_optab, "udot_prod$I$a$b")
OPTAB_CD (usdot_prod_optab, "usdot_prod$I$a$b")
+OPTAB_CD (cdot_prod_rot0_optab, "cdot_prod_rot0$I$a$b")
OPTAB_CD (while_ult_optab, "while_ult$a$b")
OPTAB_CD (select_vl_optab, "select_vl$a$b")
--
2.53.0
