Hi,
I think this is going to be the last one, I'm running out of steam :)
This patch adds autovec patterns for Zvdota as well as tests.
The implementation is pretty mechanical, in particular the usual
VLS-mode massacre that will sure contain one or two mishaps.
The actual expansion borrows from our reduction expansion and
re-uses its M1 VLA mode accumulation idea.
The patch depends on Jiawei's series and I still wanted to send it
before stage 1 closes.
Regtested on top of Jiawei's series on rva23u64.
Assisted-By: Claude Code for the tests.
Regards
Robin
gcc/ChangeLog:
* config/riscv/autovec.md (sdot_prod<mode><v_quad_trunc>): New
pattern.
(udot_prod<mode><v_quad_trunc>): Ditto.
(usdot_prod<mode><v_quad_trunc>): Ditto.
(sdot_prod<mode><v_double_trunc_bf>): Ditto.
* config/riscv/riscv-protos.h (enum insn_flags): New flag for
dot prod.
(enum insn_type): New.
(expand_dot_prod): New function to expand dot products.
* config/riscv/riscv-v.cc (expand_dot_prod): Declare.
* config/riscv/vector-dot.md: Declare.
* config/riscv/vector-iterators.md: Move VI_Q iterator
to here...
* config/riscv/xuantie-vector.md: ...From here.
gcc/testsuite/ChangeLog:
* lib/target-supports.exp: Add Zvdota plumbing.
* gcc.target/riscv/rvv/autovec/reduc/dot-bf-neg-1.c: New test.
* gcc.target/riscv/rvv/autovec/reduc/dot-bf-neg-2.c: New test.
* gcc.target/riscv/rvv/autovec/reduc/dot-bf.c: New test.
* gcc.target/riscv/rvv/autovec/reduc/dot-hi-ss.c: New test.
* gcc.target/riscv/rvv/autovec/reduc/dot-hi-us.c: New test.
* gcc.target/riscv/rvv/autovec/reduc/dot-hi-uu.c: New test.
* gcc.target/riscv/rvv/autovec/reduc/dot-noext.c: New test.
* gcc.target/riscv/rvv/autovec/reduc/dot-qi-ss.c: New test.
* gcc.target/riscv/rvv/autovec/reduc/dot-qi-su.c: New test.
* gcc.target/riscv/rvv/autovec/reduc/dot-qi-us.c: New test.
* gcc.target/riscv/rvv/autovec/reduc/dot-qi-uu.c: New test.
* gcc.target/riscv/rvv/autovec/reduc/dot.h: New test.
* gcc.target/riscv/rvv/autovec/reduc/dot_run-bf.c: New test.
* gcc.target/riscv/rvv/autovec/reduc/dot_run-hi-ss.c: New test.
* gcc.target/riscv/rvv/autovec/reduc/dot_run-hi-us.c: New test.
* gcc.target/riscv/rvv/autovec/reduc/dot_run-hi-uu.c: New test.
* gcc.target/riscv/rvv/autovec/reduc/dot_run-qi-ss.c: New test.
* gcc.target/riscv/rvv/autovec/reduc/dot_run-qi-su.c: New test.
* gcc.target/riscv/rvv/autovec/reduc/dot_run-qi-us.c: New test.
* gcc.target/riscv/rvv/autovec/reduc/dot_run-qi-uu.c: New test.
* gcc.target/riscv/rvv/autovec/reduc/dot_run.h: New test.
---
gcc/config/riscv/autovec.md | 54 +++++++++
gcc/config/riscv/riscv-protos.h | 6 +
gcc/config/riscv/riscv-v.cc | 42 +++++++
gcc/config/riscv/vector-dot.md | 4 +-
gcc/config/riscv/vector-iterators.md | 105 +++++++++++++++++
gcc/config/riscv/xuantie-vector.md | 5 -
.../riscv/rvv/autovec/reduc/dot-bf-neg-1.c | 9 ++
.../riscv/rvv/autovec/reduc/dot-bf-neg-2.c | 12 ++
.../riscv/rvv/autovec/reduc/dot-bf.c | 13 +++
.../riscv/rvv/autovec/reduc/dot-hi-ss.c | 14 +++
.../riscv/rvv/autovec/reduc/dot-hi-us.c | 14 +++
.../riscv/rvv/autovec/reduc/dot-hi-uu.c | 14 +++
.../riscv/rvv/autovec/reduc/dot-noext.c | 11 ++
.../riscv/rvv/autovec/reduc/dot-qi-ss.c | 14 +++
.../riscv/rvv/autovec/reduc/dot-qi-su.c | 14 +++
.../riscv/rvv/autovec/reduc/dot-qi-us.c | 14 +++
.../riscv/rvv/autovec/reduc/dot-qi-uu.c | 14 +++
.../gcc.target/riscv/rvv/autovec/reduc/dot.h | 20 ++++
.../riscv/rvv/autovec/reduc/dot_run-bf.c | 30 +++++
.../riscv/rvv/autovec/reduc/dot_run-hi-ss.c | 16 +++
.../riscv/rvv/autovec/reduc/dot_run-hi-us.c | 16 +++
.../riscv/rvv/autovec/reduc/dot_run-hi-uu.c | 16 +++
.../riscv/rvv/autovec/reduc/dot_run-qi-ss.c | 16 +++
.../riscv/rvv/autovec/reduc/dot_run-qi-su.c | 16 +++
.../riscv/rvv/autovec/reduc/dot_run-qi-us.c | 16 +++
.../riscv/rvv/autovec/reduc/dot_run-qi-uu.c | 16 +++
.../riscv/rvv/autovec/reduc/dot_run.h | 49 ++++++++
gcc/testsuite/lib/target-supports.exp | 110 +++++++++++++++++-
28 files changed, 672 insertions(+), 8 deletions(-)
create mode 100644
gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-bf-neg-1.c
create mode 100644
gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-bf-neg-2.c
create mode 100644 gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-bf.c
create mode 100644 gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-hi-ss.c
create mode 100644 gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-hi-us.c
create mode 100644 gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-hi-uu.c
create mode 100644 gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-noext.c
create mode 100644 gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-qi-ss.c
create mode 100644 gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-qi-su.c
create mode 100644 gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-qi-us.c
create mode 100644 gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-qi-uu.c
create mode 100644 gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot.h
create mode 100644
gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-bf.c
create mode 100644
gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-hi-ss.c
create mode 100644
gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-hi-us.c
create mode 100644
gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-hi-uu.c
create mode 100644
gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-qi-ss.c
create mode 100644
gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-qi-su.c
create mode 100644
gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-qi-us.c
create mode 100644
gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-qi-uu.c
create mode 100644 gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run.h
diff --git a/gcc/config/riscv/autovec.md b/gcc/config/riscv/autovec.md
index 607b9891405..db1f599753c 100644
--- a/gcc/config/riscv/autovec.md
+++ b/gcc/config/riscv/autovec.md
@@ -1227,6 +1227,60 @@ (define_expand "<su>sad<mode><v_quad_trunc>"
DONE;
})
+;; -------------------------------------------------------------------------
+;; ---- [INT,FP] Dot product
+;; -------------------------------------------------------------------------
+;; Includes:
+;; - vqwdotas.vv
+;; - vqwdotau.vv
+;; - vfwdota.vv
+;; -------------------------------------------------------------------------
+
+(define_expand "sdot_prod<mode><v_quad_trunc>"
+ [(match_operand:VQEXTI_DOT 0 "register_operand")
+ (match_operand:<V_QUAD_TRUNC> 1 "register_operand")
+ (match_operand:<V_QUAD_TRUNC> 2 "register_operand")
+ (match_operand:VQEXTI_DOT 3 "register_operand")]
+ "TARGET_VECTOR"
+{
+ riscv_vector::expand_dot_prod (UNSPEC_VQWDOTA_SS, operands);
+ DONE;
+})
+
+(define_expand "udot_prod<mode><v_quad_trunc>"
+ [(match_operand:VQEXTI_DOT 0 "register_operand")
+ (match_operand:<V_QUAD_TRUNC> 1 "register_operand")
+ (match_operand:<V_QUAD_TRUNC> 2 "register_operand")
+ (match_operand:VQEXTI_DOT 3 "register_operand")]
+ "TARGET_VECTOR"
+{
+ riscv_vector::expand_dot_prod (UNSPEC_VQWDOTA_UU, operands);
+ DONE;
+})
+
+(define_expand "usdot_prod<mode><v_quad_trunc>"
+ [(match_operand:VQEXTI_DOT 0 "register_operand")
+ (match_operand:<V_QUAD_TRUNC> 1 "register_operand")
+ (match_operand:<V_QUAD_TRUNC> 2 "register_operand")
+ (match_operand:VQEXTI_DOT 3 "register_operand")]
+ "TARGET_VECTOR"
+{
+ riscv_vector::expand_dot_prod (UNSPEC_VQWDOTA_US, operands);
+ DONE;
+})
+
+;; ??? Can we enable these unconditionally?
+(define_expand "sdot_prod<mode><v_double_trunc_bf>"
+ [(match_operand:VWEXTF_DOT_BF 0 "register_operand")
+ (match_operand:<V_DOUBLE_TRUNC_BF> 1 "register_operand")
+ (match_operand:<V_DOUBLE_TRUNC_BF> 2 "register_operand")
+ (match_operand:VWEXTF_DOT_BF 3 "register_operand")]
+ "TARGET_VECTOR && flag_unsafe_math_optimizations && !flag_rounding_math"
+{
+ riscv_vector::expand_dot_prod (UNSPEC_VFWDOTA, operands);
+ DONE;
+})
+
;;
-------------------------------------------------------------------------------
;; ---- [FP] Unary operations
;;
-------------------------------------------------------------------------------
diff --git a/gcc/config/riscv/riscv-protos.h b/gcc/config/riscv/riscv-protos.h
index 75d812bba64..b1acbd131f8 100644
--- a/gcc/config/riscv/riscv-protos.h
+++ b/gcc/config/riscv/riscv-protos.h
@@ -449,6 +449,8 @@ enum insn_flags : unsigned int
/* Means INSN has VXRM operand and the value is VXRM_ROD. */
VXRM_ROD_P = 1 << 23,
+
+ VTYPE_MODE_FROM_OP2_P = 1 << 24,
};
enum insn_type : unsigned int
@@ -524,6 +526,9 @@ enum insn_type : unsigned int
| TDEFAULT_POLICY_P | MDEFAULT_POLICY_P | TERNARY_OP_P,
WIDEN_TERNARY_OP_FRM_DYN = WIDEN_TERNARY_OP | FRM_DYN_P,
+ DOT_PROD_OP = HAS_DEST_P | HAS_MASK_P | USE_ALL_TRUES_MASK_P | TU_POLICY_P
+ | MDEFAULT_POLICY_P | TERNARY_OP_P | VTYPE_MODE_FROM_OP2_P,
+
/* For vmerge, no mask operand, no mask policy operand. */
MERGE_OP = __NORMAL_OP_TA2 | TERNARY_OP_P,
@@ -696,6 +701,7 @@ bool expand_vec_cmp_float (rtx, rtx_code, rtx, rtx, bool,
rtx = nullptr,
void expand_cond_len_unop (unsigned, rtx *);
void expand_cond_len_binop (unsigned, rtx *);
void expand_reduction (unsigned, unsigned, unsigned, rtx *, rtx);
+void expand_dot_prod (unsigned, rtx *);
void expand_mask_reduction (rtx *, rtx_code);
void expand_vec_ceil (rtx, rtx, machine_mode, machine_mode);
void expand_vec_floor (rtx, rtx, machine_mode, machine_mode);
diff --git a/gcc/config/riscv/riscv-v.cc b/gcc/config/riscv/riscv-v.cc
index 6ae806e01aa..b188592dc43 100644
--- a/gcc/config/riscv/riscv-v.cc
+++ b/gcc/config/riscv/riscv-v.cc
@@ -240,6 +240,8 @@ public:
machine_mode vtype_mode;
if (m_insn_flags & VTYPE_MODE_FROM_OP1_P)
vtype_mode = GET_MODE (ops[1]);
+ else if (m_insn_flags & VTYPE_MODE_FROM_OP2_P)
+ vtype_mode = GET_MODE (ops[2]);
else
vtype_mode = GET_MODE (ops[0]);
return vtype_mode;
@@ -4992,6 +4994,46 @@ expand_cond_ternop (unsigned icode, rtx *ops)
expand_cond_len_op (icode, TERNARY_OP_P, cond_ops, len);
}
+/* Expand vfqdota and vfwdota. */
+
+void
+expand_dot_prod (unsigned unspec, rtx *ops)
+{
+ machine_mode acc_mode = GET_MODE (ops[0]);
+ machine_mode src_mode = GET_MODE (ops[1]);
+ machine_mode m1_mode = get_m1_mode (GET_MODE_INNER (acc_mode)).require ();
+
+ insn_code icode;
+ if (unspec == UNSPEC_VFWDOTA)
+ icode = code_for_pred_vfwdota_vv_zvdota (src_mode);
+ else
+ icode = code_for_pred_vv_zvdota (unspec, src_mode);
+
+ /* Perform our reduction "M1" trick, i.e. accumulate into an "VLA" M1 mode
+ for VLS by mode-punning input and output. */
+ if (riscv_vls_mode_p (acc_mode)
+ && !known_le (GET_MODE_BITSIZE (acc_mode), GET_MODE_BITSIZE (m1_mode)))
+ {
+ scalar_mode inner = GET_MODE_INNER (acc_mode);
+ machine_mode pun_mode
+ = get_vector_mode (inner, TARGET_MIN_VLEN
+ / GET_MODE_BITSIZE (inner)).require ();
+ rtx tmp = gen_reg_rtx (pun_mode);
+ emit_move_insn (tmp, lowpart_subreg (pun_mode, ops[3], acc_mode));
+ rtx acc_m1 = lowpart_subreg (m1_mode, tmp, pun_mode);
+ rtx dot_ops[] = {acc_m1, acc_m1, ops[1], ops[2]};
+ emit_vlmax_insn (icode, DOT_PROD_OP, dot_ops);
+ emit_move_insn (ops[0], ops[3]);
+ emit_move_insn (lowpart_subreg (pun_mode, ops[0], acc_mode), tmp);
+ return;
+ }
+
+ emit_move_insn (ops[0], ops[3]);
+ rtx acc = lowpart_subreg (m1_mode, ops[0], acc_mode);
+ rtx dot_ops[] = {acc, acc, ops[1], ops[2]};
+ emit_vlmax_insn (icode, DOT_PROD_OP, dot_ops);
+}
+
/* Expand reduction operations.
Case 1: ops = {scalar_dest, vector_src}
Case 2: ops = {scalar_dest, vector_src, mask, vl}
diff --git a/gcc/config/riscv/vector-dot.md b/gcc/config/riscv/vector-dot.md
index b5babf4df82..1c1225b9e32 100644
--- a/gcc/config/riscv/vector-dot.md
+++ b/gcc/config/riscv/vector-dot.md
@@ -64,8 +64,8 @@ (define_insn "@pred_vfwdota_vv_zvdota<mode>"
(reg:SI VL_REGNUM)
(reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
(unspec:<V_EXT_LMUL1>
- [(match_operand:VF_ZVFBF16 3 "register_operand" "vr")
- (match_operand:VF_ZVFBF16 4 "register_operand" "vr")
+ [(match_operand:VF_ZVFWDOTA_BF 3 "register_operand" "vr")
+ (match_operand:VF_ZVFWDOTA_BF 4 "register_operand" "vr")
(match_operand:<V_EXT_LMUL1> 2 "register_operand" "0")]
UNSPEC_VFWDOTA)
(match_dup 2)] UNSPEC_REDUC))]
diff --git a/gcc/config/riscv/vector-iterators.md
b/gcc/config/riscv/vector-iterators.md
index 274f1b17d0e..f186cb6bfe3 100644
--- a/gcc/config/riscv/vector-iterators.md
+++ b/gcc/config/riscv/vector-iterators.md
@@ -1508,6 +1508,28 @@ (define_mode_iterator VI_ZVQWDOTA [
(RVVM1HI "TARGET_ZVQWDOTA16I && TARGET_VECTOR_ELEN_64")
(RVVMF2HI "TARGET_ZVQWDOTA16I && TARGET_VECTOR_ELEN_64")
(RVVMF4HI "TARGET_ZVQWDOTA16I && TARGET_VECTOR_ELEN_64")
+
+ (V1QI "TARGET_ZVQWDOTA8I && riscv_vector::vls_mode_valid_p (V1QImode)")
+ (V2QI "TARGET_ZVQWDOTA8I && riscv_vector::vls_mode_valid_p (V2QImode)")
+ (V4QI "TARGET_ZVQWDOTA8I && riscv_vector::vls_mode_valid_p (V4QImode)")
+ (V8QI "TARGET_ZVQWDOTA8I && riscv_vector::vls_mode_valid_p (V8QImode)")
+ (V16QI "TARGET_ZVQWDOTA8I && riscv_vector::vls_mode_valid_p (V16QImode)")
+ (V32QI "TARGET_ZVQWDOTA8I && riscv_vector::vls_mode_valid_p (V32QImode)")
+ (V64QI "TARGET_ZVQWDOTA8I && riscv_vector::vls_mode_valid_p (V64QImode) &&
TARGET_MIN_VLEN >= 64")
+ (V128QI "TARGET_ZVQWDOTA8I && riscv_vector::vls_mode_valid_p (V128QImode) &&
TARGET_MIN_VLEN >= 128")
+ (V256QI "TARGET_ZVQWDOTA8I && riscv_vector::vls_mode_valid_p (V256QImode) &&
TARGET_MIN_VLEN >= 256")
+ (V512QI "TARGET_ZVQWDOTA8I && riscv_vector::vls_mode_valid_p (V512QImode) &&
TARGET_MIN_VLEN >= 512")
+ (V1024QI "TARGET_ZVQWDOTA8I && riscv_vector::vls_mode_valid_p (V1024QImode)
&& TARGET_MIN_VLEN >= 1024")
+ (V1HI "TARGET_ZVQWDOTA16I && riscv_vector::vls_mode_valid_p (V1HImode) &&
TARGET_VECTOR_ELEN_64")
+ (V2HI "TARGET_ZVQWDOTA16I && riscv_vector::vls_mode_valid_p (V2HImode) &&
TARGET_VECTOR_ELEN_64")
+ (V4HI "TARGET_ZVQWDOTA16I && riscv_vector::vls_mode_valid_p (V4HImode) &&
TARGET_VECTOR_ELEN_64")
+ (V8HI "TARGET_ZVQWDOTA16I && riscv_vector::vls_mode_valid_p (V8HImode) &&
TARGET_VECTOR_ELEN_64")
+ (V16HI "TARGET_ZVQWDOTA16I && riscv_vector::vls_mode_valid_p (V16HImode) &&
TARGET_VECTOR_ELEN_64")
+ (V32HI "TARGET_ZVQWDOTA16I && riscv_vector::vls_mode_valid_p (V32HImode) &&
TARGET_VECTOR_ELEN_64 && TARGET_MIN_VLEN >= 64")
+ (V64HI "TARGET_ZVQWDOTA16I && riscv_vector::vls_mode_valid_p (V64HImode) &&
TARGET_VECTOR_ELEN_64 && TARGET_MIN_VLEN >= 128")
+ (V128HI "TARGET_ZVQWDOTA16I && riscv_vector::vls_mode_valid_p (V128HImode)
&& TARGET_VECTOR_ELEN_64 && TARGET_MIN_VLEN >= 256")
+ (V256HI "TARGET_ZVQWDOTA16I && riscv_vector::vls_mode_valid_p (V256HImode)
&& TARGET_VECTOR_ELEN_64 && TARGET_MIN_VLEN >= 512")
+ (V512HI "TARGET_ZVQWDOTA16I && riscv_vector::vls_mode_valid_p (V512HImode)
&& TARGET_VECTOR_ELEN_64 && TARGET_MIN_VLEN >= 1024")
])
;; Scalable integer accumulator modes for batched dot products.
@@ -1530,6 +1552,79 @@ (define_mode_iterator VF_S_NOFRAC [
(RVVM1SF "TARGET_VECTOR_ELEN_FP_32")
])
+(define_mode_iterator VQEXTI_DOT [
+ (RVVM8SI "TARGET_ZVQWDOTA8I")
+ (RVVM4SI "TARGET_ZVQWDOTA8I")
+ (RVVM2SI "TARGET_ZVQWDOTA8I")
+ (RVVM1SI "TARGET_ZVQWDOTA8I")
+ (RVVM8DI "TARGET_ZVQWDOTA16I && TARGET_VECTOR_ELEN_64")
+ (RVVM4DI "TARGET_ZVQWDOTA16I && TARGET_VECTOR_ELEN_64")
+ (RVVM2DI "TARGET_ZVQWDOTA16I && TARGET_VECTOR_ELEN_64")
+ (RVVM1DI "TARGET_ZVQWDOTA16I && TARGET_VECTOR_ELEN_64")
+
+ (V1SI "TARGET_ZVQWDOTA8I && riscv_vector::vls_mode_valid_p (V1SImode)")
+ (V2SI "TARGET_ZVQWDOTA8I && riscv_vector::vls_mode_valid_p (V2SImode)")
+ (V4SI "TARGET_ZVQWDOTA8I && riscv_vector::vls_mode_valid_p (V4SImode)")
+ (V8SI "TARGET_ZVQWDOTA8I && riscv_vector::vls_mode_valid_p (V8SImode)")
+ (V16SI "TARGET_ZVQWDOTA8I && riscv_vector::vls_mode_valid_p (V16SImode) &&
TARGET_MIN_VLEN >= 64")
+ (V32SI "TARGET_ZVQWDOTA8I && riscv_vector::vls_mode_valid_p (V32SImode) &&
TARGET_MIN_VLEN >= 128")
+ (V64SI "TARGET_ZVQWDOTA8I && riscv_vector::vls_mode_valid_p (V64SImode) &&
TARGET_MIN_VLEN >= 256")
+ (V128SI "TARGET_ZVQWDOTA8I && riscv_vector::vls_mode_valid_p (V128SImode) &&
TARGET_MIN_VLEN >= 512")
+ (V256SI "TARGET_ZVQWDOTA8I && riscv_vector::vls_mode_valid_p (V256SImode) &&
TARGET_MIN_VLEN >= 1024")
+ (V512SI "TARGET_ZVQWDOTA8I && riscv_vector::vls_mode_valid_p (V512SImode) &&
TARGET_MIN_VLEN >= 2048")
+ (V1024SI "TARGET_ZVQWDOTA8I && riscv_vector::vls_mode_valid_p (V1024SImode)
&& TARGET_MIN_VLEN >= 4096")
+ (V1DI "TARGET_ZVQWDOTA16I && riscv_vector::vls_mode_valid_p (V1DImode) &&
TARGET_VECTOR_ELEN_64")
+ (V2DI "TARGET_ZVQWDOTA16I && riscv_vector::vls_mode_valid_p (V2DImode) &&
TARGET_VECTOR_ELEN_64")
+ (V4DI "TARGET_ZVQWDOTA16I && riscv_vector::vls_mode_valid_p (V4DImode) &&
TARGET_VECTOR_ELEN_64")
+ (V8DI "TARGET_ZVQWDOTA16I && riscv_vector::vls_mode_valid_p (V8DImode) &&
TARGET_VECTOR_ELEN_64 && TARGET_MIN_VLEN >= 64")
+ (V16DI "TARGET_ZVQWDOTA16I && riscv_vector::vls_mode_valid_p (V16DImode) &&
TARGET_VECTOR_ELEN_64 && TARGET_MIN_VLEN >= 128")
+ (V32DI "TARGET_ZVQWDOTA16I && riscv_vector::vls_mode_valid_p (V32DImode) &&
TARGET_VECTOR_ELEN_64 && TARGET_MIN_VLEN >= 256")
+ (V64DI "TARGET_ZVQWDOTA16I && riscv_vector::vls_mode_valid_p (V64DImode) &&
TARGET_VECTOR_ELEN_64 && TARGET_MIN_VLEN >= 512")
+ (V128DI "TARGET_ZVQWDOTA16I && riscv_vector::vls_mode_valid_p (V128DImode)
&& TARGET_VECTOR_ELEN_64 && TARGET_MIN_VLEN >= 1024")
+ (V256DI "TARGET_ZVQWDOTA16I && riscv_vector::vls_mode_valid_p (V256DImode)
&& TARGET_VECTOR_ELEN_64 && TARGET_MIN_VLEN >= 2048")
+ (V512DI "TARGET_ZVQWDOTA16I && riscv_vector::vls_mode_valid_p (V512DImode)
&& TARGET_VECTOR_ELEN_64 && TARGET_MIN_VLEN >= 4096")
+])
+
+(define_mode_iterator VWEXTF_DOT_BF [
+ (RVVM8SF "TARGET_ZVFWDOTA16BF && TARGET_VECTOR_ELEN_FP_32
+ && TARGET_VECTOR_ELEN_BF_16")
+ (RVVM4SF "TARGET_ZVFWDOTA16BF && TARGET_VECTOR_ELEN_FP_32
+ && TARGET_VECTOR_ELEN_BF_16")
+ (RVVM2SF "TARGET_ZVFWDOTA16BF && TARGET_VECTOR_ELEN_FP_32
+ && TARGET_VECTOR_ELEN_BF_16")
+ (RVVM1SF "TARGET_ZVFWDOTA16BF && TARGET_VECTOR_ELEN_FP_32
+ && TARGET_VECTOR_ELEN_BF_16")
+ (V1SF "TARGET_ZVFWDOTA16BF && TARGET_VECTOR_ELEN_FP_32 &&
TARGET_VECTOR_ELEN_BF_16 && riscv_vector::vls_mode_valid_p (V1SFmode)")
+ (V2SF "TARGET_ZVFWDOTA16BF && TARGET_VECTOR_ELEN_FP_32 &&
TARGET_VECTOR_ELEN_BF_16 && riscv_vector::vls_mode_valid_p (V2SFmode)")
+ (V4SF "TARGET_ZVFWDOTA16BF && TARGET_VECTOR_ELEN_FP_32 &&
TARGET_VECTOR_ELEN_BF_16 && riscv_vector::vls_mode_valid_p (V4SFmode)")
+ (V8SF "TARGET_ZVFWDOTA16BF && TARGET_VECTOR_ELEN_FP_32 &&
TARGET_VECTOR_ELEN_BF_16 && riscv_vector::vls_mode_valid_p (V8SFmode)")
+ (V16SF "TARGET_ZVFWDOTA16BF && TARGET_VECTOR_ELEN_FP_32 &&
TARGET_VECTOR_ELEN_BF_16 && riscv_vector::vls_mode_valid_p (V16SFmode) &&
TARGET_MIN_VLEN >= 64")
+ (V32SF "TARGET_ZVFWDOTA16BF && TARGET_VECTOR_ELEN_FP_32 &&
TARGET_VECTOR_ELEN_BF_16 && riscv_vector::vls_mode_valid_p (V32SFmode) &&
TARGET_MIN_VLEN >= 128")
+ (V64SF "TARGET_ZVFWDOTA16BF && TARGET_VECTOR_ELEN_FP_32 &&
TARGET_VECTOR_ELEN_BF_16 && riscv_vector::vls_mode_valid_p (V64SFmode) &&
TARGET_MIN_VLEN >= 256")
+ (V128SF "TARGET_ZVFWDOTA16BF && TARGET_VECTOR_ELEN_FP_32 &&
TARGET_VECTOR_ELEN_BF_16 && riscv_vector::vls_mode_valid_p (V128SFmode) &&
TARGET_MIN_VLEN >= 512")
+ (V256SF "TARGET_ZVFWDOTA16BF && TARGET_VECTOR_ELEN_FP_32 &&
TARGET_VECTOR_ELEN_BF_16 && riscv_vector::vls_mode_valid_p (V256SFmode) &&
TARGET_MIN_VLEN >= 1024")
+ (V512SF "TARGET_ZVFWDOTA16BF && TARGET_VECTOR_ELEN_FP_32 &&
TARGET_VECTOR_ELEN_BF_16 && riscv_vector::vls_mode_valid_p (V512SFmode) &&
TARGET_MIN_VLEN >= 2048")
+ (V1024SF "TARGET_ZVFWDOTA16BF && TARGET_VECTOR_ELEN_FP_32 &&
TARGET_VECTOR_ELEN_BF_16 && riscv_vector::vls_mode_valid_p (V1024SFmode) &&
TARGET_MIN_VLEN >= 4096")
+])
+
+(define_mode_iterator VF_ZVFWDOTA_BF [VF_ZVFBF16 VLSF_ZVFBF16])
+
+(define_mode_attr V_DOUBLE_TRUNC_BF [
+ (RVVM8SF "RVVM4BF") (RVVM4SF "RVVM2BF") (RVVM2SF "RVVM1BF")
+ (RVVM1SF "RVVMF2BF")
+ (V1SF "V1BF") (V2SF "V2BF") (V4SF "V4BF") (V8SF "V8BF")
+ (V16SF "V16BF") (V32SF "V32BF") (V64SF "V64BF") (V128SF "V128BF")
+ (V256SF "V256BF") (V512SF "V512BF") (V1024SF "V1024BF")
+])
+
+(define_mode_attr v_double_trunc_bf [
+ (RVVM8SF "rvvm4bf") (RVVM4SF "rvvm2bf") (RVVM2SF "rvvm1bf")
+ (RVVM1SF "rvvmf2bf")
+ (V1SF "v1bf") (V2SF "v2bf") (V4SF "v4bf") (V8SF "v8bf")
+ (V16SF "v16bf") (V32SF "v32bf") (V64SF "v64bf") (V128SF "v128bf")
+ (V256SF "v256bf") (V512SF "v512bf") (V1024SF "v1024bf")
+])
+
;; Quadruple the element width and use LMUL=1.
(define_mode_attr V_QUAD_EXT_LMUL1 [
(RVVMF8QI "RVVM1SI")
@@ -1545,6 +1640,12 @@ (define_mode_attr V_QUAD_EXT_LMUL1 [
(RVVM2HI "RVVM1DI")
(RVVM4HI "RVVM1DI")
(RVVM8HI "RVVM1DI")
+ (V1QI "RVVM1SI") (V2QI "RVVM1SI") (V4QI "RVVM1SI") (V8QI "RVVM1SI")
+ (V16QI "RVVM1SI") (V32QI "RVVM1SI") (V64QI "RVVM1SI") (V128QI "RVVM1SI")
+ (V256QI "RVVM1SI") (V512QI "RVVM1SI") (V1024QI "RVVM1SI")
+ (V1HI "RVVM1DI") (V2HI "RVVM1DI") (V4HI "RVVM1DI") (V8HI "RVVM1DI")
+ (V16HI "RVVM1DI") (V32HI "RVVM1DI") (V64HI "RVVM1DI") (V128HI "RVVM1DI")
+ (V256HI "RVVM1DI") (V512HI "RVVM1DI")
])
;; Quarter the element width and select a fixed register-group size.
@@ -1872,6 +1973,10 @@ (define_mode_attr V_EXT_LMUL1 [
(RVVM2BF "RVVM1SF")
(RVVM4BF "RVVM1SF")
(RVVM8BF "RVVM1SF")
+ (V1BF "RVVM1SF") (V2BF "RVVM1SF") (V4BF "RVVM1SF") (V8BF "RVVM1SF")
+ (V16BF "RVVM1SF") (V32BF "RVVM1SF") (V64BF "RVVM1SF") (V128BF "RVVM1SF")
+ (V256BF "RVVM1SF") (V512BF "RVVM1SF") (V1024BF "RVVM1SF")
+ (V2048BF "RVVM1SF")
(RVVM8QI "RVVM1HI") (RVVM4QI "RVVM1HI") (RVVM2QI "RVVM1HI") (RVVM1QI
"RVVM1HI") (RVVMF2QI "RVVM1HI") (RVVMF4QI "RVVM1HI") (RVVMF8QI "RVVM1HI")
diff --git a/gcc/config/riscv/xuantie-vector.md
b/gcc/config/riscv/xuantie-vector.md
index 00e9659b08a..d4a62b9faba 100644
--- a/gcc/config/riscv/xuantie-vector.md
+++ b/gcc/config/riscv/xuantie-vector.md
@@ -48,11 +48,6 @@ (define_int_attr xxtvarith_ins_name [
(UNSPEC_XT_VCRCFOLDR "vcrcfoldr") (UNSPEC_XT_VCRCFOLDN "vcrcfoldn")
])
-(define_mode_iterator VI_Q [
- RVVM8QI RVVM4QI RVVM2QI RVVM1QI RVVMF2QI RVVMF4QI
- (RVVMF8QI "TARGET_MIN_VLEN > 32")
-])
-
(define_insn "@xt_pred_<xxtvarith_ins_name><mode>"
[(set (match_operand:VI_Q 0 "register_operand" "=vr")
(if_then_else:VI_Q
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-bf-neg-1.c
b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-bf-neg-1.c
new file mode 100644
index 00000000000..390892d8bd9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-bf-neg-1.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-march=rva23u64_zvfwdota16bf -mabi=lp64d
-mrvv-vector-bits=scalable -fno-associative-math -fno-vect-cost-model
-fdump-tree-vect-details" } */
+
+#include "dot.h"
+
+DEF_DOT (float, __bf16, __bf16, dot)
+
+/* { dg-final { scan-tree-dump-not "vect_recog_dot_prod_pattern: detected"
"vect" } } */
+/* { dg-final { scan-assembler-not {\tvfwdota\.vv\t} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-bf-neg-2.c
b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-bf-neg-2.c
new file mode 100644
index 00000000000..2b0b36eb54e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-bf-neg-2.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-march=rva23u64_zvfwdota16bf -mabi=lp64d
-mrvv-vector-bits=scalable -ffast-math -fno-vect-cost-model" } */
+
+float
+dot (float sum, __bf16 *__restrict a, __bf16 *__restrict b, int n)
+{
+ for (int i = 0; i < n; i++)
+ sum += (float) (__bf16) (a[i] * b[i]);
+ return sum;
+}
+
+/* { dg-final { scan-assembler-not {\tvfwdota\.vv\t} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-bf.c
b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-bf.c
new file mode 100644
index 00000000000..eb5c1c17562
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-bf.c
@@ -0,0 +1,13 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-march=rva23u64_zvfwdota16bf -mabi=lp64d
-mrvv-vector-bits=scalable -ffast-math -fno-vect-cost-model --param
vect-epilogues-nomask=0 -fdump-tree-vect-details" } */
+
+#include "dot.h"
+
+DEF_DOT (float, __bf16, __bf16, dot)
+
+/* { dg-final { scan-tree-dump "vect_recog_dot_prod_pattern: detected" "vect"
} } */
+/* { dg-final { scan-assembler {\tvfwdota\.vv\t} } } */
+/* { dg-final { scan-assembler
{vsetvli\t[a-z0-9]+,\s*[a-z0-9]+,\s*e16alt,\s*mf?[1248],\s*tu,\s*m[au]} } } */
+/* { dg-final { scan-assembler-not {\tvfwmul} } } */
+/* { dg-final { scan-assembler-not {\tvfwmacc} } } */
+/* { dg-final { scan-assembler-not {\tvfwcvt} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-hi-ss.c
b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-hi-ss.c
new file mode 100644
index 00000000000..ef662b0e641
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-hi-ss.c
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-march=rva23u64_zvqwdota16i -mabi=lp64d
-mrvv-vector-bits=scalable -fno-vect-cost-model --param vect-epilogues-nomask=0
-fdump-tree-vect-details" } */
+
+#include "dot.h"
+
+DEF_DOT (int64_t, int16_t, int16_t, dot)
+
+/* { dg-final { scan-tree-dump "vect_recog_dot_prod_pattern: detected" "vect"
} } */
+/* { dg-final { scan-assembler {\tvqwdotas\.vv\t} } } */
+/* { dg-final { scan-assembler
{vsetvli\t[a-z0-9]+,\s*[a-z0-9]+,\s*e16alt,\s*mf?[1248],\s*tu,\s*m[au]} } } */
+/* { dg-final { scan-assembler-not {\tvwmul} } } */
+/* { dg-final { scan-assembler-not {\tvwmacc} } } */
+/* { dg-final { scan-assembler-not {\tvsext} } } */
+/* { dg-final { scan-assembler-not {\tvzext} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-hi-us.c
b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-hi-us.c
new file mode 100644
index 00000000000..861d471684a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-hi-us.c
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-march=rva23u64_zvqwdota16i -mabi=lp64d
-mrvv-vector-bits=scalable -fno-vect-cost-model --param vect-epilogues-nomask=0
-fdump-tree-vect-details" } */
+
+#include "dot.h"
+
+DEF_DOT (int64_t, uint16_t, int16_t, dot)
+
+/* { dg-final { scan-tree-dump "vect_recog_dot_prod_pattern: detected" "vect"
} } */
+/* { dg-final { scan-assembler {\tvqwdotau\.vv\t} } } */
+/* { dg-final { scan-assembler
{vsetvli\t[a-z0-9]+,\s*[a-z0-9]+,\s*e16alt,\s*mf?[1248],\s*tu,\s*m[au]} } } */
+/* { dg-final { scan-assembler-not {\tvwmul} } } */
+/* { dg-final { scan-assembler-not {\tvwmacc} } } */
+/* { dg-final { scan-assembler-not {\tvsext} } } */
+/* { dg-final { scan-assembler-not {\tvzext} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-hi-uu.c
b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-hi-uu.c
new file mode 100644
index 00000000000..8d09104e5ee
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-hi-uu.c
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-march=rva23u64_zvqwdota16i -mabi=lp64d
-mrvv-vector-bits=scalable -fno-vect-cost-model --param vect-epilogues-nomask=0
-fdump-tree-vect-details" } */
+
+#include "dot.h"
+
+DEF_DOT (uint64_t, uint16_t, uint16_t, dot)
+
+/* { dg-final { scan-tree-dump "vect_recog_dot_prod_pattern: detected" "vect"
} } */
+/* { dg-final { scan-assembler {\tvqwdotau\.vv\t} } } */
+/* { dg-final { scan-assembler
{vsetvli\t[a-z0-9]+,\s*[a-z0-9]+,\s*e16,\s*mf?[1248],\s*tu,\s*m[au]} } } */
+/* { dg-final { scan-assembler-not {\tvwmul} } } */
+/* { dg-final { scan-assembler-not {\tvwmacc} } } */
+/* { dg-final { scan-assembler-not {\tvsext} } } */
+/* { dg-final { scan-assembler-not {\tvzext} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-noext.c
b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-noext.c
new file mode 100644
index 00000000000..aed97ef9ee3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-noext.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-march=rva23u64 -mabi=lp64d
-mrvv-vector-bits=scalable -ffast-math -fno-vect-cost-model" } */
+
+#include "dot.h"
+
+DEF_DOT (int32_t, int8_t, int8_t, dot_ss)
+DEF_DOT (uint64_t, uint16_t, uint16_t, dot_uu)
+DEF_DOT (float, __bf16, __bf16, dot_bf)
+
+/* { dg-final { scan-assembler-not {\tvqwdota[su]\.vv\t} } } */
+/* { dg-final { scan-assembler-not {\tvfwdota\.vv\t} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-qi-ss.c
b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-qi-ss.c
new file mode 100644
index 00000000000..6e82bc43f38
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-qi-ss.c
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-march=rva23u64_zvqwdota8i -mabi=lp64d
-mrvv-vector-bits=scalable -fno-vect-cost-model --param vect-epilogues-nomask=0
-fdump-tree-vect-details" } */
+
+#include "dot.h"
+
+DEF_DOT (int32_t, int8_t, int8_t, dot)
+
+/* { dg-final { scan-tree-dump "vect_recog_dot_prod_pattern: detected" "vect"
} } */
+/* { dg-final { scan-assembler {\tvqwdotas\.vv\t} } } */
+/* { dg-final { scan-assembler
{vsetvli\t[a-z0-9]+,\s*[a-z0-9]+,\s*e8alt,\s*mf?[1248],\s*tu,\s*m[au]} } } */
+/* { dg-final { scan-assembler-not {\tvwmul} } } */
+/* { dg-final { scan-assembler-not {\tvwmacc} } } */
+/* { dg-final { scan-assembler-not {\tvsext} } } */
+/* { dg-final { scan-assembler-not {\tvzext} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-qi-su.c
b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-qi-su.c
new file mode 100644
index 00000000000..cb739e47f58
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-qi-su.c
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-march=rva23u64_zvqwdota8i -mabi=lp64d
-mrvv-vector-bits=scalable -fno-vect-cost-model --param vect-epilogues-nomask=0
-fdump-tree-vect-details" } */
+
+#include "dot.h"
+
+DEF_DOT (int32_t, int8_t, uint8_t, dot)
+
+/* { dg-final { scan-tree-dump "vect_recog_dot_prod_pattern: detected" "vect"
} } */
+/* { dg-final { scan-assembler {\tvqwdotau\.vv\t} } } */
+/* { dg-final { scan-assembler
{vsetvli\t[a-z0-9]+,\s*[a-z0-9]+,\s*e8alt,\s*mf?[1248],\s*tu,\s*m[au]} } } */
+/* { dg-final { scan-assembler-not {\tvwmul} } } */
+/* { dg-final { scan-assembler-not {\tvwmacc} } } */
+/* { dg-final { scan-assembler-not {\tvsext} } } */
+/* { dg-final { scan-assembler-not {\tvzext} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-qi-us.c
b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-qi-us.c
new file mode 100644
index 00000000000..16d85ebed4f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-qi-us.c
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-march=rva23u64_zvqwdota8i -mabi=lp64d
-mrvv-vector-bits=scalable -fno-vect-cost-model --param vect-epilogues-nomask=0
-fdump-tree-vect-details" } */
+
+#include "dot.h"
+
+DEF_DOT (int32_t, uint8_t, int8_t, dot)
+
+/* { dg-final { scan-tree-dump "vect_recog_dot_prod_pattern: detected" "vect"
} } */
+/* { dg-final { scan-assembler {\tvqwdotau\.vv\t} } } */
+/* { dg-final { scan-assembler
{vsetvli\t[a-z0-9]+,\s*[a-z0-9]+,\s*e8alt,\s*mf?[1248],\s*tu,\s*m[au]} } } */
+/* { dg-final { scan-assembler-not {\tvwmul} } } */
+/* { dg-final { scan-assembler-not {\tvwmacc} } } */
+/* { dg-final { scan-assembler-not {\tvsext} } } */
+/* { dg-final { scan-assembler-not {\tvzext} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-qi-uu.c
b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-qi-uu.c
new file mode 100644
index 00000000000..288cbdfd0a8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot-qi-uu.c
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-march=rva23u64_zvqwdota8i -mabi=lp64d
-mrvv-vector-bits=scalable -fno-vect-cost-model --param vect-epilogues-nomask=0
-fdump-tree-vect-details" } */
+
+#include "dot.h"
+
+DEF_DOT (uint32_t, uint8_t, uint8_t, dot)
+
+/* { dg-final { scan-tree-dump "vect_recog_dot_prod_pattern: detected" "vect"
} } */
+/* { dg-final { scan-assembler {\tvqwdotau\.vv\t} } } */
+/* { dg-final { scan-assembler
{vsetvli\t[a-z0-9]+,\s*[a-z0-9]+,\s*e8,\s*mf?[1248],\s*tu,\s*m[au]} } } */
+/* { dg-final { scan-assembler-not {\tvwmul} } } */
+/* { dg-final { scan-assembler-not {\tvwmacc} } } */
+/* { dg-final { scan-assembler-not {\tvsext} } } */
+/* { dg-final { scan-assembler-not {\tvzext} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot.h
b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot.h
new file mode 100644
index 00000000000..5a1f25ea201
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot.h
@@ -0,0 +1,20 @@
+#include <stdint-gcc.h>
+
+#define DEF_DOT(ACC, TA, TB, NAME) \
+ __attribute__ ((noipa)) ACC \
+ NAME (ACC sum, TA *__restrict a, TB *__restrict b, int n) \
+ { \
+ for (int i = 0; i < n; i++)
\
+ sum += (ACC) a[i] * (ACC) b[i]; \
+ return sum;
\
+ }
+
+#define DEF_DOT_REF(ACC, TA, TB, NAME) \
+ __attribute__ ((noipa)) ACC \
+ NAME##_ref (ACC sum, TA *__restrict a, TB *__restrict b, int n) \
+ { \
+ _Pragma ("GCC novector") \
+ for (int i = 0; i < n; i++)
\
+ sum += (ACC) a[i] * (ACC) b[i]; \
+ return sum;
\
+ }
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-bf.c
b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-bf.c
new file mode 100644
index 00000000000..7771376a630
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-bf.c
@@ -0,0 +1,30 @@
+/* { dg-do run { target { riscv_v && riscv_zvfwdota16bf_ok } } } */
+/* { dg-add-options riscv_zvfwdota16bf } */
+/* { dg-additional-options "-mrvv-vector-bits=scalable -ffast-math
-fno-vect-cost-model" } */
+
+#include "dot.h"
+
+#define N 301
+
+DEF_DOT (float, __bf16, __bf16, dot)
+DEF_DOT_REF (float, __bf16, __bf16, dot)
+
+__bf16 a[N];
+__bf16 b[N];
+
+int
+main ()
+{
+ for (int i = 0; i < N; i++)
+ {
+ a[i] = (__bf16) (i % 7 - 3);
+ b[i] = (__bf16) (i % 5 - 2);
+ }
+ for (int n = 0; n < N; n++)
+ {
+ float init_sum = n * 0.5f - 17.0f;
+ if (dot (init_sum, a, b, n) != dot_ref (init_sum, a, b, n))
+ __builtin_abort ();
+ }
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-hi-ss.c
b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-hi-ss.c
new file mode 100644
index 00000000000..13c460fe0d7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-hi-ss.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { riscv_v && riscv_zvqwdota16i_ok } } } */
+/* { dg-add-options riscv_zvqwdota16i } */
+/* { dg-additional-options "-mrvv-vector-bits=scalable -fno-vect-cost-model" }
*/
+
+#include "dot.h"
+
+#define ACC int64_t
+#define TA int16_t
+#define TB int16_t
+#define DOT dot
+#define DOT_REF dot_ref
+
+DEF_DOT (ACC, TA, TB, dot)
+DEF_DOT_REF (ACC, TA, TB, dot)
+
+#include "dot_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-hi-us.c
b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-hi-us.c
new file mode 100644
index 00000000000..da19d27e958
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-hi-us.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { riscv_v && riscv_zvqwdota16i_ok } } } */
+/* { dg-add-options riscv_zvqwdota16i } */
+/* { dg-additional-options "-mrvv-vector-bits=scalable -fno-vect-cost-model" }
*/
+
+#include "dot.h"
+
+#define ACC int64_t
+#define TA uint16_t
+#define TB int16_t
+#define DOT dot
+#define DOT_REF dot_ref
+
+DEF_DOT (ACC, TA, TB, dot)
+DEF_DOT_REF (ACC, TA, TB, dot)
+
+#include "dot_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-hi-uu.c
b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-hi-uu.c
new file mode 100644
index 00000000000..cfc45aad13f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-hi-uu.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { riscv_v && riscv_zvqwdota16i_ok } } } */
+/* { dg-add-options riscv_zvqwdota16i } */
+/* { dg-additional-options "-mrvv-vector-bits=scalable -fno-vect-cost-model" }
*/
+
+#include "dot.h"
+
+#define ACC uint64_t
+#define TA uint16_t
+#define TB uint16_t
+#define DOT dot
+#define DOT_REF dot_ref
+
+DEF_DOT (ACC, TA, TB, dot)
+DEF_DOT_REF (ACC, TA, TB, dot)
+
+#include "dot_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-qi-ss.c
b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-qi-ss.c
new file mode 100644
index 00000000000..1c4bc0a518d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-qi-ss.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { riscv_v && riscv_zvqwdota8i_ok } } } */
+/* { dg-add-options riscv_zvqwdota8i } */
+/* { dg-additional-options "-mrvv-vector-bits=scalable -fno-vect-cost-model" }
*/
+
+#include "dot.h"
+
+#define ACC int32_t
+#define TA int8_t
+#define TB int8_t
+#define DOT dot
+#define DOT_REF dot_ref
+
+DEF_DOT (ACC, TA, TB, dot)
+DEF_DOT_REF (ACC, TA, TB, dot)
+
+#include "dot_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-qi-su.c
b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-qi-su.c
new file mode 100644
index 00000000000..f95145f18c8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-qi-su.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { riscv_v && riscv_zvqwdota8i_ok } } } */
+/* { dg-add-options riscv_zvqwdota8i } */
+/* { dg-additional-options "-mrvv-vector-bits=scalable -fno-vect-cost-model" }
*/
+
+#include "dot.h"
+
+#define ACC int32_t
+#define TA int8_t
+#define TB uint8_t
+#define DOT dot
+#define DOT_REF dot_ref
+
+DEF_DOT (ACC, TA, TB, dot)
+DEF_DOT_REF (ACC, TA, TB, dot)
+
+#include "dot_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-qi-us.c
b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-qi-us.c
new file mode 100644
index 00000000000..6335559507b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-qi-us.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { riscv_v && riscv_zvqwdota8i_ok } } } */
+/* { dg-add-options riscv_zvqwdota8i } */
+/* { dg-additional-options "-mrvv-vector-bits=scalable -fno-vect-cost-model" }
*/
+
+#include "dot.h"
+
+#define ACC int32_t
+#define TA uint8_t
+#define TB int8_t
+#define DOT dot
+#define DOT_REF dot_ref
+
+DEF_DOT (ACC, TA, TB, dot)
+DEF_DOT_REF (ACC, TA, TB, dot)
+
+#include "dot_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-qi-uu.c
b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-qi-uu.c
new file mode 100644
index 00000000000..1bbabd4570d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run-qi-uu.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { riscv_v && riscv_zvqwdota8i_ok } } } */
+/* { dg-add-options riscv_zvqwdota8i } */
+/* { dg-additional-options "-mrvv-vector-bits=scalable -fno-vect-cost-model" }
*/
+
+#include "dot.h"
+
+#define ACC uint32_t
+#define TA uint8_t
+#define TB uint8_t
+#define DOT dot
+#define DOT_REF dot_ref
+
+DEF_DOT (ACC, TA, TB, dot)
+DEF_DOT_REF (ACC, TA, TB, dot)
+
+#include "dot_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run.h
b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run.h
new file mode 100644
index 00000000000..5795d2fd22d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/reduc/dot_run.h
@@ -0,0 +1,49 @@
+#define N 301
+
+#define MAXV(T) ((T) -1 > 0 ? (T) -1 \
+ : (T) ((1ULL << (sizeof (T) * 8 - 1)) - 1))
+#define MINV(T) ((T) -1 > 0 ? (T) 0 : (T) (-MAXV (T) - 1))
+
+TA a[N];
+TB b[N];
+
+static void
+init (int kind)
+{
+ for (int i = 0; i < N; i++)
+ switch (kind)
+ {
+ case 0:
+ a[i] = (TA) (i * 37 + 11);
+ b[i] = (TB) (i * 101 - 7);
+ break;
+ case 1:
+ a[i] = i & 1 ? MAXV (TA) : MINV (TA);
+ b[i] = i % 3 ? MINV (TB) : MAXV (TB);
+ break;
+ case 2:
+ a[i] = MINV (TA);
+ b[i] = MINV (TB);
+ break;
+ default:
+ a[i] = MAXV (TA);
+ b[i] = MAXV (TB);
+ break;
+ }
+}
+
+int
+main ()
+{
+ for (int kind = 0; kind < 4; kind++)
+ {
+ init (kind);
+ for (int n = 0; n < N; n++)
+ {
+ ACC init_sum = (ACC) (n * 3 - 17);
+ if (DOT (init_sum, a, b, n) != DOT_REF (init_sum, a, b, n))
+ __builtin_abort ();
+ }
+ }
+ return 0;
+}
diff --git a/gcc/testsuite/lib/target-supports.exp
b/gcc/testsuite/lib/target-supports.exp
index 3ce06189d2e..71b4a83c02c 100644
--- a/gcc/testsuite/lib/target-supports.exp
+++ b/gcc/testsuite/lib/target-supports.exp
@@ -2388,6 +2388,30 @@ proc check_effective_target_riscv_zvbb { } {
}]
}
+proc check_effective_target_riscv_zvqwdota8i { } {
+ return [check_no_compiler_messages riscv_ext_zvqwdota8i assembly {
+ #ifndef __riscv_zvqwdota8i
+ #error "Not __riscv_zvqwdota8i"
+ #endif
+ }]
+}
+
+proc check_effective_target_riscv_zvqwdota16i { } {
+ return [check_no_compiler_messages riscv_ext_zvqwdota16i assembly {
+ #ifndef __riscv_zvqwdota16i
+ #error "Not __riscv_zvqwdota16i"
+ #endif
+ }]
+}
+
+proc check_effective_target_riscv_zvfwdota16bf { } {
+ return [check_no_compiler_messages riscv_ext_zvfwdota16bf assembly {
+ #ifndef __riscv_zvfwdota16bf
+ #error "Not __riscv_zvfwdota16bf"
+ #endif
+ }]
+}
+
proc check_effective_target_riscv_zvabd { } {
return [check_no_compiler_messages riscv_ext_zvabd assembly {
#ifndef __riscv_zvabd
@@ -2671,6 +2695,69 @@ proc check_effective_target_riscv_zvbb_ok { } {
return 0
}
+proc check_effective_target_riscv_zvqwdota8i_ok { } {
+ if { ![istarget riscv*-*-*] } {
+ return 0
+ }
+
+ set gcc_march [riscv_get_arch]
+ if { [check_effective_target_riscv_zvqwdota8i] } {
+ set opts ""
+ } else {
+ set opts "-march=${gcc_march}_zvqwdota8i"
+ }
+
+ return [check_runtime ${gcc_march}_zvqwdota8i_exec {
+ int main()
+ {
+ asm ("vsetivli zero,8,e8,m1,tu,ma");
+ asm ("vqwdotau.vv v8,v16,v17" : : : "v8");
+ return 0;
+ } } $opts]
+}
+
+proc check_effective_target_riscv_zvqwdota16i_ok { } {
+ if { ![istarget riscv*-*-*] } {
+ return 0
+ }
+
+ set gcc_march [riscv_get_arch]
+ if { [check_effective_target_riscv_zvqwdota16i] } {
+ set opts ""
+ } else {
+ set opts "-march=${gcc_march}_zvqwdota16i"
+ }
+
+ return [check_runtime ${gcc_march}_zvqwdota16i_exec {
+ int main()
+ {
+ asm ("vsetivli zero,4,e16,m1,tu,ma");
+ asm ("vqwdotau.vv v8,v16,v17" : : : "v8");
+ return 0;
+ } } $opts]
+}
+
+proc check_effective_target_riscv_zvfwdota16bf_ok { } {
+ if { ![istarget riscv*-*-*] } {
+ return 0
+ }
+
+ set gcc_march [riscv_get_arch]
+ if { [check_effective_target_riscv_zvfwdota16bf] } {
+ set opts ""
+ } else {
+ set opts "-march=${gcc_march}_zvfwdota16bf"
+ }
+
+ return [check_runtime ${gcc_march}_zvfwdota16bf_exec {
+ int main()
+ {
+ asm ("vsetivli zero,4,e16alt,m1,tu,ma");
+ asm ("vfwdota.vv v8,v16,v17" : : : "v8");
+ return 0;
+ } } $opts]
+}
+
proc check_effective_target_riscv_zvabd_ok { } {
if { ![istarget riscv*-*-*] } {
return 0
@@ -2968,11 +3055,32 @@ proc remove_options_for_riscv_zvbb { flags } {
return [add_options_for_riscv_multi_letter_ext zvbb $flags]
}
+proc add_options_for_riscv_zvqwdota8i { flags } {
+ if { ![check_effective_target_riscv_zvqwdota8i_ok] } {
+ return $flags
+ }
+ return [add_options_for_riscv_multi_letter_ext zvqwdota8i $flags]
+}
+
+proc add_options_for_riscv_zvqwdota16i { flags } {
+ if { ![check_effective_target_riscv_zvqwdota16i_ok] } {
+ return $flags
+ }
+ return [add_options_for_riscv_multi_letter_ext zvqwdota16i $flags]
+}
+
+proc add_options_for_riscv_zvfwdota16bf { flags } {
+ if { ![check_effective_target_riscv_zvfwdota16bf_ok] } {
+ return $flags
+ }
+ return [add_options_for_riscv_multi_letter_ext zvfwdota16bf $flags]
+}
+
proc add_options_for_riscv_zvabd { flags } {
if { ![check_effective_target_riscv_zvabd_ok] } {
return $flags
}
- return [add_options_for_riscv_z_ext zvabd $flags]
+ return [add_options_for_riscv_multi_letter_ext zvabd $flags]
}
proc add_options_for_riscv_zvfh { flags } {
--
2.56.0
