This patch implements IFN_GOMP_SIMT_REDUCE for AMD GCN, and ensures that
the instructions are handled properly in the simt_vect pass. The actual
reduction algorithm reuses the SIMD code.
The placeholder xchg_bfly instructions that were just there to allow the build
to succeed can now be removed.
Note: 64-lane reductions are not currently supported on RDNA devices in SIMD
mode, and this limitation will also apply to SIMT mode.
gcc/ChangeLog:
* config/gcn/gcn.md (UNSPECV_SIMT_REDUCE): New unspecv value.
(UNSPEC_SIMT_XCHG_BFLY): Remove.
(SIMT_REDUCE_MODE): New mode iterator.
(omp_simt_xchg_bfly): Delete.
(omp_simt_reduce): New.
(@omp_simt_xchg_bfly<mode>): Delete.
(@omp_simt_reduce<mode>): New.
* config/gcn/gcn-valu.md (V64_SIMT_REDUCE_MODE): New mode iterator.
(omp_simt_xchg_bfly<mode>): Delete.
(omp_simt_reduce<mode>): New.
* config/gcn/gcn-simt-vectorize.cc (pass_gcn_simt_vectorize::execute):
Remove the UNSPEC_SIMT_XCHG_BFLY special case. Add new
UNSPEC_VOLATILE case for UNSPECV_SIMT_REDUCE.
---
gcc/config/gcn/gcn-simt-vectorize.cc | 24 +++++++++++++++-
gcc/config/gcn/gcn-valu.md | 41 ++++++++++++++++++++++------
gcc/config/gcn/gcn.md | 31 ++++++++++++---------
3 files changed, 74 insertions(+), 22 deletions(-)
diff --git a/gcc/config/gcn/gcn-simt-vectorize.cc
b/gcc/config/gcn/gcn-simt-vectorize.cc
index 2052eb1741a..8007f6a7fbc 100644
--- a/gcc/config/gcn/gcn-simt-vectorize.cc
+++ b/gcc/config/gcn/gcn-simt-vectorize.cc
@@ -1693,7 +1693,6 @@ pass_gcn_simt_vectorize::execute (function *fn)
/* Some UNSPECs have special handling. */
switch (XINT (x, 1))
{
- case UNSPEC_SIMT_XCHG_BFLY:
case UNSPEC_DIV_FMAS:
/* The second unspec argument remains scalar, but
we do want to recurse into the first argument. */
@@ -1713,6 +1712,29 @@ pass_gcn_simt_vectorize::execute (function *fn)
}
break;
+ case UNSPEC_VOLATILE:
+ /* Some UNSPEC_VOLATILEs have special handling. */
+ switch (XINT (x, 1))
+ {
+ case UNSPECV_SIMT_REDUCE:
+ /* The second unspec argument remains scalar, but
+ we do want to recurse into the first argument. */
+ delayed_substitute = &XVECEXP (x, 0, 0);
+ do_transform = true;
+
+ if (dump_file && (dump_flags & TDF_DETAILS))
+ {
+ fprintf (dump_file, " no-op: ");
+ dump_value_with_mode (dump_file, XVECEXP (x, 0, 1));
+ fprintf (dump_file, " (special case)\n");
+ }
+ break;
+
+ default:
+ do_transform = true;
+ }
+ break;
+
case IF_THEN_ELSE:
if (pc_write)
{
diff --git a/gcc/config/gcn/gcn-valu.md b/gcc/config/gcn/gcn-valu.md
index 9672b25ea48..697d769587d 100644
--- a/gcc/config/gcn/gcn-valu.md
+++ b/gcc/config/gcn/gcn-valu.md
@@ -4693,14 +4693,39 @@ (define_insn_and_split
"*vsubreg_read<V64_MOV:mode>di_insn"
operands[4] = gcn_vec_constant (<MODE>mode, 0);
})
-(define_insn "omp_simt_xchg_bfly<mode>"
- [(set (match_operand:V_MOV 0 "register_operand" "=v")
- (unspec:V_MOV
- [(match_operand:V_MOV 1 "register_operand" "v")
- (match_operand:SI 2 "nonmemory_operand" "Sg")]
- UNSPEC_SIMT_XCHG_BFLY))]
- "flag_simt_mode"
- "; %0 = simt_xchg_bfly %1 %2")
+(define_mode_iterator V64_SIMT_REDUCE_MODE
+ [V64QI V64HI V64SI V64DI V64SF V64DF])
+
+(define_insn_and_split "omp_simt_reduce<mode>"
+ [(set (match_operand:V64_SIMT_REDUCE_MODE 0 "register_operand" "=v")
+ (unspec_volatile:V64_SIMT_REDUCE_MODE
+ [(match_operand:V64_SIMT_REDUCE_MODE 1 "register_operand" "v")
+ (match_operand:SI 2 "const_int_operand" "n")]
+ UNSPECV_SIMT_REDUCE))]
+ "flag_simt_mode && TARGET_DPP_FULL"
+ "#"
+ "can_create_pseudo_p ()"
+ [(const_int 0)]
+ {
+ enum rtx_code rcode = (enum rtx_code) INTVAL (operands[2]);
+ int unspec = (rcode == PLUS ? UNSPEC_PLUS_DPP_SHR
+ : rcode == UMIN ? UNSPEC_UMIN_DPP_SHR
+ : rcode == SMIN ? UNSPEC_SMIN_DPP_SHR
+ : rcode == UMAX ? UNSPEC_UMAX_DPP_SHR
+ : rcode == SMAX ? UNSPEC_SMAX_DPP_SHR
+ : rcode == AND ? UNSPEC_AND_DPP_SHR
+ : rcode == IOR ? UNSPEC_IOR_DPP_SHR
+ : rcode == XOR ? UNSPEC_XOR_DPP_SHR
+ : 0);
+ gcc_assert (unspec);
+
+ rtx tmp = gcn_expand_reduc_scalar (<MODE>mode, operands[1], unspec);
+ rtx last_lane = GEN_INT (GET_MODE_NUNITS (<MODE>mode) - 1);
+ rtx scalar = gen_reg_rtx (<SCALAR_MODE>mode);
+ emit_insn (gen_vec_extract<mode><scalar_mode> (scalar, tmp, last_lane));
+ emit_insn (gen_vec_duplicate<mode> (operands[0], scalar));
+ DONE;
+ })
(define_insn_and_split "omp_simt_vote_any_v64si"
[(set (match_operand:V64SI 0 "register_operand" "=v")
diff --git a/gcc/config/gcn/gcn.md b/gcc/config/gcn/gcn.md
index 4f084606363..fda958da309 100644
--- a/gcc/config/gcn/gcn.md
+++ b/gcc/config/gcn/gcn.md
@@ -65,7 +65,8 @@ (define_c_enum "unspecv" [
UNSPECV_ICACHE_INV
UNSPECV_SIMT_FORK UNSPECV_SIMT_JOIN
UNSPECV_SIMT_ENTER UNSPECV_SIMT_EXIT
- UNSPECV_SIMT_EXEC_BARRIER])
+ UNSPECV_SIMT_EXEC_BARRIER
+ UNSPECV_SIMT_REDUCE])
(define_c_enum "unspec" [
UNSPEC_ADDPTR
@@ -93,7 +94,6 @@ (define_c_enum "unspec" [
UNSPEC_LDEXP UNSPEC_FREXP_EXP UNSPEC_FREXP_MANT
UNSPEC_DIV_SCALE UNSPEC_DIV_FMAS UNSPEC_DIV_FIXUP
UNSPEC_PAIR
- UNSPEC_SIMT_XCHG_BFLY
UNSPEC_SIMT_VOTE_ANY UNSPEC_SIMT_LAST_LANE UNSPEC_SIMT_XCHG_IDX
UNSPEC_VSUBREG_WRITE UNSPEC_VSUBREG_READ
UNSPEC_ENSURE_SCALAR])
@@ -2714,26 +2714,31 @@ (define_expand "omp_simt_lane"
(reg:SI LANE_ID_REG))]
"flag_simt_mode")
+(define_mode_iterator SIMT_REDUCE_MODE [QI HI SI DI SF DF])
+
+;; Implement IFN_GOMP_SIMT_REDUCE: reduce operand 1 across all active SIMT
+;; lanes using the reduction operator (an rtx_code) given as a const_int in
+;; operand 2, leaving the result broadcast into every lane.
; This pseudo instruction should be replaced by the simt_vect pass
-(define_expand "omp_simt_xchg_bfly"
+(define_expand "omp_simt_reduce"
[(match_operand 0 "register_operand")
(match_operand 1 "register_operand")
- (match_operand:SI 2 "nonmemory_operand")]
+ (match_operand:SI 2 "const_int_operand")]
"flag_simt_mode"
{
- emit_insn (gen_omp_simt_xchg_bfly (GET_MODE (operands[0]), operands[0],
- operands[1], operands[2]));
+ emit_insn (gen_omp_simt_reduce (GET_MODE (operands[0]), operands[0],
+ operands[1], operands[2]));
DONE;
})
-(define_insn "@omp_simt_xchg_bfly<mode>"
- [(set (match_operand:MOV_MODE 0 "register_operand" "=r")
- (unspec:MOV_MODE
- [(match_operand:MOV_MODE 1 "register_operand" "r")
- (match_operand:SI 2 "nonmemory_operand" "ri")]
- UNSPEC_SIMT_XCHG_BFLY))]
+(define_insn "@omp_simt_reduce<mode>"
+ [(set (match_operand:SIMT_REDUCE_MODE 0 "register_operand" "=r")
+ (unspec_volatile:SIMT_REDUCE_MODE
+ [(match_operand:SIMT_REDUCE_MODE 1 "register_operand" "r")
+ (match_operand:SI 2 "const_int_operand" "n")]
+ UNSPECV_SIMT_REDUCE))]
"flag_simt_mode"
- "; %0 = simt_xchg_bfly %1 %2")
+ "; %0 = simt_reduce %1 %2")
;; Implement IFN_GOMP_SIMT_VOTE_ANY:
;; set operand 0 to zero iff all lanes supply zero in operand 1
--
2.54.0
