https://gcc.gnu.org/g:e418b524197a6619b2743d0ba7f8911fceb69c2a
commit e418b524197a6619b2743d0ba7f8911fceb69c2a Author: Andrew Stubbs <[email protected]> Date: Tue Oct 6 16:20:22 2026 +0000 amdgcn: Implement SIMT reductions This patch implements IFN_GOMP_SIMT_REDUCE for AMD GCN, and ensures that the instructions are handled properly in the simt_vect pass. The actual reduction algorithm reuses the SIMD code. The placeholder xchg_bfly instructions that were just there to allow the build to succeed can now be removed. Note: 64-lane reductions are not currently supported on RDNA devices in SIMD mode, and this limitation will also apply to SIMT mode. gcc/ChangeLog: * config/gcn/gcn.md (UNSPECV_SIMT_REDUCE): New unspecv value. (UNSPEC_SIMT_XCHG_BFLY): Remove. (SIMT_REDUCE_MODE): New mode iterator. (omp_simt_xchg_bfly): Delete. (omp_simt_reduce): New. (@omp_simt_xchg_bfly<mode>): Delete. (@omp_simt_reduce<mode>): New. * config/gcn/gcn-valu.md (V64_SIMT_REDUCE_MODE): New mode iterator. (omp_simt_xchg_bfly<mode>): Delete. (omp_simt_reduce<mode>): New. * config/gcn/gcn-simt-vectorize.cc (pass_gcn_simt_vectorize::execute): Remove the UNSPEC_SIMT_XCHG_BFLY special case. Add new UNSPEC_VOLATILE case for UNSPECV_SIMT_REDUCE. Diff: --- gcc/config/gcn/gcn-simt-vectorize.cc | 24 ++++++++++++++++++++- gcc/config/gcn/gcn-valu.md | 41 +++++++++++++++++++++++++++++------- gcc/config/gcn/gcn.md | 31 +++++++++++++++------------ 3 files changed, 74 insertions(+), 22 deletions(-) diff --git a/gcc/config/gcn/gcn-simt-vectorize.cc b/gcc/config/gcn/gcn-simt-vectorize.cc index 1c5f470cb530..95660fb5f802 100644 --- a/gcc/config/gcn/gcn-simt-vectorize.cc +++ b/gcc/config/gcn/gcn-simt-vectorize.cc @@ -1697,7 +1697,6 @@ pass_gcn_simt_vectorize::execute (function *fn) /* Some UNSPECs have special handling. */ switch (XINT (x, 1)) { - case UNSPEC_SIMT_XCHG_BFLY: case UNSPEC_DIV_FMAS: /* The second unspec argument remains scalar, but we do want to recurse into the first argument. */ @@ -1717,6 +1716,29 @@ pass_gcn_simt_vectorize::execute (function *fn) } break; + case UNSPEC_VOLATILE: + /* Some UNSPEC_VOLATILEs have special handling. */ + switch (XINT (x, 1)) + { + case UNSPECV_SIMT_REDUCE: + /* The second unspec argument remains scalar, but + we do want to recurse into the first argument. */ + delayed_substitute = &XVECEXP (x, 0, 0); + do_transform = true; + + if (dump_file && (dump_flags & TDF_DETAILS)) + { + fprintf (dump_file, " no-op: "); + dump_value_with_mode (dump_file, XVECEXP (x, 0, 1)); + fprintf (dump_file, " (special case)\n"); + } + break; + + default: + do_transform = true; + } + break; + case IF_THEN_ELSE: if (pc_write) { diff --git a/gcc/config/gcn/gcn-valu.md b/gcc/config/gcn/gcn-valu.md index 5a075c9638f5..8054e6e75309 100644 --- a/gcc/config/gcn/gcn-valu.md +++ b/gcc/config/gcn/gcn-valu.md @@ -4680,14 +4680,39 @@ operands[4] = gcn_vec_constant (<MODE>mode, 0); }) -(define_insn "omp_simt_xchg_bfly<mode>" - [(set (match_operand:V_MOV 0 "register_operand" "=v") - (unspec:V_MOV - [(match_operand:V_MOV 1 "register_operand" "v") - (match_operand:SI 2 "nonmemory_operand" "Sg")] - UNSPEC_SIMT_XCHG_BFLY))] - "flag_simt_mode" - "; %0 = simt_xchg_bfly %1 %2") +(define_mode_iterator V64_SIMT_REDUCE_MODE + [V64QI V64HI V64SI V64DI V64SF V64DF]) + +(define_insn_and_split "omp_simt_reduce<mode>" + [(set (match_operand:V64_SIMT_REDUCE_MODE 0 "register_operand" "=v") + (unspec_volatile:V64_SIMT_REDUCE_MODE + [(match_operand:V64_SIMT_REDUCE_MODE 1 "register_operand" "v") + (match_operand:SI 2 "const_int_operand" "n")] + UNSPECV_SIMT_REDUCE))] + "flag_simt_mode && TARGET_DPP_FULL" + "#" + "can_create_pseudo_p ()" + [(const_int 0)] + { + enum rtx_code rcode = (enum rtx_code) INTVAL (operands[2]); + int unspec = (rcode == PLUS ? UNSPEC_PLUS_DPP_SHR + : rcode == UMIN ? UNSPEC_UMIN_DPP_SHR + : rcode == SMIN ? UNSPEC_SMIN_DPP_SHR + : rcode == UMAX ? UNSPEC_UMAX_DPP_SHR + : rcode == SMAX ? UNSPEC_SMAX_DPP_SHR + : rcode == AND ? UNSPEC_AND_DPP_SHR + : rcode == IOR ? UNSPEC_IOR_DPP_SHR + : rcode == XOR ? UNSPEC_XOR_DPP_SHR + : 0); + gcc_assert (unspec); + + rtx tmp = gcn_expand_reduc_scalar (<MODE>mode, operands[1], unspec); + rtx last_lane = GEN_INT (GET_MODE_NUNITS (<MODE>mode) - 1); + rtx scalar = gen_reg_rtx (<SCALAR_MODE>mode); + emit_insn (gen_vec_extract<mode><scalar_mode> (scalar, tmp, last_lane)); + emit_insn (gen_vec_duplicate<mode> (operands[0], scalar)); + DONE; + }) (define_insn_and_split "omp_simt_vote_any_v64si" [(set (match_operand:V64SI 0 "register_operand" "=v") diff --git a/gcc/config/gcn/gcn.md b/gcc/config/gcn/gcn.md index 4868ae6d9824..cb7030e0efbf 100644 --- a/gcc/config/gcn/gcn.md +++ b/gcc/config/gcn/gcn.md @@ -65,7 +65,8 @@ UNSPECV_ICACHE_INV UNSPECV_SIMT_FORK UNSPECV_SIMT_JOIN UNSPECV_SIMT_ENTER UNSPECV_SIMT_EXIT - UNSPECV_SIMT_EXEC_BARRIER]) + UNSPECV_SIMT_EXEC_BARRIER + UNSPECV_SIMT_REDUCE]) (define_c_enum "unspec" [ UNSPEC_ADDPTR @@ -93,7 +94,6 @@ UNSPEC_LDEXP UNSPEC_FREXP_EXP UNSPEC_FREXP_MANT UNSPEC_DIV_SCALE UNSPEC_DIV_FMAS UNSPEC_DIV_FIXUP UNSPEC_PAIR - UNSPEC_SIMT_XCHG_BFLY UNSPEC_SIMT_VOTE_ANY UNSPEC_SIMT_LAST_LANE UNSPEC_SIMT_XCHG_IDX UNSPEC_VSUBREG_WRITE UNSPEC_VSUBREG_READ UNSPEC_ENSURE_SCALAR]) @@ -2714,26 +2714,31 @@ (reg:SI LANE_ID_REG))] "flag_simt_mode") +(define_mode_iterator SIMT_REDUCE_MODE [QI HI SI DI SF DF]) + +;; Implement IFN_GOMP_SIMT_REDUCE: reduce operand 1 across all active SIMT +;; lanes using the reduction operator (an rtx_code) given as a const_int in +;; operand 2, leaving the result broadcast into every lane. ; This pseudo instruction should be replaced by the simt_vect pass -(define_expand "omp_simt_xchg_bfly" +(define_expand "omp_simt_reduce" [(match_operand 0 "register_operand") (match_operand 1 "register_operand") - (match_operand:SI 2 "nonmemory_operand")] + (match_operand:SI 2 "const_int_operand")] "flag_simt_mode" { - emit_insn (gen_omp_simt_xchg_bfly (GET_MODE (operands[0]), operands[0], - operands[1], operands[2])); + emit_insn (gen_omp_simt_reduce (GET_MODE (operands[0]), operands[0], + operands[1], operands[2])); DONE; }) -(define_insn "@omp_simt_xchg_bfly<mode>" - [(set (match_operand:MOV_MODE 0 "register_operand" "=r") - (unspec:MOV_MODE - [(match_operand:MOV_MODE 1 "register_operand" "r") - (match_operand:SI 2 "nonmemory_operand" "ri")] - UNSPEC_SIMT_XCHG_BFLY))] +(define_insn "@omp_simt_reduce<mode>" + [(set (match_operand:SIMT_REDUCE_MODE 0 "register_operand" "=r") + (unspec_volatile:SIMT_REDUCE_MODE + [(match_operand:SIMT_REDUCE_MODE 1 "register_operand" "r") + (match_operand:SI 2 "const_int_operand" "n")] + UNSPECV_SIMT_REDUCE))] "flag_simt_mode" - "; %0 = simt_xchg_bfly %1 %2") + "; %0 = simt_reduce %1 %2") ;; Implement IFN_GOMP_SIMT_VOTE_ANY: ;; set operand 0 to zero iff all lanes supply zero in operand 1
