https://gcc.gnu.org/g:91d4128839bcce54e9d4e7b3887dc619cbe3bd33
commit 91d4128839bcce54e9d4e7b3887dc619cbe3bd33 Author: Andrew Stubbs <[email protected]> Date: Tue Oct 6 15:20:00 2026 +0000 nvptx, openmp: Move SIMT reductions to the backend This patch replaces IFN_GOMP_SIMT_XCHG_BFLY with a more general IFN_GOMP_SIMT_REDUCE that defers any assumptions about the hardware reduction operators to the backend. The final result is the same, on NVPTX, as before except that instead of generating a loop with a variable holding the magic number, it now unrolls that loop and emits the shuffles with the opcode numbers hard-coded. The main reason for dropping the loop was laziness, but the final instruction count is similar, and the BabelStream "Dot" performance may actually be a tiny bit faster (tested on an RTX4080, but there's quite a bit of noise in the results). Perhaps the PTX finalizer prefers the straight-line code with clearer parameters? The motivation for IFN_GOMP_SIMT_REDUCE is to allow implementing GCN reductions via a different algorithm (there's no easy bfly instruction, and the instructions that do exist need the shuffle parameter hard-coded in the assembler). gcc/ChangeLog: * config/nvptx/nvptx-protos.h (nvptx_expand_simt_reduce): New prototype. * config/nvptx/nvptx.cc (nvptx_expand_simt_reduce): New function. * config/nvptx/nvptx.md (omp_simt_xchg_bfly): Delete. (omp_simt_reduce): New. * doc/ifn.texi: Delete IFN_GOMP_SIMT_XCHG_BFLY. Add IFN_GOMP_SIMT_REDUCE. * internal-fn.def (GOMP_SIMT_XCHG_BFLY): Delete. (GOMP_SIMT_REDUCE): New. * internal-fn.cc (expand_GOMP_SIMT_XCHG_BFLY): Delete. (expand_GOMP_SIMT_REDUCE): New. * omp-low.cc (lower_rec_input_clauses): Replace IFN_GOMP_SIMT_XCHG_BFLY algorithm with IFN_GOMP_SIMT_REDUCE call. Remove the now-unused simt_lane temporary. * omp-offload.cc (execute_omp_device_lower): Update for the IFN_GOMP_SIMT_XCHG_BFLY to IFN_GOMP_SIMT_REDUCE change. * target-insns.def (omp_simt_xchg_bfly): Delete. (omp_simt_reduce): New. * tree-cfg.cc (gimple_can_duplicate_bb_p): Update for the IFN_GOMP_SIMT_XCHG_BFLY to IFN_GOMP_SIMT_REDUCE change. Diff: --- gcc/config/nvptx/nvptx-protos.h | 1 + gcc/config/nvptx/nvptx.cc | 21 ++++++++++++++++++ gcc/config/nvptx/nvptx.md | 12 +++++------ gcc/doc/ifn.texi | 8 +++---- gcc/internal-fn.cc | 31 +++++++++++++++++++------- gcc/internal-fn.def | 2 +- gcc/omp-low.cc | 48 ++++++----------------------------------- gcc/omp-offload.cc | 2 +- gcc/target-insns.def | 2 +- gcc/tree-cfg.cc | 7 +++--- 10 files changed, 69 insertions(+), 65 deletions(-) diff --git a/gcc/config/nvptx/nvptx-protos.h b/gcc/config/nvptx/nvptx-protos.h index 2875a5ddf1d7..a1d4ae601b98 100644 --- a/gcc/config/nvptx/nvptx-protos.h +++ b/gcc/config/nvptx/nvptx-protos.h @@ -52,6 +52,7 @@ extern void nvptx_expand_oacc_join (unsigned); extern void nvptx_expand_call (rtx, rtx); extern void nvptx_expand_omp_get_num_threads (rtx); extern rtx nvptx_gen_shuffle (rtx, rtx, rtx, nvptx_shuffle_kind); +extern void nvptx_expand_simt_reduce (rtx, rtx, rtx); extern rtx nvptx_expand_compare (rtx); extern const char *nvptx_ptx_type_from_mode (machine_mode, bool); extern const char *nvptx_output_mov_insn (rtx, rtx); diff --git a/gcc/config/nvptx/nvptx.cc b/gcc/config/nvptx/nvptx.cc index c4b9650ecb62..2ad8ce21468a 100644 --- a/gcc/config/nvptx/nvptx.cc +++ b/gcc/config/nvptx/nvptx.cc @@ -6456,6 +6456,27 @@ nvptx_expand_shuffle (tree exp, rtx target, machine_mode mode, int ignore) return target; } +/* Implement the "omp_simt_reduce" expander. */ + +void +nvptx_expand_simt_reduce (rtx target, rtx src, rtx code) +{ + machine_mode mode = GET_MODE (target); + rtx var = src; + + for (int shift = 1; shift < PTX_WARP_SIZE; shift <<= 1) + { + rtx idx = GEN_INT (shift); + rtx bfly = gen_reg_rtx (mode); + emit_insn (nvptx_gen_shuffle (bfly, var, idx, SHUFFLE_BFLY)); + rtx next = expand_simple_binop (mode, (enum rtx_code) INTVAL (code), + var, bfly, NULL_RTX, 1, OPTAB_DIRECT); + gcc_assert (next); + var = next; + } + emit_move_insn (target, var); +} + /* Expander for the bit reverse builtins. */ static rtx diff --git a/gcc/config/nvptx/nvptx.md b/gcc/config/nvptx/nvptx.md index 6cdb7f326a68..aadc8480b9e2 100644 --- a/gcc/config/nvptx/nvptx.md +++ b/gcc/config/nvptx/nvptx.md @@ -2101,16 +2101,16 @@ DONE; }) -;; Implement IFN_GOMP_SIMT_XCHG_BFLY: perform a "butterfly" exchange -;; across lanes -(define_expand "omp_simt_xchg_bfly" +;; Implement IFN_GOMP_SIMT_REDUCE: reduce operand 1 across all lanes of the +;; warp using the rtx_code reduction operator given as a literal constant +;; in operand 2, leaving the result in operand 0 in every lane. +(define_expand "omp_simt_reduce" [(match_operand 0 "nvptx_register_or_complex_di_df_register_operand" "=R") (match_operand 1 "nvptx_register_or_complex_di_df_register_operand" "R") - (match_operand:SI 2 "nvptx_nonmemory_operand" "Ri")] + (match_operand:SI 2 "const_int_operand" "n")] "" { - emit_insn (nvptx_gen_shuffle (operands[0], operands[1], operands[2], - SHUFFLE_BFLY)); + nvptx_expand_simt_reduce (operands[0], operands[1], operands[2]); DONE; }) diff --git a/gcc/doc/ifn.texi b/gcc/doc/ifn.texi index ada827ea5a3b..09b65086d38d 100644 --- a/gcc/doc/ifn.texi +++ b/gcc/doc/ifn.texi @@ -1161,10 +1161,10 @@ The @code{IFN_GOMP_SIMT_ORDERED_PRED} internal function is expanded by the The @code{IFN_GOMP_SIMT_VOTE_ANY} internal function is expanded by the @code{expand_GOMP_SIMT_VOTE_ANY} function. -@cindex @code{IFN_GOMP_SIMT_XCHG_BFLY} internal function -@item @samp{IFN_GOMP_SIMT_XCHG_BFLY} -The @code{IFN_GOMP_SIMT_XCHG_BFLY} internal function is expanded by the -@code{expand_GOMP_SIMT_XCHG_BFLY} function. +@cindex @code{IFN_GOMP_SIMT_REDUCE} internal function +@item @samp{IFN_GOMP_SIMT_REDUCE} +The @code{IFN_GOMP_SIMT_REDUCE} internal function is expanded by the +@code{expand_GOMP_SIMT_REDUCE} function. @cindex @code{IFN_GOMP_SIMT_XCHG_IDX} internal function @item @samp{IFN_GOMP_SIMT_XCHG_IDX} diff --git a/gcc/internal-fn.cc b/gcc/internal-fn.cc index 662d1e75d2f7..dc2dc3396791 100644 --- a/gcc/internal-fn.cc +++ b/gcc/internal-fn.cc @@ -609,26 +609,41 @@ expand_GOMP_SIMT_VOTE_ANY (internal_fn, gcall *stmt) assign_call_lhs (lhs, target, &ops[0]); } -/* Exchange between SIMT lanes with a "butterfly" pattern: source lane index - is destination lane index XOR given offset. */ +/* Expand using "omp_simt_reduce<mode>". Operand 1 of the + call is an INTEGER_CST holding the reduction operator as a tree_code, + which needs to be converted to an rtx_code for the backend. */ static void -expand_GOMP_SIMT_XCHG_BFLY (internal_fn, gcall *stmt) +expand_GOMP_SIMT_REDUCE (internal_fn, gcall *stmt) { tree lhs = gimple_call_lhs (stmt); if (!lhs) return; + tree arg0 = gimple_call_arg (stmt, 0); + tree_code code = (tree_code) TREE_INT_CST_LOW (gimple_call_arg (stmt, 1)); + enum rtx_code rcode + = (code == PLUS_EXPR ? PLUS + : code == MULT_EXPR ? MULT + : code == MIN_EXPR ? (TYPE_UNSIGNED (TREE_TYPE (arg0)) ? UMIN : SMIN) + : code == MAX_EXPR ? (TYPE_UNSIGNED (TREE_TYPE (arg0)) ? UMAX : SMAX) + : code == BIT_AND_EXPR ? AND + : code == BIT_IOR_EXPR ? IOR + : code == BIT_XOR_EXPR ? XOR + : code == TRUTH_ANDIF_EXPR ? AND + : code == TRUTH_ORIF_EXPR ? IOR + : UNKNOWN); + gcc_assert (rcode != UNKNOWN); + rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); - rtx src = expand_normal (gimple_call_arg (stmt, 0)); - rtx idx = expand_normal (gimple_call_arg (stmt, 1)); + rtx src = expand_normal (arg0); machine_mode mode = TYPE_MODE (TREE_TYPE (lhs)); class expand_operand ops[3]; create_call_lhs_operand (&ops[0], target, mode); create_input_operand (&ops[1], src, mode); - create_input_operand (&ops[2], idx, SImode); - gcc_assert (targetm.have_omp_simt_xchg_bfly ()); - expand_insn (targetm.code_for_omp_simt_xchg_bfly, 3, ops); + create_integer_operand (&ops[2], rcode); + gcc_assert (targetm.have_omp_simt_reduce ()); + expand_insn (targetm.code_for_omp_simt_reduce, 3, ops); assign_call_lhs (lhs, target, &ops[0]); } diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def index d142416725a2..2886305d663f 100644 --- a/gcc/internal-fn.def +++ b/gcc/internal-fn.def @@ -483,7 +483,7 @@ DEF_INTERNAL_FN (GOMP_MAX_VF, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL) DEF_INTERNAL_FN (GOMP_SIMT_LAST_LANE, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL) DEF_INTERNAL_FN (GOMP_SIMT_ORDERED_PRED, ECF_LEAF | ECF_NOTHROW, NULL) DEF_INTERNAL_FN (GOMP_SIMT_VOTE_ANY, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL) -DEF_INTERNAL_FN (GOMP_SIMT_XCHG_BFLY, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL) +DEF_INTERNAL_FN (GOMP_SIMT_REDUCE, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL) DEF_INTERNAL_FN (GOMP_SIMT_XCHG_IDX, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL) DEF_INTERNAL_FN (GOMP_SIMD_LANE, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL) DEF_INTERNAL_FN (GOMP_SIMD_VF, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL) diff --git a/gcc/omp-low.cc b/gcc/omp-low.cc index 837a0493c91b..bfb8cfe4eaab 100644 --- a/gcc/omp-low.cc +++ b/gcc/omp-low.cc @@ -5422,7 +5422,7 @@ lower_rec_input_clauses (tree clauses, gimple_seq *ilist, gimple_seq *dlist, bool is_simd = (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD); omplow_simd_context sctx = omplow_simd_context (); - tree simt_lane = NULL_TREE, simtrec = NULL_TREE; + tree simtrec = NULL_TREE; tree ivar = NULL_TREE, lvar = NULL_TREE, uid = NULL_TREE; gimple_seq llist[4] = { }; tree nonconst_simd_if = NULL_TREE; @@ -7282,15 +7282,10 @@ lower_rec_input_clauses (tree clauses, gimple_seq *ilist, gimple_seq *dlist, if (sctx.is_simt) { - if (!simt_lane) - simt_lane = create_tmp_var (unsigned_type_node); + tree op = build_int_cst (unsigned_type_node, code); x = build_call_expr_internal_loc - (UNKNOWN_LOCATION, IFN_GOMP_SIMT_XCHG_BFLY, - TREE_TYPE (ivar), 2, ivar, simt_lane); - /* Make sure x is evaluated unconditionally. */ - tree bfly_var = create_tmp_var (TREE_TYPE (ivar)); - gimplify_assign (bfly_var, x, &llist[2]); - x = build2 (code, TREE_TYPE (ivar), ivar, bfly_var); + (UNKNOWN_LOCATION, IFN_GOMP_SIMT_REDUCE, + TREE_TYPE (ivar), 2, ivar, op); gimplify_assign (ivar, x, &llist[2]); } tree ivar2 = ivar; @@ -7454,39 +7449,10 @@ lower_rec_input_clauses (tree clauses, gimple_seq *ilist, gimple_seq *dlist, gimple_seq_add_stmt (dlist, g); gimple_seq_add_seq (dlist, llist[3]); } - /* Emit reductions across SIMT lanes in log_2(simt_vf) steps. */ + /* Emit the IFN_GOMP_SIMT_REDUCE call built above, if any. + The actual reduction algorithm is deferred to the back-end. */ if (llist[2]) - { - tree simt_vf = create_tmp_var (unsigned_type_node); - g = gimple_build_call_internal (IFN_GOMP_SIMT_VF, 0); - gimple_call_set_lhs (g, simt_vf); - gimple_seq_add_stmt (dlist, g); - - tree t = build_int_cst (unsigned_type_node, 1); - g = gimple_build_assign (simt_lane, INTEGER_CST, t); - gimple_seq_add_stmt (dlist, g); - - t = build_int_cst (unsigned_type_node, 0); - g = gimple_build_assign (sctx.idx, INTEGER_CST, t); - gimple_seq_add_stmt (dlist, g); - - tree body = create_artificial_label (UNKNOWN_LOCATION); - tree header = create_artificial_label (UNKNOWN_LOCATION); - tree end = create_artificial_label (UNKNOWN_LOCATION); - gimple_seq_add_stmt (dlist, gimple_build_goto (header)); - gimple_seq_add_stmt (dlist, gimple_build_label (body)); - - gimple_seq_add_seq (dlist, llist[2]); - - g = gimple_build_assign (simt_lane, LSHIFT_EXPR, simt_lane, integer_one_node); - gimple_seq_add_stmt (dlist, g); - - gimple_seq_add_stmt (dlist, gimple_build_label (header)); - g = gimple_build_cond (LT_EXPR, simt_lane, simt_vf, body, end); - gimple_seq_add_stmt (dlist, g); - - gimple_seq_add_stmt (dlist, gimple_build_label (end)); - } + gimple_seq_add_seq (dlist, llist[2]); for (int i = 0; i < 2; i++) if (llist[i]) { diff --git a/gcc/omp-offload.cc b/gcc/omp-offload.cc index ef0b1650c40f..7ef3fabe4211 100644 --- a/gcc/omp-offload.cc +++ b/gcc/omp-offload.cc @@ -3385,7 +3385,7 @@ execute_omp_device_lower () unlink_stmt_vdef (stmt); break; case IFN_GOMP_SIMT_VOTE_ANY: - case IFN_GOMP_SIMT_XCHG_BFLY: + case IFN_GOMP_SIMT_REDUCE: case IFN_GOMP_SIMT_XCHG_IDX: rhs = vf == 1 ? gimple_call_arg (stmt, 0) : NULL_TREE; break; diff --git a/gcc/target-insns.def b/gcc/target-insns.def index 0ade2d0aa253..44d4f699870f 100644 --- a/gcc/target-insns.def +++ b/gcc/target-insns.def @@ -80,7 +80,7 @@ DEF_TARGET_INSN (omp_simt_lane, (rtx x0)) DEF_TARGET_INSN (omp_simt_last_lane, (rtx x0, rtx x1)) DEF_TARGET_INSN (omp_simt_ordered, (rtx x0, rtx x1)) DEF_TARGET_INSN (omp_simt_vote_any, (rtx x0, rtx x1)) -DEF_TARGET_INSN (omp_simt_xchg_bfly, (rtx x0, rtx x1, rtx x2)) +DEF_TARGET_INSN (omp_simt_reduce, (rtx x0, rtx x1, rtx x2)) DEF_TARGET_INSN (omp_simt_xchg_idx, (rtx x0, rtx x1, rtx x2)) DEF_TARGET_INSN (prefetch, (rtx x0, rtx x1, rtx x2)) DEF_TARGET_INSN (probe_stack, (rtx x0)) diff --git a/gcc/tree-cfg.cc b/gcc/tree-cfg.cc index 51858d9f2655..1a771ed592d7 100644 --- a/gcc/tree-cfg.cc +++ b/gcc/tree-cfg.cc @@ -6411,13 +6411,14 @@ gimple_can_duplicate_bb_p (const_basic_block bb) /* An IFN_GOMP_SIMT_ENTER_ALLOC/IFN_GOMP_SIMT_EXIT call must be duplicated as part of its group, or not at all. - The IFN_GOMP_SIMT_VOTE_ANY and IFN_GOMP_SIMT_XCHG_* are part of such a - group, so the same holds there. */ + The IFN_GOMP_SIMT_VOTE_ANY, IFN_GOMP_SIMT_XCHG_IDX and + IFN_GOMP_SIMT_REDUCE are part of such a group, so the same holds + there. */ if (is_gimple_call (g) && (gimple_call_internal_p (g, IFN_GOMP_SIMT_ENTER_ALLOC) || gimple_call_internal_p (g, IFN_GOMP_SIMT_EXIT) || gimple_call_internal_p (g, IFN_GOMP_SIMT_VOTE_ANY) - || gimple_call_internal_p (g, IFN_GOMP_SIMT_XCHG_BFLY) + || gimple_call_internal_p (g, IFN_GOMP_SIMT_REDUCE) || gimple_call_internal_p (g, IFN_GOMP_SIMT_XCHG_IDX))) return false; }
