This patch replaces IFN_GOMP_SIMT_XCHG_BFLY with a more general
IFN_GOMP_SIMT_REDUCE that defers any assumptions about the hardware reduction
operators to the backend. The final result is the same, on NVPTX, as before
except that instead of generating a loop with a variable holding the magic
number, it now unrolls that loop and emits the shuffles with the opcode numbers
hard-coded.
The main reason for dropping the loop was laziness, but the final instruction
count is similar, and the BabelStream "Dot" performance may actually be a tiny
bit faster (tested on an RTX4080, but there's quite a bit of noise in the
results). Perhaps the PTX finalizer prefers the straight-line code with
clearer parameters?
The motivation for IFN_GOMP_SIMT_REDUCE is to allow implementing GCN reductions
via a different algorithm (there's no easy bfly instruction, and the
instructions that do exist need the shuffle parameter hard-coded in the
assembler).
gcc/ChangeLog:
* config/nvptx/nvptx-protos.h (nvptx_expand_simt_reduce): New
prototype.
* config/nvptx/nvptx.cc (nvptx_expand_simt_reduce): New function.
* config/nvptx/nvptx.md (omp_simt_xchg_bfly): Delete.
(omp_simt_reduce): New.
* doc/ifn.texi: Delete IFN_GOMP_SIMT_XCHG_BFLY.
Add IFN_GOMP_SIMT_REDUCE.
* internal-fn.def (GOMP_SIMT_XCHG_BFLY): Delete.
(GOMP_SIMT_REDUCE): New.
* internal-fn.cc (expand_GOMP_SIMT_XCHG_BFLY): Delete.
(expand_GOMP_SIMT_REDUCE): New.
* omp-low.cc (lower_rec_input_clauses): Replace
IFN_GOMP_SIMT_XCHG_BFLY algorithm with IFN_GOMP_SIMT_REDUCE call.
Remove the now-unused simt_lane temporary.
* omp-offload.cc (execute_omp_device_lower): Update for the
IFN_GOMP_SIMT_XCHG_BFLY to IFN_GOMP_SIMT_REDUCE change.
* target-insns.def (omp_simt_xchg_bfly): Delete.
(omp_simt_reduce): New.
* tree-cfg.cc (gimple_can_duplicate_bb_p): Update for the
IFN_GOMP_SIMT_XCHG_BFLY to IFN_GOMP_SIMT_REDUCE change.
---
gcc/config/nvptx/nvptx-protos.h | 1 +
gcc/config/nvptx/nvptx.cc | 21 +++++++++++++++
gcc/config/nvptx/nvptx.md | 12 ++++-----
gcc/doc/ifn.texi | 8 +++---
gcc/internal-fn.cc | 31 +++++++++++++++------
gcc/internal-fn.def | 2 +-
gcc/omp-low.cc | 48 +++++----------------------------
gcc/omp-offload.cc | 2 +-
gcc/target-insns.def | 2 +-
gcc/tree-cfg.cc | 7 ++---
10 files changed, 69 insertions(+), 65 deletions(-)
diff --git a/gcc/config/nvptx/nvptx-protos.h b/gcc/config/nvptx/nvptx-protos.h
index 87e8647ebe1..b5bf2d5aea5 100644
--- a/gcc/config/nvptx/nvptx-protos.h
+++ b/gcc/config/nvptx/nvptx-protos.h
@@ -51,6 +51,7 @@ extern void nvptx_expand_oacc_fork (unsigned);
extern void nvptx_expand_oacc_join (unsigned);
extern void nvptx_expand_call (rtx, rtx);
extern rtx nvptx_gen_shuffle (rtx, rtx, rtx, nvptx_shuffle_kind);
+extern void nvptx_expand_simt_reduce (rtx, rtx, rtx);
extern rtx nvptx_expand_compare (rtx);
extern const char *nvptx_ptx_type_from_mode (machine_mode, bool);
extern const char *nvptx_output_mov_insn (rtx, rtx);
diff --git a/gcc/config/nvptx/nvptx.cc b/gcc/config/nvptx/nvptx.cc
index 39ea6e6f923..8fe14775310 100644
--- a/gcc/config/nvptx/nvptx.cc
+++ b/gcc/config/nvptx/nvptx.cc
@@ -6194,6 +6194,27 @@ nvptx_expand_shuffle (tree exp, rtx target, machine_mode
mode, int ignore)
return target;
}
+/* Implement the "omp_simt_reduce" expander. */
+
+void
+nvptx_expand_simt_reduce (rtx target, rtx src, rtx code)
+{
+ machine_mode mode = GET_MODE (target);
+ rtx var = src;
+
+ for (int shift = 1; shift < PTX_WARP_SIZE; shift <<= 1)
+ {
+ rtx idx = GEN_INT (shift);
+ rtx bfly = gen_reg_rtx (mode);
+ emit_insn (nvptx_gen_shuffle (bfly, var, idx, SHUFFLE_BFLY));
+ rtx next = expand_simple_binop (mode, (enum rtx_code) INTVAL (code),
+ var, bfly, NULL_RTX, 1, OPTAB_DIRECT);
+ gcc_assert (next);
+ var = next;
+ }
+ emit_move_insn (target, var);
+}
+
/* Expander for the bit reverse builtins. */
static rtx
diff --git a/gcc/config/nvptx/nvptx.md b/gcc/config/nvptx/nvptx.md
index 21731da32ce..2f2e5a34839 100644
--- a/gcc/config/nvptx/nvptx.md
+++ b/gcc/config/nvptx/nvptx.md
@@ -2026,16 +2026,16 @@ (define_expand "omp_simt_ordered"
DONE;
})
-;; Implement IFN_GOMP_SIMT_XCHG_BFLY: perform a "butterfly" exchange
-;; across lanes
-(define_expand "omp_simt_xchg_bfly"
+;; Implement IFN_GOMP_SIMT_REDUCE: reduce operand 1 across all lanes of the
+;; warp using the rtx_code reduction operator given as a literal constant
+;; in operand 2, leaving the result in operand 0 in every lane.
+(define_expand "omp_simt_reduce"
[(match_operand 0 "nvptx_register_or_complex_di_df_register_operand" "=R")
(match_operand 1 "nvptx_register_or_complex_di_df_register_operand" "R")
- (match_operand:SI 2 "nvptx_nonmemory_operand" "Ri")]
+ (match_operand:SI 2 "const_int_operand" "n")]
""
{
- emit_insn (nvptx_gen_shuffle (operands[0], operands[1], operands[2],
- SHUFFLE_BFLY));
+ nvptx_expand_simt_reduce (operands[0], operands[1], operands[2]);
DONE;
})
diff --git a/gcc/doc/ifn.texi b/gcc/doc/ifn.texi
index 7cfc80eecf2..e8dd6f63df6 100644
--- a/gcc/doc/ifn.texi
+++ b/gcc/doc/ifn.texi
@@ -1172,10 +1172,10 @@ The @code{IFN_GOMP_SIMT_ORDERED_PRED} internal function
is expanded by the
The @code{IFN_GOMP_SIMT_VOTE_ANY} internal function is expanded by the
@code{expand_GOMP_SIMT_VOTE_ANY} function.
-@cindex @code{IFN_GOMP_SIMT_XCHG_BFLY} internal function
-@item @samp{IFN_GOMP_SIMT_XCHG_BFLY}
-The @code{IFN_GOMP_SIMT_XCHG_BFLY} internal function is expanded by the
-@code{expand_GOMP_SIMT_XCHG_BFLY} function.
+@cindex @code{IFN_GOMP_SIMT_REDUCE} internal function
+@item @samp{IFN_GOMP_SIMT_REDUCE}
+The @code{IFN_GOMP_SIMT_REDUCE} internal function is expanded by the
+@code{expand_GOMP_SIMT_REDUCE} function.
@cindex @code{IFN_GOMP_SIMT_XCHG_IDX} internal function
@item @samp{IFN_GOMP_SIMT_XCHG_IDX}
diff --git a/gcc/internal-fn.cc b/gcc/internal-fn.cc
index 45ad08d2cf6..6cc3e6073d1 100644
--- a/gcc/internal-fn.cc
+++ b/gcc/internal-fn.cc
@@ -609,26 +609,41 @@ expand_GOMP_SIMT_VOTE_ANY (internal_fn, gcall *stmt)
assign_call_lhs (lhs, target, &ops[0]);
}
-/* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
- is destination lane index XOR given offset. */
+/* Expand using "omp_simt_reduce<mode>". Operand 1 of the
+ call is an INTEGER_CST holding the reduction operator as a tree_code,
+ which needs to be converted to an rtx_code for the backend. */
static void
-expand_GOMP_SIMT_XCHG_BFLY (internal_fn, gcall *stmt)
+expand_GOMP_SIMT_REDUCE (internal_fn, gcall *stmt)
{
tree lhs = gimple_call_lhs (stmt);
if (!lhs)
return;
+ tree arg0 = gimple_call_arg (stmt, 0);
+ tree_code code = (tree_code) TREE_INT_CST_LOW (gimple_call_arg (stmt, 1));
+ enum rtx_code rcode
+ = (code == PLUS_EXPR ? PLUS
+ : code == MULT_EXPR ? MULT
+ : code == MIN_EXPR ? (TYPE_UNSIGNED (TREE_TYPE (arg0)) ? UMIN : SMIN)
+ : code == MAX_EXPR ? (TYPE_UNSIGNED (TREE_TYPE (arg0)) ? UMAX : SMAX)
+ : code == BIT_AND_EXPR ? AND
+ : code == BIT_IOR_EXPR ? IOR
+ : code == BIT_XOR_EXPR ? XOR
+ : code == TRUTH_ANDIF_EXPR ? AND
+ : code == TRUTH_ORIF_EXPR ? IOR
+ : UNKNOWN);
+ gcc_assert (rcode != UNKNOWN);
+
rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
- rtx src = expand_normal (gimple_call_arg (stmt, 0));
- rtx idx = expand_normal (gimple_call_arg (stmt, 1));
+ rtx src = expand_normal (arg0);
machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
class expand_operand ops[3];
create_call_lhs_operand (&ops[0], target, mode);
create_input_operand (&ops[1], src, mode);
- create_input_operand (&ops[2], idx, SImode);
- gcc_assert (targetm.have_omp_simt_xchg_bfly ());
- expand_insn (targetm.code_for_omp_simt_xchg_bfly, 3, ops);
+ create_integer_operand (&ops[2], rcode);
+ gcc_assert (targetm.have_omp_simt_reduce ());
+ expand_insn (targetm.code_for_omp_simt_reduce, 3, ops);
assign_call_lhs (lhs, target, &ops[0]);
}
diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def
index b1c46a82a52..53244ce1bfd 100644
--- a/gcc/internal-fn.def
+++ b/gcc/internal-fn.def
@@ -506,7 +506,7 @@ DEF_INTERNAL_FN (GOMP_MAX_VF, ECF_CONST | ECF_LEAF |
ECF_NOTHROW, NULL)
DEF_INTERNAL_FN (GOMP_SIMT_LAST_LANE, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW,
NULL)
DEF_INTERNAL_FN (GOMP_SIMT_ORDERED_PRED, ECF_LEAF | ECF_NOTHROW, NULL)
DEF_INTERNAL_FN (GOMP_SIMT_VOTE_ANY, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL)
-DEF_INTERNAL_FN (GOMP_SIMT_XCHG_BFLY, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW,
NULL)
+DEF_INTERNAL_FN (GOMP_SIMT_REDUCE, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL)
DEF_INTERNAL_FN (GOMP_SIMT_XCHG_IDX, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL)
DEF_INTERNAL_FN (GOMP_SIMD_LANE, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL)
DEF_INTERNAL_FN (GOMP_SIMD_VF, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
diff --git a/gcc/omp-low.cc b/gcc/omp-low.cc
index bcc482fb839..2502967c699 100644
--- a/gcc/omp-low.cc
+++ b/gcc/omp-low.cc
@@ -4965,7 +4965,7 @@ lower_rec_input_clauses (tree clauses, gimple_seq *ilist,
gimple_seq *dlist,
bool is_simd = (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
&& gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD);
omplow_simd_context sctx = omplow_simd_context ();
- tree simt_lane = NULL_TREE, simtrec = NULL_TREE;
+ tree simtrec = NULL_TREE;
tree ivar = NULL_TREE, lvar = NULL_TREE, uid = NULL_TREE;
gimple_seq llist[4] = { };
tree nonconst_simd_if = NULL_TREE;
@@ -6825,15 +6825,10 @@ lower_rec_input_clauses (tree clauses, gimple_seq
*ilist, gimple_seq *dlist,
if (sctx.is_simt)
{
- if (!simt_lane)
- simt_lane = create_tmp_var (unsigned_type_node);
+ tree op = build_int_cst (unsigned_type_node, code);
x = build_call_expr_internal_loc
- (UNKNOWN_LOCATION, IFN_GOMP_SIMT_XCHG_BFLY,
- TREE_TYPE (ivar), 2, ivar, simt_lane);
- /* Make sure x is evaluated unconditionally. */
- tree bfly_var = create_tmp_var (TREE_TYPE (ivar));
- gimplify_assign (bfly_var, x, &llist[2]);
- x = build2 (code, TREE_TYPE (ivar), ivar, bfly_var);
+ (UNKNOWN_LOCATION, IFN_GOMP_SIMT_REDUCE,
+ TREE_TYPE (ivar), 2, ivar, op);
gimplify_assign (ivar, x, &llist[2]);
}
tree ivar2 = ivar;
@@ -6997,39 +6992,10 @@ lower_rec_input_clauses (tree clauses, gimple_seq
*ilist, gimple_seq *dlist,
gimple_seq_add_stmt (dlist, g);
gimple_seq_add_seq (dlist, llist[3]);
}
- /* Emit reductions across SIMT lanes in log_2(simt_vf) steps. */
+ /* Emit the IFN_GOMP_SIMT_REDUCE call built above, if any.
+ The actual reduction algorithm is deferred to the back-end. */
if (llist[2])
- {
- tree simt_vf = create_tmp_var (unsigned_type_node);
- g = gimple_build_call_internal (IFN_GOMP_SIMT_VF, 0);
- gimple_call_set_lhs (g, simt_vf);
- gimple_seq_add_stmt (dlist, g);
-
- tree t = build_int_cst (unsigned_type_node, 1);
- g = gimple_build_assign (simt_lane, INTEGER_CST, t);
- gimple_seq_add_stmt (dlist, g);
-
- t = build_int_cst (unsigned_type_node, 0);
- g = gimple_build_assign (sctx.idx, INTEGER_CST, t);
- gimple_seq_add_stmt (dlist, g);
-
- tree body = create_artificial_label (UNKNOWN_LOCATION);
- tree header = create_artificial_label (UNKNOWN_LOCATION);
- tree end = create_artificial_label (UNKNOWN_LOCATION);
- gimple_seq_add_stmt (dlist, gimple_build_goto (header));
- gimple_seq_add_stmt (dlist, gimple_build_label (body));
-
- gimple_seq_add_seq (dlist, llist[2]);
-
- g = gimple_build_assign (simt_lane, LSHIFT_EXPR, simt_lane,
integer_one_node);
- gimple_seq_add_stmt (dlist, g);
-
- gimple_seq_add_stmt (dlist, gimple_build_label (header));
- g = gimple_build_cond (LT_EXPR, simt_lane, simt_vf, body, end);
- gimple_seq_add_stmt (dlist, g);
-
- gimple_seq_add_stmt (dlist, gimple_build_label (end));
- }
+ gimple_seq_add_seq (dlist, llist[2]);
for (int i = 0; i < 2; i++)
if (llist[i])
{
diff --git a/gcc/omp-offload.cc b/gcc/omp-offload.cc
index f39635712e4..aa35d2394dd 100644
--- a/gcc/omp-offload.cc
+++ b/gcc/omp-offload.cc
@@ -2861,7 +2861,7 @@ execute_omp_device_lower ()
unlink_stmt_vdef (stmt);
break;
case IFN_GOMP_SIMT_VOTE_ANY:
- case IFN_GOMP_SIMT_XCHG_BFLY:
+ case IFN_GOMP_SIMT_REDUCE:
case IFN_GOMP_SIMT_XCHG_IDX:
rhs = vf == 1 ? gimple_call_arg (stmt, 0) : NULL_TREE;
break;
diff --git a/gcc/target-insns.def b/gcc/target-insns.def
index 8da504f9557..1f0cc9df88c 100644
--- a/gcc/target-insns.def
+++ b/gcc/target-insns.def
@@ -75,7 +75,7 @@ DEF_TARGET_INSN (omp_simt_lane, (rtx x0))
DEF_TARGET_INSN (omp_simt_last_lane, (rtx x0, rtx x1))
DEF_TARGET_INSN (omp_simt_ordered, (rtx x0, rtx x1))
DEF_TARGET_INSN (omp_simt_vote_any, (rtx x0, rtx x1))
-DEF_TARGET_INSN (omp_simt_xchg_bfly, (rtx x0, rtx x1, rtx x2))
+DEF_TARGET_INSN (omp_simt_reduce, (rtx x0, rtx x1, rtx x2))
DEF_TARGET_INSN (omp_simt_xchg_idx, (rtx x0, rtx x1, rtx x2))
DEF_TARGET_INSN (prefetch, (rtx x0, rtx x1, rtx x2))
DEF_TARGET_INSN (probe_stack, (rtx x0))
diff --git a/gcc/tree-cfg.cc b/gcc/tree-cfg.cc
index ba1d4a125ad..116e07f1d39 100644
--- a/gcc/tree-cfg.cc
+++ b/gcc/tree-cfg.cc
@@ -6489,13 +6489,14 @@ gimple_can_duplicate_bb_p (const_basic_block bb)
/* An IFN_GOMP_SIMT_ENTER_ALLOC/IFN_GOMP_SIMT_EXIT call must be
duplicated as part of its group, or not at all.
- The IFN_GOMP_SIMT_VOTE_ANY and IFN_GOMP_SIMT_XCHG_* are part of such a
- group, so the same holds there. */
+ The IFN_GOMP_SIMT_VOTE_ANY, IFN_GOMP_SIMT_XCHG_IDX and
+ IFN_GOMP_SIMT_REDUCE are part of such a group, so the same holds
+ there. */
if (is_gimple_call (g)
&& (gimple_call_internal_p (g, IFN_GOMP_SIMT_ENTER_ALLOC)
|| gimple_call_internal_p (g, IFN_GOMP_SIMT_EXIT)
|| gimple_call_internal_p (g, IFN_GOMP_SIMT_VOTE_ANY)
- || gimple_call_internal_p (g, IFN_GOMP_SIMT_XCHG_BFLY)
+ || gimple_call_internal_p (g, IFN_GOMP_SIMT_REDUCE)
|| gimple_call_internal_p (g, IFN_GOMP_SIMT_XCHG_IDX)))
return false;
}
--
2.54.0
