This patch lets vector branches consume the conditional compare IFNs added
earlier in the series.
The IFNs need to survive long enough for branch expansion to see them, so the
patch marks the selected internal calls as TER-replaceable. It then teaches
emit_cmp_and_jump_insns to look through IFN_COND_VEC_CMP and
IFN_COND_LEN_VEC_CMP definitions when expanding vector conditions.
For example, this source:
int any_gt (int *a, int *b, int n)
{
for (int i = 0; i < n; ++i)
if (a[i] > b[i])
return i;
return 0;
}
Before this patch the tree-outof-ssa changes would fail as the new IFNs aren't
TER-able, mainly due to them not being standard optabs. Normally
direct_internal_fn_p is enough to consider an optab available, but for
comparisons
we have to perform more checks and so direct_internal_fn_p is not enough.
That means the IFNs fail the check in ssa_is_replaceable_p. This adds a promise
essentially that if you see the IFN in the IL then the producer has checked it
and
it's TER safe.
This then updates the vec_cbranch_* code to support the new COND_* IFNs.
Without
this we regress codegen to
cmpgt p1.s, p0/z, z0.s, z1.s
ptest p0, p1.b
b.ne .Lhit
and after this patch, branch expansion can see the compare IFN directly and pass
the original compare through the normal target compare-and-branch machinery:
cmpgt p1.s, p0/z, z0.s, z1.s
b.any .Lhit
Tests for this are the existing vec_cbranch tests which fail.
Bootstrapped Regtested on aarch64-none-linux-gnu,
arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
-m32, -m64 and no issues.
-O3, profiled, profiled -O3 and -O3 -LTO bootstrap tested and no issues.
Also SPECCPU 2017 and SPECCPU 2026 tested.
OK for master?
gcc/ChangeLog:
* internal-fn.cc (ter_replaceable_internal_fn_p): New function.
* internal-fn.h (ter_replaceable_internal_fn_p): Declare.
* optabs.cc (emit_cmp_and_jump_insns): Handle IFN_COND_VEC_CMP
and IFN_COND_LEN_VEC_CMP definitions.
* tree-outof-ssa.cc (ssa_is_replaceable_p): Allow selected
internal calls that are not direct internal functions.
* tree-ssa-ter.cc: Include internal-fn.h.
(find_replaceable_in_bb): Do not count TER-replaceable internal
functions as calls.
---
diff --git a/gcc/internal-fn.cc b/gcc/internal-fn.cc
index
fbb9847b06054a39d5db7f26e293b5af909494cb..8c43d0754961d522846bd5e1f88ff8c6c85b8029
100644
--- a/gcc/internal-fn.cc
+++ b/gcc/internal-fn.cc
@@ -5685,6 +5685,28 @@ vectorized_internal_fn_supported_p (internal_fn ifn,
tree type)
return false;
}
+/* Return true if STMT is an internal function that has no side effects, does
+ not clobber registers and will be expanded to a non-library call. */
+
+bool ter_replaceable_internal_fn_p (gimple *stmt)
+{
+ if (!is_gimple_call (stmt)
+ || !gimple_call_internal_p (stmt))
+ return false;
+
+ switch (gimple_call_internal_fn (stmt))
+ {
+ case IFN_COND_VEC_CMP:
+ case IFN_COND_LEN_VEC_CMP:
+ return true;
+
+ default:
+ return false;
+ }
+
+ return false;
+}
+
void
expand_SHUFFLEVECTOR (internal_fn, gcall *)
{
diff --git a/gcc/internal-fn.h b/gcc/internal-fn.h
index
ac3a35eef62663267dd7ed8c749d02154208486b..8253f515b0fc46ccbf7486274791c4ea9bf38b9c
100644
--- a/gcc/internal-fn.h
+++ b/gcc/internal-fn.h
@@ -295,6 +295,7 @@ extern bool expand_cond_vec_cmp_p (tree_code, tree, tree,
tree);
extern bool expand_cond_len_vec_cmp_p (tree_code, tree, tree, tree, tree,
tree);
extern bool vectorized_internal_fn_supported_p (internal_fn, tree);
+extern bool ter_replaceable_internal_fn_p (gimple *);
enum {
ATOMIC_OP_FETCH_CMP_0_EQ = 0,
diff --git a/gcc/optabs.cc b/gcc/optabs.cc
index
466fa581c47120eef2e366e0d8fc789249b5bd45..cac2d64e14530d722d754436a95810a68ab3170c
100644
--- a/gcc/optabs.cc
+++ b/gcc/optabs.cc
@@ -5031,62 +5031,86 @@ emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code
comparison, rtx size,
enum insn_code icode;
if (def_stmt
- && is_gimple_assign (def_stmt)
- && TREE_CODE_CLASS (gimple_assign_rhs_code (def_stmt))
- == tcc_comparison)
+ && ((is_gimple_assign (def_stmt)
+ && TREE_CODE_CLASS (gimple_assign_rhs_code (def_stmt))
+ == tcc_comparison)
+ || gimple_call_internal_p (def_stmt, IFN_COND_VEC_CMP)
+ || gimple_call_internal_p (def_stmt, IFN_COND_LEN_VEC_CMP)))
{
class expand_operand ops[5];
rtx_insn *tmp = NULL;
start_sequence ();
- tree t_op0 = gimple_assign_rhs1 (def_stmt);
- tree t_op1 = gimple_assign_rhs2 (def_stmt);
+ tree t_op0, t_op1;
+ tree_code inner_code;
+ if (is_gimple_call (def_stmt))
+ {
+ t_op0 = gimple_call_arg (def_stmt, 2);
+ t_op1 = gimple_call_arg (def_stmt, 3);
+ tree code_arg = gimple_call_arg (def_stmt, 0);
+ inner_code = (tree_code) tree_to_uhwi (code_arg);
+ }
+ else
+ {
+ t_op0 = gimple_assign_rhs1 (def_stmt);
+ t_op1 = gimple_assign_rhs2 (def_stmt);
+ inner_code = gimple_assign_rhs_code (def_stmt);
+ }
rtx op0c = expand_normal (t_op0);
rtx op1c = expand_normal (t_op1);
machine_mode mode2 = GET_MODE (op0c);
- int nops = masked_op ? 3 : (len_op ? 5 : 2);
+ int nops = len_op ? 5 : (masked_op ? 3 : 2);
int offset = masked_op || len_op ? 1 : 0;
create_input_operand (&ops[offset + 0], op0c, mode2);
create_input_operand (&ops[offset + 1], op1c, mode2);
- if (masked_op)
- {
- auto mask_mode = TYPE_MODE (TREE_TYPE (masked_op));
- rtx mask_op = expand_normal (masked_op);
- create_input_operand (&ops[0], mask_op, mask_mode);
- }
- else if (len_op)
+ if (len_op)
{
+ machine_mode mask_mode;
+ rtx mask_op;
rtx len_rtx = expand_normal (len_op);
rtx len_bias_rtx = expand_normal (len_bias);
- tree lhs = gimple_get_lhs (def_stmt);
- auto mask_mode = TYPE_MODE (TREE_TYPE (lhs));
+ if (masked_op)
+ {
+ mask_mode = TYPE_MODE (TREE_TYPE (masked_op));
+ mask_op = expand_normal (masked_op);
+ }
+ else
+ {
+ tree lhs = gimple_get_lhs (def_stmt);
+ mask_mode = TYPE_MODE (TREE_TYPE (lhs));
+ mask_op = CONSTM1_RTX (mask_mode);
+ }
/* ??? We could use something like internal_fn's
add_mask_else_and_len_args here. Currently it
only supports a fixed, consecutive order of
mask and len, though. */
- create_input_operand (&ops[0], CONSTM1_RTX (mask_mode),
- mask_mode);
+ create_input_operand (&ops[0], mask_op, mask_mode);
create_convert_operand_from
(&ops[3], len_rtx, TYPE_MODE (TREE_TYPE (len_op)),
TYPE_UNSIGNED (TREE_TYPE (len_op)));
create_input_operand (&ops[4], len_bias_rtx, QImode);
}
+ else if (masked_op)
+ {
+ auto mask_mode = TYPE_MODE (TREE_TYPE (masked_op));
+ rtx mask_op = expand_normal (masked_op);
+ create_input_operand (&ops[0], mask_op, mask_mode);
+ }
int unsignedp2 = TYPE_UNSIGNED (TREE_TYPE (t_op0));
- auto inner_code = gimple_assign_rhs_code (def_stmt);
rtx test2 = NULL_RTX;
enum rtx_code comparison2 = get_rtx_code (inner_code, unsignedp2);
- if (unsignedp2)
- comparison2 = unsigned_condition (comparison2);
if (comparison == NE)
- optab = masked_op ? cond_vec_cbranch_any_optab
- : len_op ? cond_len_vec_cbranch_any_optab
- : vec_cbranch_any_optab;
+ optab = masked_op ? (len_op ? cond_len_vec_cbranch_any_optab
+ : cond_vec_cbranch_any_optab)
+ : (len_op ? cond_len_vec_cbranch_any_optab
+ : vec_cbranch_any_optab);
else
- optab = masked_op ? cond_vec_cbranch_all_optab
- : len_op ? cond_len_vec_cbranch_all_optab
- : vec_cbranch_all_optab;
+ optab = masked_op ? (len_op ? cond_len_vec_cbranch_all_optab
+ : cond_vec_cbranch_all_optab)
+ : (len_op ? cond_len_vec_cbranch_all_optab
+ : vec_cbranch_all_optab);
if ((icode = optab_handler (optab, mode2))
!= CODE_FOR_nothing
diff --git a/gcc/tree-outof-ssa.cc b/gcc/tree-outof-ssa.cc
index
fa7264ba832888abf18d1dc73497e856c67b6793..892f4d7dca5cd451e7051ed89f6ac62a157c4695
100644
--- a/gcc/tree-outof-ssa.cc
+++ b/gcc/tree-outof-ssa.cc
@@ -72,7 +72,8 @@ ssa_is_replaceable_p (gimple *stmt)
&& (!(call = dyn_cast <gcall *> (stmt))
|| gimple_call_tail_p (call)
|| !gimple_call_internal_p (call)
- || !direct_internal_fn_p (gimple_call_internal_fn (call))))
+ || (!direct_internal_fn_p (gimple_call_internal_fn (call))
+ && !ter_replaceable_internal_fn_p (call))))
return false;
/* If the statement may throw an exception, it cannot be replaced. */
diff --git a/gcc/tree-ssa-ter.cc b/gcc/tree-ssa-ter.cc
index
8dc26aa1907cbf25ee00b6cad524a0963a2b8839..5d6ccbb589e427f47d0392672bf1b3c41925671f
100644
--- a/gcc/tree-ssa-ter.cc
+++ b/gcc/tree-ssa-ter.cc
@@ -33,6 +33,7 @@ along with GCC; see the file COPYING3. If not see
#include "tree-ssa-ter.h"
#include "tree-outof-ssa.h"
#include "gimple-walk.h"
+#include "internal-fn.h"
/* Temporary Expression Replacement (TER)
@@ -686,6 +687,7 @@ find_replaceable_in_bb (temp_expr_table *tab, basic_block
bb)
replacement over BUILT_IN calls since many will expand to inline
insns instead of a true call. */
if (is_gimple_call (stmt)
+ && !ter_replaceable_internal_fn_p (stmt)
&& !((fndecl = gimple_call_fndecl (stmt))
&& fndecl_built_in_p (fndecl)))
cur_call_cnt++;
--
diff --git a/gcc/internal-fn.cc b/gcc/internal-fn.cc
index fbb9847b06054a39d5db7f26e293b5af909494cb..8c43d0754961d522846bd5e1f88ff8c6c85b8029 100644
--- a/gcc/internal-fn.cc
+++ b/gcc/internal-fn.cc
@@ -5685,6 +5685,28 @@ vectorized_internal_fn_supported_p (internal_fn ifn, tree type)
return false;
}
+/* Return true if STMT is an internal function that has no side effects, does
+ not clobber registers and will be expanded to a non-library call. */
+
+bool ter_replaceable_internal_fn_p (gimple *stmt)
+{
+ if (!is_gimple_call (stmt)
+ || !gimple_call_internal_p (stmt))
+ return false;
+
+ switch (gimple_call_internal_fn (stmt))
+ {
+ case IFN_COND_VEC_CMP:
+ case IFN_COND_LEN_VEC_CMP:
+ return true;
+
+ default:
+ return false;
+ }
+
+ return false;
+}
+
void
expand_SHUFFLEVECTOR (internal_fn, gcall *)
{
diff --git a/gcc/internal-fn.h b/gcc/internal-fn.h
index ac3a35eef62663267dd7ed8c749d02154208486b..8253f515b0fc46ccbf7486274791c4ea9bf38b9c 100644
--- a/gcc/internal-fn.h
+++ b/gcc/internal-fn.h
@@ -295,6 +295,7 @@ extern bool expand_cond_vec_cmp_p (tree_code, tree, tree, tree);
extern bool expand_cond_len_vec_cmp_p (tree_code, tree, tree, tree, tree, tree);
extern bool vectorized_internal_fn_supported_p (internal_fn, tree);
+extern bool ter_replaceable_internal_fn_p (gimple *);
enum {
ATOMIC_OP_FETCH_CMP_0_EQ = 0,
diff --git a/gcc/optabs.cc b/gcc/optabs.cc
index 466fa581c47120eef2e366e0d8fc789249b5bd45..cac2d64e14530d722d754436a95810a68ab3170c 100644
--- a/gcc/optabs.cc
+++ b/gcc/optabs.cc
@@ -5031,62 +5031,86 @@ emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size,
enum insn_code icode;
if (def_stmt
- && is_gimple_assign (def_stmt)
- && TREE_CODE_CLASS (gimple_assign_rhs_code (def_stmt))
- == tcc_comparison)
+ && ((is_gimple_assign (def_stmt)
+ && TREE_CODE_CLASS (gimple_assign_rhs_code (def_stmt))
+ == tcc_comparison)
+ || gimple_call_internal_p (def_stmt, IFN_COND_VEC_CMP)
+ || gimple_call_internal_p (def_stmt, IFN_COND_LEN_VEC_CMP)))
{
class expand_operand ops[5];
rtx_insn *tmp = NULL;
start_sequence ();
- tree t_op0 = gimple_assign_rhs1 (def_stmt);
- tree t_op1 = gimple_assign_rhs2 (def_stmt);
+ tree t_op0, t_op1;
+ tree_code inner_code;
+ if (is_gimple_call (def_stmt))
+ {
+ t_op0 = gimple_call_arg (def_stmt, 2);
+ t_op1 = gimple_call_arg (def_stmt, 3);
+ tree code_arg = gimple_call_arg (def_stmt, 0);
+ inner_code = (tree_code) tree_to_uhwi (code_arg);
+ }
+ else
+ {
+ t_op0 = gimple_assign_rhs1 (def_stmt);
+ t_op1 = gimple_assign_rhs2 (def_stmt);
+ inner_code = gimple_assign_rhs_code (def_stmt);
+ }
rtx op0c = expand_normal (t_op0);
rtx op1c = expand_normal (t_op1);
machine_mode mode2 = GET_MODE (op0c);
- int nops = masked_op ? 3 : (len_op ? 5 : 2);
+ int nops = len_op ? 5 : (masked_op ? 3 : 2);
int offset = masked_op || len_op ? 1 : 0;
create_input_operand (&ops[offset + 0], op0c, mode2);
create_input_operand (&ops[offset + 1], op1c, mode2);
- if (masked_op)
- {
- auto mask_mode = TYPE_MODE (TREE_TYPE (masked_op));
- rtx mask_op = expand_normal (masked_op);
- create_input_operand (&ops[0], mask_op, mask_mode);
- }
- else if (len_op)
+ if (len_op)
{
+ machine_mode mask_mode;
+ rtx mask_op;
rtx len_rtx = expand_normal (len_op);
rtx len_bias_rtx = expand_normal (len_bias);
- tree lhs = gimple_get_lhs (def_stmt);
- auto mask_mode = TYPE_MODE (TREE_TYPE (lhs));
+ if (masked_op)
+ {
+ mask_mode = TYPE_MODE (TREE_TYPE (masked_op));
+ mask_op = expand_normal (masked_op);
+ }
+ else
+ {
+ tree lhs = gimple_get_lhs (def_stmt);
+ mask_mode = TYPE_MODE (TREE_TYPE (lhs));
+ mask_op = CONSTM1_RTX (mask_mode);
+ }
/* ??? We could use something like internal_fn's
add_mask_else_and_len_args here. Currently it
only supports a fixed, consecutive order of
mask and len, though. */
- create_input_operand (&ops[0], CONSTM1_RTX (mask_mode),
- mask_mode);
+ create_input_operand (&ops[0], mask_op, mask_mode);
create_convert_operand_from
(&ops[3], len_rtx, TYPE_MODE (TREE_TYPE (len_op)),
TYPE_UNSIGNED (TREE_TYPE (len_op)));
create_input_operand (&ops[4], len_bias_rtx, QImode);
}
+ else if (masked_op)
+ {
+ auto mask_mode = TYPE_MODE (TREE_TYPE (masked_op));
+ rtx mask_op = expand_normal (masked_op);
+ create_input_operand (&ops[0], mask_op, mask_mode);
+ }
int unsignedp2 = TYPE_UNSIGNED (TREE_TYPE (t_op0));
- auto inner_code = gimple_assign_rhs_code (def_stmt);
rtx test2 = NULL_RTX;
enum rtx_code comparison2 = get_rtx_code (inner_code, unsignedp2);
- if (unsignedp2)
- comparison2 = unsigned_condition (comparison2);
if (comparison == NE)
- optab = masked_op ? cond_vec_cbranch_any_optab
- : len_op ? cond_len_vec_cbranch_any_optab
- : vec_cbranch_any_optab;
+ optab = masked_op ? (len_op ? cond_len_vec_cbranch_any_optab
+ : cond_vec_cbranch_any_optab)
+ : (len_op ? cond_len_vec_cbranch_any_optab
+ : vec_cbranch_any_optab);
else
- optab = masked_op ? cond_vec_cbranch_all_optab
- : len_op ? cond_len_vec_cbranch_all_optab
- : vec_cbranch_all_optab;
+ optab = masked_op ? (len_op ? cond_len_vec_cbranch_all_optab
+ : cond_vec_cbranch_all_optab)
+ : (len_op ? cond_len_vec_cbranch_all_optab
+ : vec_cbranch_all_optab);
if ((icode = optab_handler (optab, mode2))
!= CODE_FOR_nothing
diff --git a/gcc/tree-outof-ssa.cc b/gcc/tree-outof-ssa.cc
index fa7264ba832888abf18d1dc73497e856c67b6793..892f4d7dca5cd451e7051ed89f6ac62a157c4695 100644
--- a/gcc/tree-outof-ssa.cc
+++ b/gcc/tree-outof-ssa.cc
@@ -72,7 +72,8 @@ ssa_is_replaceable_p (gimple *stmt)
&& (!(call = dyn_cast <gcall *> (stmt))
|| gimple_call_tail_p (call)
|| !gimple_call_internal_p (call)
- || !direct_internal_fn_p (gimple_call_internal_fn (call))))
+ || (!direct_internal_fn_p (gimple_call_internal_fn (call))
+ && !ter_replaceable_internal_fn_p (call))))
return false;
/* If the statement may throw an exception, it cannot be replaced. */
diff --git a/gcc/tree-ssa-ter.cc b/gcc/tree-ssa-ter.cc
index 8dc26aa1907cbf25ee00b6cad524a0963a2b8839..5d6ccbb589e427f47d0392672bf1b3c41925671f 100644
--- a/gcc/tree-ssa-ter.cc
+++ b/gcc/tree-ssa-ter.cc
@@ -33,6 +33,7 @@ along with GCC; see the file COPYING3. If not see
#include "tree-ssa-ter.h"
#include "tree-outof-ssa.h"
#include "gimple-walk.h"
+#include "internal-fn.h"
/* Temporary Expression Replacement (TER)
@@ -686,6 +687,7 @@ find_replaceable_in_bb (temp_expr_table *tab, basic_block bb)
replacement over BUILT_IN calls since many will expand to inline
insns instead of a true call. */
if (is_gimple_call (stmt)
+ && !ter_replaceable_internal_fn_p (stmt)
&& !((fndecl = gimple_call_fndecl (stmt))
&& fndecl_built_in_p (fndecl)))
cur_call_cnt++;
