This patch forms conditional vector compare IFNs during gimple isel.
The match.pd rules recognize integer vector compare idioms where a compare is
guarded by another vector predicate. gimple isel then lowers those idioms to
IFN_COND_VEC_CMP or IFN_COND_LEN_VEC_CMP when the target reports support for
the corresponding optab. If the original compare was only used by the
converted statement, the patch removes that now-dead definition.
The tests for this lowering are kept with the later AArch64 SVE expander
patch, since they require the SVE cond_vec_cmp target support added there.
Bootstrapped Regtested on aarch64-none-linux-gnu,
arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
-m32, -m64 and no issues.
-O3, profiled, profiled -O3 and -O3 -LTO bootstrap tested and no issues.
Also SPECCPU 2017 and SPECCPU 2026 tested.
OK for master?
gcc/ChangeLog:
* gimple-isel.cc (gimple_convert_conditional_compare): Declare.
(gimple_convert_conditional_len_compare): Likewise.
(maybe_duplicate_comparison): Fix comment.
(gimple_expand_conditional_compare): New function.
(pass_gimple_isel::execute): Use it.
* match.pd (convert_conditional_compare): New match.
(convert_conditional_len_compare): Likewise.
---
diff --git a/gcc/gimple-isel.cc b/gcc/gimple-isel.cc
index
1d346ee124ffdfdccf658ea79cc72cbf80240f81..9bd523cd81be541383ea9c962a9180565cea9f0d
100644
--- a/gcc/gimple-isel.cc
+++ b/gcc/gimple-isel.cc
@@ -327,8 +327,8 @@ maybe_duplicate_comparison (gassign *stmt, basic_block bb)
tree lhs = gimple_assign_lhs (stmt);
unsigned cnt = 0;
- /* This is should not be used for -O0 nor it is not useful
- when ter is turned off. */
+ /* This should not be used for -O0 and is not useful when TER is turned
+ off. */
if (!optimize || !flag_tree_ter)
return;
@@ -376,6 +376,9 @@ maybe_duplicate_comparison (gassign *stmt, basic_block bb)
extern bool gimple_nop_atomic_bit_test_and_p (tree, tree *,
tree (*) (tree));
extern bool gimple_nop_convert (tree, tree*, tree (*) (tree));
+extern bool gimple_convert_conditional_compare (tree, tree *, tree (*) (tree));
+extern bool gimple_convert_conditional_len_compare (tree, tree *,
+ tree (*) (tree));
namespace {
@@ -488,6 +491,74 @@ convert_atomic_bit_not (enum internal_fn fn, gimple
*use_stmt,
return use_stmt;
}
+/* Try to replace STMT with a conditional vector comparison internal function.
+ The generated matchers recognize the source patterns and capture the mask,
+ operands and original comparison SSA name. If the original comparison is
+ used only by STMT, store its defining statement in *DEAD_CMP_DEF so that the
+ caller can remove it after replacing STMT. */
+
+static gimple *
+gimple_expand_conditional_compare (gimple_stmt_iterator *gsi,
+ gassign **dead_cmp_def)
+{
+ gimple *stmt = gsi_stmt (*gsi);
+ *dead_cmp_def = nullptr;
+ tree lhs = gimple_get_lhs (stmt);
+
+ if (!lhs || TREE_CODE (lhs) != SSA_NAME)
+ return nullptr;
+
+ tree ops[6] = {};
+ location_t loc = gimple_location (stmt);
+ bool len_p = false;
+ if (!gimple_convert_conditional_compare (lhs, ops, nullptr)
+ && !gimple_convert_conditional_len_compare (lhs, ops, nullptr))
+ return nullptr;
+ len_p = ops[4] != NULL_TREE;
+
+ unsigned int cmp_opno = len_p ? 5 : 3;
+ tree cmp_name = ops[cmp_opno];
+ if (TREE_CODE (cmp_name) != SSA_NAME)
+ return nullptr;
+
+ gassign *cmp_def = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (cmp_name));
+ if (!cmp_def)
+ return nullptr;
+
+ tree_code code = gimple_assign_rhs_code (cmp_def);
+ if (TREE_CODE_CLASS (code) != tcc_comparison)
+ return nullptr;
+
+ internal_fn ifn = len_p ? IFN_COND_LEN_VEC_CMP : IFN_COND_VEC_CMP;
+ bool supported_p
+ = (len_p
+ ? expand_cond_len_vec_cmp_p (code, ops[0], ops[1], ops[2],
+ ops[3], ops[4])
+ : expand_cond_vec_cmp_p (code, ops[0], ops[1], ops[2]));
+ if (!supported_p)
+ return nullptr;
+
+ auto_vec<tree, 6> args;
+ args.quick_push (build_int_cst (integer_type_node, code));
+ args.quick_push (ops[0]);
+ args.quick_push (ops[1]);
+ args.quick_push (ops[2]);
+ if (len_p)
+ {
+ args.quick_push (ops[3]);
+ args.quick_push (ops[4]);
+ }
+
+ gcall *call = gimple_build_call_internal_vec (ifn, args);
+ gimple_call_set_lhs (call, lhs);
+ gimple_set_location (call, loc);
+ use_operand_p use_p;
+ gimple *use_stmt;
+ if (single_imm_use (cmp_name, &use_p, &use_stmt) && use_stmt == stmt)
+ *dead_cmp_def = cmp_def;
+ return call;
+}
+
/* Optimize
mask_2 = 1 << cnt_1;
_4 = __atomic_fetch_or_* (ptr_6, mask_2, _3);
@@ -1369,6 +1440,20 @@ pass_gimple_isel::execute (struct function *fun)
/* Give the target first try at replacing the instruction. */
cfg_changed |= targetm.instruction_selection (fun, &gsi);
+ gimple *isel_stmt = NULL;
+ gassign *dead_cmp_def = NULL;
+ if (!gsi_end_p (gsi))
+ isel_stmt = gimple_expand_conditional_compare (&gsi, &dead_cmp_def);
+ if (isel_stmt)
+ {
+ gsi_replace (&gsi, isel_stmt, true);
+ if (dead_cmp_def)
+ {
+ gimple_stmt_iterator cmp_gsi = gsi_for_stmt (dead_cmp_def);
+ gsi_remove (&cmp_gsi, true);
+ }
+ }
+
/* Pre-expand VEC_COND_EXPRs to .VCOND* internal function
calls mapping to supported optabs. */
gimple *g = gimple_expand_vec_cond_expr (&gsi);
@@ -1411,4 +1496,3 @@ make_pass_gimple_isel (gcc::context *ctxt)
{
return new pass_gimple_isel (ctxt);
}
-
diff --git a/gcc/match.pd b/gcc/match.pd
index
c1aba410b66e38c947a4d52064fda1594934f297..db5038b7dccd030a09b67d5070a2ce2b74651184
100644
--- a/gcc/match.pd
+++ b/gcc/match.pd
@@ -13364,3 +13364,20 @@ and,
(if (INTEGRAL_TYPE_P (type))
(with { tree itype = TREE_TYPE (@2); }
(convert (minus @2 (convert:itype @1))))))
+
+#if GIMPLE
+
+(for cmp (tcc_comparison)
+ (match (convert_conditional_compare @0 @1 @2 @3)
+ (bit_and:c @0 (cmp@3 @1 @2))
+ (if (VECTOR_BOOLEAN_TYPE_P (type)
+ && !FLOAT_TYPE_P (TREE_TYPE (@1)))))
+
+/* Transform .VCOND_MASK_LEN (a `cmp` b, {-1, ...}, {0, ...}, len, bias)
+ into cond_len_vec_cmp (cmp, mask, a, b, len, bias). */
+ (match (convert_conditional_len_compare @0 @1 @2 @3 @4 @5)
+ (IFN_VCOND_MASK_LEN @0 (cmp@5 @1 @2) integer_minus_onep integer_zerop @3 @4)
+ (if (VECTOR_BOOLEAN_TYPE_P (type)
+ && !FLOAT_TYPE_P (TREE_TYPE (@1))))))
+
+#endif
--
diff --git a/gcc/gimple-isel.cc b/gcc/gimple-isel.cc
index 1d346ee124ffdfdccf658ea79cc72cbf80240f81..9bd523cd81be541383ea9c962a9180565cea9f0d 100644
--- a/gcc/gimple-isel.cc
+++ b/gcc/gimple-isel.cc
@@ -327,8 +327,8 @@ maybe_duplicate_comparison (gassign *stmt, basic_block bb)
tree lhs = gimple_assign_lhs (stmt);
unsigned cnt = 0;
- /* This is should not be used for -O0 nor it is not useful
- when ter is turned off. */
+ /* This should not be used for -O0 and is not useful when TER is turned
+ off. */
if (!optimize || !flag_tree_ter)
return;
@@ -376,6 +376,9 @@ maybe_duplicate_comparison (gassign *stmt, basic_block bb)
extern bool gimple_nop_atomic_bit_test_and_p (tree, tree *,
tree (*) (tree));
extern bool gimple_nop_convert (tree, tree*, tree (*) (tree));
+extern bool gimple_convert_conditional_compare (tree, tree *, tree (*) (tree));
+extern bool gimple_convert_conditional_len_compare (tree, tree *,
+ tree (*) (tree));
namespace {
@@ -488,6 +491,74 @@ convert_atomic_bit_not (enum internal_fn fn, gimple *use_stmt,
return use_stmt;
}
+/* Try to replace STMT with a conditional vector comparison internal function.
+ The generated matchers recognize the source patterns and capture the mask,
+ operands and original comparison SSA name. If the original comparison is
+ used only by STMT, store its defining statement in *DEAD_CMP_DEF so that the
+ caller can remove it after replacing STMT. */
+
+static gimple *
+gimple_expand_conditional_compare (gimple_stmt_iterator *gsi,
+ gassign **dead_cmp_def)
+{
+ gimple *stmt = gsi_stmt (*gsi);
+ *dead_cmp_def = nullptr;
+ tree lhs = gimple_get_lhs (stmt);
+
+ if (!lhs || TREE_CODE (lhs) != SSA_NAME)
+ return nullptr;
+
+ tree ops[6] = {};
+ location_t loc = gimple_location (stmt);
+ bool len_p = false;
+ if (!gimple_convert_conditional_compare (lhs, ops, nullptr)
+ && !gimple_convert_conditional_len_compare (lhs, ops, nullptr))
+ return nullptr;
+ len_p = ops[4] != NULL_TREE;
+
+ unsigned int cmp_opno = len_p ? 5 : 3;
+ tree cmp_name = ops[cmp_opno];
+ if (TREE_CODE (cmp_name) != SSA_NAME)
+ return nullptr;
+
+ gassign *cmp_def = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (cmp_name));
+ if (!cmp_def)
+ return nullptr;
+
+ tree_code code = gimple_assign_rhs_code (cmp_def);
+ if (TREE_CODE_CLASS (code) != tcc_comparison)
+ return nullptr;
+
+ internal_fn ifn = len_p ? IFN_COND_LEN_VEC_CMP : IFN_COND_VEC_CMP;
+ bool supported_p
+ = (len_p
+ ? expand_cond_len_vec_cmp_p (code, ops[0], ops[1], ops[2],
+ ops[3], ops[4])
+ : expand_cond_vec_cmp_p (code, ops[0], ops[1], ops[2]));
+ if (!supported_p)
+ return nullptr;
+
+ auto_vec<tree, 6> args;
+ args.quick_push (build_int_cst (integer_type_node, code));
+ args.quick_push (ops[0]);
+ args.quick_push (ops[1]);
+ args.quick_push (ops[2]);
+ if (len_p)
+ {
+ args.quick_push (ops[3]);
+ args.quick_push (ops[4]);
+ }
+
+ gcall *call = gimple_build_call_internal_vec (ifn, args);
+ gimple_call_set_lhs (call, lhs);
+ gimple_set_location (call, loc);
+ use_operand_p use_p;
+ gimple *use_stmt;
+ if (single_imm_use (cmp_name, &use_p, &use_stmt) && use_stmt == stmt)
+ *dead_cmp_def = cmp_def;
+ return call;
+}
+
/* Optimize
mask_2 = 1 << cnt_1;
_4 = __atomic_fetch_or_* (ptr_6, mask_2, _3);
@@ -1369,6 +1440,20 @@ pass_gimple_isel::execute (struct function *fun)
/* Give the target first try at replacing the instruction. */
cfg_changed |= targetm.instruction_selection (fun, &gsi);
+ gimple *isel_stmt = NULL;
+ gassign *dead_cmp_def = NULL;
+ if (!gsi_end_p (gsi))
+ isel_stmt = gimple_expand_conditional_compare (&gsi, &dead_cmp_def);
+ if (isel_stmt)
+ {
+ gsi_replace (&gsi, isel_stmt, true);
+ if (dead_cmp_def)
+ {
+ gimple_stmt_iterator cmp_gsi = gsi_for_stmt (dead_cmp_def);
+ gsi_remove (&cmp_gsi, true);
+ }
+ }
+
/* Pre-expand VEC_COND_EXPRs to .VCOND* internal function
calls mapping to supported optabs. */
gimple *g = gimple_expand_vec_cond_expr (&gsi);
@@ -1411,4 +1496,3 @@ make_pass_gimple_isel (gcc::context *ctxt)
{
return new pass_gimple_isel (ctxt);
}
-
diff --git a/gcc/match.pd b/gcc/match.pd
index c1aba410b66e38c947a4d52064fda1594934f297..db5038b7dccd030a09b67d5070a2ce2b74651184 100644
--- a/gcc/match.pd
+++ b/gcc/match.pd
@@ -13364,3 +13364,20 @@ and,
(if (INTEGRAL_TYPE_P (type))
(with { tree itype = TREE_TYPE (@2); }
(convert (minus @2 (convert:itype @1))))))
+
+#if GIMPLE
+
+(for cmp (tcc_comparison)
+ (match (convert_conditional_compare @0 @1 @2 @3)
+ (bit_and:c @0 (cmp@3 @1 @2))
+ (if (VECTOR_BOOLEAN_TYPE_P (type)
+ && !FLOAT_TYPE_P (TREE_TYPE (@1)))))
+
+/* Transform .VCOND_MASK_LEN (a `cmp` b, {-1, ...}, {0, ...}, len, bias)
+ into cond_len_vec_cmp (cmp, mask, a, b, len, bias). */
+ (match (convert_conditional_len_compare @0 @1 @2 @3 @4 @5)
+ (IFN_VCOND_MASK_LEN @0 (cmp@5 @1 @2) integer_minus_onep integer_zerop @3 @4)
+ (if (VECTOR_BOOLEAN_TYPE_P (type)
+ && !FLOAT_TYPE_P (TREE_TYPE (@1))))))
+
+#endif
