When predicating a load we implicitly assume that the else value is
zero. In order to formalize this this patch queries the target for
its supported else operand and uses that for the maskload call.
Subsequently, if the else operand is nonzero, a cond_expr enforcing
a zero else value is emitted.
gcc/ChangeLog:
* tree-if-conv.cc (predicate_load_or_store): Enforce zero else
value.
(predicate_statements): Use sequence instead of statement.
---
gcc/tree-if-conv.cc | 78 +++++++++++++++++++++++++++++++++++----------
1 file changed, 62 insertions(+), 16 deletions(-)
diff --git a/gcc/tree-if-conv.cc b/gcc/tree-if-conv.cc
index 57992b6deca..54cb9ef0ef1 100644
--- a/gcc/tree-if-conv.cc
+++ b/gcc/tree-if-conv.cc
@@ -2452,10 +2452,12 @@ mask_exists (int size, const vec<int> &vec)
write and it needs to be predicated by MASK. Return a statement
that does so. */
-static gimple *
-predicate_load_or_store (gimple_stmt_iterator *gsi, gassign *stmt, tree mask)
+static gimple_seq
+predicate_load_or_store (gimple_stmt_iterator *gsi, gassign *stmt, tree mask,
+ hash_set<tree_ssa_name_hash> *ssa_names)
{
- gcall *new_stmt;
+ gimple_seq stmts = NULL;
+ gcall *call_stmt;
tree lhs = gimple_assign_lhs (stmt);
tree rhs = gimple_assign_rhs1 (stmt);
@@ -2471,21 +2473,59 @@ predicate_load_or_store (gimple_stmt_iterator *gsi,
gassign *stmt, tree mask)
ref);
if (TREE_CODE (lhs) == SSA_NAME)
{
- new_stmt
- = gimple_build_call_internal (IFN_MASK_LOAD, 3, addr,
- ptr, mask);
- gimple_call_set_lhs (new_stmt, lhs);
- gimple_set_vuse (new_stmt, gimple_vuse (stmt));
+ /* Get the preferred vector mode and its corresponding mask for the
+ masked load. We need this to query the target's supported else
+ operands. */
+ machine_mode mode = TYPE_MODE (TREE_TYPE (addr));
+ scalar_mode smode = as_a <scalar_mode> (mode);
+
+ machine_mode vmode = targetm.vectorize.preferred_simd_mode (smode);
+ machine_mode mask_mode
+ = targetm.vectorize.get_mask_mode (vmode).require ();
+
+ int elsval;
+ internal_fn ifn;
+ target_supports_mask_load_store_p (vmode, mask_mode, true, &ifn,
&elsval);
+ tree els = vect_get_mask_load_else (elsval, TREE_TYPE (lhs));
+
+ call_stmt
+ = gimple_build_call_internal (IFN_MASK_LOAD, 4, addr,
+ ptr, mask, els);
+
+ /* Build the load call and, if the else value is nonzero,
+ a COND_EXPR that enforces it. */
+ tree loadlhs;
+ if (elsval == MASK_LOAD_ELSE_ZERO)
+ gimple_call_set_lhs (call_stmt, gimple_get_lhs (stmt));
+ else
+ {
+ loadlhs = make_temp_ssa_name (TREE_TYPE (lhs), NULL, "_ifc_");
+ ssa_names->add (loadlhs);
+ gimple_call_set_lhs (call_stmt, loadlhs);
+ }
+ gimple_set_vuse (call_stmt, gimple_vuse (stmt));
+ gimple_seq_add_stmt (&stmts, call_stmt);
+
+ if (elsval != MASK_LOAD_ELSE_ZERO)
+ {
+ tree cond_rhs
+ = fold_build_cond_expr (TREE_TYPE (loadlhs), mask, loadlhs,
+ build_zero_cst (TREE_TYPE (loadlhs)));
+ gassign *cond_stmt
+ = gimple_build_assign (gimple_get_lhs (stmt), cond_rhs);
+ gimple_seq_add_stmt (&stmts, cond_stmt);
+ }
}
else
{
- new_stmt
+ call_stmt
= gimple_build_call_internal (IFN_MASK_STORE, 4, addr, ptr,
mask, rhs);
- gimple_move_vops (new_stmt, stmt);
+ gimple_move_vops (call_stmt, stmt);
+ gimple_seq_add_stmt (&stmts, call_stmt);
}
- gimple_call_set_nothrow (new_stmt, true);
- return new_stmt;
+ gimple_call_set_nothrow (call_stmt, true);
+ return stmts;
}
/* STMT uses OP_LHS. Check whether it is equivalent to:
@@ -2789,11 +2829,17 @@ predicate_statements (loop_p loop)
vect_masks.safe_push (mask);
}
if (gimple_assign_single_p (stmt))
- new_stmt = predicate_load_or_store (&gsi, stmt, mask);
- else
- new_stmt = predicate_rhs_code (stmt, mask, cond, &ssa_names);
+ {
+ gimple_seq call_seq
+ = predicate_load_or_store (&gsi, stmt, mask, &ssa_names);
- gsi_replace (&gsi, new_stmt, true);
+ gsi_replace_with_seq (&gsi, call_seq, true);
+ }
+ else
+ {
+ new_stmt = predicate_rhs_code (stmt, mask, cond, &ssa_names);
+ gsi_replace (&gsi, new_stmt, true);
+ }
}
else if (((lhs = gimple_assign_lhs (stmt)), true)
&& (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
--
2.45.2
