Allow IFN_MASK_COMPRESS_SCATTER_STORE in masked DR analysis and admit its
data reference in vect_find_stmt_data_reference. Select compress-scatter
when the store offset is the predicated-index PHI.
Gate predicated-index PHI relevance on both phase-2 pattern rewrites
(predicated_index_update and compress_scatter) in
vect_mark_stmts_to_be_vectorized,
vect_analyze_stmt, and vect_is_simple_use.
---
gcc/tree-vect-data-refs.cc | 14 ++++++++++++--
gcc/tree-vect-stmts.cc | 33 +++++++++++++++++++++++++--------
2 files changed, 37 insertions(+), 10 deletions(-)
diff --git a/gcc/tree-vect-data-refs.cc b/gcc/tree-vect-data-refs.cc
index 3b27beb9c5b..d6491366dbf 100644
--- a/gcc/tree-vect-data-refs.cc
+++ b/gcc/tree-vect-data-refs.cc
@@ -4860,7 +4860,11 @@ vect_check_gather_scatter (stmt_vec_info stmt_info, tree
vectype,
supports_vec_gather_load_p (TYPE_MODE (vectype), elsvals);
return true;
}
- masked_p = (ifn == IFN_MASK_LOAD || ifn == IFN_MASK_STORE);
+ /* TODO: IFN_MASK_COMPRESS_SCATTER_STORE is usually handled above via
+ internal_gather_scatter_fn_p; remove from this check once confirmed
+ unnecessary. */
+ masked_p = (ifn == IFN_MASK_LOAD || ifn == IFN_MASK_STORE
+ || ifn == IFN_MASK_COMPRESS_SCATTER_STORE);
}
/* True if we should aim to use internal functions rather than
@@ -5116,6 +5120,11 @@ vect_check_gather_scatter (stmt_vec_info stmt_info, tree
vectype,
&tmp_offset_vectype,
elsvals))
ifn = IFN_LAST;
+ /* TODO: May be redundant once phase-2 pattern recog has already
+ rewritten the store to IFN_MASK_COMPRESS_SCATTER_STORE. */
+ if (!DR_IS_READ (dr) && masked_p && loop_vinfo->predicated_index_phi
+ && off == PHI_RESULT (loop_vinfo->predicated_index_phi))
+ ifn = IFN_MASK_COMPRESS_SCATTER_STORE;
decl = NULL_TREE;
}
else
@@ -5200,7 +5209,8 @@ vect_find_stmt_data_reference (loop_p loop, gimple *stmt,
if (gcall *call = dyn_cast <gcall *> (stmt))
if (!gimple_call_internal_p (call)
|| (gimple_call_internal_fn (call) != IFN_MASK_LOAD
- && gimple_call_internal_fn (call) != IFN_MASK_STORE))
+ && gimple_call_internal_fn (call) != IFN_MASK_STORE
+ && gimple_call_internal_fn (call) !=
IFN_MASK_COMPRESS_SCATTER_STORE))
{
free_data_ref (dr);
return opt_result::failure_at (stmt,
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index 4c5fb462f2e..84e5e701057 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -705,13 +705,16 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info
loop_vinfo, bool *fatal)
if (vect_stmt_relevant_p (phi_info, loop_vinfo, &relevant, &live_p))
{
- /* TODO: Till predicated-index PHI is handled, exit
- gracefully. */
+ /* Allow predicated-index PHI after both pattern rewrites. */
if (STMT_VINFO_DEF_TYPE (phi_info) == vect_predicated_index_def)
- return opt_result::failure_at
- (*si, "not vectorized: predicated-index PHI "
- "not supported yet: %G", *si);
- if (STMT_VINFO_DEF_TYPE (phi_info) == vect_unknown_def_type)
+ {
+ if (!loop_vinfo->predicated_index_update_ifn_p
+ || !loop_vinfo->predicated_index_compress_scatter_ifn_p)
+ return opt_result::failure_at
+ (*si, "not vectorized: predicated-index PHI "
+ "not supported yet: %G", *si);
+ }
+ else if (STMT_VINFO_DEF_TYPE (phi_info) == vect_unknown_def_type)
return opt_result::failure_at
(*si, "not vectorized: unhandled relevant PHI: %G", *si);
vect_mark_relevant (&worklist, phi_info, relevant, live_p);
@@ -13321,8 +13324,18 @@ vect_analyze_stmt (vec_info *vinfo,
case vect_external_def:
case vect_unknown_def_type:
gcc_unreachable ();
- /* TODO: Till predicated-index PHI is handled, exit gracefully. */
+
case vect_predicated_index_def:
+ {
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
+ if (loop_vinfo
+ && loop_vinfo->predicated_index_update_ifn_p
+ && loop_vinfo->predicated_index_compress_scatter_ifn_p)
+ {
+ gcc_assert (!bb_vinfo);
+ break;
+ }
+ }
return opt_result::failure_at
(stmt_info->stmt,
"not vectorized: predicated-index PHI not supported yet: %G\n",
@@ -13972,9 +13985,13 @@ vect_is_simple_use (tree operand, vec_info *vinfo,
enum vect_def_type *dt,
}
}
- /* TODO: Till predicated-index PHI is handled, exit gracefully. */
if (*dt == vect_predicated_index_def)
{
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
+ if (loop_vinfo
+ && loop_vinfo->predicated_index_update_ifn_p
+ && loop_vinfo->predicated_index_compress_scatter_ifn_p)
+ return true;
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
"unsupported: predicated-index PHI.\n");
--
2.34.1
