In preparation for enabling tail-predication for basic block SLP
vectorization, the function used to check whether a load or store
statement in a loop is possible using partial vectors needs to be
updated to work for BB SLP too.
This preparatory commit does not change any of the callers of
check_load_store_for_partial_vectors, therefore loop_vinfo is
expected to be non-null within the modified function (for now).
gcc/ChangeLog:
* tree-vect-stmts.cc (check_load_store_for_partial_vectors):
Change parameter type from loop_vec_info to vec_info *. Use
dyn_cast to get a loop_vec_info or null. Return false if doing
BB SLP and SLP_TREE_LANES might exceed the number of subparts in
the passed-in vector type. Pass vec_info * instead of
loop_vec_info to vect_get_num_copies, vect_record_len and
vect_record_mask. Pass the SLP node through to vect_record_len
and vect_record_mask to let them get SLP_TREE_LANES.
When calculating the number of vectors, get the group size from
SLP_TREE_LANES instead of a parameter (e.g., DR_GROUP_SIZE) if
doing BB SLP vectorization.
---
gcc/tree-vect-stmts.cc | 44 +++++++++++++++++++++++++-----------------
1 file changed, 26 insertions(+), 18 deletions(-)
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index 0b411a6797e..27d13ddf524 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -1443,10 +1443,10 @@ vect_get_load_store_partial_vector_style (tree vectype,
bool is_load,
return vect_load_store_partial_vectors_none;
}
-/* Check whether a load or store statement in the loop described by
- LOOP_VINFO is possible in a loop using partial vectors. This is
- testing whether the vectorizer pass has the appropriate support,
- as well as whether the target does.
+/* Check whether a load or store statement in the loop or SLP region described
+ by VINFO is possible using partial vectors. This is testing whether the
+ vectorizer pass has the appropriate support, as well as whether the target
+ does.
VLS_TYPE says whether the statement is a load or store and VECTYPE
is the type of the vector being loaded or stored. SLP_NODE is the SLP
@@ -1457,14 +1457,14 @@ vect_get_load_store_partial_vector_style (tree vectype,
bool is_load,
its arguments. If the load or store is conditional, SCALAR_MASK is the
condition under which it occurs.
- Return false if a loop using partial vectors is not supported, otherwise
- record the required rgroup control types.
+ Return false if a loop or SLP vectorization using partial vectors is not
+ supported, otherwise record the required rgroup control types.
If partial vectors can be used and ELSVALS is nonzero the supported
else values will be added to the vector ELSVALS points to. */
static bool
-check_load_store_for_partial_vectors (loop_vec_info loop_vinfo, tree vectype,
+check_load_store_for_partial_vectors (vec_info *vinfo, tree vectype,
slp_tree slp_node,
vec_load_store_type vls_type,
unsigned int group_size,
@@ -1472,8 +1472,15 @@ check_load_store_for_partial_vectors (loop_vec_info
loop_vinfo, tree vectype,
slp_tree mask_node,
vec<int> *elsvals = nullptr)
{
+ loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
vect_memory_access_type memory_access_type = ls->memory_access_type;
+ /* For SLP vectorization, we cannot use partial vectors if the
+ group could be wider than the vector type. */
+ poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
+ if (!loop_vinfo && maybe_gt (SLP_TREE_LANES (slp_node), nunits))
+ return false;
+
/* Invariant loads need no special support. */
if (memory_access_type == VMAT_INVARIANT)
return true;
@@ -1495,7 +1502,7 @@ check_load_store_for_partial_vectors (loop_vec_info
loop_vinfo, tree vectype,
scalar_mask = def;
}
- unsigned int nvectors = vect_get_num_copies (loop_vinfo, slp_node);
+ unsigned int nvectors = vect_get_num_copies (vinfo, slp_node);
machine_mode vecmode = TYPE_MODE (vectype);
bool is_load = (vls_type == VLS_LOAD);
if (memory_access_type == VMAT_LOAD_STORE_LANES)
@@ -1506,9 +1513,9 @@ check_load_store_for_partial_vectors (loop_vec_info
loop_vinfo, tree vectype,
elsvals)
: vect_store_lanes_supported (vectype, group_size, true));
if (ifn == IFN_MASK_LEN_LOAD_LANES || ifn == IFN_MASK_LEN_STORE_LANES)
- vect_record_len (loop_vinfo, nvectors, vectype, 1);
+ vect_record_len (vinfo, nvectors, vectype, 1, slp_node);
else if (ifn == IFN_MASK_LOAD_LANES || ifn == IFN_MASK_STORE_LANES)
- vect_record_mask (loop_vinfo, nvectors, vectype, scalar_mask);
+ vect_record_mask (vinfo, nvectors, vectype, scalar_mask, slp_node);
else
{
if (dump_enabled_p ())
@@ -1550,13 +1557,13 @@ check_load_store_for_partial_vectors (loop_vec_info
loop_vinfo, tree vectype,
memory_type,
off_vectype, scale,
elsvals))
- vect_record_len (loop_vinfo, nvectors, vectype, 1);
+ vect_record_len (vinfo, nvectors, vectype, 1, slp_node);
else if (internal_gather_scatter_fn_supported_p (ifn, vectype,
memory_type,
off_vectype, scale,
elsvals)
|| memory_access_type == VMAT_GATHER_SCATTER_LEGACY)
- vect_record_mask (loop_vinfo, nvectors, vectype, scalar_mask);
+ vect_record_mask (vinfo, nvectors, vectype, scalar_mask, slp_node);
else
{
if (dump_enabled_p ())
@@ -1600,8 +1607,9 @@ check_load_store_for_partial_vectors (loop_vec_info
loop_vinfo, tree vectype,
gcc_unreachable ();
};
- poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
- poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+ poly_uint64 size = loop_vinfo
+ ? group_size * LOOP_VINFO_VECT_FACTOR (loop_vinfo)
+ : SLP_TREE_LANES (slp_node);
unsigned factor;
vect_load_store_partial_vector_style partial_vector_style
= vect_get_load_store_partial_vector_style (vectype, is_load, &factor,
@@ -1609,13 +1617,13 @@ check_load_store_for_partial_vectors (loop_vec_info
loop_vinfo, tree vectype,
if (partial_vector_style == vect_load_store_partial_vectors_len)
{
- nvectors = group_memory_nvectors (group_size * vf, nunits);
- vect_record_len (loop_vinfo, nvectors, vectype, factor);
+ nvectors = group_memory_nvectors (size, nunits);
+ vect_record_len (vinfo, nvectors, vectype, factor, slp_node);
}
else if (partial_vector_style == vect_load_store_partial_vectors_mask)
{
- nvectors = group_memory_nvectors (group_size * vf, nunits);
- vect_record_mask (loop_vinfo, nvectors, vectype, scalar_mask);
+ nvectors = group_memory_nvectors (size, nunits);
+ vect_record_mask (vinfo, nvectors, vectype, scalar_mask, slp_node);
}
else
{
--
2.43.0
