In preparation for enabling tail-predication for basic block SLP
vectorization, the function used to check whether a load or store
statement in a loop is possible using partial vectors needs to be
updated to work for BB SLP too.
This preparatory commit does not change any of the callers of
check_load_store_for_partial_vectors, therefore loop_vinfo is
expected to be non-null within the modified function (for now).
gcc/ChangeLog:
* tree-vect-stmts.cc (check_load_store_for_partial_vectors):
Change parameter type from loop_vec_info to vec_info *. Use
dyn_cast to get a loop_vec_info or null. Return false if doing
BB SLP and SLP_TREE_LANES might exceed the number of subparts in
the passed-in vector type. Pass vec_info * instead of
loop_vec_info to vect_get_num_copies, vect_record_len and
vect_record_mask. Pass the group size to vect_record_len and
vect_record_mask.
When calculating the number of vectors for BB SLP, get the group
size from SLP_TREE_LANES except in the case of permuted loads,
for which the containing load group size (DR_GROUP_SIZE) is used
instead. Change the type of group_size to unsigned int to match
DR_GROUP_SIZE and SLP_TREE_LANES.
---
gcc/tree-vect-stmts.cc | 52 +++++++++++++++++++++++++++---------------
1 file changed, 33 insertions(+), 19 deletions(-)
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index 2252bcff0ec..c897166e8ef 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -1443,10 +1443,10 @@ vect_get_load_store_partial_vector_style (tree vectype,
bool is_load,
return vect_load_store_partial_vectors_none;
}
-/* Check whether a load or store statement in the loop described by
- LOOP_VINFO is possible in a loop using partial vectors. This is
- testing whether the vectorizer pass has the appropriate support,
- as well as whether the target does.
+/* Check whether a load or store statement in the loop or SLP region described
+ by VINFO is possible using partial vectors. This is testing whether the
+ vectorizer pass has the appropriate support, as well as whether the target
+ does.
VLS_TYPE says whether the statement is a load or store and VECTYPE
is the type of the vector being loaded or stored. SLP_NODE is the SLP
@@ -1457,23 +1457,30 @@ vect_get_load_store_partial_vector_style (tree vectype,
bool is_load,
its arguments. If the load or store is conditional, SCALAR_MASK is the
condition under which it occurs.
- Return false if a loop using partial vectors is not supported, otherwise
- record the required rgroup control types.
+ Return false if a loop or SLP vectorization using partial vectors is not
+ supported, otherwise record the required rgroup control types.
If partial vectors can be used and ELSVALS is nonzero the supported
else values will be added to the vector ELSVALS points to. */
static bool
-check_load_store_for_partial_vectors (loop_vec_info loop_vinfo, tree vectype,
+check_load_store_for_partial_vectors (vec_info *vinfo, tree vectype,
slp_tree slp_node,
vec_load_store_type vls_type,
- int group_size,
+ unsigned int group_size,
vect_load_store_data *ls,
slp_tree mask_node,
vec<int> *elsvals = nullptr)
{
+ loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
vect_memory_access_type memory_access_type = ls->memory_access_type;
+ /* For SLP vectorization, we cannot use partial vectors if the
+ group could be wider than the vector type. */
+ poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
+ if (!loop_vinfo && maybe_gt (SLP_TREE_LANES (slp_node), nunits))
+ return false;
+
/* Invariant loads need no special support. */
if (memory_access_type == VMAT_INVARIANT)
return true;
@@ -1495,7 +1502,7 @@ check_load_store_for_partial_vectors (loop_vec_info
loop_vinfo, tree vectype,
scalar_mask = def;
}
- unsigned int nvectors = vect_get_num_copies (loop_vinfo, slp_node);
+ unsigned int nvectors = vect_get_num_copies (vinfo, slp_node);
machine_mode vecmode = TYPE_MODE (vectype);
bool is_load = (vls_type == VLS_LOAD);
if (memory_access_type == VMAT_LOAD_STORE_LANES)
@@ -1506,9 +1513,11 @@ check_load_store_for_partial_vectors (loop_vec_info
loop_vinfo, tree vectype,
elsvals)
: vect_store_lanes_supported (vectype, group_size, true));
if (ifn == IFN_MASK_LEN_LOAD_LANES || ifn == IFN_MASK_LEN_STORE_LANES)
- vect_record_len (loop_vinfo, nvectors, vectype, 1);
+ vect_record_len (vinfo, nvectors, vectype, 1,
+ SLP_TREE_LANES (slp_node));
else if (ifn == IFN_MASK_LOAD_LANES || ifn == IFN_MASK_STORE_LANES)
- vect_record_mask (loop_vinfo, nvectors, vectype, scalar_mask);
+ vect_record_mask (vinfo, nvectors, vectype, scalar_mask,
+ SLP_TREE_LANES (slp_node));
else
{
if (dump_enabled_p ())
@@ -1550,13 +1559,15 @@ check_load_store_for_partial_vectors (loop_vec_info
loop_vinfo, tree vectype,
memory_type,
off_vectype, scale,
elsvals))
- vect_record_len (loop_vinfo, nvectors, vectype, 1);
+ vect_record_len (vinfo, nvectors, vectype, 1,
+ SLP_TREE_LANES (slp_node));
else if (internal_gather_scatter_fn_supported_p (ifn, vectype,
memory_type,
off_vectype, scale,
elsvals)
|| memory_access_type == VMAT_GATHER_SCATTER_LEGACY)
- vect_record_mask (loop_vinfo, nvectors, vectype, scalar_mask);
+ vect_record_mask (vinfo, nvectors, vectype, scalar_mask,
+ SLP_TREE_LANES (slp_node));
else
{
if (dump_enabled_p ())
@@ -1600,8 +1611,11 @@ check_load_store_for_partial_vectors (loop_vec_info
loop_vinfo, tree vectype,
gcc_unreachable ();
};
- poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
- poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+ if (!loop_vinfo && !ls->slp_perm)
+ group_size = SLP_TREE_LANES (slp_node);
+
+ poly_uint64 size = group_size * vinfo->vectorization_factor;
+
unsigned factor;
vect_load_store_partial_vector_style partial_vector_style
= vect_get_load_store_partial_vector_style (vectype, is_load, &factor,
@@ -1609,13 +1623,13 @@ check_load_store_for_partial_vectors (loop_vec_info
loop_vinfo, tree vectype,
if (partial_vector_style == vect_load_store_partial_vectors_len)
{
- nvectors = group_memory_nvectors (group_size * vf, nunits);
- vect_record_len (loop_vinfo, nvectors, vectype, factor);
+ nvectors = group_memory_nvectors (size, nunits);
+ vect_record_len (vinfo, nvectors, vectype, factor, group_size);
}
else if (partial_vector_style == vect_load_store_partial_vectors_mask)
{
- nvectors = group_memory_nvectors (group_size * vf, nunits);
- vect_record_mask (loop_vinfo, nvectors, vectype, scalar_mask);
+ nvectors = group_memory_nvectors (size, nunits);
+ vect_record_mask (vinfo, nvectors, vectype, scalar_mask, group_size);
}
else
{
--
2.43.0
