This is part of preparation for enabling BB SLP with predicated
vector tails.
Previously, if this function were constructing a permutation mask
for an SLP node with a four-subpart vector type but only three
active lanes, it would have only put three elements into its
vec_perm_builder, which would have never satisfied the
index == count condition in its loop over each element in
nelts_to_build. Consequently, the vec_perm_builder was never used
to make a new instance of vec_perm_indices, the vec_perm_indices
was never converted to the equivalent VECTOR_CST mask, and the
SLP node was left with a missing vectorized def.
The solution is to invent a value for any missing elements. The
corresponding lanes are inactive, so the value is irrelevant.
gcc/ChangeLog:
* tree-vect-slp.cc (vect_transform_slp_perm_load_1): Pad
the elements supplied to a vec_perm_builder if necessary
to complete it for an SLP node that has a partial vector.
---
gcc/tree-vect-slp.cc | 74 ++++++++++++++++++++++++++------------------
1 file changed, 44 insertions(+), 30 deletions(-)
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index 63c8731fa2b..eb96962edec 100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -11270,48 +11270,62 @@ vect_transform_slp_perm_load_1 (vec_info *vinfo,
slp_tree node,
unsigned int count = mask.encoded_nelts ();
mask.quick_grow (count);
+
+ unsigned int active_nelts_to_build = nelts_to_build;
+ if (!repeating_p)
+ nelts_to_build = CEIL (nelts_to_build, count) * count;
vec_perm_indices indices;
for (unsigned int j = 0; j < nelts_to_build; j++)
{
- unsigned int iter_num = j / group_size;
- unsigned int stmt_num = j % group_size;
- unsigned int i = (iter_num * dr_group_size + perm[stmt_num]);
- bitmap_set_bit (used_in_lanes, i);
- if (repeating_p)
+ if (j >= active_nelts_to_build)
{
- first_vec_index = 0;
- mask_element = i;
+ /* Fill unused lanes of a partial final SLP vector. Their
+ values are irrelevant, but the permutation mask itself
+ must describe a complete vector. */
+ mask_element = 0;
}
else
{
- /* Enforced before the loop when !repeating_p. */
- unsigned int const_nunits = nunits.to_constant ();
- vec_index = i / const_nunits;
- mask_element = i % const_nunits;
- if (vec_index == first_vec_index
- || first_vec_index == -1)
+ unsigned int iter_num = j / group_size;
+ unsigned int stmt_num = j % group_size;
+ unsigned int i = (iter_num * dr_group_size + perm[stmt_num]);
+ bitmap_set_bit (used_in_lanes, i);
+ if (repeating_p)
{
- first_vec_index = vec_index;
- }
- else if (vec_index == second_vec_index
- || second_vec_index == -1)
- {
- second_vec_index = vec_index;
- mask_element += const_nunits;
+ first_vec_index = 0;
+ mask_element = i;
}
else
{
- if (dump_p)
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "permutation requires at "
- "least three vectors %G",
- stmt_info->stmt);
- gcc_assert (analyze_only);
- return false;
- }
+ /* Enforced before the loop when !repeating_p. */
+ unsigned int const_nunits = nunits.to_constant ();
+ vec_index = i / const_nunits;
+ mask_element = i % const_nunits;
+ if (vec_index == first_vec_index
+ || first_vec_index == -1)
+ {
+ first_vec_index = vec_index;
+ }
+ else if (vec_index == second_vec_index
+ || second_vec_index == -1)
+ {
+ second_vec_index = vec_index;
+ mask_element += const_nunits;
+ }
+ else
+ {
+ if (dump_p)
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "permutation requires at "
+ "least three vectors %G",
+ stmt_info->stmt);
+ gcc_assert (analyze_only);
+ return false;
+ }
- gcc_assert (mask_element < 2 * const_nunits);
+ gcc_assert (mask_element < 2 * const_nunits);
+ }
}
if (mask_element != index)
@@ -11329,7 +11343,7 @@ vect_transform_slp_perm_load_1 (vec_info *vinfo,
slp_tree node,
{
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
"unsupported vect permute { ");
- for (i = 0; i < count; ++i)
+ for (unsigned i = 0; i < count; ++i)
{
dump_dec (MSG_MISSED_OPTIMIZATION, mask[i]);
dump_printf (MSG_MISSED_OPTIMIZATION, " ");
--
2.43.0
