When BB SLP chooses a variable-length vector type for an invariant or
constant node, the scalar elements represented by the node give only
the initial elements of the vector. Any remaining elements (whose
number is unknown at compile time) must be zero rather than repeating
the initial elements.
This required a related helper, vect_get_num_copies_for_invariant, to
be updated to handle basic block vectorization with variable-length
vector types.
Unlike in the loop vectorization case, there is no polynomial vectorization
factor for BB SLP; therefore, that factor cannot cancel out the polynomial
count of subparts in the vector type to produce a non-polynomial quotient
(the number of vectors). Instead, the minimum number of subparts in a
variable-length vector type is used to calculate the minimum number of
excess elements for BB SLP if and only if the SLP group fits within it.
For BB SLP, only the non-excess elements in the last vector come from
scalar operands; the remainder are assigned constant zero when the
vector is constructed. If the vector type is variable-length, the vector
builder uses two elements per pattern to encode an infinite-length tail
of constant zeros.
gcc/ChangeLog:
* gimple-fold.cc (gimple_build_vector): Check that the
passed-in vector builder produces element values that
are constant zero if they lie outside the minimum length
of a variable-length vector type, and also a repeating tail
of constant zeros.
* tree-vect-slp.cc (vect_get_num_copies_for_invariant):
Handle BB SLP with a variable-length vector type when the
SLP group fits within the minimum vector length. Calculate
the number of excess elements from the minimum vector length
in such cases.
(vect_create_constant_vectors): Initialize the vector builder
with two elements per pattern instead of one, if doing BB SLP
vectorization and the vector type is variable-length. Only
the non-excess elements in the last vector come from scalar
operands. Assign zero constants to the remaining elements
and to the second (repeating) element of each pattern.
---
gcc/gimple-fold.cc | 33 ++++++++++++-
gcc/tree-vect-slp.cc | 112 +++++++++++++++++++++++++++++++------------
2 files changed, 112 insertions(+), 33 deletions(-)
diff --git a/gcc/gimple-fold.cc b/gcc/gimple-fold.cc
index e3df0d1163c..74cc8b92be5 100644
--- a/gcc/gimple-fold.cc
+++ b/gcc/gimple-fold.cc
@@ -11534,7 +11534,13 @@ gimple_build_vector_from_val (gimple_stmt_iterator
*gsi,
BUILDER must not have a stepped encoding on entry. This is because
the function is not geared up to handle the arithmetic that would
be needed in the variable case, and any code building a vector that
- is known to be constant should use BUILDER->build () directly. */
+ is known to be constant should use BUILDER->build () directly.
+
+ If at least one element is non-constant and the type associated with BUILDER
+ is a variable-length vector type then the encoding used by BUILDER must have
+ two elements per pattern; the second (repeating) element of each pattern
+ must be constant zero, as must the first if it falls outside the minimum
+ vector length. */
tree
gimple_build_vector (gimple_stmt_iterator *gsi,
@@ -11548,7 +11554,30 @@ gimple_build_vector (gimple_stmt_iterator *gsi,
{
gimple_seq seq = NULL;
tree type = builder->type ();
- unsigned int nelts = TYPE_VECTOR_SUBPARTS (type).to_constant ();
+ unsigned int min_nelts
+ = constant_lower_bound (TYPE_VECTOR_SUBPARTS (type));
+
+ /* All vector builders produce an infinitely repeating sequence; we only
+ care about truncation of that sequence if the vector type has a
+ variable length. */
+ unsigned HOST_WIDE_INT nelts;
+ if (!TYPE_VECTOR_SUBPARTS (type).is_constant (&nelts))
+ {
+ /* Initial elements outside the minimum vector length must be
+ constant zero because that is the only implicit value that a
+ CONSTRUCTOR node can have. */
+ for (i = min_nelts; i < builder->npatterns (); ++i)
+ gcc_assert (zerop ((*builder)[i]));
+
+ /* The builder must also produce an infinitely repeating tail of
+ constant zeros. */
+ gcc_assert (builder->nelts_per_pattern () == 2);
+ for (i = builder->npatterns (); i < encoded_nelts; ++i)
+ gcc_assert (zerop ((*builder)[i]));
+
+ nelts = min_nelts;
+ }
+
vec<constructor_elt, va_gc> *v;
vec_alloc (v, nelts);
for (i = 0; i < nelts; ++i)
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index eb96962edec..134e8072267 100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -8926,7 +8926,9 @@ vect_scalar_ops_slice_hash::equal (const value_type &s1,
/* Like vect_get_num_copies but N copies of the vector might have
excess elements in the last vector. Stores the number of excess
- elements in the last vector in *EXCESS_ELTS.
+ elements in the last vector in *EXCESS_ELTS. For BB SLP with a
+ variable-length vector type, use the minimum vector length to
+ calculate the minimum number of excess elements.
Returns false if *NVECTORS or *EXCESS_ELTS cannot be computed. */
bool
@@ -8934,23 +8936,35 @@ vect_get_num_copies_for_invariant (vec_info *vinfo,
slp_tree node,
unsigned *nvectors,
unsigned *excess_elts)
{
- poly_uint64 vf;
+ tree vectype = SLP_TREE_VECTYPE (node);
+ poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
+ poly_uint64 nelts;
if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
- vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+ nelts = LOOP_VINFO_VECT_FACTOR (loop_vinfo) * SLP_TREE_LANES (node);
else
- vf = 1;
- vf *= SLP_TREE_LANES (node);
+ {
+ unsigned int group_size = SLP_TREE_LANES (node);
+ if (!nunits.is_constant () && known_le (group_size, nunits))
+ {
+ *nvectors = 1;
+ *excess_elts = constant_lower_bound (nunits) - group_size;
+ return true;
+ }
+ nelts = group_size;
+ }
- tree vectype = SLP_TREE_VECTYPE (node);
- uint64_t rem;
- if (can_div_away_from_zero_p (vf, TYPE_VECTOR_SUBPARTS (vectype),
- nvectors)
- && (TYPE_VECTOR_SUBPARTS (vectype) * *nvectors - vf) .is_constant (&rem))
+ if (can_div_away_from_zero_p (nelts, nunits, nvectors))
{
- *excess_elts = rem;
- return true;
+ poly_uint64 excess = (nunits * *nvectors) - nelts;
+ uint64_t const_excess;
+ if (excess.is_constant (&const_excess))
+ {
+ *excess_elts = const_excess;
+ return true;
+ }
}
+
return false;
}
@@ -10892,10 +10906,10 @@ vect_create_constant_vectors (vec_info *vinfo,
slp_tree op_node)
{
unsigned HOST_WIDE_INT nunits;
tree vec_cst;
- unsigned j, number_of_places_left_in_vector;
+ unsigned j, number_of_places_in_vector, number_of_places_left_in_vector;
tree vector_type;
tree vop;
- int group_size = op_node->ops.length ();
+ unsigned int group_size = op_node->ops.length ();
unsigned int vec_num, i;
unsigned number_of_copies = 1;
bool constant_p;
@@ -10929,21 +10943,48 @@ vect_create_constant_vectors (vec_info *vinfo,
slp_tree op_node)
(s1, s2, ..., s8). We will create two vectors {s1, s2, s3, s4} and
{s5, s6, s7, s8}. */
- /* When using duplicate_and_interleave, we just need one element for
- each scalar statement. */
- if (!TYPE_VECTOR_SUBPARTS (vector_type).is_constant (&nunits))
- nunits = group_size;
+ unsigned int npatterns, nelts_per_pattern = 1;
+ poly_uint64 type_nunits = TYPE_VECTOR_SUBPARTS (vector_type);
+ if (is_a <bb_vec_info> (vinfo))
+ {
+ /* We don't use duplicate_and_interleave for basic block vectorization.
+ If the vector length is unknown then we need two elements per pattern
+ to encode an infinite-length tail of constant zeros. */
+ if (!type_nunits.is_constant (&nunits))
+ {
+ nunits = constant_lower_bound (type_nunits);
+ nelts_per_pattern = 2;
+ }
- number_of_copies = nunits * number_of_vectors / group_size;
+ npatterns = nunits;
+ }
+ else
+ {
+ /* When using duplicate_and_interleave, we just need one element for
+ each scalar statement. */
+ if (!type_nunits.is_constant (&nunits))
+ nunits = group_size;
+
+ npatterns = nunits;
+ }
+
+ number_of_copies = ((nunits * number_of_vectors) - excess_elts) / group_size;
constant_p = true;
tree uniform_elt = NULL_TREE;
- tree_vector_builder elts (vector_type, nunits, 1);
- elts.quick_grow (nunits);
- /* Zero-pad the last vector if necessary. */
- number_of_places_left_in_vector = nunits - excess_elts;
- for (i = nunits; i > number_of_places_left_in_vector; --i)
- elts[i-1] = build_zero_cst (TREE_TYPE (vector_type));
+ tree_vector_builder elts (vector_type, npatterns, nelts_per_pattern);
+ elts.quick_grow (elts.encoded_nelts ());
+
+ number_of_places_in_vector = nunits - excess_elts;
+ number_of_places_left_in_vector = number_of_places_in_vector;
+
+ if (!is_a <bb_vec_info> (vinfo))
+ {
+ /* Zero-pad the last vector if necessary. */
+ for (i = nunits; i > number_of_places_left_in_vector; --i)
+ elts[i - 1] = build_zero_cst (TREE_TYPE (vector_type));
+ }
+
stmt_vec_info insert_after = NULL;
for (j = 0; j < number_of_copies; j++)
{
@@ -11027,10 +11068,18 @@ vect_create_constant_vectors (vec_info *vinfo,
slp_tree op_node)
if (number_of_places_left_in_vector == 0)
{
- auto type_nunits = TYPE_VECTOR_SUBPARTS (vector_type);
if (uniform_elt)
vec_cst = gimple_build_vector_from_val (&ctor_seq, vector_type,
elts[0]);
+ else if (is_a <bb_vec_info> (vinfo))
+ {
+ unsigned int encoded_nelts = elts.encoded_nelts ();
+ tree zero_val = build_zero_cst (TREE_TYPE (vector_type));
+ for (unsigned int k = number_of_places_in_vector;
+ k < encoded_nelts; ++k)
+ elts[k] = zero_val;
+ vec_cst = gimple_build_vector (&ctor_seq, &elts);
+ }
else if (constant_p
? multiple_p (type_nunits, nunits)
: known_eq (type_nunits, nunits))
@@ -11078,12 +11127,13 @@ vect_create_constant_vectors (vec_info *vinfo,
slp_tree op_node)
}
voprnds.quick_push (vec_cst);
insert_after = NULL;
- number_of_places_left_in_vector = nunits;
+ number_of_places_in_vector = nunits;
+ number_of_places_left_in_vector = nunits;
constant_p = true;
- elts.new_vector (vector_type, nunits, 1);
- elts.quick_grow (nunits);
- }
- }
+ elts.new_vector (vector_type, npatterns, nelts_per_pattern);
+ elts.quick_grow (elts.encoded_nelts ());
+ }
+ }
}
/* Since the vectors are created in the reverse order, we should invert
--
2.43.0
