gcc/ChangeLog:
* gimple-fold.cc (gimple_build_vector): Check that the
passed-in vector builder produces element values that
are constant zero if they lie outside the minimum length
of a variable-length vector type, and also a repeating tail
of constant zeros.
* tree-vect-slp.cc (vect_create_constant_vectors):
Initialize the vector builder with two elements per pattern
instead of one, if doing BB SLP vectorization and the vector
type is variable-length or the SLP group has fewer lanes than
needed to fill a fixed-length vector type. Assign a zero
constant to the second (repeating) element of each pattern.
---
gcc/gimple-fold.cc | 33 +++++++++++++++++++++--
gcc/tree-vect-slp.cc | 62 +++++++++++++++++++++++++++++++++++---------
2 files changed, 81 insertions(+), 14 deletions(-)
diff --git a/gcc/gimple-fold.cc b/gcc/gimple-fold.cc
index fdb9b419766..132205c6ba9 100644
--- a/gcc/gimple-fold.cc
+++ b/gcc/gimple-fold.cc
@@ -11487,7 +11487,13 @@ gimple_build_vector_from_val (gimple_stmt_iterator
*gsi,
BUILDER must not have a stepped encoding on entry. This is because
the function is not geared up to handle the arithmetic that would
be needed in the variable case, and any code building a vector that
- is known to be constant should use BUILDER->build () directly. */
+ is known to be constant should use BUILDER->build () directly.
+
+ If at least one element is non-constant and the type associated with BUILDER
+ is a variable-length vector type then the encoding used by BUILDER must have
+ two elements per pattern; the second (repeating) element of each pattern
+ must be constant zero, as must the first if it falls outside the minimum
+ vector length. */
tree
gimple_build_vector (gimple_stmt_iterator *gsi,
@@ -11501,7 +11507,30 @@ gimple_build_vector (gimple_stmt_iterator *gsi,
{
gimple_seq seq = NULL;
tree type = builder->type ();
- unsigned int nelts = TYPE_VECTOR_SUBPARTS (type).to_constant ();
+ unsigned int min_nelts
+ = constant_lower_bound (TYPE_VECTOR_SUBPARTS (type));
+
+ /* All vector builders produce an infinitely repeating sequence; we only
+ care about truncation of that sequence if the vector type has a
+ variable length. */
+ unsigned HOST_WIDE_INT nelts;
+ if (!TYPE_VECTOR_SUBPARTS (type).is_constant (&nelts))
+ {
+ /* Initial elements outside the minimum vector length must be
+ constant zero because that is the only implicit value that a
+ CONSTRUCTOR node can have. */
+ for (i = min_nelts; i < builder->npatterns (); ++i)
+ gcc_assert (zerop ((*builder)[i]));
+
+ /* The builder must also produce an infinitely repeating tail of
+ constant zeros. */
+ gcc_assert (builder->nelts_per_pattern () == 2);
+ for (i = builder->npatterns (); i < encoded_nelts; ++i)
+ gcc_assert (zerop ((*builder)[i]));
+
+ nelts = min_nelts;
+ }
+
vec<constructor_elt, va_gc> *v;
vec_alloc (v, nelts);
for (i = 0; i < nelts; ++i)
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index 850cb1efacc..9331f25286f 100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -10773,7 +10773,7 @@ vect_create_constant_vectors (vec_info *vinfo, slp_tree
op_node)
unsigned j, number_of_places_left_in_vector;
tree vector_type;
tree vop;
- int group_size = op_node->ops.length ();
+ unsigned int group_size = op_node->ops.length ();
unsigned int vec_num, i;
unsigned number_of_copies = 1;
bool constant_p;
@@ -10803,18 +10803,49 @@ vect_create_constant_vectors (vec_info *vinfo,
slp_tree op_node)
(s1, s2, ..., s8). We will create two vectors {s1, s2, s3, s4} and
{s5, s6, s7, s8}. */
- /* When using duplicate_and_interleave, we just need one element for
- each scalar statement. */
- if (!TYPE_VECTOR_SUBPARTS (vector_type).is_constant (&nunits))
- nunits = group_size;
+ unsigned int npatterns, nelts_per_pattern = 1;
+ poly_uint64 type_nunits = TYPE_VECTOR_SUBPARTS (vector_type);
+ if (is_a <bb_vec_info> (vinfo))
+ {
+ /* We don't use duplicate_and_interleave for basic block vectorization.
+ We know that either the group size is exactly divisible by the vector
+ length or it fits within a single vector. */
+ gcc_checking_assert (multiple_p (group_size, type_nunits)
+ || known_le (group_size, type_nunits));
+
+ /* If the vector length is unknown then we need two elements per pattern
+ to encode an infinite-length tail of constant zeros. */
+ if (!type_nunits.is_constant (&nunits))
+ {
+ nunits = constant_lower_bound (type_nunits);
+ nelts_per_pattern = 2;
+ }
+
+ npatterns = nunits;
+
+ /* If the group does not occupy a full vector then only the first
+ GROUP_SIZE elements come from scalar operands. The remainder are
+ assigned constant zero below. */
+ if (group_size < nunits)
+ nunits = group_size;
+ }
+ else
+ {
+ /* When using duplicate_and_interleave, we just need one element for
+ each scalar statement. */
+ if (!type_nunits.is_constant (&nunits))
+ nunits = group_size;
+
+ npatterns = nunits;
+ }
number_of_copies = nunits * number_of_vectors / group_size;
number_of_places_left_in_vector = nunits;
constant_p = true;
tree uniform_elt = NULL_TREE;
- tree_vector_builder elts (vector_type, nunits, 1);
- elts.quick_grow (nunits);
+ tree_vector_builder elts (vector_type, npatterns, nelts_per_pattern);
+ elts.quick_grow (elts.encoded_nelts());
stmt_vec_info insert_after = NULL;
for (j = 0; j < number_of_copies; j++)
{
@@ -10898,10 +10929,17 @@ vect_create_constant_vectors (vec_info *vinfo,
slp_tree op_node)
if (number_of_places_left_in_vector == 0)
{
- auto type_nunits = TYPE_VECTOR_SUBPARTS (vector_type);
if (uniform_elt)
vec_cst = gimple_build_vector_from_val (&ctor_seq, vector_type,
elts[0]);
+ else if (is_a <bb_vec_info> (vinfo))
+ {
+ unsigned int encoded_nelts = elts.encoded_nelts ();
+ tree zero_val = build_zero_cst (TREE_TYPE (vector_type));
+ for (unsigned int k = nunits; k < encoded_nelts; k++)
+ elts[k] = zero_val;
+ vec_cst = gimple_build_vector (&ctor_seq, &elts);
+ }
else if (constant_p
? multiple_p (type_nunits, nunits)
: known_eq (type_nunits, nunits))
@@ -10951,10 +10989,10 @@ vect_create_constant_vectors (vec_info *vinfo,
slp_tree op_node)
insert_after = NULL;
number_of_places_left_in_vector = nunits;
constant_p = true;
- elts.new_vector (vector_type, nunits, 1);
- elts.quick_grow (nunits);
- }
- }
+ elts.new_vector (vector_type, npatterns, nelts_per_pattern);
+ elts.quick_grow (elts.encoded_nelts ());
+ }
+ }
}
/* Since the vectors are created in the reverse order, we should invert
--
2.43.0
