The following fixes wrong excess number of lanes computation in
vect_create_constant_vectors, integrating that into
vect_get_num_copies_for_invariant because the poly computation
has to be constant.
Bootstrapped and tested on x86_64-unknown-linux-gnu, pushed.
PR tree-optimization/126998
* tree-vectorizer.h (vect_get_num_copies_for_invariant):
Adjust API.
* tree-vect-slp.cc (vect_get_num_copies_for_invariant): Also
compute and output the constant number of excess lanes produced.
(vect_create_constant_vectors): Adjust.
(vect_slp_analyze_node_operations): Likewise.
* tree-vect-loop.cc (vectorizable_lane_reducing): Likewise.
* gcc.dg/vect/vect-reduc-dot-pr126998-2.c: New testcase.
---
.../gcc.dg/vect/vect-reduc-dot-pr126998-2.c | 37 +++++++++++++++++++
gcc/tree-vect-loop.cc | 5 ++-
gcc/tree-vect-slp.cc | 29 +++++++++------
gcc/tree-vectorizer.h | 2 +-
4 files changed, 59 insertions(+), 14 deletions(-)
create mode 100644 gcc/testsuite/gcc.dg/vect/vect-reduc-dot-pr126998-2.c
diff --git a/gcc/testsuite/gcc.dg/vect/vect-reduc-dot-pr126998-2.c
b/gcc/testsuite/gcc.dg/vect/vect-reduc-dot-pr126998-2.c
new file mode 100644
index 00000000000..66f018c291d
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-reduc-dot-pr126998-2.c
@@ -0,0 +1,37 @@
+/* { dg-additional-options "-O3" } */
+/* { dg-additional-options "-msse4" { target sse4 } } */
+
+#include "tree-vect.h"
+
+unsigned a, c;
+int b, d;
+signed char e;
+
+void __attribute__((noipa))
+f (void)
+{
+ for (; b; ++b)
+ {
+ c = 0;
+ for (; c < 10; c++)
+ {
+ d = 0;
+ for (; d < c; d++)
+ a += e;
+ }
+ }
+}
+
+int
+main ()
+{
+ check_vect ();
+
+ b = -3;
+ e = 2;
+ f ();
+ if (a != 3 * 45 * 2)
+ __builtin_abort ();
+
+ return 0;
+}
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index 78f84c6269a..cb961440656 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -6854,9 +6854,10 @@ vectorizable_lane_reducing (loop_vec_info loop_vinfo,
stmt_vec_info stmt_info,
/* Compute number of effective vector statements for costing from the
number of input lanes allow for excess lanes in the last input vector. */
- unsigned int ncopies_for_cost;
+ unsigned int ncopies_for_cost, excess_elts;
bool res = vect_get_num_copies_for_invariant (loop_vinfo, node_in,
- &ncopies_for_cost);
+ &ncopies_for_cost,
+ &excess_elts);
gcc_assert (res && ncopies_for_cost >= 1);
if (vect_is_emulated_mixed_dot_prod (slp_node))
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index 35b319e95f0..52c623f75a2 100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -8937,12 +8937,14 @@ vect_scalar_ops_slice_hash::equal (const value_type &s1,
}
/* Like vect_get_num_copies but N copies of the vector might have
- excess elements in the last vector. Returns false if *NVECTORS
- cannot be computed. */
+ excess elements in the last vector. Stores the number of excess
+ elements in the last vector in *EXCESS_ELTS.
+ Returns false if *NVECTORS or *EXCESS_ELTS cannot be computed. */
bool
vect_get_num_copies_for_invariant (vec_info *vinfo, slp_tree node,
- unsigned *nvectors)
+ unsigned *nvectors,
+ unsigned *excess_elts)
{
poly_uint64 vf;
@@ -8953,8 +8955,12 @@ vect_get_num_copies_for_invariant (vec_info *vinfo,
slp_tree node,
vf *= SLP_TREE_LANES (node);
tree vectype = SLP_TREE_VECTYPE (node);
- bool res = can_div_away_from_zero_p (vf, TYPE_VECTOR_SUBPARTS (vectype),
- nvectors);
+ uint64_t rem;
+ bool res = (can_div_away_from_zero_p (vf, TYPE_VECTOR_SUBPARTS (vectype),
+ nvectors)
+ && ((TYPE_VECTOR_SUBPARTS (vectype) * *nvectors - vf)
+ .is_constant (&rem)));
+ *excess_elts = rem;
return res;
}
@@ -9138,8 +9144,9 @@ vect_slp_analyze_node_operations (vec_info *vinfo,
slp_tree node,
}
/* Make sure we can generate them and then cost them. */
- unsigned nvectors;
- if (!vect_get_num_copies_for_invariant (vinfo, node, &nvectors))
+ unsigned nvectors, excess_elts;
+ if (!vect_get_num_copies_for_invariant (vinfo, node, &nvectors,
+ &excess_elts))
return false;
vect_prologue_cost_for_slp (child, nvectors, cost_vec);
}
@@ -10908,9 +10915,10 @@ vect_create_constant_vectors (vec_info *vinfo,
slp_tree op_node)
/* We always want SLP_TREE_VECTYPE (op_node) here correctly set. */
vector_type = SLP_TREE_VECTYPE (op_node);
- unsigned int number_of_vectors;
+ unsigned int number_of_vectors, excess_elts;
bool res = vect_get_num_copies_for_invariant (vinfo, op_node,
- &number_of_vectors);
+ &number_of_vectors,
+ &excess_elts);
gcc_assert (res);
SLP_TREE_VEC_DEFS (op_node).create (number_of_vectors);
auto_vec<tree> voprnds (number_of_vectors);
@@ -10943,8 +10951,7 @@ vect_create_constant_vectors (vec_info *vinfo, slp_tree
op_node)
tree_vector_builder elts (vector_type, nunits, 1);
elts.quick_grow (nunits);
/* Zero-pad the last vector if necessary. */
- number_of_places_left_in_vector
- = nunits - nunits * number_of_vectors % group_size;
+ number_of_places_left_in_vector = nunits - excess_elts;
for (i = nunits; i > number_of_places_left_in_vector; --i)
elts[i-1] = build_zero_cst (TREE_TYPE (vector_type));
stmt_vec_info insert_after = NULL;
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 4d542d57e6d..b3fe5acb60a 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -2790,7 +2790,7 @@ extern tree prepare_vec_mask (loop_vec_info, tree, tree,
tree,
extern tree vect_get_mask_load_else (int, tree);
extern bool vect_load_perm_consecutive_p (slp_tree, unsigned = UINT_MAX);
extern bool vect_get_num_copies_for_invariant (vec_info *, slp_tree,
- unsigned *);
+ unsigned *, unsigned *);
/* In tree-vect-patterns.cc. */
extern void
--
2.51.0
