The following simplifies the earlier RFC for making it easier
for the target to correlate multiple cost events created from
a single SLP operation. Instead of changing where we record
costs this patch only adjusts the submission part.
Targets wanting to take advantage of this can implement
add_slp_cost and handle all or select cases and resort
to the default implementation to get add_stmt_cost events
for unhandled groups.
Bootstrapped and tested on x86_64-unknown-linux-gnu, pushed.
v2 integrates Tamars comment on better handling end + 1
* tree-vectorizer.h (vector_costs::add_slp_cost): New.
* tree-vectorizer.cc (vector_costs::add_slp_cost): New
default version.
* tree-vect-slp.cc (add_slp_costs): Helper for dispatching
a cost vector in SLP chunks.
(vect_slp_analyze_operations): Adjust.
(li_cost_vec_cmp): Likewise.
(vect_bb_vectorization_profitable_p): Likewise.
---
gcc/tree-vect-slp.cc | 30 +++++++++++++++++++++++++-----
gcc/tree-vectorizer.cc | 11 +++++++++++
gcc/tree-vectorizer.h | 5 +++++
3 files changed, 41 insertions(+), 5 deletions(-)
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index 8a052c9baf1..ea49c32b780 100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -9167,6 +9167,24 @@ vect_slp_prune_covered_roots (slp_tree node,
hash_set<stmt_vec_info> &roots,
vect_slp_prune_covered_roots (child, roots, visited);
}
+/* Hand over COST_VEC to the target COSTS grouped by SLP node. */
+
+static void
+add_slp_costs (vector_costs *costs, stmt_vector_for_cost& cost_vec)
+{
+ for (unsigned start = 0; start < cost_vec.length ();)
+ {
+ unsigned end = start + 1;
+ while (end < cost_vec.length ()
+ && cost_vec[start].node == cost_vec[end].node)
+ end++;
+ costs->add_slp_cost (cost_vec[start].node,
+ array_slice<stmt_info_for_cost>
+ (cost_vec.begin () + start, end - start));
+ start = end;
+ }
+}
+
/* Analyze statements in SLP instances of VINFO. Return true if the
operations are supported. */
@@ -9252,7 +9270,7 @@ vect_slp_analyze_operations (vec_info *vinfo)
i++;
if (loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo))
{
- add_stmt_costs (loop_vinfo->vector_costs, &cost_vec);
+ add_slp_costs (loop_vinfo->vector_costs, cost_vec);
cost_vec.release ();
}
else
@@ -9520,7 +9538,7 @@ vect_bb_slp_scalar_cost (bb_vec_info vinfo,
/* Comparator for the loop-index sorted cost vectors. */
static int
-li_cost_vec_cmp (const void *a_, const void *b_)
+li_cost_vec_cmp (const void *a_, const void *b_, void *)
{
auto *a = (const std::pair<unsigned, stmt_info_for_cost *> *)a_;
auto *b = (const std::pair<unsigned, stmt_info_for_cost *> *)b_;
@@ -9610,8 +9628,8 @@ vect_bb_vectorization_profitable_p (bb_vec_info bb_vinfo,
l = gimple_bb (cost->stmt_info->stmt)->loop_father->num;
li_vector_costs.quick_push (std::make_pair (l, cost));
}
- li_scalar_costs.qsort (li_cost_vec_cmp);
- li_vector_costs.qsort (li_cost_vec_cmp);
+ li_scalar_costs.stablesort (li_cost_vec_cmp, NULL);
+ li_vector_costs.stablesort (li_cost_vec_cmp, NULL);
/* Now cost the portions individually. */
unsigned vi = 0;
@@ -9651,13 +9669,15 @@ vect_bb_vectorization_profitable_p (bb_vec_info
bb_vinfo,
/* Complete the target-specific vector cost calculation. */
class vector_costs *vect_target_cost_data = init_cost (bb_vinfo, false);
+ auto_vec<stmt_info_for_cost> tem;
do
{
- add_stmt_cost (vect_target_cost_data, li_vector_costs[vi].second);
+ tem.safe_push (*li_vector_costs[vi].second);
vi++;
}
while (vi < li_vector_costs.length ()
&& li_vector_costs[vi].first == vl);
+ add_slp_costs (vect_target_cost_data, tem);
vect_target_cost_data->finish_cost (scalar_target_cost_data);
vec_prologue_cost = vect_target_cost_data->prologue_cost ();
vec_inside_cost = vect_target_cost_data->body_cost ();
diff --git a/gcc/tree-vectorizer.cc b/gcc/tree-vectorizer.cc
index 9e71f71fab7..205a07b0be5 100644
--- a/gcc/tree-vectorizer.cc
+++ b/gcc/tree-vectorizer.cc
@@ -1845,6 +1845,17 @@ vector_costs::add_stmt_cost (int count,
vect_cost_for_stmt kind,
return record_stmt_cost (stmt_info, where, cost);
}
+unsigned int
+vector_costs::add_slp_cost (slp_tree,
+ const array_slice<stmt_info_for_cost> &cost_vec)
+{
+ unsigned int sum = 0;
+ for (auto item : cost_vec)
+ sum += ::add_stmt_cost (this, item.count, item.kind, item.stmt_info,
+ item.node, item.vectype, item.misalign, item.where);
+ return sum;
+}
+
/* See the comment above the declaration for details. */
void
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 67cc099c43f..14d31cfe0fd 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -1777,6 +1777,11 @@ public:
tree vectype, int misalign,
vect_cost_model_location where);
+ /* Update the costs in response to adding costs in V which are all from
+ vectorizing NODE to the respective part. */
+ virtual unsigned int add_slp_cost (slp_tree node,
+ const array_slice<stmt_info_for_cost> &v);
+
/* Finish calculating the cost of the code. The results can be
read back using the functions below.
--
2.51.0
