The following adds vec_decostruct to replace nunits * vec_to_scalar
which allows for more precise costing whenever vectorization
needs to decompose a vector to pieces, like for emulated gather/scatter
but also for strided [SLP] loads/stores.
This requires changes across targets, both for the different kind
but also for the difference in expected count.
I've done x86 adjustments but refrained from trying to understand
the complex aarch64 and riscv cost models (nor arm or ppc). I'd
welcome patches or a simple "OK, we'll fix it" though ;)
That is, does this look OK?
The series bootstrapped / tested ok on x86_64-unknown-linux-gnu.
Richard.
* target.h (vect_cost_for_stmt::vec_deconstruct): New kind.
* tree-vect-loop.cc (vect_model_reduction_cost): Use it.
* tree-vect-stmts.cc (vectorizable_store): Likewise.
(vectorizable_load): Likewise.
* tree-vectorizer.cc (dump_stmt_cost): Handle vec_deconstruct.
---
gcc/target.h | 3 ++-
gcc/tree-vect-loop.cc | 11 +++++------
gcc/tree-vect-stmts.cc | 15 +++++++--------
gcc/tree-vectorizer.cc | 3 +++
4 files changed, 17 insertions(+), 15 deletions(-)
diff --git a/gcc/target.h b/gcc/target.h
index 68ba626da6a..67b04f90f45 100644
--- a/gcc/target.h
+++ b/gcc/target.h
@@ -212,7 +212,8 @@ enum vect_cost_for_stmt
cond_branch_taken,
vec_perm,
vec_promote_demote,
- vec_construct
+ vec_construct,
+ vec_deconstruct
};
/* Separate locations for which the vectorizer cost model should
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index 4d9b691c329..0167d52b28b 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -4752,10 +4752,10 @@ vect_model_reduction_cost (loop_vec_info loop_vinfo,
node, 0, vect_body);
else
{
- /* Use NELEMENTS extracts and NELEMENTS scalar ops. */
+ /* Use NCOPIES deconstructs and NELEMENTS scalar ops. */
unsigned int nelements = ncopies * vect_nunits_for_cost (vectype);
- inside_cost = record_stmt_cost (cost_vec, nelements,
- vec_to_scalar, node, 0,
+ inside_cost = record_stmt_cost (cost_vec, ncopies,
+ vec_deconstruct, node, 0,
vect_body);
inside_cost += record_stmt_cost (cost_vec, nelements,
scalar_stmt, node, 0,
@@ -4817,9 +4817,8 @@ vect_model_reduction_cost (loop_vec_info loop_vinfo,
{
unsigned estimated_nunits = vect_nunits_for_cost (vectype);
/* Extraction of scalar elements. */
- epilogue_cost += record_stmt_cost (cost_vec,
- 2 * estimated_nunits,
- vec_to_scalar, node, 0,
+ epilogue_cost += record_stmt_cost (cost_vec, 2,
+ vec_deconstruct, node, 0,
vect_epilogue);
/* Scalar max reductions via COND_EXPR / MAX_EXPR. */
epilogue_cost += record_stmt_cost (cost_vec,
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index 4e7b45f5a6c..e991e861525 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -8654,12 +8654,12 @@ vectorizable_store (vec_info *vinfo,
inside_cost
+= record_stmt_cost (cost_vec, n_adjacent_stores,
scalar_store, slp_node, 0, vect_body);
- /* Only need vector extracting when there are more
- than one stores. */
+ /* Only need vector deconstruction when there is more
+ than one store. */
if (nstores > 1)
inside_cost
- += record_stmt_cost (cost_vec, n_adjacent_stores,
- vec_to_scalar, slp_node, 0, vect_body);
+ += record_stmt_cost (cost_vec, ncopies,
+ vec_deconstruct, slp_node, 0, vect_body);
}
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
@@ -9222,11 +9222,11 @@ vectorizable_store (vec_info *vinfo,
(we assume the scalar scaling and ptr + offset add is
consumed by the load). */
inside_cost
- += record_stmt_cost (cost_vec, cnunits, vec_to_scalar,
+ += record_stmt_cost (cost_vec, 1, vec_deconstruct,
slp_node, 0, vect_body);
/* N scalar stores plus extracting the elements. */
inside_cost
- += record_stmt_cost (cost_vec, cnunits, vec_to_scalar,
+ += record_stmt_cost (cost_vec, 1, vec_deconstruct,
slp_node, 0, vect_body);
inside_cost
+= record_stmt_cost (cost_vec, cnunits, scalar_store,
@@ -11201,8 +11201,7 @@ vectorizable_load (vec_info *vinfo,
{
/* For emulated gathers N offset vector element
offset add is consumed by the load). */
- inside_cost = record_stmt_cost (cost_vec, const_nunits,
- vec_to_scalar,
+ inside_cost = record_stmt_cost (cost_vec, 1, vec_deconstruct,
slp_node, 0, vect_body);
/* N scalar loads plus gathering them into a
vector. */
diff --git a/gcc/tree-vectorizer.cc b/gcc/tree-vectorizer.cc
index ab455c08349..8c824e4ee33 100644
--- a/gcc/tree-vectorizer.cc
+++ b/gcc/tree-vectorizer.cc
@@ -169,6 +169,9 @@ dump_stmt_cost (FILE *f, int count, enum vect_cost_for_stmt
kind,
case vec_construct:
ks = "vec_construct";
break;
+ case vec_deconstruct:
+ ks = "vec_deconstruct";
+ break;
}
fprintf (f, "%s ", ks);
if (kind == unaligned_load || kind == unaligned_store)
--
2.51.0
