The following adds vec_decostruct to replace nunits * vec_to_scalar
which allows for more precise costing whenever vectorization
needs to decompose a vector to pieces, like for emulated gather/scatter
but also for strided [SLP] loads/stores.
This requires changes across targets, both for the different kind
but also for the difference in expected count. For now this adjusts
them all to handle vec_deconstruct like vec_construct to avoid ICEing.
Bootstrapped/tested on x86_64-unknown-linux-gnu and pushed.
So I got yelled at from the pre-checkin CI for not handling for
arm and aarch64 which will the ICE in their builtin_vectorization_cost
hook. So will all other targets. Meaning I have amended this patch
by handling of vec_deconstruct like vec_construct for _all_ targets.
It's of a similar ballpark cost so not too obviously wrong. You
still might want to double-check explicit matching on vec_to_scalar
for parts of gather/scatter or strided load/store pattern matching
in your add_stmt costing hook (if you have that).
Richard.
* target.h (vect_cost_for_stmt::vec_deconstruct): New kind.
* tree-vect-loop.cc (vect_model_reduction_cost): Use it.
* tree-vect-stmts.cc (vectorizable_store): Likewise.
(vectorizable_load): Likewise.
* tree-vectorizer.cc (dump_stmt_cost): Handle vec_deconstruct.
* config/aarch64/aarch64.cc (aarch64_builtin_vectorization_cost):
Handle vec_deconstruct like vec_construct.
* config/i386/i386.cc (ix86_default_vector_cost): Likewise.
* config/arm/arm.cc (arm_builtin_vectorization_cost): Likewise.
* config/loongarch/loongarch.cc (loongarch_builtin_vectorization_cost):
Likewise.
* config/riscv/riscv.cc (riscv_builtin_vectorization_cost):
Likewise.
* config/rs6000/rs6000.cc (rs6000_builtin_vectorization_cost):
Likewise.
* config/s390/s390.cc (s390_builtin_vectorization_cost): Likewise.
* targhooks.cc (default_builtin_vectorization_cost): Likewise.
---
gcc/config/aarch64/aarch64.cc | 1 +
gcc/config/arm/arm.cc | 1 +
gcc/config/i386/i386.cc | 1 +
gcc/config/loongarch/loongarch.cc | 1 +
gcc/config/riscv/riscv.cc | 1 +
gcc/config/rs6000/rs6000.cc | 1 +
gcc/config/s390/s390.cc | 1 +
gcc/target.h | 3 ++-
gcc/targhooks.cc | 1 +
gcc/tree-vect-loop.cc | 11 +++++------
gcc/tree-vect-stmts.cc | 15 +++++++--------
gcc/tree-vectorizer.cc | 3 +++
12 files changed, 25 insertions(+), 15 deletions(-)
diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index 42e56512c61..c9403ed94e7 100644
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -17766,6 +17766,7 @@ aarch64_builtin_vectorization_cost (enum
vect_cost_for_stmt type_of_cost,
: simd_costs->int_stmt_cost;
case vec_construct:
+ case vec_deconstruct:
elements = estimated_poly_value (TYPE_VECTOR_SUBPARTS (vectype));
return elements / 2 + 1;
diff --git a/gcc/config/arm/arm.cc b/gcc/config/arm/arm.cc
index d24b06a6445..0bc66abe238 100644
--- a/gcc/config/arm/arm.cc
+++ b/gcc/config/arm/arm.cc
@@ -12732,6 +12732,7 @@ arm_builtin_vectorization_cost (enum vect_cost_for_stmt
type_of_cost,
return current_tune->vec_costs->vec_stmt_cost;
case vec_construct:
+ case vec_deconstruct:
elements = TYPE_VECTOR_SUBPARTS (vectype);
return elements / 2 + 1;
diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc
index b1fd86c2b32..1734996c68a 100644
--- a/gcc/config/i386/i386.cc
+++ b/gcc/config/i386/i386.cc
@@ -25699,6 +25699,7 @@ ix86_default_vector_cost (enum vect_cost_for_stmt
type_of_cost,
return ix86_vec_cost (mode, ix86_cost->sse_op);
case vec_construct:
+ case vec_deconstruct:
{
int n = GET_MODE_NUNITS (mode);
/* N - 1 element inserts into an SSE vector, the possible
diff --git a/gcc/config/loongarch/loongarch.cc
b/gcc/config/loongarch/loongarch.cc
index 40111971501..131f7b3f57b 100644
--- a/gcc/config/loongarch/loongarch.cc
+++ b/gcc/config/loongarch/loongarch.cc
@@ -4547,6 +4547,7 @@ loongarch_builtin_vectorization_cost (enum
vect_cost_for_stmt type_of_cost,
return 2;
case vec_construct:
+ case vec_deconstruct:
elements = TYPE_VECTOR_SUBPARTS (vectype);
if (LASX_SUPPORTED_MODE_P (mode) && !LSX_SUPPORTED_MODE_P (mode))
return elements / 2 + 3;
diff --git a/gcc/config/riscv/riscv.cc b/gcc/config/riscv/riscv.cc
index 3be5606ba01..29d9aaf1ffd 100644
--- a/gcc/config/riscv/riscv.cc
+++ b/gcc/config/riscv/riscv.cc
@@ -14065,6 +14065,7 @@ riscv_builtin_vectorization_cost (enum
vect_cost_for_stmt type_of_cost,
return fp ? common_costs->fp_stmt_cost : common_costs->int_stmt_cost;
case vec_construct:
+ case vec_deconstruct:
{
/* TODO: This is too pessimistic in case we can splat. */
int regmove_cost = fp ? get_fr2vr_cost () : get_gr2vr_cost ();
diff --git a/gcc/config/rs6000/rs6000.cc b/gcc/config/rs6000/rs6000.cc
index 5562d612b22..afe5d45c125 100644
--- a/gcc/config/rs6000/rs6000.cc
+++ b/gcc/config/rs6000/rs6000.cc
@@ -5113,6 +5113,7 @@ rs6000_builtin_vectorization_cost (enum
vect_cost_for_stmt type_of_cost,
return 2;
case vec_construct:
+ case vec_deconstruct:
/* This is a rough approximation assuming non-constant elements
constructed into a vector via element insertion. FIXME:
vec_construct is not granular enough for uniformly good
diff --git a/gcc/config/s390/s390.cc b/gcc/config/s390/s390.cc
index bd2adf6bfa1..1bddd5e454a 100644
--- a/gcc/config/s390/s390.cc
+++ b/gcc/config/s390/s390.cc
@@ -4385,6 +4385,7 @@ s390_builtin_vectorization_cost (enum vect_cost_for_stmt
type_of_cost,
return 3;
case vec_construct:
+ case vec_deconstruct:
return TYPE_VECTOR_SUBPARTS (vectype) - 1;
default:
diff --git a/gcc/target.h b/gcc/target.h
index 68ba626da6a..67b04f90f45 100644
--- a/gcc/target.h
+++ b/gcc/target.h
@@ -212,7 +212,8 @@ enum vect_cost_for_stmt
cond_branch_taken,
vec_perm,
vec_promote_demote,
- vec_construct
+ vec_construct,
+ vec_deconstruct
};
/* Separate locations for which the vectorizer cost model should
diff --git a/gcc/targhooks.cc b/gcc/targhooks.cc
index 670bc7aa22c..113d8836bea 100644
--- a/gcc/targhooks.cc
+++ b/gcc/targhooks.cc
@@ -777,6 +777,7 @@ default_builtin_vectorization_cost (enum vect_cost_for_stmt
type_of_cost,
return 3;
case vec_construct:
+ case vec_deconstruct:
return estimated_poly_value (TYPE_VECTOR_SUBPARTS (vectype)) - 1;
default:
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index b6e5a864108..d9c218449c0 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -4757,10 +4757,10 @@ vect_model_reduction_cost (loop_vec_info loop_vinfo,
node, 0, vect_body);
else
{
- /* Use NELEMENTS extracts and NELEMENTS scalar ops. */
+ /* Use NCOPIES deconstructs and NELEMENTS scalar ops. */
unsigned int nelements = ncopies * vect_nunits_for_cost (vectype);
- inside_cost = record_stmt_cost (cost_vec, nelements,
- vec_to_scalar, node, 0,
+ inside_cost = record_stmt_cost (cost_vec, ncopies,
+ vec_deconstruct, node, 0,
vect_body);
inside_cost += record_stmt_cost (cost_vec, nelements,
scalar_stmt, node, 0,
@@ -4822,9 +4822,8 @@ vect_model_reduction_cost (loop_vec_info loop_vinfo,
{
unsigned estimated_nunits = vect_nunits_for_cost (vectype);
/* Extraction of scalar elements. */
- epilogue_cost += record_stmt_cost (cost_vec,
- 2 * estimated_nunits,
- vec_to_scalar, node, 0,
+ epilogue_cost += record_stmt_cost (cost_vec, 2,
+ vec_deconstruct, node, 0,
vect_epilogue);
/* Scalar max reductions via COND_EXPR / MAX_EXPR. */
epilogue_cost += record_stmt_cost (cost_vec,
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index 4e7b45f5a6c..e991e861525 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -8654,12 +8654,12 @@ vectorizable_store (vec_info *vinfo,
inside_cost
+= record_stmt_cost (cost_vec, n_adjacent_stores,
scalar_store, slp_node, 0, vect_body);
- /* Only need vector extracting when there are more
- than one stores. */
+ /* Only need vector deconstruction when there is more
+ than one store. */
if (nstores > 1)
inside_cost
- += record_stmt_cost (cost_vec, n_adjacent_stores,
- vec_to_scalar, slp_node, 0, vect_body);
+ += record_stmt_cost (cost_vec, ncopies,
+ vec_deconstruct, slp_node, 0, vect_body);
}
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
@@ -9222,11 +9222,11 @@ vectorizable_store (vec_info *vinfo,
(we assume the scalar scaling and ptr + offset add is
consumed by the load). */
inside_cost
- += record_stmt_cost (cost_vec, cnunits, vec_to_scalar,
+ += record_stmt_cost (cost_vec, 1, vec_deconstruct,
slp_node, 0, vect_body);
/* N scalar stores plus extracting the elements. */
inside_cost
- += record_stmt_cost (cost_vec, cnunits, vec_to_scalar,
+ += record_stmt_cost (cost_vec, 1, vec_deconstruct,
slp_node, 0, vect_body);
inside_cost
+= record_stmt_cost (cost_vec, cnunits, scalar_store,
@@ -11201,8 +11201,7 @@ vectorizable_load (vec_info *vinfo,
{
/* For emulated gathers N offset vector element
offset add is consumed by the load). */
- inside_cost = record_stmt_cost (cost_vec, const_nunits,
- vec_to_scalar,
+ inside_cost = record_stmt_cost (cost_vec, 1, vec_deconstruct,
slp_node, 0, vect_body);
/* N scalar loads plus gathering them into a
vector. */
diff --git a/gcc/tree-vectorizer.cc b/gcc/tree-vectorizer.cc
index ab455c08349..8c824e4ee33 100644
--- a/gcc/tree-vectorizer.cc
+++ b/gcc/tree-vectorizer.cc
@@ -169,6 +169,9 @@ dump_stmt_cost (FILE *f, int count, enum vect_cost_for_stmt
kind,
case vec_construct:
ks = "vec_construct";
break;
+ case vec_deconstruct:
+ ks = "vec_deconstruct";
+ break;
}
fprintf (f, "%s ", ks);
if (kind == unaligned_load || kind == unaligned_store)
--
2.51.0
