This adds a vectype parameter to add_stmt_cost which avoids the need
to pass down a (wrong) stmt_info just to carry this information.
Useful for invariants which do not have a stmt_info associated.
Bootstrap and regtest running on x86_64-unknown-linux-gnu.
Richard.
2020-05-13 Richard Biener <rguent...@suse.de>
* target.def (add_stmt_cost): Add new vectype parameter.
* targhooks.c (default_add_stmt_cost): Adjust.
* targhooks.h (default_add_stmt_cost): Likewise.
* config/aarch64/aarch64.c (aarch64_add_stmt_cost): Take new
vectype parameter.
* config/arm/arm.c (arm_add_stmt_cost): Likewise.
* config/i386/i386.c (ix86_add_stmt_cost): Likewise.
* config/rs6000/rs6000.c (rs6000_add_stmt_cost): Likewise.
* tree-vectorizer.h (stmt_info_for_cost::vectype): Add.
(dump_stmt_cost): Add new vectype parameter.
(add_stmt_cost): Likewise.
(record_stmt_cost): Likewise.
(record_stmt_cost): Add overload with old signature.
* tree-vect-loop.c (vect_compute_single_scalar_iteration_cost):
Adjust.
(vect_get_known_peeling_cost): Likewise.
(vect_estimate_min_profitable_iters): Likewise.
* tree-vectorizer.c (dump_stmt_cost): Add new vectype parameter.
* tree-vect-stmts.c (record_stmt_cost): Likewise.
(vect_prologue_cost_for_slp_op): Remove stmt_vec_info parameter
and pass down correct vectype and NULL stmt_info.
(vect_model_simple_cost): Adjust.
(vect_model_store_cost): Likewise.
---
gcc/config/aarch64/aarch64.c | 5 ++---
gcc/config/arm/arm.c | 7 +++----
gcc/config/i386/i386.c | 5 ++---
gcc/config/rs6000/rs6000.c | 5 ++---
gcc/target.def | 2 +-
gcc/targhooks.c | 5 ++---
gcc/targhooks.h | 2 +-
gcc/tree-vect-loop.c | 46 +++++++++++++++++++++++---------------------
gcc/tree-vect-stmts.c | 17 ++++++----------
gcc/tree-vectorizer.c | 2 +-
gcc/tree-vectorizer.h | 27 ++++++++++++++++++++------
11 files changed, 65 insertions(+), 58 deletions(-)
diff --git a/gcc/config/aarch64/aarch64.c b/gcc/config/aarch64/aarch64.c
index 434e095cb66..70aa2f752b5 100644
--- a/gcc/config/aarch64/aarch64.c
+++ b/gcc/config/aarch64/aarch64.c
@@ -13753,15 +13753,14 @@ aarch64_sve_adjust_stmt_cost (class vec_info *vinfo,
vect_cost_for_stmt kind,
static unsigned
aarch64_add_stmt_cost (class vec_info *vinfo, void *data, int count,
enum vect_cost_for_stmt kind,
- struct _stmt_vec_info *stmt_info, int misalign,
- enum vect_cost_model_location where)
+ struct _stmt_vec_info *stmt_info, tree vectype,
+ int misalign, enum vect_cost_model_location where)
{
unsigned *cost = (unsigned *) data;
unsigned retval = 0;
if (flag_vect_cost_model)
{
- tree vectype = stmt_info ? stmt_vectype (stmt_info) : NULL_TREE;
int stmt_cost =
aarch64_builtin_vectorization_cost (kind, vectype, misalign);
diff --git a/gcc/config/arm/arm.c b/gcc/config/arm/arm.c
index d50781953c0..56d6be02996 100644
--- a/gcc/config/arm/arm.c
+++ b/gcc/config/arm/arm.c
@@ -309,7 +309,7 @@ static int arm_builtin_vectorization_cost (enum
vect_cost_for_stmt type_of_cost,
static unsigned arm_add_stmt_cost (vec_info *vinfo, void *data, int count,
enum vect_cost_for_stmt kind,
struct _stmt_vec_info *stmt_info,
- int misalign,
+ tree vectype, int misalign,
enum vect_cost_model_location where);
static void arm_canonicalize_comparison (int *code, rtx *op0, rtx *op1,
@@ -12133,15 +12133,14 @@ arm_builtin_vectorization_cost (enum
vect_cost_for_stmt type_of_cost,
static unsigned
arm_add_stmt_cost (vec_info *vinfo, void *data, int count,
enum vect_cost_for_stmt kind,
- struct _stmt_vec_info *stmt_info, int misalign,
- enum vect_cost_model_location where)
+ struct _stmt_vec_info *stmt_info, tree vectype,
+ int misalign, enum vect_cost_model_location where)
{
unsigned *cost = (unsigned *) data;
unsigned retval = 0;
if (flag_vect_cost_model)
{
- tree vectype = stmt_info ? stmt_vectype (stmt_info) : NULL_TREE;
int stmt_cost = arm_builtin_vectorization_cost (kind, vectype, misalign);
/* Statements in an inner loop relative to the loop being
diff --git a/gcc/config/i386/i386.c b/gcc/config/i386/i386.c
index f7a4bae49bb..060e2df62ea 100644
--- a/gcc/config/i386/i386.c
+++ b/gcc/config/i386/i386.c
@@ -21886,15 +21886,14 @@ ix86_init_cost (class loop *)
static unsigned
ix86_add_stmt_cost (class vec_info *vinfo, void *data, int count,
enum vect_cost_for_stmt kind,
- class _stmt_vec_info *stmt_info, int misalign,
+ class _stmt_vec_info *stmt_info, tree vectype,
+ int misalign,
enum vect_cost_model_location where)
{
unsigned *cost = (unsigned *) data;
unsigned retval = 0;
bool scalar_p
= (kind == scalar_stmt || kind == scalar_load || kind == scalar_store);
-
- tree vectype = stmt_info ? stmt_vectype (stmt_info) : NULL_TREE;
int stmt_cost = - 1;
bool fp = false;
diff --git a/gcc/config/rs6000/rs6000.c b/gcc/config/rs6000/rs6000.c
index 273a7215bc5..8435bc15d72 100644
--- a/gcc/config/rs6000/rs6000.c
+++ b/gcc/config/rs6000/rs6000.c
@@ -5049,15 +5049,14 @@ adjust_vectorization_cost (enum vect_cost_for_stmt kind,
static unsigned
rs6000_add_stmt_cost (class vec_info *vinfo, void *data, int count,
enum vect_cost_for_stmt kind,
- struct _stmt_vec_info *stmt_info, int misalign,
- enum vect_cost_model_location where)
+ struct _stmt_vec_info *stmt_info, tree vectype,
+ int misalign, enum vect_cost_model_location where)
{
rs6000_cost_data *cost_data = (rs6000_cost_data*) data;
unsigned retval = 0;
if (flag_vect_cost_model)
{
- tree vectype = stmt_info ? stmt_vectype (stmt_info) : NULL_TREE;
int stmt_cost = rs6000_builtin_vectorization_cost (kind, vectype,
misalign);
stmt_cost += adjust_vectorization_cost (kind, stmt_info);
diff --git a/gcc/target.def b/gcc/target.def
index f36aebb535c..07059a87caf 100644
--- a/gcc/target.def
+++ b/gcc/target.def
@@ -2031,7 +2031,7 @@ DEFHOOK
"revised.",
unsigned,
(class vec_info *, void *data, int count, enum vect_cost_for_stmt kind,
- class _stmt_vec_info *stmt_info, int misalign,
+ class _stmt_vec_info *stmt_info, tree vectype, int misalign,
enum vect_cost_model_location where),
default_add_stmt_cost)
diff --git a/gcc/targhooks.c b/gcc/targhooks.c
index e0982328d8e..0113c7b0ce2 100644
--- a/gcc/targhooks.c
+++ b/gcc/targhooks.c
@@ -1350,13 +1350,12 @@ default_init_cost (class loop *loop_info
ATTRIBUTE_UNUSED)
unsigned
default_add_stmt_cost (class vec_info *vinfo, void *data, int count,
enum vect_cost_for_stmt kind,
- class _stmt_vec_info *stmt_info, int misalign,
+ class _stmt_vec_info *stmt_info, tree vectype,
+ int misalign,
enum vect_cost_model_location where)
{
unsigned *cost = (unsigned *) data;
unsigned retval = 0;
-
- tree vectype = stmt_info ? stmt_vectype (stmt_info) : NULL_TREE;
int stmt_cost = targetm.vectorize.builtin_vectorization_cost (kind, vectype,
misalign);
/* Statements in an inner loop relative to the loop being
diff --git a/gcc/targhooks.h b/gcc/targhooks.h
index 9bab77bc638..b572a36e8cf 100644
--- a/gcc/targhooks.h
+++ b/gcc/targhooks.h
@@ -120,7 +120,7 @@ extern bool default_empty_mask_is_expensive (unsigned);
extern void *default_init_cost (class loop *);
extern unsigned default_add_stmt_cost (class vec_info *, void *, int,
enum vect_cost_for_stmt,
- class _stmt_vec_info *, int,
+ class _stmt_vec_info *, tree, int,
enum vect_cost_model_location);
extern void default_finish_cost (void *, unsigned *, unsigned *, unsigned *);
extern void default_destroy_cost_data (void *);
diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c
index 180790abf42..a579eb469a5 100644
--- a/gcc/tree-vect-loop.c
+++ b/gcc/tree-vect-loop.c
@@ -1124,8 +1124,8 @@ vect_compute_single_scalar_iteration_cost (loop_vec_info
loop_vinfo)
FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo),
j, si)
(void) add_stmt_cost (loop_vinfo, target_cost_data, si->count,
- si->kind, si->stmt_info, si->misalign,
- vect_body);
+ si->kind, si->stmt_info, si->vectype,
+ si->misalign, vect_body);
unsigned dummy, body_cost = 0;
finish_cost (target_cost_data, &dummy, &body_cost, &dummy);
destroy_cost_data (target_cost_data);
@@ -3324,9 +3324,9 @@ vect_get_known_peeling_cost (loop_vec_info loop_vinfo,
int peel_iters_prologue,
/* If peeled iterations are known but number of scalar loop
iterations are unknown, count a taken branch per peeled loop. */
retval = record_stmt_cost (prologue_cost_vec, 1, cond_branch_taken,
- NULL, 0, vect_prologue);
+ NULL, NULL_TREE, 0, vect_prologue);
retval += record_stmt_cost (epilogue_cost_vec, 1, cond_branch_taken,
- NULL, 0, vect_epilogue);
+ NULL, NULL_TREE, 0, vect_epilogue);
}
else
{
@@ -3407,7 +3407,7 @@ vect_estimate_min_profitable_iters (loop_vec_info
loop_vinfo,
/* FIXME: Make cost depend on complexity of individual check. */
unsigned len = LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).length ();
(void) add_stmt_cost (loop_vinfo, target_cost_data, len, vector_stmt,
- NULL, 0, vect_prologue);
+ NULL, NULL_TREE, 0, vect_prologue);
if (dump_enabled_p ())
dump_printf (MSG_NOTE,
"cost model: Adding cost of checks for loop "
@@ -3420,12 +3420,12 @@ vect_estimate_min_profitable_iters (loop_vec_info
loop_vinfo,
/* FIXME: Make cost depend on complexity of individual check. */
unsigned len = LOOP_VINFO_COMP_ALIAS_DDRS (loop_vinfo).length ();
(void) add_stmt_cost (loop_vinfo, target_cost_data, len, vector_stmt,
- NULL, 0, vect_prologue);
+ NULL, NULL_TREE, 0, vect_prologue);
len = LOOP_VINFO_CHECK_UNEQUAL_ADDRS (loop_vinfo).length ();
if (len)
/* Count LEN - 1 ANDs and LEN comparisons. */
(void) add_stmt_cost (loop_vinfo, target_cost_data, len * 2 - 1,
- scalar_stmt, NULL, 0, vect_prologue);
+ scalar_stmt, NULL, NULL_TREE, 0, vect_prologue);
len = LOOP_VINFO_LOWER_BOUNDS (loop_vinfo).length ();
if (len)
{
@@ -3436,7 +3436,7 @@ vect_estimate_min_profitable_iters (loop_vec_info
loop_vinfo,
if (!LOOP_VINFO_LOWER_BOUNDS (loop_vinfo)[i].unsigned_p)
nstmts += 1;
(void) add_stmt_cost (loop_vinfo, target_cost_data, nstmts,
- scalar_stmt, NULL, 0, vect_prologue);
+ scalar_stmt, NULL, NULL_TREE, 0, vect_prologue);
}
if (dump_enabled_p ())
dump_printf (MSG_NOTE,
@@ -3449,7 +3449,7 @@ vect_estimate_min_profitable_iters (loop_vec_info
loop_vinfo,
{
/* FIXME: Make cost depend on complexity of individual check. */
(void) add_stmt_cost (loop_vinfo, target_cost_data, 1, vector_stmt,
- NULL, 0, vect_prologue);
+ NULL, NULL_TREE, 0, vect_prologue);
if (dump_enabled_p ())
dump_printf (MSG_NOTE,
"cost model: Adding cost of checks for loop "
@@ -3458,7 +3458,7 @@ vect_estimate_min_profitable_iters (loop_vec_info
loop_vinfo,
if (LOOP_REQUIRES_VERSIONING (loop_vinfo))
(void) add_stmt_cost (loop_vinfo, target_cost_data, 1, cond_branch_taken,
- NULL, 0, vect_prologue);
+ NULL, NULL_TREE, 0, vect_prologue);
/* Count statements in scalar loop. Using this as scalar cost for a single
iteration for now.
@@ -3494,8 +3494,8 @@ vect_estimate_min_profitable_iters (loop_vec_info
loop_vinfo,
FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo),
j, si)
(void) add_stmt_cost (loop_vinfo, target_cost_data, si->count,
- si->kind, si->stmt_info, si->misalign,
- vect_epilogue);
+ si->kind, si->stmt_info, si->vectype,
+ si->misalign, vect_epilogue);
}
/* Calculate how many masks we need to generate. */
@@ -3521,10 +3521,10 @@ vect_estimate_min_profitable_iters (loop_vec_info
loop_vinfo,
probably better not to vectorize. */
(void) add_stmt_cost (loop_vinfo,
target_cost_data, num_masks, vector_stmt,
- NULL, 0, vect_prologue);
+ NULL, NULL_TREE, 0, vect_prologue);
(void) add_stmt_cost (loop_vinfo,
target_cost_data, num_masks - 1, vector_stmt,
- NULL, 0, vect_body);
+ NULL, NULL_TREE, 0, vect_body);
}
else if (npeel < 0)
{
@@ -3546,26 +3546,28 @@ vect_estimate_min_profitable_iters (loop_vec_info
loop_vinfo,
vector iterations are not known since peeled prologue iterations are
not known. Hence guards remain the same. */
(void) add_stmt_cost (loop_vinfo, target_cost_data, 1, cond_branch_taken,
- NULL, 0, vect_prologue);
+ NULL, NULL_TREE, 0, vect_prologue);
(void) add_stmt_cost (loop_vinfo,
target_cost_data, 1, cond_branch_not_taken,
- NULL, 0, vect_prologue);
+ NULL, NULL_TREE, 0, vect_prologue);
(void) add_stmt_cost (loop_vinfo, target_cost_data, 1, cond_branch_taken,
- NULL, 0, vect_epilogue);
+ NULL, NULL_TREE, 0, vect_epilogue);
(void) add_stmt_cost (loop_vinfo,
target_cost_data, 1, cond_branch_not_taken,
- NULL, 0, vect_epilogue);
+ NULL, NULL_TREE, 0, vect_epilogue);
stmt_info_for_cost *si;
int j;
FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo), j, si)
{
(void) add_stmt_cost (loop_vinfo, target_cost_data,
si->count * peel_iters_prologue,
- si->kind, si->stmt_info, si->misalign,
+ si->kind, si->stmt_info, si->vectype,
+ si->misalign,
vect_prologue);
(void) add_stmt_cost (loop_vinfo, target_cost_data,
si->count * peel_iters_epilogue,
- si->kind, si->stmt_info, si->misalign,
+ si->kind, si->stmt_info, si->vectype,
+ si->misalign,
vect_epilogue);
}
}
@@ -3590,12 +3592,12 @@ vect_estimate_min_profitable_iters (loop_vec_info
loop_vinfo,
FOR_EACH_VEC_ELT (prologue_cost_vec, j, si)
(void) add_stmt_cost (loop_vinfo,
data, si->count, si->kind, si->stmt_info,
- si->misalign, vect_prologue);
+ si->vectype, si->misalign, vect_prologue);
FOR_EACH_VEC_ELT (epilogue_cost_vec, j, si)
(void) add_stmt_cost (loop_vinfo,
data, si->count, si->kind, si->stmt_info,
- si->misalign, vect_epilogue);
+ si->vectype, si->misalign, vect_epilogue);
prologue_cost_vec.release ();
epilogue_cost_vec.release ();
diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c
index 9a715b82f1a..0ebecc43d14 100644
--- a/gcc/tree-vect-stmts.c
+++ b/gcc/tree-vect-stmts.c
@@ -92,7 +92,8 @@ stmt_in_inner_loop_p (vec_info *vinfo, class _stmt_vec_info
*stmt_info)
unsigned
record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
enum vect_cost_for_stmt kind, stmt_vec_info stmt_info,
- int misalign, enum vect_cost_model_location where)
+ tree vectype, int misalign,
+ enum vect_cost_model_location where)
{
if ((kind == vector_load || kind == unaligned_load)
&& STMT_VINFO_GATHER_SCATTER_P (stmt_info))
@@ -101,10 +102,9 @@ record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int
count,
&& STMT_VINFO_GATHER_SCATTER_P (stmt_info))
kind = vector_scatter_store;
- stmt_info_for_cost si = { count, kind, where, stmt_info, misalign };
+ stmt_info_for_cost si = { count, kind, where, stmt_info, vectype, misalign };
body_cost_vec->safe_push (si);
- tree vectype = stmt_info ? stmt_vectype (stmt_info) : NULL_TREE;
return (unsigned)
(builtin_vectorization_cost (kind, vectype, misalign) * count);
}
@@ -790,7 +790,7 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo,
bool *fatal)
static unsigned
vect_prologue_cost_for_slp_op (vec_info *vinfo,
- slp_tree node, stmt_vec_info stmt_info,
+ slp_tree node,
unsigned opno, enum vect_def_type dt,
stmt_vector_for_cost *cost_vec)
{
@@ -836,14 +836,11 @@ vect_prologue_cost_for_slp_op (vec_info *vinfo,
nelt++;
if (nelt == nelt_limit)
{
- /* ??? We need to pass down stmt_info for a vector type
- even if it points to the wrong stmt. */
prologue_cost += record_stmt_cost
(cost_vec, 1,
dt == vect_external_def
- ? (elt ? scalar_to_vec : vec_construct)
- : vector_load,
- stmt_info, 0, vect_prologue);
+ ? (elt ? scalar_to_vec : vec_construct) : vector_load,
+ NULL, vectype, 0, vect_prologue);
nelt = 0;
}
}
@@ -890,7 +887,6 @@ vect_model_simple_cost (vec_info *vinfo,
if (vect_is_simple_use (op, vinfo, &dt)
&& (dt == vect_constant_def || dt == vect_external_def))
prologue_cost += vect_prologue_cost_for_slp_op (vinfo, node,
- stmt_info,
i, dt, cost_vec);
}
}
@@ -1012,7 +1008,6 @@ vect_model_store_cost (vec_info *vinfo, stmt_vec_info
stmt_info, int ncopies,
{
if (slp_node)
prologue_cost += vect_prologue_cost_for_slp_op (vinfo, slp_node,
- stmt_info,
1, dt, cost_vec);
else
prologue_cost += record_stmt_cost (cost_vec, 1, scalar_to_vec,
diff --git a/gcc/tree-vectorizer.c b/gcc/tree-vectorizer.c
index 41ff6791966..584f13095ab 100644
--- a/gcc/tree-vectorizer.c
+++ b/gcc/tree-vectorizer.c
@@ -99,7 +99,7 @@ auto_purge_vect_location::~auto_purge_vect_location ()
void
dump_stmt_cost (FILE *f, void *data, int count, enum vect_cost_for_stmt kind,
- stmt_vec_info stmt_info, int misalign, unsigned cost,
+ stmt_vec_info stmt_info, tree, int misalign, unsigned cost,
enum vect_cost_model_location where)
{
fprintf (f, "%p ", data);
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index aa8bd33b9d1..09fc5b0ac2f 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -99,6 +99,7 @@ struct stmt_info_for_cost {
enum vect_cost_for_stmt kind;
enum vect_cost_model_location where;
stmt_vec_info stmt_info;
+ tree vectype;
int misalign;
};
@@ -1355,7 +1356,7 @@ init_cost (class loop *loop_info)
}
extern void dump_stmt_cost (FILE *, void *, int, enum vect_cost_for_stmt,
- stmt_vec_info, int, unsigned,
+ stmt_vec_info, tree, int, unsigned,
enum vect_cost_model_location);
/* Alias targetm.vectorize.add_stmt_cost. */
@@ -1363,13 +1364,14 @@ extern void dump_stmt_cost (FILE *, void *, int, enum
vect_cost_for_stmt,
static inline unsigned
add_stmt_cost (vec_info *vinfo, void *data, int count,
enum vect_cost_for_stmt kind,
- stmt_vec_info stmt_info, int misalign,
+ stmt_vec_info stmt_info, tree vectype, int misalign,
enum vect_cost_model_location where)
{
unsigned cost = targetm.vectorize.add_stmt_cost (vinfo, data, count, kind,
- stmt_info, misalign, where);
+ stmt_info, vectype,
+ misalign, where);
if (dump_file && (dump_flags & TDF_DETAILS))
- dump_stmt_cost (dump_file, data, count, kind, stmt_info, misalign,
+ dump_stmt_cost (dump_file, data, count, kind, stmt_info, vectype, misalign,
cost, where);
return cost;
}
@@ -1398,7 +1400,7 @@ add_stmt_costs (vec_info *vinfo, void *data,
stmt_vector_for_cost *cost_vec)
unsigned i;
FOR_EACH_VEC_ELT (*cost_vec, i, cost)
add_stmt_cost (vinfo, data, cost->count, cost->kind, cost->stmt_info,
- cost->misalign, cost->where);
+ cost->vectype, cost->misalign, cost->where);
}
/*-----------------------------------------------------------------*/
@@ -1699,9 +1701,22 @@ extern bool supportable_widening_operation (vec_info *,
extern bool supportable_narrowing_operation (enum tree_code, tree, tree,
enum tree_code *, int *,
vec<tree> *);
+
extern unsigned record_stmt_cost (stmt_vector_for_cost *, int,
enum vect_cost_for_stmt, stmt_vec_info,
- int, enum vect_cost_model_location);
+ tree, int, enum vect_cost_model_location);
+
+/* Overload of record_stmt_cost with VECTYPE derived from STMT_INFO. */
+
+static inline unsigned
+record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
+ enum vect_cost_for_stmt kind, stmt_vec_info stmt_info,
+ int misalign, enum vect_cost_model_location where)
+{
+ return record_stmt_cost (body_cost_vec, count, kind, stmt_info,
+ STMT_VINFO_VECTYPE (stmt_info), misalign, where);
+}
+
extern stmt_vec_info vect_finish_replace_stmt (vec_info *,
stmt_vec_info, gimple *);
extern stmt_vec_info vect_finish_stmt_generation (vec_info *,
--
2.13.7
