Continuing the work started in GCC 12 with g:a0dae768c6f78eba
this patch replaces uses of SLP_TREE_SCALAR_STMTS with SLP_TREE_LANES when used
to determine the number of lanes in the SLP tree. This de-couples the need to
have the same number of scalar statements as lanes in the SLP tree.
The change is to help prepare the vectorizer for being able to handle vector
statements and so some cases already handle (where safe) the case where
SLP_TREE_SCALAR_STMTS.length () != SLP_TREE_LANES.
In some places it just removes the use of .length() so it's easier to see that
the length amount of statements was never actually used as a property directly.
i.e. it was a numbered for each.
Bootstrapped Regtested on aarch64-none-linux-gnu,
arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
-m32, -m64 and no issues.
Any feedback?
Thanks,
Tamar
gcc/ChangeLog:
* tree-vect-data-refs.cc (vect_slp_analyze_store_dependences,
vect_slp_analyze_load_dependences,
vect_slp_analyze_instance_dependence): Replace uses of .length ().
* tree-vect-slp-patterns.cc (compatible_complex_nodes_p): Support
mismatched group sizes and use group size to iterate.
* tree-vect-slp.cc (vect_slp_tree_uniform_p): Consider mismatched ops
lanes as non uniform.
(vect_mark_slp_stmts_relevant, vect_find_last_scalar_stmt_in_slp,
vect_find_first_scalar_stmt_in_slp): Use for-each iterator.
(vect_build_slp_store_interleaving): Check SLP Lanes instead.
(vect_build_slp_instance, vect_analyze_slp_reduc_chain,
vect_analyze_slp_reduction, vect_analyze_slp_reduction_group,
vect_analyze_slp_instance, vect_analyze_slp): Compare against lanes
instead of .length ().
(vect_optimize_slp_pass::get_result_with_layout): Reject layouts where
lanes and scalar ops don't match for now.
(vect_prologue_cost_for_slp): Cost mismatched lanes and ops as a
vec_constructor.
(vect_slp_prune_covered_roots, vect_bb_partition_graph_r): Replace
iterators.
(vect_get_slp_scalar_def): Safely handle cases where LANES and scalar
stmts or ops len don't match.
(vect_transform_slp_perm_load_1): Use lanes instead of .length ().
(vect_remove_slp_scalar_calls): Replace iterators.
* tree-vect-stmts.cc (vectorizable_shift): Use group size over ops
length.
(vect_is_simple_use): Support getting scalar def from lane outside of
ops vect.
---
diff --git a/gcc/tree-vect-data-refs.cc b/gcc/tree-vect-data-refs.cc
index
ab850de024bf41ea7635ce7381bcc6acd8fe0c28..80f7539614c2437d888611098b32467b894bacb0
100644
--- a/gcc/tree-vect-data-refs.cc
+++ b/gcc/tree-vect-data-refs.cc
@@ -1037,10 +1037,10 @@ vect_slp_analyze_store_dependences (vec_info *vinfo,
slp_tree node)
stmt_vec_info last_access_info = vect_find_last_scalar_stmt_in_slp (node);
gcc_assert (DR_IS_WRITE (STMT_VINFO_DATA_REF (last_access_info)));
- for (unsigned k = 0; k < SLP_TREE_SCALAR_STMTS (node).length (); ++k)
+ for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (node))
{
stmt_vec_info access_info
- = vect_orig_stmt (SLP_TREE_SCALAR_STMTS (node)[k]);
+ = vect_orig_stmt (stmt_vinfo);
if (access_info == last_access_info)
continue;
data_reference *dr_a = STMT_VINFO_DATA_REF (access_info);
@@ -1103,12 +1103,12 @@ vect_slp_analyze_load_dependences (vec_info *vinfo,
slp_tree node,
stmt_vec_info first_access_info = vect_find_first_scalar_stmt_in_slp (node);
gcc_assert (DR_IS_READ (STMT_VINFO_DATA_REF (first_access_info)));
- for (unsigned k = 0; k < SLP_TREE_SCALAR_STMTS (node).length (); ++k)
+ for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (node))
{
- if (! SLP_TREE_SCALAR_STMTS (node)[k])
+ if (! stmt_vinfo)
continue;
stmt_vec_info access_info
- = vect_orig_stmt (SLP_TREE_SCALAR_STMTS (node)[k]);
+ = vect_orig_stmt (stmt_vinfo);
if (access_info == first_access_info)
continue;
data_reference *dr_a = STMT_VINFO_DATA_REF (access_info);
@@ -1245,8 +1245,8 @@ vect_slp_analyze_instance_dependence (vec_info *vinfo,
slp_instance instance)
/* Mark stores in this instance and remember the last one. */
last_store_info = vect_find_last_scalar_stmt_in_slp (store);
- for (unsigned k = 0; k < SLP_TREE_SCALAR_STMTS (store).length (); ++k)
- gimple_set_visited (SLP_TREE_SCALAR_STMTS (store)[k]->stmt, true);
+ for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (store))
+ gimple_set_visited (STMT_VINFO_STMT (stmt_vinfo), true);
}
bool res = true;
@@ -1265,8 +1265,8 @@ vect_slp_analyze_instance_dependence (vec_info *vinfo,
slp_instance instance)
/* Unset the visited flag. */
if (store)
- for (unsigned k = 0; k < SLP_TREE_SCALAR_STMTS (store).length (); ++k)
- gimple_set_visited (SLP_TREE_SCALAR_STMTS (store)[k]->stmt, false);
+ for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (store))
+ gimple_set_visited (STMT_VINFO_STMT (stmt_vinfo), false);
/* If this is a SLP instance with a store check if there's a dependent
load that cannot be forwarded from a previous iteration of a loop
diff --git a/gcc/tree-vect-slp-patterns.cc b/gcc/tree-vect-slp-patterns.cc
index
181281c2154bedfc8e8c6b1bde1226e194ba99e5..82aec27498432d4634f4cc135bef901d700236bb
100644
--- a/gcc/tree-vect-slp-patterns.cc
+++ b/gcc/tree-vect-slp-patterns.cc
@@ -762,7 +762,11 @@ compatible_complex_nodes_p (slp_compat_nodes_map_t
*compat_cache,
are externals. */
if (SLP_TREE_DEF_TYPE (a) != vect_internal_def)
{
- for (unsigned i = 0; i < SLP_TREE_SCALAR_OPS (a).length (); i++)
+ unsigned group_size = SLP_TREE_LANES (a);
+ if (SLP_TREE_SCALAR_OPS (a).length () != group_size
+ || SLP_TREE_SCALAR_OPS (b).length () != group_size)
+ return false;
+ for (unsigned i = 0; i < group_size; i++)
{
tree op1 = SLP_TREE_SCALAR_OPS (a)[pa[i % 2]];
tree op2 = SLP_TREE_SCALAR_OPS (b)[pb[i % 2]];
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index
2250f6f74a1ae63b0b9e42559955c0ab77ff3135..97a46ac929be16c3516fdbef80ad960954a8fafa
100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -378,6 +378,9 @@ vect_slp_tree_uniform_p (slp_tree node)
if (SLP_TREE_SCALAR_OPS (node).is_empty ())
return false;
+ if (SLP_TREE_SCALAR_OPS (node).length () != SLP_TREE_LANES (node))
+ return false;
+
unsigned i;
tree op, first = NULL_TREE;
FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_OPS (node), i, op)
@@ -3601,17 +3604,13 @@ vect_bb_slp_mark_stmts_vectorized (bb_vec_info vinfo)
static void
vect_mark_slp_stmts_relevant (slp_tree node, hash_set<slp_tree> &visited)
{
- int i;
- stmt_vec_info stmt_info;
- slp_tree child;
-
if (SLP_TREE_DEF_TYPE (node) != vect_internal_def)
return;
if (visited.add (node))
return;
- FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
+ for (auto stmt_info : SLP_TREE_SCALAR_STMTS (node))
if (stmt_info)
{
gcc_assert (!STMT_VINFO_RELEVANT (stmt_info)
@@ -3619,7 +3618,7 @@ vect_mark_slp_stmts_relevant (slp_tree node,
hash_set<slp_tree> &visited)
STMT_VINFO_RELEVANT (stmt_info) = vect_used_in_scope;
}
- FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
+ for (auto child: SLP_TREE_CHILDREN (node))
if (child)
vect_mark_slp_stmts_relevant (child, visited);
}
@@ -3665,9 +3664,7 @@ stmt_vec_info
vect_find_last_scalar_stmt_in_slp (slp_tree node)
{
stmt_vec_info last = NULL;
- stmt_vec_info stmt_vinfo;
-
- for (int i = 0; SLP_TREE_SCALAR_STMTS (node).iterate (i, &stmt_vinfo); i++)
+ for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (node))
if (stmt_vinfo)
{
stmt_vinfo = vect_orig_stmt (stmt_vinfo);
@@ -3683,9 +3680,8 @@ stmt_vec_info
vect_find_first_scalar_stmt_in_slp (slp_tree node)
{
stmt_vec_info first = NULL;
- stmt_vec_info stmt_vinfo;
- for (int i = 0; SLP_TREE_SCALAR_STMTS (node).iterate (i, &stmt_vinfo); i++)
+ for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (node))
if (stmt_vinfo)
{
stmt_vinfo = vect_orig_stmt (stmt_vinfo);
@@ -4025,7 +4021,7 @@ vect_build_slp_store_interleaving (vec<slp_tree>
&rhs_nodes,
.quick_push (SLP_TREE_CHILDREN (rhs_nodes[j])[l]);
SLP_TREE_CHILDREN (rhs_nodes[j])[l]->refcnt++;
for (unsigned k = 0;
- k < SLP_TREE_SCALAR_STMTS (rhs_nodes[j]).length (); ++k)
+ k < SLP_TREE_LANES (rhs_nodes[j]); ++k)
{
/* ??? We should populate SLP_TREE_SCALAR_STMTS
or SLP_TREE_SCALAR_OPS but then we might have
@@ -4292,8 +4288,8 @@ vect_build_slp_instance (vec_info *vinfo,
/* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
the number of scalar stmts in the root in a few places.
Verify that assumption holds. */
- gcc_assert (SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (new_instance))
- .length () == group_size);
+ gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
+ == group_size);
if (dump_enabled_p ())
{
@@ -4727,8 +4723,8 @@ vect_analyze_slp_reduc_chain (loop_vec_info vinfo,
/* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
the number of scalar stmts in the root in a few places.
Verify that assumption holds. */
- gcc_assert (SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (new_instance))
- .length () == group_size);
+ gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
+ == group_size);
if (dump_enabled_p ())
{
@@ -4816,8 +4812,8 @@ vect_analyze_slp_reduction (loop_vec_info vinfo,
/* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
the number of scalar stmts in the root in a few places.
Verify that assumption holds. */
- gcc_assert (SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (new_instance))
- .length () == group_size);
+ gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
+ == group_size);
if (dump_enabled_p ())
{
@@ -4889,8 +4885,8 @@ vect_analyze_slp_reduction_group (loop_vec_info
loop_vinfo,
/* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
the number of scalar stmts in the root in a few places.
Verify that assumption holds. */
- gcc_assert (SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (new_instance))
- .length () == group_size);
+ gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
+ == group_size);
if (dump_enabled_p ())
{
@@ -5145,8 +5141,8 @@ vect_analyze_slp_instance (vec_info *vinfo,
/* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
the number of scalar stmts in the root in a few places.
Verify that assumption holds. */
- gcc_assert (SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (new_instance))
- .length () == group_size);
+ gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
+ == group_size);
if (dump_enabled_p ())
{
@@ -5392,8 +5388,8 @@ vect_analyze_slp_instance (vec_info *vinfo,
/* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
the number of scalar stmts in the root in a few places.
Verify that assumption holds. */
- gcc_assert (SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (new_instance))
- .length () == group_size);
+ gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
+ == group_size);
if (dump_enabled_p ())
{
@@ -6087,9 +6083,7 @@ vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size,
{
if (!SLP_TREE_LOAD_PERMUTATION (load_node).exists ())
continue;
- unsigned k;
- stmt_vec_info load_info;
- FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (load_node), k, load_info)
+ for (unsigned k = 0; k < SLP_TREE_LANES (load_node); k++)
if (SLP_TREE_LOAD_PERMUTATION (load_node)[k] != k)
{
loads_permuted = true;
@@ -7865,6 +7859,8 @@ vect_optimize_slp_pass::get_result_with_layout (slp_tree
node,
/* If the vector is uniform or unchanged, there's nothing to do. */
if (to_layout_i == 0 || vect_slp_tree_uniform_p (node))
result = node;
+ else if (SLP_TREE_SCALAR_OPS (node).length () != SLP_TREE_LANES (node))
+ result = node;
else
{
auto scalar_ops = SLP_TREE_SCALAR_OPS (node).copy ();
@@ -7915,7 +7911,7 @@ vect_optimize_slp_pass::get_result_with_layout (slp_tree
node,
unsigned int num_lanes = SLP_TREE_LANES (node);
result = vect_create_new_slp_node (num_inputs, VEC_PERM_EXPR);
- if (SLP_TREE_SCALAR_STMTS (node).length ())
+ if (SLP_TREE_SCALAR_STMTS (node).exists ())
{
auto &stmts = SLP_TREE_SCALAR_STMTS (result);
stmts.safe_splice (SLP_TREE_SCALAR_STMTS (node));
@@ -8828,7 +8824,14 @@ vect_prologue_cost_for_slp (vec_info *vinfo, slp_tree
node,
constants can be implemented as load from the constant pool.
When all elements are the same we can use a splat. */
tree vectype = SLP_TREE_VECTYPE (node);
- unsigned group_size = SLP_TREE_SCALAR_OPS (node).length ();
+ unsigned group_size = SLP_TREE_LANES (node);
+ if (SLP_TREE_SCALAR_OPS (node).length () != group_size)
+ {
+ record_stmt_cost (cost_vec, vect_get_num_copies (vinfo, node),
+ vec_construct, nullptr, node, vectype, 0,
+ vect_prologue);
+ return;
+ }
unsigned HOST_WIDE_INT const_nunits;
unsigned nelt_limit;
unsigned nvectors = vect_get_num_copies (vinfo, node);
@@ -9194,14 +9197,11 @@ vect_slp_prune_covered_roots (slp_tree node,
hash_set<stmt_vec_info> &roots,
|| visited.add (node))
return;
- stmt_vec_info stmt;
- unsigned i;
- FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt)
+ for (auto stmt : SLP_TREE_SCALAR_STMTS (node))
if (stmt)
roots.remove (vect_orig_stmt (stmt));
- slp_tree child;
- FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
+ for (auto child : SLP_TREE_CHILDREN (node))
if (child)
vect_slp_prune_covered_roots (child, roots, visited);
}
@@ -9411,10 +9411,7 @@ vect_bb_partition_graph_r (bb_vec_info bb_vinfo,
hash_map<slp_tree, slp_instance> &node_to_instance,
hash_map<slp_instance, slp_instance>
&instance_leader)
{
- stmt_vec_info stmt_info;
- unsigned i;
-
- FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
+ for (auto stmt_info : SLP_TREE_SCALAR_STMTS (node))
if (stmt_info)
vect_map_to_instance (instance, stmt_info, stmt_to_instance,
instance_leader);
@@ -9423,8 +9420,7 @@ vect_bb_partition_graph_r (bb_vec_info bb_vinfo,
instance_leader))
return;
- slp_tree child;
- FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
+ for (auto child : SLP_TREE_CHILDREN (node))
if (child && SLP_TREE_DEF_TYPE (child) == vect_internal_def)
vect_bb_partition_graph_r (bb_vinfo, instance, child, stmt_to_instance,
node_to_instance, instance_leader);
@@ -10942,13 +10938,19 @@ vect_get_slp_scalar_def (slp_tree slp_node, unsigned
n)
{
if (!SLP_TREE_SCALAR_STMTS (slp_node).exists ())
return NULL_TREE;
+ if (n >= SLP_TREE_SCALAR_STMTS (slp_node).length ())
+ return NULL_TREE;
stmt_vec_info def = SLP_TREE_SCALAR_STMTS (slp_node)[n];
if (!def)
return NULL_TREE;
return gimple_get_lhs (STMT_VINFO_STMT (def));
}
else
- return SLP_TREE_SCALAR_OPS (slp_node)[n];
+ {
+ if (n >= SLP_TREE_SCALAR_OPS (slp_node).length ())
+ return NULL_TREE;
+ return SLP_TREE_SCALAR_OPS (slp_node)[n];
+ }
}
/* Get the Ith vectorized definition from SLP_NODE. */
@@ -11003,7 +11005,7 @@ vect_transform_slp_perm_load_1 (vec_info *vinfo,
slp_tree node,
stmt_vec_info stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
int vec_index = 0;
tree vectype = SLP_TREE_VECTYPE (node);
- unsigned int group_size = SLP_TREE_SCALAR_STMTS (node).length ();
+ unsigned int group_size = SLP_TREE_LANES (node);
unsigned int mask_element;
unsigned dr_group_size;
machine_mode mode;
@@ -12057,10 +12059,7 @@ vect_remove_slp_scalar_calls (vec_info *vinfo,
{
gimple *new_stmt;
gimple_stmt_iterator gsi;
- int i;
- slp_tree child;
tree lhs;
- stmt_vec_info stmt_info;
if (!node || SLP_TREE_DEF_TYPE (node) != vect_internal_def)
return;
@@ -12068,10 +12067,10 @@ vect_remove_slp_scalar_calls (vec_info *vinfo,
if (visited.add (node))
return;
- FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
+ for (auto child : SLP_TREE_CHILDREN (node))
vect_remove_slp_scalar_calls (vinfo, child, visited);
- FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
+ for (auto stmt_info : SLP_TREE_SCALAR_STMTS (node))
{
if (!stmt_info)
continue;
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index
76e24b2c1699b022e4480fc4aac87ff0b6ddf650..0c7787e8939f2d8b040988584a2213de946eee1b
100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -6388,15 +6388,19 @@ vectorizable_shift (vec_info *vinfo,
/* Now adjust the constant shift amount in place. */
if (incompatible_op1_vectype_p
&& dt[1] == vect_constant_def)
- for (unsigned i = 0;
- i < SLP_TREE_SCALAR_OPS (slp_op1).length (); ++i)
- {
- SLP_TREE_SCALAR_OPS (slp_op1)[i]
- = fold_convert (TREE_TYPE (vectype),
- SLP_TREE_SCALAR_OPS (slp_op1)[i]);
- gcc_assert ((TREE_CODE (SLP_TREE_SCALAR_OPS (slp_op1)[i])
- == INTEGER_CST));
- }
+ {
+ unsigned group_size = SLP_TREE_LANES (slp_op1);
+ if (SLP_TREE_SCALAR_OPS (slp_op1).length () != group_size)
+ return false;
+ for (unsigned i = 0; i < group_size; ++i)
+ {
+ SLP_TREE_SCALAR_OPS (slp_op1)[i]
+ = fold_convert (TREE_TYPE (vectype),
+ SLP_TREE_SCALAR_OPS (slp_op1)[i]);
+ gcc_assert ((TREE_CODE (SLP_TREE_SCALAR_OPS (slp_op1)[i])
+ == INTEGER_CST));
+ }
+ }
SLP_TREE_TYPE (slp_node) = shift_vec_info_type;
DUMP_VECT_SCOPE ("vectorizable_shift");
vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
@@ -13997,7 +14001,9 @@ vect_is_simple_use (vec_info *vinfo, slp_tree slp_node,
{
if (def_stmt_info_out)
*def_stmt_info_out = NULL;
- *op = SLP_TREE_SCALAR_OPS (child)[0];
+ *op = vect_get_slp_scalar_def (child, 0);
+ if (!*op)
+ return false;
*dt = SLP_TREE_DEF_TYPE (child);
return true;
}
@@ -14899,4 +14905,3 @@ vect_gen_len (tree len, tree start_index, tree
end_index, tree len_limit)
return stmts;
}
-
--
diff --git a/gcc/tree-vect-data-refs.cc b/gcc/tree-vect-data-refs.cc
index ab850de024bf41ea7635ce7381bcc6acd8fe0c28..80f7539614c2437d888611098b32467b894bacb0 100644
--- a/gcc/tree-vect-data-refs.cc
+++ b/gcc/tree-vect-data-refs.cc
@@ -1037,10 +1037,10 @@ vect_slp_analyze_store_dependences (vec_info *vinfo, slp_tree node)
stmt_vec_info last_access_info = vect_find_last_scalar_stmt_in_slp (node);
gcc_assert (DR_IS_WRITE (STMT_VINFO_DATA_REF (last_access_info)));
- for (unsigned k = 0; k < SLP_TREE_SCALAR_STMTS (node).length (); ++k)
+ for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (node))
{
stmt_vec_info access_info
- = vect_orig_stmt (SLP_TREE_SCALAR_STMTS (node)[k]);
+ = vect_orig_stmt (stmt_vinfo);
if (access_info == last_access_info)
continue;
data_reference *dr_a = STMT_VINFO_DATA_REF (access_info);
@@ -1103,12 +1103,12 @@ vect_slp_analyze_load_dependences (vec_info *vinfo, slp_tree node,
stmt_vec_info first_access_info = vect_find_first_scalar_stmt_in_slp (node);
gcc_assert (DR_IS_READ (STMT_VINFO_DATA_REF (first_access_info)));
- for (unsigned k = 0; k < SLP_TREE_SCALAR_STMTS (node).length (); ++k)
+ for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (node))
{
- if (! SLP_TREE_SCALAR_STMTS (node)[k])
+ if (! stmt_vinfo)
continue;
stmt_vec_info access_info
- = vect_orig_stmt (SLP_TREE_SCALAR_STMTS (node)[k]);
+ = vect_orig_stmt (stmt_vinfo);
if (access_info == first_access_info)
continue;
data_reference *dr_a = STMT_VINFO_DATA_REF (access_info);
@@ -1245,8 +1245,8 @@ vect_slp_analyze_instance_dependence (vec_info *vinfo, slp_instance instance)
/* Mark stores in this instance and remember the last one. */
last_store_info = vect_find_last_scalar_stmt_in_slp (store);
- for (unsigned k = 0; k < SLP_TREE_SCALAR_STMTS (store).length (); ++k)
- gimple_set_visited (SLP_TREE_SCALAR_STMTS (store)[k]->stmt, true);
+ for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (store))
+ gimple_set_visited (STMT_VINFO_STMT (stmt_vinfo), true);
}
bool res = true;
@@ -1265,8 +1265,8 @@ vect_slp_analyze_instance_dependence (vec_info *vinfo, slp_instance instance)
/* Unset the visited flag. */
if (store)
- for (unsigned k = 0; k < SLP_TREE_SCALAR_STMTS (store).length (); ++k)
- gimple_set_visited (SLP_TREE_SCALAR_STMTS (store)[k]->stmt, false);
+ for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (store))
+ gimple_set_visited (STMT_VINFO_STMT (stmt_vinfo), false);
/* If this is a SLP instance with a store check if there's a dependent
load that cannot be forwarded from a previous iteration of a loop
diff --git a/gcc/tree-vect-slp-patterns.cc b/gcc/tree-vect-slp-patterns.cc
index 181281c2154bedfc8e8c6b1bde1226e194ba99e5..82aec27498432d4634f4cc135bef901d700236bb 100644
--- a/gcc/tree-vect-slp-patterns.cc
+++ b/gcc/tree-vect-slp-patterns.cc
@@ -762,7 +762,11 @@ compatible_complex_nodes_p (slp_compat_nodes_map_t *compat_cache,
are externals. */
if (SLP_TREE_DEF_TYPE (a) != vect_internal_def)
{
- for (unsigned i = 0; i < SLP_TREE_SCALAR_OPS (a).length (); i++)
+ unsigned group_size = SLP_TREE_LANES (a);
+ if (SLP_TREE_SCALAR_OPS (a).length () != group_size
+ || SLP_TREE_SCALAR_OPS (b).length () != group_size)
+ return false;
+ for (unsigned i = 0; i < group_size; i++)
{
tree op1 = SLP_TREE_SCALAR_OPS (a)[pa[i % 2]];
tree op2 = SLP_TREE_SCALAR_OPS (b)[pb[i % 2]];
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index 2250f6f74a1ae63b0b9e42559955c0ab77ff3135..97a46ac929be16c3516fdbef80ad960954a8fafa 100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -378,6 +378,9 @@ vect_slp_tree_uniform_p (slp_tree node)
if (SLP_TREE_SCALAR_OPS (node).is_empty ())
return false;
+ if (SLP_TREE_SCALAR_OPS (node).length () != SLP_TREE_LANES (node))
+ return false;
+
unsigned i;
tree op, first = NULL_TREE;
FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_OPS (node), i, op)
@@ -3601,17 +3604,13 @@ vect_bb_slp_mark_stmts_vectorized (bb_vec_info vinfo)
static void
vect_mark_slp_stmts_relevant (slp_tree node, hash_set<slp_tree> &visited)
{
- int i;
- stmt_vec_info stmt_info;
- slp_tree child;
-
if (SLP_TREE_DEF_TYPE (node) != vect_internal_def)
return;
if (visited.add (node))
return;
- FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
+ for (auto stmt_info : SLP_TREE_SCALAR_STMTS (node))
if (stmt_info)
{
gcc_assert (!STMT_VINFO_RELEVANT (stmt_info)
@@ -3619,7 +3618,7 @@ vect_mark_slp_stmts_relevant (slp_tree node, hash_set<slp_tree> &visited)
STMT_VINFO_RELEVANT (stmt_info) = vect_used_in_scope;
}
- FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
+ for (auto child: SLP_TREE_CHILDREN (node))
if (child)
vect_mark_slp_stmts_relevant (child, visited);
}
@@ -3665,9 +3664,7 @@ stmt_vec_info
vect_find_last_scalar_stmt_in_slp (slp_tree node)
{
stmt_vec_info last = NULL;
- stmt_vec_info stmt_vinfo;
-
- for (int i = 0; SLP_TREE_SCALAR_STMTS (node).iterate (i, &stmt_vinfo); i++)
+ for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (node))
if (stmt_vinfo)
{
stmt_vinfo = vect_orig_stmt (stmt_vinfo);
@@ -3683,9 +3680,8 @@ stmt_vec_info
vect_find_first_scalar_stmt_in_slp (slp_tree node)
{
stmt_vec_info first = NULL;
- stmt_vec_info stmt_vinfo;
- for (int i = 0; SLP_TREE_SCALAR_STMTS (node).iterate (i, &stmt_vinfo); i++)
+ for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (node))
if (stmt_vinfo)
{
stmt_vinfo = vect_orig_stmt (stmt_vinfo);
@@ -4025,7 +4021,7 @@ vect_build_slp_store_interleaving (vec<slp_tree> &rhs_nodes,
.quick_push (SLP_TREE_CHILDREN (rhs_nodes[j])[l]);
SLP_TREE_CHILDREN (rhs_nodes[j])[l]->refcnt++;
for (unsigned k = 0;
- k < SLP_TREE_SCALAR_STMTS (rhs_nodes[j]).length (); ++k)
+ k < SLP_TREE_LANES (rhs_nodes[j]); ++k)
{
/* ??? We should populate SLP_TREE_SCALAR_STMTS
or SLP_TREE_SCALAR_OPS but then we might have
@@ -4292,8 +4288,8 @@ vect_build_slp_instance (vec_info *vinfo,
/* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
the number of scalar stmts in the root in a few places.
Verify that assumption holds. */
- gcc_assert (SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (new_instance))
- .length () == group_size);
+ gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
+ == group_size);
if (dump_enabled_p ())
{
@@ -4727,8 +4723,8 @@ vect_analyze_slp_reduc_chain (loop_vec_info vinfo,
/* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
the number of scalar stmts in the root in a few places.
Verify that assumption holds. */
- gcc_assert (SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (new_instance))
- .length () == group_size);
+ gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
+ == group_size);
if (dump_enabled_p ())
{
@@ -4816,8 +4812,8 @@ vect_analyze_slp_reduction (loop_vec_info vinfo,
/* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
the number of scalar stmts in the root in a few places.
Verify that assumption holds. */
- gcc_assert (SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (new_instance))
- .length () == group_size);
+ gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
+ == group_size);
if (dump_enabled_p ())
{
@@ -4889,8 +4885,8 @@ vect_analyze_slp_reduction_group (loop_vec_info loop_vinfo,
/* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
the number of scalar stmts in the root in a few places.
Verify that assumption holds. */
- gcc_assert (SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (new_instance))
- .length () == group_size);
+ gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
+ == group_size);
if (dump_enabled_p ())
{
@@ -5145,8 +5141,8 @@ vect_analyze_slp_instance (vec_info *vinfo,
/* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
the number of scalar stmts in the root in a few places.
Verify that assumption holds. */
- gcc_assert (SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (new_instance))
- .length () == group_size);
+ gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
+ == group_size);
if (dump_enabled_p ())
{
@@ -5392,8 +5388,8 @@ vect_analyze_slp_instance (vec_info *vinfo,
/* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
the number of scalar stmts in the root in a few places.
Verify that assumption holds. */
- gcc_assert (SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (new_instance))
- .length () == group_size);
+ gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
+ == group_size);
if (dump_enabled_p ())
{
@@ -6087,9 +6083,7 @@ vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size,
{
if (!SLP_TREE_LOAD_PERMUTATION (load_node).exists ())
continue;
- unsigned k;
- stmt_vec_info load_info;
- FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (load_node), k, load_info)
+ for (unsigned k = 0; k < SLP_TREE_LANES (load_node); k++)
if (SLP_TREE_LOAD_PERMUTATION (load_node)[k] != k)
{
loads_permuted = true;
@@ -7865,6 +7859,8 @@ vect_optimize_slp_pass::get_result_with_layout (slp_tree node,
/* If the vector is uniform or unchanged, there's nothing to do. */
if (to_layout_i == 0 || vect_slp_tree_uniform_p (node))
result = node;
+ else if (SLP_TREE_SCALAR_OPS (node).length () != SLP_TREE_LANES (node))
+ result = node;
else
{
auto scalar_ops = SLP_TREE_SCALAR_OPS (node).copy ();
@@ -7915,7 +7911,7 @@ vect_optimize_slp_pass::get_result_with_layout (slp_tree node,
unsigned int num_lanes = SLP_TREE_LANES (node);
result = vect_create_new_slp_node (num_inputs, VEC_PERM_EXPR);
- if (SLP_TREE_SCALAR_STMTS (node).length ())
+ if (SLP_TREE_SCALAR_STMTS (node).exists ())
{
auto &stmts = SLP_TREE_SCALAR_STMTS (result);
stmts.safe_splice (SLP_TREE_SCALAR_STMTS (node));
@@ -8828,7 +8824,14 @@ vect_prologue_cost_for_slp (vec_info *vinfo, slp_tree node,
constants can be implemented as load from the constant pool.
When all elements are the same we can use a splat. */
tree vectype = SLP_TREE_VECTYPE (node);
- unsigned group_size = SLP_TREE_SCALAR_OPS (node).length ();
+ unsigned group_size = SLP_TREE_LANES (node);
+ if (SLP_TREE_SCALAR_OPS (node).length () != group_size)
+ {
+ record_stmt_cost (cost_vec, vect_get_num_copies (vinfo, node),
+ vec_construct, nullptr, node, vectype, 0,
+ vect_prologue);
+ return;
+ }
unsigned HOST_WIDE_INT const_nunits;
unsigned nelt_limit;
unsigned nvectors = vect_get_num_copies (vinfo, node);
@@ -9194,14 +9197,11 @@ vect_slp_prune_covered_roots (slp_tree node, hash_set<stmt_vec_info> &roots,
|| visited.add (node))
return;
- stmt_vec_info stmt;
- unsigned i;
- FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt)
+ for (auto stmt : SLP_TREE_SCALAR_STMTS (node))
if (stmt)
roots.remove (vect_orig_stmt (stmt));
- slp_tree child;
- FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
+ for (auto child : SLP_TREE_CHILDREN (node))
if (child)
vect_slp_prune_covered_roots (child, roots, visited);
}
@@ -9411,10 +9411,7 @@ vect_bb_partition_graph_r (bb_vec_info bb_vinfo,
hash_map<slp_tree, slp_instance> &node_to_instance,
hash_map<slp_instance, slp_instance> &instance_leader)
{
- stmt_vec_info stmt_info;
- unsigned i;
-
- FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
+ for (auto stmt_info : SLP_TREE_SCALAR_STMTS (node))
if (stmt_info)
vect_map_to_instance (instance, stmt_info, stmt_to_instance,
instance_leader);
@@ -9423,8 +9420,7 @@ vect_bb_partition_graph_r (bb_vec_info bb_vinfo,
instance_leader))
return;
- slp_tree child;
- FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
+ for (auto child : SLP_TREE_CHILDREN (node))
if (child && SLP_TREE_DEF_TYPE (child) == vect_internal_def)
vect_bb_partition_graph_r (bb_vinfo, instance, child, stmt_to_instance,
node_to_instance, instance_leader);
@@ -10942,13 +10938,19 @@ vect_get_slp_scalar_def (slp_tree slp_node, unsigned n)
{
if (!SLP_TREE_SCALAR_STMTS (slp_node).exists ())
return NULL_TREE;
+ if (n >= SLP_TREE_SCALAR_STMTS (slp_node).length ())
+ return NULL_TREE;
stmt_vec_info def = SLP_TREE_SCALAR_STMTS (slp_node)[n];
if (!def)
return NULL_TREE;
return gimple_get_lhs (STMT_VINFO_STMT (def));
}
else
- return SLP_TREE_SCALAR_OPS (slp_node)[n];
+ {
+ if (n >= SLP_TREE_SCALAR_OPS (slp_node).length ())
+ return NULL_TREE;
+ return SLP_TREE_SCALAR_OPS (slp_node)[n];
+ }
}
/* Get the Ith vectorized definition from SLP_NODE. */
@@ -11003,7 +11005,7 @@ vect_transform_slp_perm_load_1 (vec_info *vinfo, slp_tree node,
stmt_vec_info stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
int vec_index = 0;
tree vectype = SLP_TREE_VECTYPE (node);
- unsigned int group_size = SLP_TREE_SCALAR_STMTS (node).length ();
+ unsigned int group_size = SLP_TREE_LANES (node);
unsigned int mask_element;
unsigned dr_group_size;
machine_mode mode;
@@ -12057,10 +12059,7 @@ vect_remove_slp_scalar_calls (vec_info *vinfo,
{
gimple *new_stmt;
gimple_stmt_iterator gsi;
- int i;
- slp_tree child;
tree lhs;
- stmt_vec_info stmt_info;
if (!node || SLP_TREE_DEF_TYPE (node) != vect_internal_def)
return;
@@ -12068,10 +12067,10 @@ vect_remove_slp_scalar_calls (vec_info *vinfo,
if (visited.add (node))
return;
- FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
+ for (auto child : SLP_TREE_CHILDREN (node))
vect_remove_slp_scalar_calls (vinfo, child, visited);
- FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
+ for (auto stmt_info : SLP_TREE_SCALAR_STMTS (node))
{
if (!stmt_info)
continue;
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index 76e24b2c1699b022e4480fc4aac87ff0b6ddf650..0c7787e8939f2d8b040988584a2213de946eee1b 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -6388,15 +6388,19 @@ vectorizable_shift (vec_info *vinfo,
/* Now adjust the constant shift amount in place. */
if (incompatible_op1_vectype_p
&& dt[1] == vect_constant_def)
- for (unsigned i = 0;
- i < SLP_TREE_SCALAR_OPS (slp_op1).length (); ++i)
- {
- SLP_TREE_SCALAR_OPS (slp_op1)[i]
- = fold_convert (TREE_TYPE (vectype),
- SLP_TREE_SCALAR_OPS (slp_op1)[i]);
- gcc_assert ((TREE_CODE (SLP_TREE_SCALAR_OPS (slp_op1)[i])
- == INTEGER_CST));
- }
+ {
+ unsigned group_size = SLP_TREE_LANES (slp_op1);
+ if (SLP_TREE_SCALAR_OPS (slp_op1).length () != group_size)
+ return false;
+ for (unsigned i = 0; i < group_size; ++i)
+ {
+ SLP_TREE_SCALAR_OPS (slp_op1)[i]
+ = fold_convert (TREE_TYPE (vectype),
+ SLP_TREE_SCALAR_OPS (slp_op1)[i]);
+ gcc_assert ((TREE_CODE (SLP_TREE_SCALAR_OPS (slp_op1)[i])
+ == INTEGER_CST));
+ }
+ }
SLP_TREE_TYPE (slp_node) = shift_vec_info_type;
DUMP_VECT_SCOPE ("vectorizable_shift");
vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
@@ -13997,7 +14001,9 @@ vect_is_simple_use (vec_info *vinfo, slp_tree slp_node,
{
if (def_stmt_info_out)
*def_stmt_info_out = NULL;
- *op = SLP_TREE_SCALAR_OPS (child)[0];
+ *op = vect_get_slp_scalar_def (child, 0);
+ if (!*op)
+ return false;
*dt = SLP_TREE_DEF_TYPE (child);
return true;
}
@@ -14899,4 +14905,3 @@ vect_gen_len (tree len, tree start_index, tree end_index, tree len_limit)
return stmts;
}
-
