The following re-wires vect_build_slp_tree_1 to compute more than
one set of matching stmts from the candidate set by changing how
matches[] operates, making elements an int rather than a bool
and indicating the lane it is matching with. Special values -1
denote "not matching/matched with another lane" and -2
"not vectorizable lane". The latter is fully computed by the
first sweep while further match sets are computed by re-trying
for -1 valued lanes from the first sweep and starting with the
first one.
two_operators handling needs adjustment if we want to be able to
take a match subset as-is for further processing.
There is also vector type selection to be considered, and at least
still partially relevant for shift handling.
Failure due to BB SLP unrolling is noted as problem in PR126306
and also predicated tail vectorization will touch that. I'd like
to get rid of this entirely.
As is the patch below dumbs down to the matches[] we know at the
end of vect_build_slp_tree_1, but I intend to push this upwards.
This one is just for comments for now, I'm looking for applications,
PR126080 is where I want to make use of it, but it's nice for
all the code that does splitting after discovery fail I think.
Thanks,
Richard.
* tree-vect-slp.cc (vect_build_slp_tree_12): Make
re-startable at start_i, change representation of
matches[] to int[].
(vect_build_slp_tree_1): Pre-initialize matches[] and swap[]
here, iteratively resolve not matched lanes.
---
gcc/tree-vect-slp.cc | 109 +++++++++++++++++++++++++++++++------------
1 file changed, 79 insertions(+), 30 deletions(-)
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index 70140d72fdf..73cd4786f9d 100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -1148,11 +1148,10 @@ vect_record_max_nunits (vec_info *vinfo, stmt_vec_info
stmt_info,
static bool
vect_build_slp_tree_12 (vec_info *vinfo, unsigned char *swap,
- vec<stmt_vec_info> stmts, bool *matches,
- bool *two_operators, tree vectype)
+ vec<stmt_vec_info> stmts, int *matches,
+ bool *two_operators, tree vectype, unsigned start_i)
{
- unsigned int i;
- stmt_vec_info first_stmt_info = stmts[0];
+ stmt_vec_info first_stmt_info = stmts[start_i];
code_helper first_stmt_code = ERROR_MARK;
code_helper alt_stmt_code = ERROR_MARK;
code_helper first_cond_code = ERROR_MARK;
@@ -1164,19 +1163,20 @@ vect_build_slp_tree_12 (vec_info *vinfo, unsigned char
*swap,
bool first_stmt_phi_p = false;
int first_reduc_idx = -1;
- stmt_vec_info stmt_info;
- FOR_EACH_VEC_ELT (stmts, i, stmt_info)
+ for (unsigned i = start_i; i < stmts.length (); ++i)
{
+ stmt_vec_info stmt_info = stmts[i];
bool ldst_p = false;
bool ldst_masklen_p = false;
bool phi_p = false;
code_helper rhs_code = ERROR_MARK;
- swap[i] = 0;
- matches[i] = false;
+ if (matches[i] != -1)
+ continue;
+
if (!stmt_info)
{
- matches[i] = true;
+ matches[i] = start_i;
continue;
}
@@ -1198,8 +1198,11 @@ vect_build_slp_tree_12 (vec_info *vinfo, unsigned char
*swap,
to shuffle all unvectorizable defs into one operand and have
the other still vectorized. The following doesn't reliably
work for this though but it's the easiest we can do here. */
- if (is_a <bb_vec_info> (vinfo) && i != 0)
- continue;
+ if (is_a <bb_vec_info> (vinfo) && i != start_i)
+ {
+ matches[i] = -2;
+ continue;
+ }
return false;
}
@@ -1211,8 +1214,11 @@ vect_build_slp_tree_12 (vec_info *vinfo, unsigned char
*swap,
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
"Build SLP failed: not GIMPLE_ASSIGN nor "
"GIMPLE_CALL %G", stmt);
- if (is_a <bb_vec_info> (vinfo) && i != 0)
- continue;
+ if (is_a <bb_vec_info> (vinfo) && i != start_i)
+ {
+ matches[i] = -2;
+ continue;
+ }
return false;
}
@@ -1256,8 +1262,11 @@ vect_build_slp_tree_12 (vec_info *vinfo, unsigned char
*swap,
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
"Build SLP failed: unsupported call type %G",
(gimple *) call_stmt);
- if (is_a <bb_vec_info> (vinfo) && i != 0)
- continue;
+ if (is_a <bb_vec_info> (vinfo) && i != start_i)
+ {
+ matches[i] = -2;
+ continue;
+ }
return false;
}
}
@@ -1288,8 +1297,11 @@ vect_build_slp_tree_12 (vec_info *vinfo, unsigned char
*swap,
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
"Build SLP failed: operation unsupported %G",
stmt);
- if (is_a <bb_vec_info> (vinfo) && i != 0)
- continue;
+ if (is_a <bb_vec_info> (vinfo) && i != start_i)
+ {
+ matches[i] = -2;
+ continue;
+ }
return false;
}
@@ -1310,14 +1322,17 @@ vect_build_slp_tree_12 (vec_info *vinfo, unsigned char
*swap,
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
"Build SLP failed: "
"BIT_FIELD_REF not supported\n");
- if (i != 0)
- continue;
+ if (i != start_i)
+ {
+ matches[i] = -2;
+ continue;
+ }
return false;
}
}
/* Check the operation. */
- if (i == 0)
+ if (i == start_i)
{
first_lhs = lhs;
first_stmt_code = rhs_code;
@@ -1566,8 +1581,11 @@ vect_build_slp_tree_12 (vec_info *vinfo, unsigned char
*swap,
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
"Build SLP failed: not grouped load %G", stmt);
- if (i != 0)
- continue;
+ if (i != start_i)
+ {
+ matches[i] = -2;
+ continue;
+ }
return false;
}
}
@@ -1581,7 +1599,7 @@ vect_build_slp_tree_12 (vec_info *vinfo, unsigned char
*swap,
enum tree_code swap_code = ERROR_MARK;
enum tree_code invert_code = ERROR_MARK;
- if (i == 0)
+ if (i == start_i)
first_cond_code = TREE_CODE (cond_expr);
else if (TREE_CODE_CLASS (cond_code) == tcc_comparison)
{
@@ -1609,7 +1627,7 @@ vect_build_slp_tree_12 (vec_info *vinfo, unsigned char
*swap,
}
}
- if (i != 0
+ if (i != start_i
&& first_stmt_code != rhs_code
&& first_stmt_code.is_tree_code ()
&& rhs_code.is_tree_code ()
@@ -1618,7 +1636,7 @@ vect_build_slp_tree_12 (vec_info *vinfo, unsigned char
*swap,
== (tree_code)rhs_code))
swap[i] = 1;
- if (i != 0
+ if (i != start_i
&& first_reduc_idx != STMT_VINFO_REDUC_IDX (stmt_info)
&& first_reduc_idx != -1
&& STMT_VINFO_REDUC_IDX (stmt_info) != -1
@@ -1628,7 +1646,7 @@ vect_build_slp_tree_12 (vec_info *vinfo, unsigned char
*swap,
swap[i] = 1;
}
- matches[i] = true;
+ matches[i] = start_i;
}
/* If we allowed a two-operation SLP node verify the target can cope
@@ -1707,17 +1725,48 @@ vect_build_slp_tree_1 (vec_info *vinfo, unsigned char
*swap,
gcc_assert (vectype || !gimple_get_lhs (first_stmt_info->stmt));
*node_vectype = vectype;
- if (!vect_build_slp_tree_12 (vinfo, swap, stmts, matches, two_operators,
- vectype))
+ memset (swap, 0, group_size);
+
+ int *matches_ = XALLOCAVEC (int, group_size);
+ for (unsigned i = 0; i < group_size; ++i)
+ matches_[i] = -1;
+ unsigned start_i = 0;
+ if (!vect_build_slp_tree_12 (vinfo, swap, stmts, matches_, two_operators,
+ vectype, start_i))
{
/* Fatal mismatch. */
matches[0] = false;
return false;
}
+ gcc_assert (matches_[0] != -1);
+ unsigned num_sets = 1;
+ if (!*two_operators)
+ for (unsigned i = start_i + 1; i < group_size; ++i)
+ if (matches_[i] == -1)
+ {
+ bool tem_two_operators = false;
+ bool res = vect_build_slp_tree_12 (vinfo, swap, stmts, matches_,
+ &tem_two_operators, vectype, i);
+ gcc_assert (res && matches_[i] > 0);
+ if (tem_two_operators)
+ /* *two_operators would need to be per start index, or only
+ considered valid for start_i == 0, meaning other starts would
+ need to go through separate full discovery? Alternatively
+ make two_operators a vec<> where we push one element per
+ start? */
+ ;
+ num_sets++;
+ }
+ bool res = true;
for (unsigned i = 0; i < group_size; ++i)
- if (!matches[i])
- return false;
+ {
+ matches[i] = matches_[i] == matches_[0];
+ if (!matches[i])
+ res = false;
+ }
+ if (!res)
+ return false;
if (maybe_soft_fail)
{
--
2.51.0
