This involves:
- Before scheduling SLP, inserting a HSSR state read for eash HSSR region
to be a scheduling barrier for the SLP nodes governed by the region,
and the loads defining the region.
- While scheduling nodes, if they are goverened by a HSSR region, then
using the loop mask defined by the region, and making sure they are
scheduled after the region read point.
- Using the number of successfully loaded elements from the last HSSR
region in the loop for all IV increments.
- After scheduling SLP, inserting "fixup" basic blocks and control flow
at each HSSR regions read point. These fixup blocks should only be
taken when a partial read happens (which we assume is rare) and
calculates the number of successfully loaded iterations, and
does the permutations required to create all the required loop masks.
gcc/ChangeLog:
* tree-vect-loop-manip.cc (vect_get_loop_iv_increment):
Add logic for HSSR invrement.
(vect_add_hssr_read): New function.
(vect_add_hssr_fixup_controls): New function.
* tree-vect-loop.cc (vectorize_fold_left_reduction): Add
HSSR region index argument to vect_get_loop_mask.
(vect_transform_reduction): Likewise.
(vectorizable_live_operation_1): Likewise.
(vect_get_loop_mask): Add hssr region index and logic for
fetching it.
(vect_transform_loop): Add logic for saving and restoring HSSR
state before HSSR vectorized function.
* tree-vect-slp.cc (vect_schedule_slp_node): Add logic to ensure
nodes that need a hssr mask come after the mask read point.
(vect_schedule_slp): Add logic to insert a HSSR state read for
each region.
* tree-vect-stmts.cc (vectorizable_call): Add HSSR region index
argument to vect_get_loop_mask.
(vectorizable_simd_clone_call): Likewise.
(vectorizable_operation): Likewise.
(vectorizable_store): Likewise.
(vectorizable_condition): Likewise.
(vectorizable_early_exit): Likewise.
(vectorizable_load): Add logic for emitting HSSR loads when
required and to get the correct mask when governed by a region.
* tree-vectorizer.h (LOOP_VINFO_IV_INCREMENT_INVARIANT_P):
Update logic for HSSR.
(vect_add_hssr_fixup_controls): New function.
(vect_add_hssr_read): New function.
(vect_get_loop_mask): Add hssr_region_index argument.
---
gcc/tree-vect-loop-manip.cc | 248 ++++++++++++++++++++++++++++++++++++
gcc/tree-vect-loop.cc | 83 +++++++++++-
gcc/tree-vect-slp.cc | 19 +++
gcc/tree-vect-stmts.cc | 163 ++++++++++++++++++------
gcc/tree-vectorizer.h | 6 +-
5 files changed, 469 insertions(+), 50 deletions(-)
diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc
index 3147de99b5f..fa7a411ace9 100644
--- a/gcc/tree-vect-loop-manip.cc
+++ b/gcc/tree-vect-loop-manip.cc
@@ -3208,6 +3208,23 @@ vect_get_loop_iv_increment (loop_vec_info loop_vinfo)
return iv_increment;
}
+ if (LOOP_VINFO_USING_HSSR_P (loop_vinfo))
+ {
+ hssr_region *final_region
+ = &LOOP_VINFO_MASKS (loop_vinfo).region_vec.last ();
+ gcc_assert (!final_region->num_iter);
+ final_region->num_iter
+ = make_temp_ssa_name (LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo), NULL,
+ "hssr_num_iters");
+
+ gcall *varying_call = gimple_build_call_internal (IFN_VARYING, 0);
+ gimple_call_set_lhs (varying_call, final_region->num_iter);
+
+ gimple_stmt_iterator gsi = gsi_after_labels (loop_vinfo->loop->header);
+ gsi_insert_before (&gsi, varying_call, GSI_NEW_STMT);
+
+ return final_region->num_iter;
+ }
else
return build_int_cst (sizetype, LOOP_VINFO_VECT_FACTOR (loop_vinfo));
}
@@ -4953,3 +4970,234 @@ vect_can_add_hssr_controls (loop_vec_info loop_vinfo)
return true;
}
+
+/* Insert the HSSR read. */
+void
+vect_add_hssr_read (vec_info *vinfo, hssr_region *region)
+{
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
+ gcc_assert (loop_vinfo);
+
+ gimple_stmt_iterator insert_point = gsi_for_stmt (region->hssr_read_point);
+
+ gcall *hssr_read_call = gimple_build_call_internal (IFN_READ_HSSR_STATE, 0);
+ gsi_insert_after (&insert_point, hssr_read_call, GSI_SAME_STMT);
+
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "add new stmt for region: %G", (gimple *) hssr_read_call);
+
+ region->hssr_read_point = hssr_read_call;
+
+ /* We need to update the SSA after inserting the call as it has a VDEF and
+ VUSE. */
+ update_ssa (TODO_update_ssa_only_virtuals);
+}
+
+/* Inserts fixup block for HSSR controls.
+
+ This inserts the following logic:
+
+ <bb 0>:
+ v1 = .MASK_FF_HSSR_LOAD (src1, 64B, loop_mask, { 0, ... });
+ v2 = .MASK_FF_HSSR_LOAD (src2, 64B, loop_mask, { 0, ... });
+ hssr_mask = .READ_FAULT_STATE ();
+ if (hssr_mask == { -1, ... })
+ goto <bb 2>; [99.95%]
+ else
+ goto <bb 1>; [0.05%]
+
+ <bb 1>:
+ .SET_FAULT_STATE ({ -1, ... });
+ hssr_loop_mask = loop_mask & hssr_mask;
+ num_iters = .COUNT_ACTIVE (hssr_mask);
+
+ <bb 2>:
+ # loop_mask_1 = PHI <loop_mask(0), hssr_loop_mask(1)>
+ # num_iters_56 = PHI <POLY_INT_CST [4, 4](0), num_iters (1)>
+
+ ... Logic using loop_mask_1 and incrementing IV's by num_iters_56
+ */
+void
+vect_add_hssr_fixup_controls (loop_vec_info loop_vinfo)
+{
+ gcc_assert (LOOP_VINFO_USING_HSSR_P (loop_vinfo));
+ gcc_assert (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo));
+ int index;
+ hssr_region *region;
+ FOR_EACH_VEC_ELT (LOOP_VINFO_MASKS (loop_vinfo).region_vec, index, region)
+ {
+ /* At the very least the GCOND must use the HSSR mask so it can't be that
+ there are no uses. */
+ gcc_assert (!region->controls.is_empty ());
+
+ vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
+ rgroup_controls *base_rgm = &masks->rgc_vec[0];
+ tree mask_type = base_rgm->type;
+
+ /* The base mask is established by reading from hssr state. */
+ tree hssr_read_val = make_temp_ssa_name (mask_type, NULL, "hssr_mask");
+ gimple_call_set_lhs (region->hssr_read_point, hssr_read_val);
+
+ gcall *detect_fault_call
+ = gimple_build_call_internal (IFN_DETECT_HSSR_FAULT, 1,
+ hssr_read_val);
+ tree fault_detected_p = make_temp_ssa_name (boolean_type_node,
+ detect_fault_call,
+ "partial_read_p");
+ gimple_call_set_lhs (detect_fault_call, fault_detected_p);
+ gimple_stmt_iterator gsi_read = gsi_for_stmt (region->hssr_read_point);
+ gsi_insert_after (&gsi_read, detect_fault_call, GSI_NEW_STMT);
+
+ /* The fixup is taken if the load is partial. */
+ gcond *cond = gimple_build_cond (EQ_EXPR, fault_detected_p,
+ boolean_false_node, NULL_TREE,
+ NULL_TREE);
+ gsi_insert_after (&gsi_read, cond, GSI_SAME_STMT);
+
+ /* Split the BB after the FF reads and setup control flow for the fixup
+ block. */
+ basic_block read_bb = cond->bb;
+ edge non_hssr_edge = split_block (read_bb, cond);
+ non_hssr_edge->flags = EDGE_TRUE_VALUE;
+ non_hssr_edge->probability = profile_probability::very_likely ();
+
+ basic_block continuation_bb = non_hssr_edge->dest;
+ continuation_bb->count = read_bb->count;
+
+ basic_block fixup_bb = create_empty_bb (read_bb);
+ fixup_bb->count = read_bb->count.apply_scale (1, 100);
+
+ edge fixup_edge = make_edge (read_bb, fixup_bb, EDGE_FALSE_VALUE);
+ fixup_edge->probability = profile_probability::very_unlikely ();
+
+ edge fixup_cont_edge = make_edge (fixup_bb, continuation_bb,
+ EDGE_FALLTHRU);
+ fixup_cont_edge->probability = profile_probability::always ();
+
+ set_immediate_dominator (CDI_DOMINATORS, fixup_bb, read_bb);
+ set_immediate_dominator (CDI_DOMINATORS, continuation_bb, read_bb);
+
+ add_bb_to_loop (fixup_bb, LOOP_VINFO_LOOP (loop_vinfo));
+
+ /* Mask nodes in the fixup. */
+ vec<vec<tree>> temp_masks = {};
+ temp_masks.safe_grow_cleared (masks->rgc_vec.length ());
+ temp_masks[0].safe_grow (1);
+
+ gimple_seq fixup_logic = NULL;
+
+ /* We always need the loop to have the nV=1 mask. */
+ if (base_rgm->controls.is_empty ())
+ {
+ base_rgm->controls.safe_grow_cleared (1, true);
+ tree mask = make_temp_ssa_name (mask_type, NULL, "loop_mask");
+ /* Provide a dummy definition until the real one is available. */
+ base_rgm->controls[0] = mask;
+ }
+ if (region->controls[0].is_empty ())
+ {
+ region->controls[0].safe_grow_cleared (1, true);
+ region->controls[0][0]
+ = make_temp_ssa_name (mask_type, NULL, "loop_mask_AFTER_HSSR");
+ }
+
+ tree previous_base_mask
+ = region->prev_region
+ ? vect_get_hssr_region (loop_vinfo, region->prev_region)
+ ->controls[0][0]
+ : masks->rgc_vec[0].controls[0];
+
+ gcall *call = gimple_build_call_internal (IFN_SET_HSSR_STATE, 1,
+ build_all_ones_cst (mask_type));
+ gimple_seq_add_stmt (&fixup_logic, call);
+
+ tree num_iters = gimple_build (&fixup_logic, IFN_COND_COUNT_ACTIVE,
+ sizetype, previous_base_mask,
+ hssr_read_val, build_zero_cst (mask_type));
+ temp_masks[0][0] = make_temp_ssa_name (mask_type, NULL,
"hssr_loop_mask");
+ gimple *hssr_and_loop
+ = gimple_build_assign (temp_masks[0][0], BIT_AND_EXPR,
+ previous_base_mask, hssr_read_val);
+ gimple_seq_add_stmt (&fixup_logic, hssr_and_loop);
+
+ /* PHI node for the NV=1 mask in the region. */
+ gphi *phi = create_phi_node (region->controls[0][0], continuation_bb);
+ add_phi_arg (phi, previous_base_mask, non_hssr_edge, UNKNOWN_LOCATION);
+ add_phi_arg (phi, temp_masks[0][0], fixup_cont_edge, UNKNOWN_LOCATION);
+
+ int i;
+ rgroup_controls *rgc;
+ FOR_EACH_VEC_ELT (masks->rgc_vec, i, rgc)
+ {
+ /* The base case is handled above. */
+ if (i == 0)
+ continue;
+
+ unsigned int nmasks = i + 1;
+ vec<tree> this_masks = region->controls[i];
+ vec<tree> previous_masks
+ = region->prev_region
+ ? vect_get_hssr_region (loop_vinfo, region->prev_region)
+ ->controls[i]
+ : rgc->controls;
+
+ vec<tree> half_controls_temp = temp_masks[nmasks / 2 - 1];
+ rgroup_controls *half_controls_rgc = &masks->rgc_vec[nmasks / 2 - 1];
+
+ vec<tree> this_temps = temp_masks[i];
+ this_temps.safe_grow_cleared (nmasks);
+ for (unsigned j = 0; j < nmasks; j++)
+ if (!this_temps[j])
+ this_temps[j] = make_temp_ssa_name (rgc->type, NULL,
+ "hssr_mask");
+
+ /* Permute the masks from the half rgoup to this one. */
+ gcc_assert (vect_maybe_permute_loop_masks (&fixup_logic,
+ rgc,
+ half_controls_rgc,
+ this_temps,
+ half_controls_temp));
+
+ /* Create PHI nodes for the masks. */
+ for (unsigned int j = 0; j < nmasks; ++j)
+ {
+ gphi *mask_phi
+ = create_phi_node (this_masks[j], continuation_bb);
+ add_phi_arg (mask_phi, previous_masks[j], non_hssr_edge,
+ UNKNOWN_LOCATION);
+ add_phi_arg (mask_phi, this_temps[j], fixup_cont_edge,
+ UNKNOWN_LOCATION);
+ }
+ }
+
+ /* Initialize the num_iter nodes. */
+ if (!region->num_iter)
+ region->num_iter
+ = make_temp_ssa_name (LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo), NULL,
+ "hssr_num_iters");
+ else
+ {
+ gimple *def = SSA_NAME_DEF_STMT (region->num_iter);
+ gcc_assert (gimple_call_internal_p (def, IFN_VARYING));
+ gimple_stmt_iterator gsi = gsi_for_stmt (def);
+ gsi_remove (&gsi, true);
+ }
+
+ gphi *num_iter_phi = create_phi_node (region->num_iter, continuation_bb);
+ SSA_NAME_DEF_STMT (region->num_iter) = num_iter_phi;
+ add_phi_arg (
+ num_iter_phi,
+ region->prev_region
+ ? vect_get_hssr_region (loop_vinfo, region->prev_region)->num_iter
+ : build_int_cst (LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo),
+ LOOP_VINFO_VECT_FACTOR (loop_vinfo)),
+ non_hssr_edge, UNKNOWN_LOCATION);
+ add_phi_arg (num_iter_phi,
+ num_iters,
+ fixup_cont_edge, UNKNOWN_LOCATION);
+
+ gimple_stmt_iterator fixup_gsi = gsi_start_bb (fixup_bb);
+ gsi_insert_seq_before (&fixup_gsi, fixup_logic, GSI_SAME_STMT);
+ }
+}
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index e5c94125c7e..9cbb170a8ea 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -60,6 +60,7 @@ along with GCC; see the file COPYING3. If not see
#include "langhooks.h"
#include "opts.h"
#include "hierarchical_discriminator.h"
+#include "tree-into-ssa.h"
/* Loop Vectorization Pass.
@@ -6805,7 +6806,8 @@ vectorize_fold_left_reduction (loop_vec_info loop_vinfo,
if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
{
tree loop_mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
- vec_num, vectype_in, i);
+ vec_num, vectype_in, i,
+ SLP_TREE_HSSR_REGION (slp_node));
if (is_cond_op)
mask = prepare_vec_mask (loop_vinfo, TREE_TYPE (loop_mask),
loop_mask, vec_opmask[i], gsi);
@@ -8524,7 +8526,8 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
tree mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
vec_in_num, vectype_in,
- mask_index++);
+ mask_index++,
+ SLP_TREE_HSSR_REGION (slp_node));
gcall *call;
if (code.is_internal_fn () && cond_fn_p)
{
@@ -8554,7 +8557,8 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
{
tree mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
vec_in_num, vectype_in,
- mask_index++);
+ mask_index++,
+ SLP_TREE_HSSR_REGION (slp_node));
build_vect_cond_expr (code, vop, mask, gsi);
}
@@ -10454,7 +10458,8 @@ vectorizable_live_operation_1 (loop_vec_info
loop_vinfo, basic_block exit_bb,
gimple_stmt_iterator gsi = gsi_last (tem);
tree mask = vect_get_loop_mask (loop_vinfo, &gsi,
&LOOP_VINFO_MASKS (loop_vinfo),
- 1, vectype, 0);
+ 1, vectype, 0,
+ SLP_TREE_HSSR_REGION (slp_node));
tree scalar_res;
gimple_seq_add_seq (&stmts, tem);
@@ -10912,12 +10917,17 @@ vect_record_loop_mask (loop_vec_info loop_vinfo,
vec_loop_masks *masks,
where 0 <= INDEX < NVECTORS. Insert any set-up statements before GSI.
See the comment above vec_loop_masks for more details about the mask
- arrangement. */
+ arrangement.
+
+ REGION_INDEX is the index of the HSSR region governing this operation
+ which should be used to get the loop mask. Or it is 0 to use the loop
+ control mask. */
tree
vect_get_loop_mask (loop_vec_info loop_vinfo,
gimple_stmt_iterator *gsi, vec_loop_masks *masks,
- unsigned int nvectors, tree vectype, unsigned int index)
+ unsigned int nvectors, tree vectype, unsigned int index,
+ unsigned int region_index)
{
if (LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo)
== vect_partial_vectors_while_ult)
@@ -10937,9 +10947,40 @@ vect_get_loop_mask (loop_vec_info loop_vinfo,
SSA_NAME_DEF_STMT (mask) = gimple_build_nop ();
rgm->controls[i] = mask;
}
+
+ if (LOOP_VINFO_USING_HSSR_P (loop_vinfo))
+ for (hssr_region ®ion : &masks->region_vec)
+ {
+ if (region.controls.length () < nvectors)
+ region.controls.safe_grow_cleared (nvectors);
+
+ vec<tree> *controls = ®ion.controls[nvectors - 1];
+ if (controls->is_empty ())
+ {
+ controls->safe_grow_cleared (nvectors, true);
+ for (unsigned int i = 0; i < nvectors; ++i)
+ {
+ tree mask = make_temp_ssa_name (mask_type, NULL,
+ "hssr_loop_mask");
+ /* Provide a dummy definition until the real one is
+ available. */
+ SSA_NAME_DEF_STMT (mask) = gimple_build_nop ();
+ (*controls)[i] = mask;
+ }
+ }
+ }
}
tree mask = rgm->controls[index];
+ if (LOOP_VINFO_USING_HSSR_P (loop_vinfo)
+ && region_index)
+ mask = vect_get_hssr_region (loop_vinfo,
+ region_index)
+ ->controls[nvectors - 1][index];
+
+ /* If we are in an HSSR region, also populate all the other required
+ masks, and use the HSSR mask. */
+
if (maybe_ne (TYPE_VECTOR_SUBPARTS (mask_type),
TYPE_VECTOR_SUBPARTS (vectype)))
{
@@ -11894,6 +11935,36 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple
*loop_vectorized_call)
" variable-length vectorization factor\n");
}
+ /* Save and restore the HSSR state. */
+ if (LOOP_VINFO_USING_HSSR_P (loop_vinfo))
+ {
+ tree hssr_type = LOOP_VINFO_MASKS (loop_vinfo).rgc_vec[0].type;
+
+ tree hssr_ssa = make_temp_ssa_name (hssr_type, NULL,
"hssr_preservation");
+
+ gcall *call = gimple_build_call_internal (IFN_READ_HSSR_STATE, 0);
+ gimple_set_lhs (call, hssr_ssa);
+ gsi_insert_on_edge_immediate (loop_preheader_edge (loop), call);
+
+
+ /* Initializze the hssr state to all true. */
+ gcall *set_hssr_call
+ = gimple_build_call_internal (IFN_SET_HSSR_STATE, 1,
+ build_all_ones_cst (hssr_type));
+ gsi_insert_on_edge_immediate (loop_preheader_edge (loop), set_hssr_call);
+
+ auto_vec<edge> exits = get_loop_exit_edges (loop);
+ for (edge e: exits)
+ {
+ gcall *call = gimple_build_call_internal (IFN_SET_HSSR_STATE, 1,
+ hssr_ssa);
+ gsi_insert_on_edge_immediate (e, call);
+ }
+
+ vect_add_hssr_fixup_controls (loop_vinfo);
+ update_ssa (TODO_update_ssa_only_virtuals);
+ }
+
/* When we have unrolled the loop due to a user requested value we should
leave it up to the RTL unroll heuristics to determine if it's still worth
while to unroll more. */
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index 81409a6a9ab..8335bfe6bd6 100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -12266,6 +12266,16 @@ vect_schedule_slp_node (vec_info *vinfo,
gcc_checking_assert (!last_stmt
|| !is_ctrl_altering_stmt (last_stmt));
+ loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
+
+ /* If the statement is governed by a HSSR region, then is must come after
+ the point that the HSSR region is . */
+ hssr_region *region
+ = vect_get_hssr_region (loop_vinfo, SLP_TREE_HSSR_REGION (node));
+ if (loop_vinfo && LOOP_VINFO_USING_HSSR_P (loop_vinfo) && region)
+ if (vect_stmt_dominates_stmt_p (last_stmt, region->hssr_read_point))
+ last_stmt = region->hssr_read_point;
+
if (is_a <bb_vec_info> (vinfo)
&& !SLP_TREE_PERMUTE_P (node)
&& (!last_stmt
@@ -12807,6 +12817,15 @@ vect_schedule_slp (vec_info *vinfo, vec<slp_instance>
&slp_instances,
slp_instance instance;
unsigned int i;
+ loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
+
+ /* Insert the call to read the HSSR now as we need it for data
+ dependencies when scheduling, the actual fixup logic will be added
+ later. */
+ if (loop_vinfo && LOOP_VINFO_USING_HSSR_P (loop_vinfo))
+ for (auto ®ion : LOOP_VINFO_MASKS (loop_vinfo).region_vec)
+ vect_add_hssr_read (vinfo, ®ion);
+
hash_map<slp_tree, slp_scc_info> scc_info;
int maxdfs = 0;
for (i = 0; slp_instances.iterate (i, &instance); )
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index 0fd2d5a4f0f..362e9d3af5d 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -58,6 +58,7 @@ along with GCC; see the file COPYING3. If not see
#include "attribs.h"
#include "optabs-libfuncs.h"
#include "tree-dfa.h"
+#include "tree-into-ssa.h"
/* For lang_hooks.types.type_for_mode. */
#include "langhooks.h"
@@ -3876,9 +3877,10 @@ vectorizable_call (vec_info *vinfo,
if (masked_loop_p)
{
unsigned int vec_num = vec_oprnds0.length ();
- vargs[varg++] = vect_get_loop_mask (loop_vinfo, gsi,
- masks, vec_num,
- vectype_out, i);
+ vargs[varg++]
+ = vect_get_loop_mask (loop_vinfo, gsi, masks, vec_num,
+ vectype_out, i,
+ SLP_TREE_HSSR_REGION (slp_node));
}
else
{
@@ -3948,8 +3950,10 @@ vectorizable_call (vec_info *vinfo,
else if (mask_opno >= 0 && masked_loop_p)
{
unsigned int vec_num = vec_oprnds0.length ();
- tree mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
- vec_num, vectype_out, i);
+ tree mask = vect_get_loop_mask
+ (loop_vinfo, gsi, masks, vec_num,
+ vectype_out, i,
+ SLP_TREE_HSSR_REGION (slp_node));
vargs[mask_opno]
= prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
vargs[mask_opno], gsi);
@@ -4756,9 +4760,10 @@ vectorizable_simd_clone_call (vec_info *vinfo,
stmt_vec_info stmt_info,
vec_loop_masks *loop_masks
= &LOOP_VINFO_MASKS (loop_vinfo);
tree loop_mask
- = vect_get_loop_mask (loop_vinfo, gsi,
- loop_masks, ncopies_in,
- vectype, j);
+ = vect_get_loop_mask
+ (loop_vinfo, gsi, loop_masks, ncopies_in,
+ vectype, j,
+ SLP_TREE_HSSR_REGION (slp_node));
vec_oprnd0
= prepare_vec_mask (loop_vinfo,
TREE_TYPE (loop_mask),
@@ -4817,9 +4822,10 @@ vectorizable_simd_clone_call (vec_info *vinfo,
stmt_vec_info stmt_info,
vec_loop_masks *loop_masks
= &LOOP_VINFO_MASKS (loop_vinfo);
tree loop_mask
- = vect_get_loop_mask (loop_vinfo, gsi,
- loop_masks, ncopies_in,
- vectype, j);
+ = vect_get_loop_mask
+ (loop_vinfo, gsi, loop_masks, ncopies_in,
+ vectype, j,
+ SLP_TREE_HSSR_REGION (slp_node));
vec_oprnd0
= prepare_vec_mask (loop_vinfo,
TREE_TYPE (loop_mask),
@@ -4969,7 +4975,8 @@ vectorizable_simd_clone_call (vec_info *vinfo,
stmt_vec_info stmt_info,
{
vec_loop_masks *loop_masks = &LOOP_VINFO_MASKS (loop_vinfo);
mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
- ncopies * o, mask_vectype, m);
+ ncopies * o, mask_vectype, m,
+ SLP_TREE_HSSR_REGION (slp_node));
}
else
mask = vect_build_all_ones_mask (vinfo, stmt_info,
@@ -7160,7 +7167,8 @@ vectorizable_operation (vec_info *vinfo,
tree mask;
if (masked_loop_p)
mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
- vec_num, vectype, i);
+ vec_num, vectype, i,
+ SLP_TREE_HSSR_REGION (slp_node));
else
/* Dummy mask. */
mask = build_minus_one_cst (truth_type_for (vectype));
@@ -7217,7 +7225,8 @@ vectorizable_operation (vec_info *vinfo,
if (loop_vinfo->scalar_cond_masked_set.contains ({ op0, vec_num
}))
{
mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
- vec_num, vectype, i);
+ vec_num, vectype, i,
+ SLP_TREE_HSSR_REGION (slp_node));
vop0 = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
vop0, gsi);
@@ -7226,7 +7235,8 @@ vectorizable_operation (vec_info *vinfo,
if (loop_vinfo->scalar_cond_masked_set.contains ({ op1, vec_num
}))
{
mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
- vec_num, vectype, i);
+ vec_num, vectype, i,
+ SLP_TREE_HSSR_REGION (slp_node));
vop1 = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
vop1, gsi);
@@ -8918,7 +8928,8 @@ vectorizable_store (vec_info *vinfo,
tree bias = NULL;
if (loop_masks)
final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
- ncopies, vectype, j);
+ ncopies, vectype, j,
+ SLP_TREE_HSSR_REGION (slp_node));
if (vec_mask)
final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
vec_mask, gsi);
@@ -9059,9 +9070,10 @@ vectorizable_store (vec_info *vinfo,
if (!costing_p)
{
if (loop_masks)
- final_mask = vect_get_loop_mask (loop_vinfo, gsi,
- loop_masks, num_stmts,
- vectype, j);
+ final_mask
+ = vect_get_loop_mask (loop_vinfo, gsi, loop_masks, num_stmts,
+ vectype, j,
+ SLP_TREE_HSSR_REGION (slp_node));
if (vec_mask)
final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
final_mask, vec_mask, gsi);
@@ -9478,7 +9490,8 @@ vectorizable_store (vec_info *vinfo,
tree bias = NULL_TREE;
if (loop_masks)
final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
- vec_num, vectype, i);
+ vec_num, vectype, i,
+ SLP_TREE_HSSR_REGION (slp_node));
if (vec_mask)
vec_mask = vec_masks[i];
if (vec_mask)
@@ -10111,9 +10124,17 @@ vectorizable_load (vec_info *vinfo,
if (!costing_p)
ensure_base_align (dr_info);
+ /* Does this load require HSSR. */
+ bool hssr_required = stmt_info->requires_hssr_p
+ || (loop_vinfo
+ && LOOP_VINFO_USING_HSSR_P (loop_vinfo)
+ && ls.defines_region);
+ if (hssr_required && !LOOP_VINFO_CAN_USE_HSSR_P (loop_vinfo))
+ return false;
+
if (memory_access_type == VMAT_INVARIANT)
{
- gcc_assert (!grouped_load && !mask_node && !bb_vinfo);
+ gcc_assert (!grouped_load && !mask_node && !bb_vinfo && !hssr_required);
/* If we have versioned for aliasing or the loop doesn't
have any data dependencies that would preclude this,
then we are sure this is a loop invariant load and
@@ -10278,6 +10299,7 @@ vectorizable_load (vec_info *vinfo,
gcc_assert (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo));
gcc_assert (!nested_in_vect_loop);
+ gcc_assert (!hssr_required);
if (grouped_load)
{
@@ -10623,6 +10645,8 @@ vectorizable_load (vec_info *vinfo,
if (grouped_load
|| SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
{
+ /* HSSR dost not support permutations or grouped loads. */
+ gcc_assert (!hssr_required);
if (grouped_load)
{
first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
@@ -10798,6 +10822,7 @@ vectorizable_load (vec_info *vinfo,
|| alignment_support_scheme == dr_explicit_realign)
&& !compute_in_loop)
{
+ gcc_assert (!hssr_required);
/* If we have different first_stmt_info, we can't set up realignment
here, since we can't guarantee first_stmt_info DR has been
initialized yet, use first_stmt_info_for_drptr DR by bumping the
@@ -10844,6 +10869,7 @@ vectorizable_load (vec_info *vinfo,
gcc_assert (alignment_support_scheme == dr_aligned
|| alignment_support_scheme == dr_unaligned_supported);
+ gcc_assert (!hssr_required);
aggr_type = build_array_type_nelts (elem_type, group_size * nunits);
if (!costing_p)
@@ -10916,7 +10942,8 @@ vectorizable_load (vec_info *vinfo,
tree bias = NULL_TREE;
if (loop_masks)
final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
- ncopies, vectype, j);
+ ncopies, vectype, j,
+ SLP_TREE_HSSR_REGION (slp_node));
if (vec_mask)
final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
vec_mask, gsi);
@@ -11016,6 +11043,8 @@ vectorizable_load (vec_info *vinfo,
if (mat_gather_scatter_p (memory_access_type))
{
gcc_assert ((!grouped_load && !ls.slp_perm) || ls.ls_type);
+ /* TODO: Add support for HSSR gather. */
+ gcc_assert (!hssr_required);
auto_vec<tree> dr_chain (vec_num);
@@ -11065,8 +11094,10 @@ vectorizable_load (vec_info *vinfo,
if (mask_node)
vec_mask = vec_masks[i];
if (loop_masks)
- final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
- vec_num, vectype, i);
+ final_mask
+ = vect_get_loop_mask (loop_vinfo, gsi, loop_masks, vec_num,
+ vectype, i,
+ SLP_TREE_HSSR_REGION (slp_node));
if (vec_mask)
final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
final_mask, vec_mask, gsi);
@@ -11527,6 +11558,9 @@ vectorizable_load (vec_info *vinfo,
if (grouped_load || ls.slp_perm)
dr_chain.create (vec_num);
+ /* HSSR only supports vec_num == 1. */
+ gcc_assert (!hssr_required || vec_num == 1);
+
gimple *new_stmt = NULL;
for (i = 0; i < vec_num; i++)
{
@@ -11540,7 +11574,8 @@ vectorizable_load (vec_info *vinfo,
vec_mask = vec_masks[i];
if (loop_masks)
final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
- vec_num, vectype, i);
+ vec_num, vectype, i,
+ SLP_TREE_HSSR_REGION (slp_node));
if (vec_mask)
final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
final_mask, vec_mask, gsi);
@@ -11627,6 +11662,7 @@ vectorizable_load (vec_info *vinfo,
if (final_len)
{
+ gcc_assert (!hssr_required);
tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
gcall *call;
@@ -11676,21 +11712,48 @@ vectorizable_load (vec_info *vinfo,
}
else if (final_mask)
{
- tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
- vec_els = vect_get_mask_load_else (maskload_elsval, vectype);
- if (type_mode_padding_p
- && maskload_elsval != MASK_LOAD_ELSE_ZERO)
- need_zeroing = true;
- gcall *call = gimple_build_call_internal (IFN_MASK_LOAD, 4,
- dataref_ptr, ptr,
- final_mask,
- vec_els);
- gimple_call_set_nothrow (call, true);
- new_stmt = call;
- data_ref = NULL_TREE;
+ if (hssr_required)
+ {
+ /* HSSR only supports vec_num == 1. */
+ gcc_assert (vec_num == 1);
+
+ /* Do the HSSR read, and then read the HSSR and save it for
+ this r-group. There should only be one, so we should be
+ safe... */
+ tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
+ vec_els = vect_get_mask_load_else (maskload_elsval,
+ vectype);
+ if (type_mode_padding_p
+ && maskload_elsval != MASK_LOAD_ELSE_ZERO)
+ need_zeroing = true;
+ gcall *call
+ = gimple_build_call_internal (IFN_MASK_FF_HSSR_LOAD, 4,
+ dataref_ptr, ptr,
+ final_mask, vec_els);
+ gimple_call_set_nothrow (call, true);
+ new_stmt = call;
+ data_ref = NULL_TREE;
+ }
+ else
+ {
+ tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
+ vec_els = vect_get_mask_load_else (maskload_elsval,
+ vectype);
+ if (type_mode_padding_p
+ && maskload_elsval != MASK_LOAD_ELSE_ZERO)
+ need_zeroing = true;
+ gcall *call = gimple_build_call_internal (IFN_MASK_LOAD, 4,
+ dataref_ptr, ptr,
+ final_mask,
+ vec_els);
+ gimple_call_set_nothrow (call, true);
+ new_stmt = call;
+ data_ref = NULL_TREE;
+ }
}
else
{
+ gcc_assert (!hssr_required);
tree ltype = vectype;
tree new_vtype = NULL_TREE;
unsigned HOST_WIDE_INT gap = DR_GROUP_GAP (first_stmt_info);
@@ -11855,6 +11918,7 @@ vectorizable_load (vec_info *vinfo,
}
case dr_explicit_realign:
{
+ gcc_assert (!hssr_required);
if (costing_p)
break;
tree ptr, bump;
@@ -11909,6 +11973,7 @@ vectorizable_load (vec_info *vinfo,
}
case dr_explicit_realign_optimized:
{
+ gcc_assert (!hssr_required);
if (costing_p)
break;
if (TREE_CODE (dataref_ptr) == SSA_NAME)
@@ -11993,6 +12058,7 @@ vectorizable_load (vec_info *vinfo,
&& (alignment_support_scheme == dr_explicit_realign_optimized
|| alignment_support_scheme == dr_explicit_realign))
{
+ gcc_assert (!hssr_required);
lsq = gimple_assign_lhs (new_stmt);
if (!realignment_token)
realignment_token = dataref_ptr;
@@ -12015,6 +12081,8 @@ vectorizable_load (vec_info *vinfo,
if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
{
+ gcc_assert (!hssr_required);
+
if (costing_p)
inside_cost = record_stmt_cost (cost_vec, 1, vec_perm,
slp_node, 0, vect_body);
@@ -12033,7 +12101,11 @@ vectorizable_load (vec_info *vinfo,
dr_chain.quick_push (new_temp);
/* Store vector loads in the corresponding SLP_NODE. */
- if (!costing_p && !ls.slp_perm)
+ if (!costing_p && hssr_required)
+ /* For HSSR we cannot use push_vec_def (stmt) as it also defines a
+ vdef for the HSSR state, so push the result directly. */
+ slp_node->push_vec_def (new_temp);
+ else if (!costing_p && !ls.slp_perm)
slp_node->push_vec_def (new_stmt);
/* With SLP permutation we load the gaps as well, without
@@ -12112,6 +12184,11 @@ vectorizable_load (vec_info *vinfo,
slp_node->data = new vect_load_store_data (std::move (ls));
}
+ /* If we used a HSSR on this, we need to update the virtual ssa nodes because
+ of the HSSR read. */
+ if (loop_vinfo && LOOP_VINFO_USING_HSSR_P (loop_vinfo)
+ && dr_safe_speculative_read_required (stmt_info))
+ update_ssa (TODO_update_ssa_only_virtuals);
return true;
}
@@ -12423,7 +12500,8 @@ vectorizable_condition (vec_info *vinfo,
{
tree loop_mask
= vect_get_loop_mask (loop_vinfo, gsi, masks, vec_num,
- vectype, i);
+ vectype, i,
+ SLP_TREE_HSSR_REGION (slp_node));
tree tmp2 = make_ssa_name (vec_cmp_type);
gassign *g
= gimple_build_assign (tmp2, BIT_AND_EXPR, vec_compare,
@@ -13008,7 +13086,8 @@ vectorizable_early_exit (loop_vec_info loop_vinfo,
stmt_vec_info stmt_info,
{
tree stmt_mask
= vect_get_loop_mask (loop_vinfo, gsi, masks, vec_num,
- vectype, i);
+ vectype, i,
+ SLP_TREE_HSSR_REGION (slp_node));
stmt_mask
= prepare_vec_mask (loop_vinfo, TREE_TYPE (stmt_mask), stmt_mask,
stmts[i], &cond_gsi);
@@ -13054,8 +13133,8 @@ vectorizable_early_exit (loop_vec_info loop_vinfo,
stmt_vec_info stmt_info,
new_temp = stmts[0];
if (masked_loop_p)
{
- tree mask
- = vect_get_loop_mask (loop_vinfo, gsi, masks, 1, vectype, 0);
+ tree mask = vect_get_loop_mask (loop_vinfo, gsi, masks, 1, vectype, 0,
+ SLP_TREE_HSSR_REGION (slp_node));
new_temp = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
new_temp, &cond_gsi);
}
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 272703c1d17..0877104236a 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -1377,7 +1377,7 @@ public:
#define LOOP_VINFO_VECT_FACTOR(L) (L)->vectorization_factor
#define LOOP_VINFO_IV_INCREMENT(L) (L)->iv_increment
#define LOOP_VINFO_IV_INCREMENT_INVARIANT_P(L) \
- (!LOOP_VINFO_USING_SELECT_VL_P (L))
+ (!LOOP_VINFO_USING_SELECT_VL_P (L) && !LOOP_VINFO_USING_HSSR_P (L))
#define LOOP_VINFO_MAX_VECT_FACTOR(L) (L)->max_vectorization_factor
#define LOOP_VINFO_MASKS(L) (L)->masks
#define LOOP_VINFO_LENS(L) (L)->lens
@@ -2555,6 +2555,8 @@ class loop *slpeel_tree_duplicate_loop_to_edge_cfg (class
loop *, edge,
bool = true);
class loop *vect_loop_versioning (loop_vec_info, gimple *);
extern bool vect_can_add_hssr_controls (loop_vec_info);
+extern void vect_add_hssr_fixup_controls (loop_vec_info);
+extern void vect_add_hssr_read (vec_info *, hssr_region *);
extern class loop *vect_do_peeling (loop_vec_info, tree, tree,
tree *, tree *, tree *, int, bool, bool,
tree *);
@@ -2750,7 +2752,7 @@ extern void vect_record_loop_mask (loop_vec_info,
vec_loop_masks *,
unsigned int, tree, tree);
extern tree vect_get_loop_mask (loop_vec_info, gimple_stmt_iterator *,
vec_loop_masks *,
- unsigned int, tree, unsigned int);
+ unsigned int, tree, unsigned int, unsigned int);
extern hssr_region *vect_get_hssr_region (loop_vec_info, unsigned);
extern void vect_record_loop_len (loop_vec_info, vec_loop_lens *, unsigned int,
tree, unsigned int);
--
2.43.0
