Checks if a loop could use first faulting reads.
Also establishes FFR regions, a region is attached to an early break SLP
instance where safe speculative reads would be required. The vector loads
that are required at the start of the instance define the region, and
then the nodes that use the values from those loads are part of the
region that need to use the mask of successfully loaded elements.
gcc/ChangeLog:
* config/aarch64/aarch64.cc (better_main_loop_than_p):
Add code to optionally prefer FFR.
* optabs-tree.cc (target_supports_ffl_mask_load_p):
New function.
(can_vec_ff_mask_load_p):
New function.
* optabs-tree.h (can_vec_ff_mask_load_p):
New function.
* tree-vect-data-refs.cc (vect_supportable_dr_alignment):
Add logic that unaligned safe speculative loads are supported
when using FFR.
* tree-vect-loop-manip.cc (vect_can_add_ffr_controls):
New function.
* tree-vect-loop.cc (_loop_vec_info::_loop_vec_info):
Add must_use_ffr_p, can_use_ffr_p, using_ffr_p.
(vect_verify_full_masking): Add call to check
vect_can_add_ffr_controls.
(set_ffr_regions): New function.
(vect_determine_ffr_and_versioning): New function.
(vectorizable_induction): Add logic to diable FFR for multilane
inductions.
* tree-vect-slp.cc (_slp_tree::_slp_tree): Add initialization for
ffr_region and ffr_mask_needed.
(vect_free_slp_instance): Add handling of ffr_region.
(vect_build_slp_instance): Ditto.
(vect_analyze_slp_reduc_chain): Ditto.
(vect_analyze_slp_reduction): Ditto.
(vect_analyze_slp_reduction_group): Ditto.
(vect_analyze_slp_instance): Ditto.
* tree-vect-stmts.cc (check_load_for_ffr): New function.
(get_load_store_type): Add handling for FFR.
(vectorizable_conversion): Add setting of
SLP_TREE_FFR_MASK_NEEDED to false.
(vectorizable_load): Add call to check_load_for_ffr.
* tree-vectorizer.h (struct ffr_region): New struct.
(struct vect_load_store_data): Add ffr_ifn and defines_region
members.
(struct _slp_tree): Add ffr_region and ffr_mask_needed members.
(SLP_INSTANCE_FFR_REGION): New macro.
(SLP_TREE_FFR_REGION): Ditto.
(SLP_TREE_FFR_MASK_NEEDED): Ditto.
(LOOP_VINFO_CAN_USE_FFR_P): Ditto.
(LOOP_VINFO_MUST_USE_FFR_P): Ditto.
(LOOP_VINFO_USING_FFR_P): Ditto.
(vect_can_add_ffr_controls): New function.
---
gcc/config/aarch64/aarch64.cc | 10 ++
gcc/optabs-tree.cc | 57 ++++++++++
gcc/optabs-tree.h | 2 +
gcc/tree-vect-data-refs.cc | 3 +-
gcc/tree-vect-loop-manip.cc | 33 ++++++
gcc/tree-vect-loop.cc | 190 ++++++++++++++++++++++++++++++++++
gcc/tree-vect-slp.cc | 11 +-
gcc/tree-vect-stmts.cc | 65 ++++++++++++
gcc/tree-vectorizer.h | 52 ++++++++++
9 files changed, 421 insertions(+), 2 deletions(-)
diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index c9403ed94e7..f26832eac7d 100644
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -19352,6 +19352,16 @@ better_main_loop_than_p (const vector_costs
*uncast_other) const
auto other_estimated_vf = (vect_vf_for_cost (other_loop_vinfo)
* other->m_ops[i].vf_factor ());
+ if ((LOOP_VINFO_USING_FFR_P (this_loop_vinfo)
+ != LOOP_VINFO_USING_FFR_P (other_loop_vinfo))
+ && param_vect_ffr_usage == 2)
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "FFR: Preferring loop with FFR\n");
+ return LOOP_VINFO_USING_FFR_P (this_loop_vinfo);
+ }
+
/* If it appears that one loop could process the same amount of data
in fewer cycles, prefer that loop over the other one. */
fractional_cost this_cost
diff --git a/gcc/optabs-tree.cc b/gcc/optabs-tree.cc
index 1b80cac85c7..8aa27be3379 100644
--- a/gcc/optabs-tree.cc
+++ b/gcc/optabs-tree.cc
@@ -600,6 +600,63 @@ can_vec_mask_load_store_p (machine_mode mode,
return false;
}
+/* Return true if the target has support for ffl masked load. */
+
+bool
+target_supports_ffl_mask_load_p (machine_mode mode, machine_mode mask_mode,
+ internal_fn *ifn)
+{
+ optab op = mask_firstfault_load_optab;
+ enum insn_code icode;
+ if ((icode = convert_optab_handler (op, mode, mask_mode)) !=
CODE_FOR_nothing)
+ {
+ if (ifn)
+ *ifn = IFN_MASK_FIRSTFAULT_LOAD;
+ return true;
+ }
+ return false;
+}
+
+/* Return true if target supports vector masked first fault load for mode.
+ An additional output in the last argument which is the IFN pointer.
+ We set IFN as MASK_FIRSTFAULT_LOAD. */
+
+bool
+can_vec_ff_mask_load_p (machine_mode mode, machine_mode mask_mode,
+ internal_fn *ifn)
+{
+ machine_mode vmode;
+
+ /* If mode is vector mode, check it directly. */
+ if (VECTOR_MODE_P (mode))
+ return target_supports_ffl_mask_load_p (mode, mask_mode, ifn);
+
+ /* Otherwise, return true if there is some vector mode with
+ the mask load/store supported. */
+
+ /* See if there is any chance the mask load or store might be
+ vectorized. If not, punt. */
+ scalar_mode smode;
+ if (!is_a<scalar_mode> (mode, &smode))
+ return false;
+
+ vmode = targetm.vectorize.preferred_simd_mode (smode);
+ if (VECTOR_MODE_P (vmode)
+ && targetm.vectorize.get_mask_mode (vmode).exists (&mask_mode)
+ && target_supports_ffl_mask_load_p (vmode, mask_mode, ifn))
+ return true;
+
+ auto_vector_modes vector_modes;
+ targetm.vectorize.autovectorize_vector_modes (&vector_modes, true);
+ for (machine_mode base_mode : vector_modes)
+ if (related_vector_mode (base_mode, smode).exists (&vmode)
+ && targetm.vectorize.get_mask_mode (vmode).exists (&mask_mode)
+ && target_supports_ffl_mask_load_p (vmode, mask_mode, ifn))
+ return true;
+
+ return false;
+}
+
/* Return true if the target has support for len load/store.
We can support len load/store by either len_{load,store}
or mask_len_{load,store}.
diff --git a/gcc/optabs-tree.h b/gcc/optabs-tree.h
index dad9ed9b0ba..a5098a4e8c3 100644
--- a/gcc/optabs-tree.h
+++ b/gcc/optabs-tree.h
@@ -52,6 +52,8 @@ bool target_supports_mask_load_store_p (machine_mode,
machine_mode,
bool can_vec_mask_load_store_p (machine_mode, machine_mode, bool,
internal_fn * = nullptr,
vec<int> * = nullptr);
+bool can_vec_ff_mask_load_p (machine_mode, machine_mode,
+ internal_fn * = nullptr);
opt_machine_mode get_len_load_store_mode (machine_mode, bool,
internal_fn * = nullptr,
vec<int> * = nullptr);
diff --git a/gcc/tree-vect-data-refs.cc b/gcc/tree-vect-data-refs.cc
index 2d80d97c7db..216666359dc 100644
--- a/gcc/tree-vect-data-refs.cc
+++ b/gcc/tree-vect-data-refs.cc
@@ -6789,7 +6789,8 @@ vect_supportable_dr_alignment (vec_info *vinfo,
dr_vec_info *dr_info,
if (misalignment == 0)
return dr_aligned;
- else if (dr_safe_speculative_read_required (stmt_info))
+ else if (dr_safe_speculative_read_required (stmt_info)
+ && !(loop_vinfo && LOOP_VINFO_MUST_USE_FFR_P (loop_vinfo)))
return dr_unaligned_unsupported;
if (loop_vinfo)
diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc
index c42e101824d..bbf1beb6f0a 100644
--- a/gcc/tree-vect-loop-manip.cc
+++ b/gcc/tree-vect-loop-manip.cc
@@ -4818,3 +4818,36 @@ vect_loop_versioning (loop_vec_info loop_vinfo,
return nloop;
}
+
+/* Checks if it's possible to create the necessary controls for an FFR loop.
+
+ For loops with FFR we need to generate all the required masks from the
+ successfully loaded element mask. */
+
+
+bool
+vect_can_add_ffr_controls (loop_vec_info loop_vinfo)
+{
+ if (!LOOP_VINFO_CAN_USE_FFR_P (loop_vinfo))
+ return false;
+
+ int i;
+ rgroup_controls *rgc;
+ FOR_EACH_VEC_ELT (LOOP_VINFO_MASKS (loop_vinfo).rgc_vec, i, rgc)
+ {
+ /* The base case is handled by reading the FFR value. */
+ if (i == 0)
+ continue;
+
+ unsigned int nmasks = i + 1;
+
+ rgroup_controls *half_controls_rgc
+ = &LOOP_VINFO_MASKS (loop_vinfo).rgc_vec[nmasks / 2 - 1];
+
+ /* Permute the masks from the half rgoup to this one. */
+ if (!vect_maybe_permute_loop_masks (NULL, rgc, half_controls_rgc))
+ return false;
+ }
+
+ return true;
+}
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index 7aff5e8971f..4d1215b2c37 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -755,6 +755,9 @@ _loop_vec_info::_loop_vec_info (class loop *loop_in,
vec_info_shared *shared)
vectorizable (false),
can_use_partial_vectors_p (true),
must_use_partial_vectors_p (false),
+ must_use_ffr_p (false),
+ can_use_ffr_p (false),
+ using_ffr_p (false),
using_partial_vectors_p (false),
using_decrementing_iv_p (false),
using_select_vl_p (false),
@@ -1099,6 +1102,17 @@ vect_verify_full_masking (loop_vec_info loop_vinfo)
return false;
}
+ /* Check we can generate all the masks we need for FFR. */
+ if (LOOP_VINFO_CAN_USE_FFR_P (loop_vinfo)
+ && !vect_can_add_ffr_controls (loop_vinfo))
+ {
+ LOOP_VINFO_CAN_USE_FFR_P (loop_vinfo) = false;
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "not using FFR: Cannot generate the masks from the "
+ "FFR mask.\n");
+ }
+
LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo) = cmp_type;
LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo) = iv_type;
LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo) =
vect_partial_vectors_while_ult;
@@ -2140,6 +2154,165 @@ vect_determine_partial_vectors_and_peeling
(loop_vec_info loop_vinfo,
return opt_result::success ();
}
+/* Sets the ffr region for NODE and its children to be REGION. */
+
+static void
+set_ffr_regions (loop_vec_info loop_vinfo, slp_tree node, ffr_region *region,
+ hash_set<slp_tree> &visited)
+{
+ /* If the node is an FFR load, then return the region it defines. */
+ if (!node || SLP_TREE_TYPE (node) == load_vec_info_type
+ || SLP_TREE_TYPE (node) == phi_info_type || SLP_TREE_FFR_REGION (node)
+ || visited.add (node))
+ return;
+
+ if (SLP_TREE_FFR_MASK_NEEDED (node))
+ SLP_TREE_FFR_REGION (node) = region;
+
+ /* Propagate the region to all unset children. */
+ for (slp_tree child : SLP_TREE_CHILDREN (node))
+ set_ffr_regions (loop_vinfo, child, region, visited);
+}
+
+/* Checks if first faulting loads can be used for this loop and makes a
+ decision enabling them.
+
+ Establishes ffr-regions. */
+
+static opt_result
+vect_determine_ffr_and_versioning (loop_vec_info loop_vinfo)
+{
+ if (LOOP_VINFO_CAN_USE_FFR_P (loop_vinfo))
+ {
+ if (param_vect_ffr_usage == 0)
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "FFR: Disabled.\n");
+ LOOP_VINFO_CAN_USE_FFR_P (loop_vinfo) = false;
+ goto exit;
+ }
+
+ if (param_vect_ffr_usage != 2
+ && !LOOP_VINFO_MUST_USE_FFR_P (loop_vinfo)
+ && LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).length () == 0)
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "FFR: Decided not to.\n");
+ LOOP_VINFO_CAN_USE_FFR_P (loop_vinfo) = false;
+ goto exit;
+ }
+
+ if (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "FFR: Loop not fully masked.\n");
+ LOOP_VINFO_CAN_USE_FFR_P (loop_vinfo) = false;
+ goto exit;
+ }
+
+ /* Work out the FFR regions. */
+ ffr_region *last_region = NULL;
+ slp_instance instance;
+ int i;
+ hash_set<slp_tree> visited;
+ FOR_EACH_VEC_ELT_REVERSE (LOOP_VINFO_SLP_INSTANCES (loop_vinfo), i,
+ instance)
+ {
+ /* Check if the instance needs an FFR region.
+
+ NOTE: This assumes all FFR loads are leaf nodes.
+
+ This is a safe assumption as FFR loads can't sensibly be made to
+ do conditional loads so can never take a mask, and we don't yet
+ support gather FF loads. */
+ if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_gcond)
+ {
+ SLP_INSTANCE_FFR_REGION (instance) = new ffr_region ();
+ for (slp_tree load : SLP_INSTANCE_LOADS (instance))
+ {
+ gcc_assert (load->children.length () == 0);
+
+ /* Set the load "defines region" to this one. */
+ vect_load_store_data _ls_data{};
+ vect_load_store_data &ls = load->get_data (_ls_data);
+
+ /* The number of elements of the load must equal the
+ vectorization factor, as we must only fault if the next
+ scalar iteration is a fault, and optionally load later
+ iterations.
+
+ This is checked by vectoriable_load.
+
+ TODO: We could support other cases and grain sizes here.
+ However, this would likely require separate FFR regions
+ which would have high runtime cost. */
+ gcc_assert (
+ known_eq (GET_MODE_NUNITS (TYPE_MODE (load->vectype)),
+ LOOP_VINFO_VECT_FACTOR (loop_vinfo)));
+
+ /* If this load is already handled by another region, we don't
+ need to handle it in this one. */
+ if (ls.defines_region)
+ continue;
+
+ ls.defines_region = SLP_INSTANCE_FFR_REGION (instance);
+
+ /* This load will be part of the region for this break. */
+ SLP_INSTANCE_FFR_REGION (instance)->loads.safe_push (load);
+ SLP_INSTANCE_FFR_REGION (instance)->last_region = last_region;
+
+ /* Set the region it uses for its mask to be the previous one.
+ */
+ SLP_TREE_FFR_REGION (load) = last_region;
+ SLP_INSTANCE_FFR_REGION (instance)->ffr_read_point
+ = SLP_TREE_REPRESENTATIVE (load)->stmt;
+ }
+ /* If this region doesnt handle any loads, it isnt needed. */
+ if (SLP_INSTANCE_FFR_REGION (instance)->loads.is_empty ())
+ SLP_INSTANCE_FFR_REGION (instance) = NULL;
+ else
+ last_region = SLP_INSTANCE_FFR_REGION (instance);
+ }
+
+ /* Populate the ffr_regions for the nodes in the tree. */
+ set_ffr_regions (loop_vinfo, SLP_INSTANCE_TREE (instance),
+ last_region, visited);
+ }
+
+ /* If none of the instances require loads, then FFR is pointless. */
+ if (!last_region)
+ {
+ LOOP_VINFO_CAN_USE_FFR_P (loop_vinfo) = false;
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "FFR: No early break instances require FFR.\n");
+ goto exit;
+ }
+
+ /* We can and will use FFR. */
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location, "FFR: Will use ffr\n");
+
+ /* We no longer need a bunch of versioning. */
+ LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true;
+ LOOP_VINFO_USING_FFR_P (loop_vinfo) = true;
+ LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo) = vNULL;
+ LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT (loop_vinfo) = false;
+ LOOP_VINFO_MAX_SPEC_READ_AMOUNT (loop_vinfo) = 0;
+ }
+exit:
+ if (LOOP_VINFO_MUST_USE_FFR_P (loop_vinfo)
+ && !LOOP_VINFO_USING_FFR_P (loop_vinfo))
+ return opt_result::failure_at (vect_location,
+ "not vectorized: loop needs but cannot "
+ "use ffr\n");
+
+ return opt_result::success ();
+}
+
/* Function vect_analyze_loop_2.
Apply a set of analyses on LOOP specified by LOOP_VINFO, the different
@@ -2409,6 +2582,11 @@ start_over:
if (!ok)
return ok;
+ /* Check if we can use FFR instead of peeling and/or versioning. */
+ ok = vect_determine_ffr_and_versioning (loop_vinfo);
+ if (!ok)
+ return ok;
+
/* If we're vectorizing a loop that uses length "controls" and
can iterate more than once, we apply decrementing IV approach
in loop control. */
@@ -2731,6 +2909,9 @@ again:
LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
= saved_can_use_partial_vectors_p;
LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ LOOP_VINFO_USING_FFR_P (loop_vinfo) = false;
+ LOOP_VINFO_MUST_USE_FFR_P (loop_vinfo) = false;
+ LOOP_VINFO_CAN_USE_FFR_P (loop_vinfo) = false;
LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo) = false;
LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo) = false;
LOOP_VINFO_USING_DECREMENTING_IV_P (loop_vinfo) = false;
@@ -9605,6 +9786,15 @@ vectorizable_induction (loop_vec_info loop_vinfo,
return false;
}
+ if (SLP_TREE_LANES (slp_node) != 1)
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "FFR: Cannot use FFR as multiple-lane SLP inductions"
+ "aren't supported.\n");
+ LOOP_VINFO_CAN_USE_FFR_P (loop_vinfo) = false;
+ }
+
if (FLOAT_TYPE_P (vectype) && !param_vect_induction_float)
{
if (dump_enabled_p ())
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index 773040ba46f..8b0a4348d0f 100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -135,6 +135,8 @@ _slp_tree::_slp_tree ()
this->lanes = 0;
SLP_TREE_TYPE (this) = undef_vec_info_type;
this->data = NULL;
+ SLP_TREE_FFR_REGION (this) = NULL;
+ SLP_TREE_FFR_MASK_NEEDED (this) = true;
}
/* Tear down a SLP node. */
@@ -217,6 +219,7 @@ vect_free_slp_instance (slp_instance instance)
SLP_INSTANCE_REMAIN_DEFS (instance).release ();
instance->subgraph_entries.release ();
instance->cost_vec.release ();
+ delete SLP_INSTANCE_FFR_REGION (instance);
free (instance);
}
@@ -4287,6 +4290,7 @@ vect_build_slp_instance (vec_info *vinfo,
new_instance->reduc_phis = NULL;
new_instance->cost_vec = vNULL;
new_instance->subgraph_entries = vNULL;
+ SLP_INSTANCE_FFR_REGION (new_instance) = NULL;
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
@@ -4605,7 +4609,7 @@ vect_analyze_slp_reduc_chain (loop_vec_info vinfo,
new_instance->reduc_phis = NULL;
new_instance->cost_vec = vNULL;
new_instance->subgraph_entries = vNULL;
-
+ SLP_INSTANCE_FFR_REGION (new_instance) = NULL;
vinfo->slp_instances.safe_push (new_instance);
if (dump_enabled_p ())
@@ -4671,6 +4675,7 @@ vect_analyze_slp_reduc_chain (loop_vec_info vinfo,
new_instance->reduc_phis = NULL;
new_instance->cost_vec = vNULL;
new_instance->subgraph_entries = vNULL;
+ SLP_INSTANCE_FFR_REGION (new_instance) = NULL;
vect_reduc_info reduc_info = info_for_reduction (vinfo, node);
reduc_info->is_reduc_chain = true;
@@ -4811,6 +4816,7 @@ vect_analyze_slp_reduction (loop_vec_info vinfo,
new_instance->reduc_phis = NULL;
new_instance->cost_vec = vNULL;
new_instance->subgraph_entries = vNULL;
+ SLP_INSTANCE_FFR_REGION (new_instance) = NULL;
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
@@ -4884,6 +4890,7 @@ vect_analyze_slp_reduction_group (loop_vec_info
loop_vinfo,
new_instance->reduc_phis = NULL;
new_instance->cost_vec = vNULL;
new_instance->subgraph_entries = vNULL;
+ SLP_INSTANCE_FFR_REGION (new_instance) = NULL;
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
@@ -5140,6 +5147,7 @@ vect_analyze_slp_instance (vec_info *vinfo,
new_instance->reduc_phis = NULL;
new_instance->cost_vec = vNULL;
new_instance->subgraph_entries = vNULL;
+ SLP_INSTANCE_FFR_REGION (new_instance) = NULL;
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
@@ -5387,6 +5395,7 @@ vect_analyze_slp_instance (vec_info *vinfo,
new_instance->reduc_phis = NULL;
new_instance->cost_vec = vNULL;
new_instance->subgraph_entries = vNULL;
+ SLP_INSTANCE_FFR_REGION (new_instance) = NULL;
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index 03746d6c44c..427adce4594 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -1583,6 +1583,44 @@ check_load_store_for_partial_vectors (loop_vec_info
loop_vinfo, tree vectype,
}
}
+/* Checks if a load can support FFR. If supported sets ls->ffr_ifn to the
+ first faulting load internal function. */
+
+static void
+check_load_for_ffr (loop_vec_info loop_vinfo, tree vectype, slp_tree slp_node,
+ vect_load_store_data *ls, int num_vec)
+{
+ vect_memory_access_type memory_access_type = ls->memory_access_type;
+
+ /* Never need FFR for invariant addresses. */
+ if (memory_access_type == VMAT_INVARIANT)
+ return;
+
+ if (!LOOP_VINFO_CAN_USE_FFR_P (loop_vinfo))
+ return;
+
+ machine_mode vecmode = TYPE_MODE (vectype);
+ machine_mode mask_mode;
+ /* For now, dont handle group_size != 1 or vec_num != 1. */
+ if (num_vec != 1 || slp_node->lanes != 1)
+ {
+ LOOP_VINFO_CAN_USE_FFR_P (loop_vinfo) = false;
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "FFR: multiple lane loads aren't supported\n");
+ }
+
+ /* If we can't do a ff load, remember that. */
+ if (targetm.vectorize.get_mask_mode (vecmode).exists (&mask_mode))
+ if (!can_vec_ff_mask_load_p (vecmode, mask_mode, &ls->ffr_ifn))
+ {
+ LOOP_VINFO_CAN_USE_FFR_P (loop_vinfo) = false;
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "FFR: load not supported\n");
+ }
+}
+
/* Return the mask input to a masked load or store. VEC_MASK is the vectorized
form of the scalar mask condition and LOOP_MASK, if nonnull, is the mask
that needs to be applied to all loads and stores in a vectorized loop.
@@ -2520,6 +2558,19 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info
stmt_info,
= vect_supportable_dr_alignment
(vinfo, first_dr_info, vectype, *misalignment,
mat_gather_scatter_p (*memory_access_type));
+
+ /* If we would currently fail, but could be fine with FFR, use FFR. */
+ if (loop_vinfo
+ && *alignment_support_scheme == dr_unaligned_unsupported
+ && LOOP_VINFO_CAN_USE_FFR_P (loop_vinfo)
+ && !LOOP_VINFO_MUST_USE_FFR_P (loop_vinfo))
+ {
+ LOOP_VINFO_MUST_USE_FFR_P (loop_vinfo) = true;
+ *alignment_support_scheme = vect_supportable_dr_alignment
+ (vinfo, first_dr_info, vectype, *misalignment,
+ mat_gather_scatter_p (*memory_access_type));
+ }
+
if (grouped_gather_fallback != VMAT_UNINITIALIZED
&& *alignment_support_scheme != dr_aligned
&& *alignment_support_scheme != dr_unaligned_supported)
@@ -5694,6 +5745,14 @@ vectorizable_conversion (vec_info *vinfo,
if (cost_vec) /* transformation not required. */
{
+ /* If this is not a conversion that could trap, then we do not require
+ a totally accurate FFR mask.
+
+ This allows conversions to be emitted next to loads and be combined
+ into widening loads. */
+ if (!gimple_could_trap_p (stmt))
+ SLP_TREE_FFR_MASK_NEEDED (slp_node) = false;
+
if (!vect_maybe_update_slp_op_vectype (slp_op0, vectype_in)
|| !vect_maybe_update_slp_op_vectype (slp_op1, vectype_in))
{
@@ -9964,6 +10023,12 @@ vectorizable_load (vec_info *vinfo,
return false;
}
+ if (loop_vinfo
+ && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
+ && LOOP_VINFO_CAN_USE_FFR_P (loop_vinfo)
+ && dr_safe_speculative_read_required (stmt_info))
+ check_load_for_ffr (loop_vinfo, vectype, slp_node, &ls, vec_num);
+
if (dump_enabled_p ()
&& memory_access_type != VMAT_ELEMENTWISE
&& !mat_gather_scatter_p (memory_access_type)
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 45d0441f53a..8116a82ba2f 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -275,6 +275,31 @@ struct vect_simd_clone_data : vect_data {
auto_vec<tree> simd_clone_info;
};
+struct ffr_region {
+ ffr_region () = default;
+
+ /* The FFR loads that create the mask that defines this region. */
+ vec<slp_tree> loads;
+
+ /* Where the FFR and the fixup code for this region read should be
+ inserted. */
+ gimple* ffr_read_point;
+
+ /* The SSA node to store how much IV's should be incremented by. */
+ tree num_iter;
+
+ /* The SSA node for the mask of elements not processed by this vectorized
+ iteration that will need to be re-processed. */
+ tree next_iter_mask;
+
+ /* The SSA nodes for the masks that operations in this region will use. */
+ vec<vec<tree>> controls;
+
+ /* The previous region before this one, or NULL if this is the first
+ region. */
+ ffr_region *last_region;
+};
+
/* Analysis data from vectorizable_load and vectorizable_store for
load_vec_info_type and store_vec_info_type. */
struct vect_load_store_data : vect_data {
@@ -286,6 +311,7 @@ struct vect_load_store_data : vect_data {
dr_alignment_support alignment_support_scheme;
int misalignment;
internal_fn lanes_ifn; // VMAT_LOAD_STORE_LANES
+ internal_fn ffr_ifn;
poly_int64 poffset;
union {
internal_fn ifn; // VMAT_GATHER_SCATTER_IFN
@@ -313,6 +339,10 @@ struct vect_load_store_data : vect_data {
unsigned n_loads; // SLP_TREE_LOAD_PERMUTATION
/* Whether the load permutation is consecutive and simple. */
bool subchain_p; // VMAT_STRIDED_SLP and VMAT_GATHER_SCATTER
+
+ /* If this is a first faulting load, then the region that the resultant
+ mask will define. */
+ struct ffr_region *defines_region;
};
/* A computation tree of an SLP instance. Each node corresponds to a group of
@@ -405,6 +435,14 @@ struct _slp_tree {
/* Linked list of nodes to release when we free the slp_tree_pool. */
slp_tree next_node;
slp_tree prev_node;
+
+ /* The ffr region this node is goverened by and gets it's mask from.
+ Will be NULL if there is no ffr region for this node and it should use the
+ normal loop masks. */
+ struct ffr_region *ffr_region;
+ /* If this FFR node needs to have an accurate FFR mask, or can make do with
+ a possibly inaccurate superset mask. */
+ bool ffr_mask_needed;
};
/* The enum describes the type of operations that an SLP instance
@@ -450,6 +488,9 @@ public:
/* The type of operation the SLP instance is performing. */
slp_instance_kind kind;
+ /* The FFR region defining the mask for this node. */
+ ffr_region *region;
+
dump_user_location_t location () const;
} *slp_instance;
@@ -460,6 +501,7 @@ public:
#define SLP_INSTANCE_ROOT_STMTS(S) (S)->root_stmts
#define SLP_INSTANCE_REMAIN_DEFS(S) (S)->remain_defs
#define SLP_INSTANCE_KIND(S) (S)->kind
+#define SLP_INSTANCE_FFR_REGION(S) (S)->region
#define SLP_TREE_CHILDREN(S) (S)->children
#define SLP_TREE_SCALAR_STMTS(S) (S)->stmts
@@ -479,6 +521,8 @@ public:
#define SLP_TREE_GS_BASE(S) (S)->gs_base
#define SLP_TREE_REDUC_IDX(S) (S)->cycle_info.reduc_idx
#define SLP_TREE_PERMUTE_P(S) ((S)->code == VEC_PERM_EXPR)
+#define SLP_TREE_FFR_REGION(S) (S)->ffr_region
+#define SLP_TREE_FFR_MASK_NEEDED(S) (S)->ffr_mask_needed
inline vect_memory_access_type
SLP_TREE_MEMORY_ACCESS_TYPE (slp_tree node)
@@ -1140,6 +1184,10 @@ public:
/* Records whether we must use niter masking for correctness reasons. */
bool must_use_partial_vectors_p;
+ bool must_use_ffr_p;
+ bool can_use_ffr_p;
+ bool using_ffr_p;
+
/* True if we've decided to use partially-populated vectors, so that
the vector loop can handle fewer than VF scalars. */
bool using_partial_vectors_p;
@@ -1301,6 +1349,9 @@ public:
#define LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P(L) (L)->can_use_partial_vectors_p
#define LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P(L)
(L)->must_use_partial_vectors_p
#define LOOP_VINFO_USING_PARTIAL_VECTORS_P(L) (L)->using_partial_vectors_p
+#define LOOP_VINFO_CAN_USE_FFR_P(L) (L)->can_use_ffr_p
+#define LOOP_VINFO_MUST_USE_FFR_P(L) (L)->must_use_ffr_p
+#define LOOP_VINFO_USING_FFR_P(L) (L)->using_ffr_p
#define LOOP_VINFO_USING_DECREMENTING_IV_P(L) (L)->using_decrementing_iv_p
#define LOOP_VINFO_USING_SELECT_VL_P(L) (L)->using_select_vl_p
#define LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT(L) (L)->allow_mutual_alignment
@@ -2504,6 +2555,7 @@ class loop *slpeel_tree_duplicate_loop_to_edge_cfg (class
loop *, edge,
bool = false, bool = false,
bool = true);
class loop *vect_loop_versioning (loop_vec_info, gimple *);
+extern bool vect_can_add_ffr_controls (loop_vec_info);
extern class loop *vect_do_peeling (loop_vec_info, tree, tree,
tree *, tree *, tree *, int, bool, bool,
tree *);
--
2.34.1
