gcc/ChangeLog:
* tree-vect-loop-manip.cc (vect_set_loop_controls_directly):
Add logic for setting the next mask from FFR controls.
(vect_get_loop_iv_increment): Add case for
(vect_add_ffr_read): New function.
(vect_add_ffr_fixup_controls): New function.
* tree-vect-loop.cc (vect_get_loop_mask): Add handling for
getting the FFR mask.
(vect_transform_loop): Add preservation of the FFR value before
vectorization.
* tree-vect-slp.cc (vect_schedule_slp_node): Add FFR
consideration scheduling in.
(vect_schedule_slp): Add FFR read call when scheduling FFR
instance.
* tree-vect-stmts.cc (vectorizable_load): Add FFR generation.
* tree-vectorizer.h (LOOP_VINFO_IV_INCREMENT_INVARIANT_P): Add
case for FFR.
(vect_add_ffr_fixup_controls): New function.
(vect_add_ffr_read): New function.
---
gcc/tree-vect-loop-manip.cc | 263 +++++++++++++++++++++++++++++++++++-
gcc/tree-vect-loop.cc | 64 ++++++++-
gcc/tree-vect-slp.cc | 11 ++
gcc/tree-vect-stmts.cc | 61 +++++++--
gcc/tree-vectorizer.h | 7 +-
5 files changed, 387 insertions(+), 19 deletions(-)
diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc
index bbf1beb6f0a..7e46f4879a5 100644
--- a/gcc/tree-vect-loop-manip.cc
+++ b/gcc/tree-vect-loop-manip.cc
@@ -821,6 +821,31 @@ vect_set_loop_controls_directly (class loop *loop,
loop_vec_info loop_vinfo,
gimple_seq stmts = NULL;
next_ctrl = vect_gen_while (&stmts, ctrl_type, test_index,
this_test_limit, "next_mask");
+ if (LOOP_VINFO_USING_FFR_P (loop_vinfo))
+ {
+ tree ffr_mask = NULL_TREE;
+ slp_instance instance;
+ int i;
+ FOR_EACH_VEC_ELT_REVERSE (LOOP_VINFO_SLP_INSTANCES (loop_vinfo),
+ i, instance)
+ {
+ if (!SLP_INSTANCE_FFR_REGION (instance))
+ continue;
+ SLP_INSTANCE_FFR_REGION (instance)->next_iter_mask
+ = make_temp_ssa_name (ctrl_type, NULL, "next_mask_ffr");
+ ffr_mask = SLP_INSTANCE_FFR_REGION (instance)->next_iter_mask;
+ break;
+ }
+ gcc_assert (ffr_mask != NULL_TREE);
+ tree next_ctrl_after_ffr = make_temp_ssa_name (ctrl_type, NULL,
+ "next_mask_ffr");
+ gimple *assign = gimple_build_assign (next_ctrl_after_ffr,
+ BIT_AND_EXPR, next_ctrl,
+ ffr_mask);
+
+ gimple_seq_add_stmt (&stmts, assign);
+ next_ctrl = next_ctrl_after_ffr;
+ }
gsi_insert_seq_before (test_gsi, stmts, GSI_SAME_STMT);
}
else
@@ -841,8 +866,8 @@ vect_set_loop_controls_directly (class loop *loop,
loop_vec_info loop_vinfo,
gassign *minus = gimple_build_assign (adjusted_len, PLUS_EXPR,
rgc->controls[0],
build_int_cst
- (TREE_TYPE (rgc->controls[0]),
- partial_load_bias));
+ (TREE_TYPE (rgc->controls[0]),
+ partial_load_bias));
gimple_seq_add_stmt (header_seq, minus);
}
@@ -3108,6 +3133,23 @@ vect_get_loop_iv_increment (loop_vec_info loop_vinfo)
SSA_NAME_DEF_STMT (iv_increment) = gimple_build_nop ();
return iv_increment;
}
+ if (LOOP_VINFO_USING_FFR_P (loop_vinfo))
+ {
+ slp_instance instance;
+ int i;
+ FOR_EACH_VEC_ELT_REVERSE (LOOP_VINFO_SLP_INSTANCES (loop_vinfo), i,
+ instance)
+ if (SLP_INSTANCE_FFR_REGION (instance))
+ {
+ SLP_INSTANCE_FFR_REGION (instance)->num_iter
+ = make_temp_ssa_name (LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo),
+ NULL, "ffr_num_iters");
+ SSA_NAME_DEF_STMT (SLP_INSTANCE_FFR_REGION (instance)->num_iter)
+ = gimple_build_nop ();
+ return SLP_INSTANCE_FFR_REGION (instance)->num_iter;
+ }
+ gcc_unreachable ();
+ }
else
return build_int_cst (sizetype, LOOP_VINFO_VECT_FACTOR (loop_vinfo));
}
@@ -4851,3 +4893,220 @@ vect_can_add_ffr_controls (loop_vec_info loop_vinfo)
return true;
}
+
+/* Insert the FFR read. */
+void
+vect_add_ffr_read (vec_info *vinfo, slp_instance instance)
+{
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
+ gcc_assert (loop_vinfo);
+
+ gimple_stmt_iterator insert_point
+ = gsi_for_stmt (SLP_INSTANCE_FFR_REGION (instance)->ffr_read_point);
+
+ gcall *ffr_read_call = gimple_build_call_internal (IFN_READ_FAULT_STATE, 0);
+ gsi_insert_after (&insert_point, ffr_read_call, GSI_SAME_STMT);
+
+ SLP_INSTANCE_FFR_REGION (instance)->ffr_read_point = ffr_read_call;
+}
+
+/* Inserts fixup block for FFR controls.
+
+ This inserts the following logic:
+
+ <bb 0>:
+ v1 = .MASK_FIRSTFAULT_LOAD (src1, 64B, loop_mask, { 0, ... });
+ v2 = .MASK_FIRSTFAULT_LOAD (src2, 64B, loop_mask, { 0, ... });
+ ffr_mask = .READ_FAULT_STATE ();
+ if (ffr_mask == { -1, ... })
+ goto <bb 2>; [99.95%]
+ else
+ goto <bb 1>; [0.05%]
+
+ <bb 1>:
+ .SET_FAULT_STATE ({ -1, ... });
+ ffr_mask_invert = ~ffr_mask;
+ ffr_loop_mask = loop_mask & ffr_mask;
+
+ <bb 2>:
+ # loop_mask_1 = PHI <loop_mask(0), ffr_loop_mask(1)>
+ # num_iters_56 = PHI <POLY_INT_CST [4, 4](0), 0(1)>
+ # next_mask = PHI <{ -1, ... }(0), ffr_mask_invert(1)>
+
+ ... Logic using loop_mask_1 and incrementing IV's by num_iters_56
+ */
+void
+vect_add_ffr_fixup_controls (loop_vec_info loop_vinfo)
+{
+ gcc_assert (LOOP_VINFO_USING_FFR_P (loop_vinfo));
+ gcc_assert (LOOP_VINFO_USING_PARTIAL_VECTORS_P(loop_vinfo));
+ slp_instance instance;
+ int index;
+ FOR_EACH_VEC_ELT_REVERSE (LOOP_VINFO_SLP_INSTANCES (loop_vinfo), index,
+ instance)
+ {
+ /* This instance does not define an FFR region. No work to do. */
+ if (!SLP_INSTANCE_FFR_REGION (instance))
+ continue;
+
+ /* At the very least the GCOND must use the FFR mask so it can't be that
+ there are no uses. */
+ gcc_assert (!SLP_INSTANCE_FFR_REGION (instance)->controls.is_empty ());
+
+ ffr_region *region = SLP_INSTANCE_FFR_REGION (instance);
+ vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
+ rgroup_controls *base_rgm = &masks->rgc_vec[0];
+ tree mask_type = base_rgm->type;
+
+ /* The base mask is established by reading from ffr. */
+ tree ffr_read_val = make_temp_ssa_name (mask_type, NULL, "ffr_mask");
+ gimple_call_set_lhs (region->ffr_read_point, ffr_read_val);
+
+ /* The fixup is taken if the FFR read is not all true. */
+ tree true_node = build_all_ones_cst (mask_type);
+ gcond *cond = gimple_build_cond (EQ_EXPR, ffr_read_val, true_node,
+ NULL_TREE, NULL_TREE);
+ gimple_stmt_iterator gsi_read = gsi_for_stmt (region->ffr_read_point);
+ gsi_insert_after (&gsi_read, cond, GSI_SAME_STMT);
+
+ /* Split the BB after the FF reads and setup control flow for the fixup
+ block. */
+ basic_block read_bb = cond->bb;
+ edge non_ffr_edge = split_block (read_bb, cond);
+ non_ffr_edge->flags = EDGE_TRUE_VALUE;
+ non_ffr_edge->probability = profile_probability::very_likely ();
+
+ basic_block continuation_bb = non_ffr_edge->dest;
+ continuation_bb->count = read_bb->count;
+ continuation_bb->loop_father = read_bb->loop_father;
+
+ basic_block fixup_bb = create_empty_bb (read_bb);
+ fixup_bb->loop_father = read_bb->loop_father;
+ fixup_bb->count = read_bb->count.apply_scale (1, 100);
+
+ edge fixup_edge = make_edge (read_bb, fixup_bb, EDGE_FALSE_VALUE);
+ fixup_edge->probability = profile_probability::very_unlikely ();
+
+ edge fixup_cont_edge = make_edge (fixup_bb, continuation_bb,
EDGE_FALLTHRU);
+ fixup_cont_edge->probability = profile_probability::always ();
+
+ set_immediate_dominator (CDI_DOMINATORS, fixup_bb, read_bb);
+ set_immediate_dominator (CDI_DOMINATORS, continuation_bb, read_bb);
+
+ /* Mask nodes in the fixup. */
+ vec<vec<tree>> temp_masks = {};
+ temp_masks.safe_grow_cleared (masks->rgc_vec.length ());
+ temp_masks[0].safe_grow (1);
+
+ gimple_seq fixup_logic = NULL;
+
+ /* We always need the loop to have the nV=1 mask. */
+ if (base_rgm->controls.is_empty ())
+ {
+ base_rgm->controls.safe_grow_cleared (1, true);
+ tree mask = make_temp_ssa_name (mask_type, NULL, "loop_mask");
+ /* Provide a dummy definition until the real one is available. */
+ base_rgm->controls[0] = mask;
+ }
+ if (region->controls[0].is_empty ())
+ {
+ region->controls[0].safe_grow_cleared (1, true);
+ region->controls[0][0]
+ = make_temp_ssa_name (mask_type, NULL, "loop_mask_AFTER_FFR");
+ }
+ tree previous_mask = (region->last_region)
+ ? region->last_region->controls[0][0]
+ : masks->rgc_vec[0].controls[0];
+
+ gcall *call = gimple_build_call_internal (IFN_SET_FAULT_STATE, 1,
+ build_all_ones_cst (mask_type));
+ gimple_seq_add_stmt (&fixup_logic, call);
+
+ tree invert_mask = gimple_build (&fixup_logic, BIT_NOT_EXPR, mask_type,
+ ffr_read_val);
+ temp_masks[0][0] = make_temp_ssa_name (mask_type, NULL, "ffr_loop_mask");
+ gimple *ffr_and_loop
+ = gimple_build_assign (temp_masks[0][0], BIT_AND_EXPR, previous_mask,
+ ffr_read_val);
+ gimple_seq_add_stmt (&fixup_logic, ffr_and_loop);
+
+ /* PHI node for the NV=1 mask in the region. */
+ gphi *phi = create_phi_node (region->controls[0][0], continuation_bb);
+ add_phi_arg (phi, previous_mask, non_ffr_edge, UNKNOWN_LOCATION);
+ add_phi_arg (phi, temp_masks[0][0], fixup_cont_edge, UNKNOWN_LOCATION);
+
+ int i;
+ rgroup_controls *rgc;
+ FOR_EACH_VEC_ELT (masks->rgc_vec, i, rgc)
+ {
+ /* The base case is handled above. */
+ if (i == 0)
+ continue;
+
+ unsigned int nmasks = i + 1;
+ vec<tree> *this_masks = ®ion->controls[i];
+ vec<tree> *previous_masks = region->last_region
+ ? ®ion->last_region->controls[i]
+ : &rgc->controls;
+
+ vec<tree> *half_controls_temp = &(temp_masks[nmasks / 2 - 1]);
+ rgroup_controls *half_controls_rgc = &masks->rgc_vec[nmasks / 2 - 1];
+
+ vec<tree> *this_temps = &temp_masks[i];
+ this_temps->safe_grow_cleared (nmasks);
+ for (unsigned i = 0; i < nmasks; i++)
+ if (!(*this_temps)[i])
+ (*this_temps)[i] = make_temp_ssa_name (rgc->type, NULL,
+ "ffr_mask");
+
+ /* Permute the masks from the half rgoup to this one. */
+ gcc_assert (vect_maybe_permute_loop_masks (&fixup_logic, rgc,
+ half_controls_rgc,
+ this_temps,
+ half_controls_temp));
+
+ /* Create PHI nodes for the masks. */
+ for (unsigned int j = 0; j < nmasks; ++j)
+ {
+ gphi *mask_phi
+ = create_phi_node ((*this_masks)[j], continuation_bb);
+ add_phi_arg (mask_phi, (*previous_masks)[j], non_ffr_edge,
+ UNKNOWN_LOCATION);
+ add_phi_arg (mask_phi, (*this_temps)[j], fixup_cont_edge,
+ UNKNOWN_LOCATION);
+ }
+ }
+
+ /* Initialize the num_iter nodes. */
+ if (!region->num_iter)
+ region->num_iter
+ = make_temp_ssa_name (LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo), NULL,
+ "ffr_num_iters");
+
+ gphi *num_iter_phi = create_phi_node (region->num_iter, continuation_bb);
+ add_phi_arg (num_iter_phi,
+ region->last_region
+ ? region->last_region->num_iter
+ : build_int_cst (LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo),
+ LOOP_VINFO_VECT_FACTOR (loop_vinfo)),
+ non_ffr_edge, UNKNOWN_LOCATION);
+ add_phi_arg (num_iter_phi,
+ build_zero_cst (LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo)),
+ fixup_cont_edge, UNKNOWN_LOCATION);
+
+ if (!region->next_iter_mask)
+ region->next_iter_mask = make_temp_ssa_name (mask_type, NULL,
+ "ffr_next_mask");
+ gphi *next_mask_phi = create_phi_node (region->next_iter_mask,
+ continuation_bb);
+ add_phi_arg (next_mask_phi,
+ region->last_region ? region->last_region->next_iter_mask
+ : build_all_ones_cst (mask_type),
+ non_ffr_edge, UNKNOWN_LOCATION);
+ add_phi_arg (next_mask_phi, invert_mask, fixup_cont_edge,
+ UNKNOWN_LOCATION);
+
+ gimple_stmt_iterator fixup_gsi = gsi_start_bb (fixup_bb);
+ gsi_insert_seq_before (&fixup_gsi, fixup_logic, GSI_SAME_STMT);
+ }
+}
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index 4d1215b2c37..afebd7e5fa7 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -60,6 +60,7 @@ along with GCC; see the file COPYING3. If not see
#include "langhooks.h"
#include "opts.h"
#include "hierarchical_discriminator.h"
+#include "tree-into-ssa.h"
/* Loop Vectorization Pass.
@@ -10747,7 +10748,7 @@ tree
vect_get_loop_mask (loop_vec_info loop_vinfo,
gimple_stmt_iterator *gsi, vec_loop_masks *masks,
unsigned int nvectors, tree vectype, unsigned int index,
- slp_tree slp_node ATTRIBUTE_UNUSED)
+ slp_tree slp_node)
{
if (LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo)
== vect_partial_vectors_while_ult)
@@ -10770,6 +10771,37 @@ vect_get_loop_mask (loop_vec_info loop_vinfo,
}
tree mask = rgm->controls[index];
+
+ /* If we are in an FFR region, also populate all the other required
+ masks, and use the FFR mask */
+ if (LOOP_VINFO_USING_FFR_P (loop_vinfo) && SLP_TREE_FFR_REGION
(slp_node))
+ {
+ ffr_region *region = SLP_TREE_FFR_REGION (slp_node);
+ while (region)
+ {
+ if (region->controls.length() < nvectors)
+ region->controls.safe_grow_cleared(nvectors);
+
+ vec<tree> *controls = ®ion->controls[nvectors - 1];
+ if (controls->is_empty ())
+ {
+ controls->safe_grow_cleared (nvectors, true);
+ for (unsigned int i = 0; i < nvectors; ++i)
+ {
+ tree mask = make_temp_ssa_name (mask_type, NULL,
+ "ffr_loop_mask");
+ /* Provide a dummy definition until the real one is
+ available. */
+ SSA_NAME_DEF_STMT (mask) = gimple_build_nop ();
+ (*controls)[i] = mask;
+ }
+ }
+ region = region->last_region;
+ }
+
+ mask = SLP_TREE_FFR_REGION (slp_node)->controls[nvectors-1][index];
+ }
+
if (maybe_ne (TYPE_VECTOR_SUBPARTS (mask_type),
TYPE_VECTOR_SUBPARTS (vectype)))
{
@@ -11708,6 +11740,36 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple
*loop_vectorized_call)
" variable-length vectorization factor\n");
}
+ /* Save and restore the FFR. */
+ if (LOOP_VINFO_USING_FFR_P (loop_vinfo))
+ {
+ tree ffr_type = LOOP_VINFO_MASKS (loop_vinfo).rgc_vec[0].type;
+
+ tree ffr_ssa = make_temp_ssa_name (ffr_type, NULL, "ffr_preservation");
+
+ gcall *call = gimple_build_call_internal (IFN_READ_FAULT_STATE, 0);
+ gimple_set_lhs (call, ffr_ssa);
+ gsi_insert_on_edge_immediate (loop_preheader_edge (loop), call);
+
+
+ /* Initializze the first fault state to all true. */
+ gcall *setffrcall
+ = gimple_build_call_internal (IFN_SET_FAULT_STATE, 1,
+ build_all_ones_cst (ffr_type));
+ gsi_insert_on_edge_immediate (loop_preheader_edge (loop), setffrcall);
+
+ auto_vec<edge> exits = get_loop_exit_edges (loop);
+ for (edge e: exits) {
+ gcall *call
+ = gimple_build_call_internal (IFN_SET_FAULT_STATE, 1, ffr_ssa);
+
+ gsi_insert_on_edge_immediate (e, call);
+ }
+
+ vect_add_ffr_fixup_controls (loop_vinfo);
+ update_ssa (TODO_update_ssa_only_virtuals);
+ }
+
/* When we have unrolled the loop due to a user requested value we should
leave it up to the RTL unroll heuristics to determine if it's still worth
while to unroll more. */
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index 8b0a4348d0f..729e6566af2 100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -11971,6 +11971,12 @@ vect_schedule_slp_node (vec_info *vinfo,
}
}
}
+
+ if (SLP_TREE_FFR_REGION (node))
+ if (vect_stmt_dominates_stmt_p
+ (last_stmt, SLP_TREE_FFR_REGION (node)->ffr_read_point))
+ last_stmt = SLP_TREE_FFR_REGION (node)->ffr_read_point;
+
/* This can happen when all children are pre-existing vectors or
constants. */
if (!last_stmt)
@@ -12434,6 +12440,11 @@ vect_schedule_slp (vec_info *vinfo, const
vec<slp_instance> &slp_instances)
int maxdfs = 0;
FOR_EACH_VEC_ELT (slp_instances, i, instance)
{
+ /* Insert the call to read the FFR now as we need it for data
dependencies
+ when scheduling, the actual fixup logic will be added later. */
+ if (SLP_INSTANCE_FFR_REGION (instance))
+ vect_add_ffr_read (vinfo, instance);
+
slp_tree node = SLP_INSTANCE_TREE (instance);
if (dump_enabled_p ())
{
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index 427adce4594..acb2b289fd0 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -58,6 +58,7 @@ along with GCC; see the file COPYING3. If not see
#include "attribs.h"
#include "optabs-libfuncs.h"
#include "tree-dfa.h"
+#include "tree-into-ssa.h"
/* For lang_hooks.types.type_for_mode. */
#include "langhooks.h"
@@ -1302,7 +1303,9 @@ vect_finish_replace_stmt (vec_info *vinfo,
}
/* Add VEC_STMT to the vectorized implementation of STMT_INFO and insert it
- before *GSI. Create and return a stmt_vec_info for VEC_STMT. */
+ before *GSI. Create and return a stmt_vec_info for VEC_STMT.
+
+ If this is a FFR, insert the new load at the FFR insertion point. */
void
vect_finish_stmt_generation (vec_info *vinfo,
@@ -11630,18 +11633,43 @@ vectorizable_load (vec_info *vinfo,
}
else if (final_mask)
{
- tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
- vec_els = vect_get_mask_load_else (maskload_elsval, vectype);
- if (type_mode_padding_p
- && maskload_elsval != MASK_LOAD_ELSE_ZERO)
- need_zeroing = true;
- gcall *call = gimple_build_call_internal (IFN_MASK_LOAD, 4,
- dataref_ptr, ptr,
- final_mask,
- vec_els);
- gimple_call_set_nothrow (call, true);
- new_stmt = call;
- data_ref = NULL_TREE;
+ if (LOOP_VINFO_USING_FFR_P (loop_vinfo)
+ && dr_safe_speculative_read_required (stmt_info)
+ && ls.defines_region)
+ {
+ /* Do the FFR read, and then read the FFR and save it for
+ this r-group. There should only be one, so we should be
+ safe... */
+ tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
+ vec_els = vect_get_mask_load_else (maskload_elsval,
+ vectype);
+ if (type_mode_padding_p
+ && maskload_elsval != MASK_LOAD_ELSE_ZERO)
+ need_zeroing = true;
+ gcall *call
+ = gimple_build_call_internal (IFN_MASK_FIRSTFAULT_LOAD, 4,
+ dataref_ptr, ptr,
+ final_mask, vec_els);
+ gimple_call_set_nothrow (call, true);
+ new_stmt = call;
+ data_ref = NULL_TREE;
+ }
+ else
+ {
+ tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
+ vec_els = vect_get_mask_load_else (maskload_elsval,
+ vectype);
+ if (type_mode_padding_p
+ && maskload_elsval != MASK_LOAD_ELSE_ZERO)
+ need_zeroing = true;
+ gcall *call = gimple_build_call_internal (IFN_MASK_LOAD, 4,
+ dataref_ptr, ptr,
+ final_mask,
+ vec_els);
+ gimple_call_set_nothrow (call, true);
+ new_stmt = call;
+ data_ref = NULL_TREE;
+ }
}
else
{
@@ -11988,7 +12016,7 @@ vectorizable_load (vec_info *vinfo,
/* Store vector loads in the corresponding SLP_NODE. */
if (!costing_p && !ls.slp_perm)
- slp_node->push_vec_def (new_stmt);
+ slp_node->push_vec_def (new_stmt, true);
/* With SLP permutation we load the gaps as well, without
we need to skip the gaps after we manage to fully load
@@ -12066,6 +12094,11 @@ vectorizable_load (vec_info *vinfo,
slp_node->data = new vect_load_store_data (std::move (ls));
}
+ /* If we used a FFR on this, we need to update the virtual ssa nodes because
+ of the FFR read. */
+ if (loop_vinfo && LOOP_VINFO_USING_FFR_P (loop_vinfo)
+ && dr_safe_speculative_read_required (stmt_info))
+ update_ssa (TODO_update_ssa_only_virtuals);
return true;
}
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 8116a82ba2f..807452a0478 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -1358,8 +1358,9 @@ public:
#define LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS(L) (L)->partial_load_store_bias
#define LOOP_VINFO_VECT_FACTOR(L) (L)->vectorization_factor
#define LOOP_VINFO_IV_INCREMENT(L) (L)->iv_increment
-#define LOOP_VINFO_IV_INCREMENT_INVARIANT_P(L)\
- (!(L) || !LOOP_VINFO_USING_SELECT_VL_P (L))
+#define LOOP_VINFO_IV_INCREMENT_INVARIANT_P(L) (!(L)\
+ || ((!LOOP_VINFO_USING_SELECT_VL_P (L))\
+ && (!LOOP_VINFO_USING_FFR_P (L))))
#define LOOP_VINFO_MAX_VECT_FACTOR(L) (L)->max_vectorization_factor
#define LOOP_VINFO_MASKS(L) (L)->masks
#define LOOP_VINFO_LENS(L) (L)->lens
@@ -2555,6 +2556,8 @@ class loop *slpeel_tree_duplicate_loop_to_edge_cfg (class
loop *, edge,
bool = false, bool = false,
bool = true);
class loop *vect_loop_versioning (loop_vec_info, gimple *);
+extern void vect_add_ffr_fixup_controls (loop_vec_info);
+extern void vect_add_ffr_read (vec_info *, slp_instance);
extern bool vect_can_add_ffr_controls (loop_vec_info);
extern class loop *vect_do_peeling (loop_vec_info, tree, tree,
tree *, tree *, tree *, int, bool, bool,
--
2.34.1
