diff --git a/gcc/tree-vect-data-refs.cc b/gcc/tree-vect-data-refs.cc
index 6311e0d03df..4de1a15ae2c 100644
--- a/gcc/tree-vect-data-refs.cc
+++ b/gcc/tree-vect-data-refs.cc
@@ -6015,7 +6015,8 @@ vect_create_data_ref_ptr (vec_info *vinfo,
stmt_vec_info stmt_info,
create_iv (aggr_ptr_init, PLUS_EXPR,
iv_step, aggr_ptr, loop, &incr_gsi, insert_after,
- &indx_before_incr, &indx_after_incr);
+ &indx_before_incr, &indx_after_incr,
+ LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo));
incr = gsi_stmt (incr_gsi);
/* Copy the points-to information if it exists. */
@@ -6059,30 +6060,12 @@ vect_create_data_ref_ptr (vec_info *vinfo,
stmt_vec_info stmt_info,
/* Function bump_vector_ptr
- Increment a pointer (to a vector type) by vector-size. If requested,
- i.e. if PTR-INCR is given, then also connect the new increment stmt
- to the existing def-use update-chain of the pointer, by modifying
- the PTR_INCR as illustrated below:
-
- The pointer def-use update-chain before this function:
- DATAREF_PTR = phi (p_0, p_2)
- ....
- PTR_INCR: p_2 = DATAREF_PTR + step
-
- The pointer def-use update-chain after this function:
- DATAREF_PTR = phi (p_0, p_2)
- ....
- NEW_DATAREF_PTR = DATAREF_PTR + BUMP
- ....
- PTR_INCR: p_2 = NEW_DATAREF_PTR + step
+ Increment DATAREF_PTR by UPDATE.
Input:
DATAREF_PTR - ssa_name of a pointer (to vector type) that is being updated
in the loop.
- PTR_INCR - optional. The stmt that updates the pointer in each iteration of
- the loop. The increment amount across iterations is expected
- to be vector_size.
- BSI - location where the new update stmt is to be placed.
+ GSI - location where the new update stmt is to be placed.
STMT_INFO - the original scalar memory-access stmt that is being
vectorized.
UPDATE - The offset by which to bump the pointer.
@@ -6092,13 +6075,11 @@ vect_create_data_ref_ptr (vec_info *vinfo,
stmt_vec_info stmt_info,
tree
bump_vector_ptr (vec_info *vinfo,
- tree dataref_ptr, gimple *ptr_incr, gimple_stmt_iterator *gsi,
+ tree dataref_ptr, gimple_stmt_iterator *gsi,
stmt_vec_info stmt_info, tree update)
{
struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
gimple *incr_stmt;
- ssa_op_iter iter;
- use_operand_p use_p;
tree new_dataref_ptr;
if (TREE_CODE (dataref_ptr) == SSA_NAME)
@@ -6129,20 +6110,6 @@ bump_vector_ptr (vec_info *vinfo,
/* Copy the points-to information if it exists. */
duplicate_ssa_name_ptr_info (new_dataref_ptr, DR_PTR_INFO (dr));
- if (!ptr_incr)
- return new_dataref_ptr;
-
- /* Update the vector-pointer's cross-iteration increment. */
- FOR_EACH_SSA_USE_OPERAND (use_p, ptr_incr, iter, SSA_OP_USE)
- {
- tree use = USE_FROM_PTR (use_p);
-
- if (use == dataref_ptr)
- SET_USE (use_p, new_dataref_ptr);
- else
- gcc_assert (operand_equal_p (use, update, 0));
- }
-
return new_dataref_ptr;
}
diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc
index a64a79d3299..dd5b20f5a0e 100644
--- a/gcc/tree-vect-loop-manip.cc
+++ b/gcc/tree-vect-loop-manip.cc
@@ -514,7 +514,6 @@ vect_set_loop_controls_directly (class loop *loop,
loop_vec_info loop_vinfo,
tree ctrl_type = rgc->type;
unsigned int nitems_per_iter = rgc->max_nscalars_per_iter * rgc->factor;
poly_uint64 nitems_per_ctrl = TYPE_VECTOR_SUBPARTS (ctrl_type) * rgc-
factor;
- poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
tree length_limit = NULL_TREE;
/* For length, we need length_limit to ensure length in range. */
if (!use_masks_p)
@@ -525,7 +524,15 @@ vect_set_loop_controls_directly (class loop *loop,
loop_vec_info loop_vinfo,
of the vector loop, and the number that it should skip during the
first iteration of the vector loop. */
tree nitems_total = niters;
- tree nitems_step = build_int_cst (iv_type, vf);
+ tree nitems_vf
+ = build_int_cst (iv_type, LOOP_VINFO_VECT_FACTOR (loop_vinfo));
+ tree nitems_step
+ = LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)
+ ? gimple_convert (preheader_seq, iv_type,
+ LOOP_VINFO_IV_INCREMENT (loop_vinfo))
+ : gimple_convert (&loop_cond_gsi, true, GSI_SAME_STMT,
UNKNOWN_LOCATION,
+ iv_type, LOOP_VINFO_IV_INCREMENT (loop_vinfo));
+
tree nitems_skip = niters_skip;
if (nitems_per_iter != 1)
{
@@ -535,8 +542,14 @@ vect_set_loop_controls_directly (class loop *loop,
loop_vec_info loop_vinfo,
tree iv_factor = build_int_cst (iv_type, nitems_per_iter);
nitems_total = gimple_build (preheader_seq, MULT_EXPR, compare_type,
nitems_total, compare_factor);
- nitems_step = gimple_build (preheader_seq, MULT_EXPR, iv_type,
- nitems_step, iv_factor);
+ nitems_vf = gimple_build (preheader_seq, MULT_EXPR, iv_type,
+ nitems_vf, iv_factor);
+ nitems_step = LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)
+ ? gimple_build (preheader_seq, MULT_EXPR, iv_type,
+ nitems_step, iv_factor)
+ : gimple_build (&loop_cond_gsi, true, GSI_SAME_STMT,
+ UNKNOWN_LOCATION, MULT_EXPR,
iv_type,
+ nitems_step, iv_factor);
if (nitems_skip)
nitems_skip = gimple_build (preheader_seq, MULT_EXPR,
compare_type,
nitems_skip, compare_factor);
@@ -578,9 +591,21 @@ vect_set_loop_controls_directly (class loop *loop,
loop_vec_info loop_vinfo,
insert_after, &index_before_incr, &index_after_incr);
tree vectype = build_zero_cst (rgc->type);
tree len = gimple_build (header_seq, IFN_SELECT_VL, iv_type,
- index_before_incr, nitems_step,
+ index_before_incr, nitems_vf,
vectype);
gimple_seq_add_stmt (header_seq, gimple_build_assign (step, len));
+ len = gimple_convert (header_seq, sizetype, len);
+
+ /* Remove the previous initialiation of IV_INCREMENT to VARYING.
*/
+ gimple *varying_def
+ = SSA_NAME_DEF_STMT (LOOP_VINFO_IV_INCREMENT
(loop_vinfo));
+ auto def_gsi = gsi_for_stmt (varying_def);
+ gsi_remove (&def_gsi, true);
+
+ /* Set the LOOP_VINFO_IV_INCREMENT to be len. */
+ gassign* assign_iv_increment
+ = gimple_build_assign (LOOP_VINFO_IV_INCREMENT (loop_vinfo),
len);
+ gimple_seq_add_stmt (header_seq, assign_iv_increment);
}
else
{
@@ -593,7 +618,7 @@ vect_set_loop_controls_directly (class loop *loop,
loop_vec_info loop_vinfo,
nitems_step));
}
*iv_step = step;
- *compare_step = nitems_step;
+ *compare_step = nitems_vf;
return LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo) ? index_after_incr
: index_before_incr;
}
@@ -601,7 +626,8 @@ vect_set_loop_controls_directly (class loop *loop,
loop_vec_info loop_vinfo,
/* Create increment IV. */
create_iv (build_int_cst (iv_type, 0), PLUS_EXPR, nitems_step, NULL_TREE,
loop, &incr_gsi, insert_after, &index_before_incr,
- &index_after_incr);
+ &index_after_incr,
+ LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo));
tree zero_index = build_int_cst (compare_type, 0);
tree test_index, test_limit, first_limit;
@@ -636,7 +662,7 @@ vect_set_loop_controls_directly (class loop *loop,
loop_vec_info loop_vinfo,
COMPARE_TYPE. */
test_index = index_before_incr;
tree adjust = gimple_convert (preheader_seq, compare_type,
- nitems_step);
+ nitems_vf);
if (nitems_skip)
adjust = gimple_build (preheader_seq, MINUS_EXPR, compare_type,
adjust, nitems_skip);
@@ -1040,14 +1066,16 @@
vect_set_loop_condition_partial_vectors_avx512 (class loop *loop,
iv_type, niters, skip);
}
- /* The iteration step is the vectorization factor. */
- tree iv_step = build_int_cst (iv_type, vf);
-
- /* Create the decrement IV. */
- tree index_before_incr, index_after_incr;
gimple_stmt_iterator incr_gsi;
+ tree index_before_incr, index_after_incr;
bool insert_after;
vect_iv_increment_position (exit_edge, &incr_gsi, &insert_after);
+
+ /* The iteration step is the vectorization factor. */
+ tree iv_step = gimple_convert (&preheader_seq, iv_type,
+ LOOP_VINFO_IV_INCREMENT (loop_vinfo));
+
+ /* Create the decrement IV. */
create_iv (niters_adj, MINUS_EXPR, iv_step, NULL_TREE, loop,
&incr_gsi, insert_after, &index_before_incr,
&index_after_incr);
@@ -1230,7 +1258,7 @@ vect_set_loop_condition_partial_vectors_avx512
(class loop *loop,
loop handles exactly VF scalars per iteration. */
static gcond *
-vect_set_loop_condition_normal (loop_vec_info /* loop_vinfo */, edge
exit_edge,
+vect_set_loop_condition_normal (loop_vec_info loop_vinfo, edge exit_edge,
class loop *loop, tree niters, tree step,
tree final_iv, bool niters_maybe_zero,
gimple_stmt_iterator loop_cond_gsi)
@@ -1325,7 +1353,10 @@ vect_set_loop_condition_normal (loop_vec_info
/* loop_vinfo */, edge exit_edge,
vect_iv_increment_position (exit_edge, &incr_gsi, &insert_after);
create_iv (init, PLUS_EXPR, step, NULL_TREE, loop,
- &incr_gsi, insert_after, &indx_before_incr, &indx_after_incr);
+ &incr_gsi, insert_after,
+ &indx_before_incr, &indx_after_incr,
+ LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo));
+
indx_after_incr = force_gimple_operand_gsi (&loop_cond_gsi,
indx_after_incr,
true, NULL_TREE, true,
GSI_SAME_STMT);
@@ -2979,7 +3010,8 @@ vect_gen_vector_loop_niters (loop_vec_info
loop_vinfo, tree niters,
/* To silence some unexpected warnings, simply initialize to 0. */
unsigned HOST_WIDE_INT const_vf = 0;
if (vf.is_constant (&const_vf)
- && !LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
+ && !LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
+ && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
{
/* Create: niters / vf, which is equivalent to niters >> log2(vf) when
vf is a power of two, and when not we approximate using a
@@ -3054,6 +3086,30 @@ vect_gen_vector_loop_niters (loop_vec_info
loop_vinfo, tree niters,
return;
}
+/* Finds the amount IV's should be incremented by each iteration.
+ Stored in LOOP_VINFO_IV_INCREMENT. */
+
+tree
+vect_get_loop_iv_increment (loop_vec_info loop_vinfo)
+{
+ if (LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo))
+ {
+ /* For now, set this as varying.
+ Fill this in later when building the loop controls. */
+ tree iv_increment = make_temp_ssa_name (sizetype, NULL,
"iv_increment");
+
+ gcall *varying_call = gimple_build_call_internal (IFN_VARYING, 0);
+ gimple_call_set_lhs (varying_call, iv_increment);
+
+ gimple_stmt_iterator gsi = gsi_after_labels (loop_vinfo->loop->header);
+ gsi_insert_before (&gsi, varying_call, GSI_NEW_STMT);
+
+ return iv_increment;
+ }
+ else
+ return build_int_cst (sizetype, LOOP_VINFO_VECT_FACTOR (loop_vinfo));
+}
+
/* Given NITERS_VECTOR which is the number of iterations for vectorized
loop specified by LOOP_VINFO after vectorization, compute the number
of iterations before vectorization (niters_vector * vf) and store it
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index 0167d52b28b..931b942d7cb 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -9524,9 +9524,7 @@ vectorizable_induction (loop_vec_info loop_vinfo,
tree new_name;
gphi *induction_phi;
tree induc_def, vec_dest;
- poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
unsigned i;
- tree expr;
tree index_vectype = NULL_TREE;
gimple_stmt_iterator si;
enum vect_induction_op_type induction_type
@@ -9633,7 +9631,8 @@ vectorizable_induction (loop_vec_info loop_vinfo,
if (!target_supports_op_p (step_vectype, PLUS_EXPR, optab_default)
|| !target_supports_op_p (step_vectype, MINUS_EXPR, optab_default))
return false;
- if (!nunits.is_constant ())
+ if (!nunits.is_constant ()
+ || !LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
{
if (!target_supports_op_p (step_vectype, MULT_EXPR, optab_default))
return false;
@@ -9734,7 +9733,8 @@ vectorizable_induction (loop_vec_info loop_vinfo,
unsigned HOST_WIDE_INT const_nunits;
if (nested_in_vect_loop)
nivs = nvects;
- else if (nunits.is_constant (&const_nunits))
+ else if (nunits.is_constant (&const_nunits)
+ && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
{
/* Compute the number of distinct IVs we need. First reduce
group_size if it is a multiple of const_nunits so we get
@@ -9750,10 +9750,12 @@ vectorizable_induction (loop_vec_info
loop_vinfo,
nivs = 1;
}
gimple_seq init_stmts = NULL;
+ gimple_seq lupdate_mul_stmts = NULL;
tree lupdate_mul = NULL_TREE;
if (!nested_in_vect_loop)
{
- if (nunits.is_constant (&const_nunits))
+ if (nunits.is_constant (&const_nunits)
+ && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
{
/* The number of iterations covered in one vector iteration. */
unsigned lup_mul = (nvects * const_nunits) / group_size;
@@ -9766,14 +9768,23 @@ vectorizable_induction (loop_vec_info
loop_vinfo,
}
else
{
+ gimple_seq *update_stmts
+ = LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)
+ ? &init_stmts
+ : &lupdate_mul_stmts;
if (SCALAR_FLOAT_TYPE_P (stept))
{
- tree tem = build_int_cst (integer_type_node, vf);
- lupdate_mul = gimple_build (&init_stmts, FLOAT_EXPR, stept,
tem);
+ tree increment
+ = gimple_convert (update_stmts, integer_type_node,
+ LOOP_VINFO_IV_INCREMENT (loop_vinfo));
+ lupdate_mul = gimple_build (update_stmts, FLOAT_EXPR, stept,
+ increment);
}
else
- lupdate_mul = build_int_cst (stept, vf);
- lupdate_mul = gimple_build_vector_from_val (&init_stmts,
step_vectype,
+ lupdate_mul = gimple_convert (update_stmts, stept,
+ LOOP_VINFO_IV_INCREMENT
(loop_vinfo));
+ lupdate_mul = gimple_build_vector_from_val (update_stmts,
+ step_vectype,
lupdate_mul);
}
}
@@ -9796,7 +9807,8 @@ vectorizable_induction (loop_vec_info loop_vinfo,
{
gimple_seq stmts = NULL;
bool invariant = true;
- if (nunits.is_constant (&const_nunits))
+ if (nunits.is_constant (&const_nunits)
+ && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
{
tree_vector_builder step_elts (step_vectype, const_nunits, 1);
tree_vector_builder init_elts (vectype, const_nunits, 1);
@@ -9904,36 +9916,12 @@ vectorizable_induction (loop_vec_info
loop_vinfo,
/* Create the iv update inside the loop */
tree up = vec_step;
if (lupdate_mul)
- {
- if (LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo))
- {
- /* When we're using loop_len produced by SELEC_VL, the
- non-final iterations are not always processing VF
- elements. So vectorize induction variable instead of
-
- _21 = vect_vec_iv_.6_22 + { VF, ... };
-
- We should generate:
-
- _35 = .SELECT_VL (ivtmp_33, VF);
- vect_cst__22 = [vec_duplicate_expr] _35;
- _21 = vect_vec_iv_.6_22 + vect_cst__22; */
- vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
- tree len = vect_get_loop_len (loop_vinfo, NULL, lens, 1,
- vectype, 0, 0, false);
- if (SCALAR_FLOAT_TYPE_P (stept))
- expr = gimple_build (&stmts, FLOAT_EXPR, stept, len);
- else
- expr = gimple_convert (&stmts, stept, len);
- lupdate_mul = gimple_build_vector_from_val (&stmts,
step_vectype,
- expr);
- up = gimple_build (&stmts, MULT_EXPR,
- step_vectype, vec_step, lupdate_mul);
- }
- else
- up = gimple_build (&init_stmts, MULT_EXPR, step_vectype,
- vec_step, lupdate_mul);
- }
+ {
+ if (lupdate_mul_stmts)
+ gimple_seq_add_seq (&stmts, lupdate_mul_stmts);
+ up = gimple_build (&stmts, MULT_EXPR, step_vectype, vec_step,
+ lupdate_mul);
+ }
vec_def = gimple_convert (&stmts, step_vectype, induc_def);
vec_def = gimple_build (&stmts, PLUS_EXPR, step_vectype, vec_def, up);
vec_def = gimple_convert (&stmts, vectype, vec_def);
@@ -9964,7 +9952,8 @@ vectorizable_induction (loop_vec_info loop_vinfo,
if (!nested_in_vect_loop)
{
/* Fill up to the number of vectors we need for the whole group. */
- if (nunits.is_constant (&const_nunits))
+ if (nunits.is_constant (&const_nunits)
+ && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
nivs = least_common_multiple (group_size, const_nunits) /
const_nunits;
else
nivs = 1;
@@ -9980,7 +9969,8 @@ vectorizable_induction (loop_vec_info loop_vinfo,
stmts by adding VF' * stride to the IVs generated above. */
if (ivn < nvects)
{
- if (nunits.is_constant (&const_nunits))
+ if (nunits.is_constant (&const_nunits)
+ && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
{
unsigned vfp = (least_common_multiple (group_size, const_nunits)
/ group_size);
@@ -11073,7 +11063,6 @@
vect_update_ivs_after_vectorizer_for_early_breaks (loop_vec_info loop_vinfo)
tree phi_var = LOOP_VINFO_EARLY_BRK_NITERS_VAR (loop_vinfo);
tree niters_skip = LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo);
- poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
tree ty_var = TREE_TYPE (phi_var);
auto loop = LOOP_VINFO_LOOP (loop_vinfo);
tree induc_var = niters_skip ? copy_ssa_name (phi_var) : phi_var;
@@ -11090,27 +11079,15 @@
vect_update_ivs_after_vectorizer_for_early_breaks (loop_vec_info loop_vinfo)
gimple_seq init_stmts = NULL;
gimple_seq stmts = NULL;
gimple_seq iv_stmts = NULL;
- tree tree_vf = build_int_cst (ty_var, vf);
-
- /* For loop len targets we have to use .SELECT_VL (ivtmp_33, VF); instead of
- just += VF as the VF can change in between two loop iterations. */
- if (LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo))
- {
- vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
- tree_vf = vect_get_loop_len (loop_vinfo, NULL, lens, 1,
- NULL_TREE, 0, 0, true);
- }
+ tree tree_iv_incr = LOOP_VINFO_IV_INCREMENT (loop_vinfo);
tree iter_var;
if (POINTER_TYPE_P (ty_var))
- {
- tree offset = gimple_convert (&stmts, sizetype, tree_vf);
- iter_var = gimple_build (&stmts, POINTER_PLUS_EXPR, ty_var, induc_def,
- gimple_convert (&stmts, sizetype, offset));
- }
+ iter_var = gimple_build (&stmts, POINTER_PLUS_EXPR, ty_var, induc_def,
+ tree_iv_incr);
else
{
- tree offset = gimple_convert (&stmts, ty_var, tree_vf);
+ tree offset = gimple_convert (&stmts, ty_var, tree_iv_incr);
iter_var = gimple_build (&stmts, PLUS_EXPR, ty_var, induc_def, offset);
}
@@ -11157,10 +11134,17 @@
vect_update_ivs_after_vectorizer_for_early_breaks (loop_vec_info loop_vinfo)
/* Write the init_stmts in the loop-preheader block. */
auto psi = gsi_last_nondebug_bb (pe->src);
gsi_insert_seq_after (&psi, init_stmts, GSI_LAST_NEW_STMT);
- /* Write the adjustments in the header block. */
- basic_block bb = loop->header;
- auto si = gsi_after_labels (bb);
- gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT);
+
+ /* Write the adjustments at the end of the iv increment. */
+ bool insert_after;
+ gimple_stmt_iterator incr_gsi;
+ vect_iv_increment_position (LOOP_VINFO_MAIN_EXIT (loop_vinfo),
&incr_gsi,
+ &insert_after);
+
+ if (insert_after)
+ gsi_insert_seq_after (&incr_gsi, stmts, GSI_NEW_STMT);
+ else
+ gsi_insert_seq_before (&incr_gsi, stmts, GSI_NEW_STMT);
}
/* Function vect_transform_loop.
@@ -11259,6 +11243,9 @@ vect_transform_loop (loop_vec_info loop_vinfo,
gimple *loop_vectorized_call)
check_profitability, niters_no_overflow,
&advance);
+ LOOP_VINFO_IV_INCREMENT (loop_vinfo)
+ = vect_get_loop_iv_increment (loop_vinfo);
+
/* Assign hierarchical discriminators to the vectorized loop. */
poly_uint64 vf_val = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
unsigned int vf_int = constant_lower_bound (vf_val);
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index b69f9de5de8..bc1a99bf571 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -3170,44 +3170,35 @@ vect_get_gather_scatter_ops (class loop *loop,
slp_tree slp_node,
the gather load or scatter store operation described by GS_INFO.
STMT_INFO is the load or store statement.
- Set *DATAREF_BUMP to the amount that should be added to the base
- address after each copy of the vectorized statement. Set *VEC_OFFSET
- to an invariant offset vector in which element I has the value
- I * DR_STEP / SCALE. */
+ Set *DR_INCREMENT to the amount that should be added to pointer base
address
+ to get to the next iteration's base address.
+ Set *DR_BUMP to the amount that should be added to the base
+ address after each copy of the vectorized statement in a grouped read.
+ Set *VEC_OFFSET to an invariant offset vector in which element I has the
+ value I * DR_STEP / SCALE. */
static void
vect_get_strided_load_store_ops (stmt_vec_info stmt_info, slp_tree node,
tree vectype, tree offset_vectype,
loop_vec_info loop_vinfo,
gimple_stmt_iterator *gsi,
- tree *dataref_bump, tree *vec_offset,
- vec_loop_lens *loop_lens)
+ tree *dr_step, tree *dr_bump,
+ tree *vec_offset)
{
struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
- if (LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo))
- {
- /* _31 = .SELECT_VL (ivtmp_29, POLY_INT_CST [4, 4]);
- ivtmp_8 = _31 * 16 (step in bytes);
- .MASK_LEN_SCATTER_STORE (vectp_a.9_7, ... );
- vectp_a.9_26 = vectp_a.9_7 + ivtmp_8; */
- tree loop_len
- = vect_get_loop_len (loop_vinfo, gsi, loop_lens, 1, vectype, 0, 0,
true);
- tree tmp
- = fold_build2 (MULT_EXPR, sizetype,
- fold_convert (sizetype, unshare_expr (DR_STEP (dr))),
- loop_len);
- *dataref_bump = force_gimple_operand_gsi (gsi, tmp, true, NULL_TREE,
true,
- GSI_SAME_STMT);
- }
- else
- {
- tree bump
- = size_binop (MULT_EXPR,
- fold_convert (sizetype, unshare_expr (DR_STEP (dr))),
- size_int (TYPE_VECTOR_SUBPARTS (vectype)));
- *dataref_bump = cse_and_gimplify_to_preheader (loop_vinfo, bump);
- }
+ tree dr_step_temp
+ = size_binop (MULT_EXPR,
+ fold_convert (sizetype, unshare_expr (DR_STEP (dr))),
+ LOOP_VINFO_IV_INCREMENT (loop_vinfo));
+ *dr_step = LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)
+ ? cse_and_gimplify_to_preheader (loop_vinfo, dr_step_temp)
+ : force_gimple_operand_gsi (gsi, dr_step_temp, false, NULL_TREE,
+ true, GSI_SAME_STMT);
+ tree bump = size_binop (MULT_EXPR,
+ fold_convert (sizetype, unshare_expr (DR_STEP
(dr))),
+ size_int (TYPE_VECTOR_SUBPARTS (vectype)));
+ *dr_bump = cse_and_gimplify_to_preheader (loop_vinfo, bump);
internal_fn ifn
= DR_IS_READ (dr) ? IFN_MASK_LEN_STRIDED_LOAD :
IFN_MASK_LEN_STRIDED_STORE;
@@ -3233,47 +3224,31 @@ vect_get_strided_load_store_ops
(stmt_vec_info stmt_info, slp_tree node,
*vec_offset = cse_and_gimplify_to_preheader (loop_vinfo, offset);
}
-/* Prepare the pointer IVs which needs to be updated by a variable amount.
- Such variable amount is the outcome of .SELECT_VL. In this case, we can
- allow each iteration process the flexible number of elements as long as
- the number <= vf elements.
-
- Return data reference according to SELECT_VL.
- If new statements are needed, insert them before GSI. */
+/* Return the amount that should be added to a vector pointer, represented
by
+ DR_INFO, to increment to the next vectorized iteration. */
static tree
-vect_get_loop_variant_data_ptr_increment (
- vec_info *vinfo, tree aggr_type, gimple_stmt_iterator *gsi,
- vec_loop_lens *loop_lens, dr_vec_info *dr_info,
- vect_memory_access_type memory_access_type)
+vect_get_data_ptr_step (vec_info *vinfo, dr_vec_info *dr_info,
+ vect_memory_access_type memory_access_type)
{
+ if (memory_access_type == VMAT_INVARIANT)
+ return size_zero_node;
+
loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
- tree step = vect_dr_behavior (vinfo, dr_info)->step;
+
+ /* For BB SLP there is no next iteration. */
+ if (!loop_vinfo)
+ return build_zero_cst (sizetype);
+
+ tree step = vect_dr_behavior (loop_vinfo, dr_info)->step;
/* gather/scatter never reach here. */
gcc_assert (!mat_gather_scatter_p (memory_access_type));
- /* When we support SELECT_VL pattern, we dynamic adjust
- the memory address by .SELECT_VL result.
+ tree iv_increment = LOOP_VINFO_IV_INCREMENT (loop_vinfo);
- The result of .SELECT_VL is the number of elements to
- be processed of each iteration. So the memory address
- adjustment operation should be:
-
- addr = addr + .SELECT_VL (ARG..) * step;
- */
- tree loop_len
- = vect_get_loop_len (loop_vinfo, gsi, loop_lens, 1, aggr_type, 0, 0, true);
- tree len_type = TREE_TYPE (loop_len);
- /* Since the outcome of .SELECT_VL is element size, we should adjust
- it into bytesize so that it can be used in address pointer variable
- amount IVs adjustment. */
- tree tmp = fold_build2 (MULT_EXPR, len_type, loop_len,
- wide_int_to_tree (len_type, wi::to_widest (step)));
- tree bump = make_temp_ssa_name (len_type, NULL, "ivtmp");
- gassign *assign = gimple_build_assign (bump, tmp);
- gsi_insert_before (gsi, assign, GSI_SAME_STMT);
- return bump;
+ return fold_build2 (MULT_EXPR, sizetype, iv_increment,
+ fold_convert (sizetype, step));
}
/* Return the amount that should be added to a vector pointer to move
@@ -3282,19 +3257,18 @@ vect_get_loop_variant_data_ptr_increment (
vectorization. */
static tree
-vect_get_data_ptr_increment (vec_info *vinfo, gimple_stmt_iterator *gsi,
- dr_vec_info *dr_info, tree aggr_type,
- vect_memory_access_type memory_access_type,
- vec_loop_lens *loop_lens)
+vect_get_data_ptr_bump (vec_info *vinfo,
+ dr_vec_info *dr_info, tree aggr_type,
+ vect_memory_access_type memory_access_type)
{
if (memory_access_type == VMAT_INVARIANT)
return size_zero_node;
loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
+ /* We do not support SLP loads where num_vec != 1 with SELECT_VL so this
value
+ should never be needed. */
if (loop_vinfo && LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo))
- return vect_get_loop_variant_data_ptr_increment (vinfo, aggr_type, gsi,
- loop_lens, dr_info,
- memory_access_type);
+ return NULL_TREE;
tree iv_step = TYPE_SIZE_UNIT (aggr_type);
tree step = vect_dr_behavior (vinfo, dr_info)->step;
@@ -7945,19 +7919,12 @@ vectorizable_scan_store (vec_info *vinfo,
stmt_vec_info stmt_info,
perms[i] = vect_gen_perm_mask_checked (vectype, indices);
}
- vec_loop_lens *loop_lens
- = (loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)
- ? &LOOP_VINFO_LENS (loop_vinfo)
- : NULL);
-
tree vec_oprnd1 = NULL_TREE;
tree vec_oprnd2 = NULL_TREE;
tree vec_oprnd3 = NULL_TREE;
tree dataref_ptr = DR_BASE_ADDRESS (dr_info->dr);
tree dataref_offset = build_int_cst (ref_type, 0);
- tree bump = vect_get_data_ptr_increment (vinfo, gsi, dr_info,
- vectype, VMAT_CONTIGUOUS,
- loop_lens);
+ tree bump = vect_get_data_ptr_bump (vinfo, dr_info, vectype,
VMAT_CONTIGUOUS);
tree ldataref_ptr = NULL_TREE;
tree orig = NULL_TREE;
if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 &&
!inscan_var_store)
@@ -8131,7 +8098,6 @@ vectorizable_store (vec_info *vinfo,
enum vect_def_type mask_dt = vect_unknown_def_type;
tree dataref_ptr = NULL_TREE;
tree dataref_offset = NULL_TREE;
- gimple *ptr_incr = NULL;
int j;
stmt_vec_info first_stmt_info;
bool grouped_store;
@@ -8558,15 +8524,26 @@ vectorizable_store (vec_info *vinfo,
if (!costing_p)
{
ivstep = stride_step;
+
+ tree increment = fold_convert (TREE_TYPE (ivstep),
+ LOOP_VINFO_IV_INCREMENT
(loop_vinfo));
+
ivstep = fold_build2 (MULT_EXPR, TREE_TYPE (ivstep), ivstep,
- build_int_cst (TREE_TYPE (ivstep), vf));
+ increment);
standard_iv_increment_position (loop, &incr_gsi, &insert_after);
stride_base = cse_and_gimplify_to_preheader (loop_vinfo,
stride_base);
- ivstep = cse_and_gimplify_to_preheader (loop_vinfo, ivstep);
+ if (LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
+ ivstep = force_gimple_operand_gsi (&incr_gsi, unshare_expr
(ivstep),
+ true, NULL_TREE, true,
+ GSI_SAME_STMT);
+ else
+ ivstep = cse_and_gimplify_to_preheader (loop_vinfo, ivstep);
+
create_iv (stride_base, PLUS_EXPR, ivstep, NULL, loop, &incr_gsi,
- insert_after, &offvar, NULL);
+ insert_after, &offvar, NULL,
+ LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo));
stride_step = cse_and_gimplify_to_preheader (loop_vinfo,
stride_step);
}
@@ -8705,12 +8682,15 @@ vectorizable_store (vec_info *vinfo,
if (!known_eq (poffset, 0))
offset = size_int (poffset);
- tree bump;
+ tree dr_increment;
+ tree dr_bump;
+
tree vec_offset = NULL_TREE;
if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
{
aggr_type = NULL_TREE;
- bump = NULL_TREE;
+ dr_increment = NULL_TREE;
+ dr_bump = NULL_TREE;
}
else if (mat_gather_scatter_p (memory_access_type))
{
@@ -8721,7 +8701,8 @@ vectorizable_store (vec_info *vinfo,
vect_get_strided_load_store_ops (stmt_info, slp_node, vtype,
ls.strided_offset_vectype,
loop_vinfo, gsi,
- &bump, &vec_offset, loop_lens);
+ &dr_increment, &dr_bump,
+ &vec_offset);
}
}
else
@@ -8731,8 +8712,12 @@ vectorizable_store (vec_info *vinfo,
else
aggr_type = vectype;
if (!costing_p)
- bump = vect_get_data_ptr_increment (vinfo, gsi, dr_info, aggr_type,
- memory_access_type, loop_lens);
+ {
+ dr_increment = vect_get_data_ptr_step (vinfo, dr_info,
+ memory_access_type);
+ dr_bump = vect_get_data_ptr_bump (vinfo, dr_info, aggr_type,
+ memory_access_type);
+ }
}
if (loop_vinfo && mask_node && !costing_p)
@@ -8785,7 +8770,7 @@ vectorizable_store (vec_info *vinfo,
dataref_ptr
= vect_create_data_ref_ptr (vinfo, first_stmt_info,
aggr_type, NULL, offset,
&dummy,
- gsi, &ptr_incr, false, bump);
+ gsi, NULL, false,
dr_increment);
}
}
else if (!costing_p)
@@ -8793,8 +8778,8 @@ vectorizable_store (vec_info *vinfo,
gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
if (mask_node)
vec_mask = vec_masks[j];
- dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
- stmt_info, bump);
+ dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
+ stmt_info, dr_bump);
}
if (costing_p)
@@ -8941,16 +8926,16 @@ vectorizable_store (vec_info *vinfo,
dataref_ptr
= vect_create_data_ref_ptr (vinfo, first_stmt_info,
aggr_type, NULL, offset,
- &dummy, gsi, &ptr_incr,
false,
- bump);
+ &dummy, gsi, NULL, false,
+ dr_increment);
}
}
else if (!costing_p)
{
gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
if (!STMT_VINFO_GATHER_SCATTER_P (stmt_info))
- dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr,
- gsi, stmt_info, bump);
+ dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr,
+ gsi, stmt_info, dr_bump);
}
new_stmt = NULL;
@@ -9347,8 +9332,8 @@ vectorizable_store (vec_info *vinfo,
else if (!costing_p)
dataref_ptr = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
simd_lane_access_p ? loop : NULL,
- offset, &dummy, gsi, &ptr_incr,
- simd_lane_access_p, bump);
+ offset, &dummy, gsi, NULL,
+ simd_lane_access_p, dr_increment);
new_stmt = NULL;
gcc_assert (!grouped_store);
@@ -9398,8 +9383,8 @@ vectorizable_store (vec_info *vinfo,
if (i > 0)
/* Bump the vector pointer. */
- dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
- stmt_info, bump);
+ dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info,
+ dr_bump);
unsigned misalign;
unsigned HOST_WIDE_INT align;
@@ -9722,7 +9707,6 @@ vectorizable_load (vec_info *vinfo,
tree dummy;
tree dataref_ptr = NULL_TREE;
tree dataref_offset = NULL_TREE;
- gimple *ptr_incr = NULL;
int i, j;
unsigned int group_size;
poly_uint64 group_gap_adj;
@@ -10231,16 +10215,22 @@ vectorizable_load (vec_info *vinfo,
vectemp = {tmp1, tmp2, ...}
*/
- ivstep = fold_build2 (MULT_EXPR, TREE_TYPE (stride_step),
stride_step,
- build_int_cst (TREE_TYPE (stride_step), vf));
+ tree increment = fold_convert (TREE_TYPE (stride_step),
+ LOOP_VINFO_IV_INCREMENT
(loop_vinfo));
+ ivstep = fold_build2 (MULT_EXPR, TREE_TYPE (stride_step),
+ stride_step, increment);
standard_iv_increment_position (loop, &incr_gsi, &insert_after);
stride_base = cse_and_gimplify_to_preheader (loop_vinfo,
stride_base);
- ivstep = cse_and_gimplify_to_preheader (loop_vinfo, ivstep);
- create_iv (stride_base, PLUS_EXPR, ivstep, NULL,
- loop, &incr_gsi, insert_after,
- &offvar, NULL);
+ if (LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
+ ivstep = cse_and_gimplify_to_preheader (loop_vinfo, ivstep);
+ else
+ ivstep = force_gimple_operand_gsi (&incr_gsi, unshare_expr
(ivstep),
+ true, NULL_TREE, true,
+ GSI_SAME_STMT);
+ create_iv (stride_base, PLUS_EXPR, ivstep, NULL, loop, &incr_gsi,
+ insert_after, &offvar, NULL, true);
stride_step = cse_and_gimplify_to_preheader (loop_vinfo,
stride_step);
}
@@ -10710,7 +10700,8 @@ vectorizable_load (vec_info *vinfo,
? size_binop (PLUS_EXPR, offset, size_int (poffset))
: size_int (poffset));
- tree bump;
+ tree dr_increment;
+ tree dr_bump;
tree vec_offset = NULL_TREE;
auto_vec<tree> vec_offsets;
@@ -10730,8 +10721,12 @@ vectorizable_load (vec_info *vinfo,
aggr_type = build_array_type_nelts (elem_type, group_size * nunits);
if (!costing_p)
- bump = vect_get_data_ptr_increment (vinfo, gsi, dr_info, aggr_type,
- memory_access_type, loop_lens);
+ {
+ dr_increment = vect_get_data_ptr_step (vinfo, dr_info,
+ memory_access_type);
+ dr_bump = vect_get_data_ptr_bump (vinfo, dr_info, aggr_type,
+ memory_access_type);
+ }
unsigned int inside_cost = 0, prologue_cost = 0;
/* For costing some adjacent vector loads, we'd like to cost with
@@ -10778,12 +10773,12 @@ vectorizable_load (vec_info *vinfo,
dataref_ptr
= vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
at_loop, offset, &dummy, gsi,
- &ptr_incr, false, bump);
+ NULL, false, dr_increment);
else
{
gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
- dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
- stmt_info, bump);
+ dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
+ stmt_info, dr_bump);
}
if (mask_node)
vec_mask = vec_masks[j];
@@ -10909,7 +10904,7 @@ vectorizable_load (vec_info *vinfo,
if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
{
aggr_type = NULL_TREE;
- bump = NULL_TREE;
+ dr_increment = NULL_TREE;
if (!costing_p)
vect_get_gather_scatter_ops (loop, slp_node, &dataref_ptr,
&vec_offsets);
@@ -10922,11 +10917,12 @@ vectorizable_load (vec_info *vinfo,
vect_get_strided_load_store_ops (stmt_info, slp_node, vectype,
ls.strided_offset_vectype,
loop_vinfo, gsi,
- &bump, &vec_offset, loop_lens);
+ &dr_increment, &dr_bump,
+ &vec_offset);
dataref_ptr
= vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
at_loop, offset, &dummy, gsi,
- &ptr_incr, false, bump);
+ NULL, false, dr_increment);
}
}
@@ -10950,8 +10946,8 @@ vectorizable_load (vec_info *vinfo,
final_mask, vec_mask, gsi);
if (i > 0 && !STMT_VINFO_GATHER_SCATTER_P (stmt_info))
- dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr,
- gsi, stmt_info, bump);
+ dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
+ stmt_info, dr_bump);
}
/* 2. Create the vector-load in the loop. */
@@ -11336,8 +11332,12 @@ vectorizable_load (vec_info *vinfo,
aggr_type = vectype;
if (!costing_p)
- bump = vect_get_data_ptr_increment (vinfo, gsi, dr_info, aggr_type,
- memory_access_type, loop_lens);
+ {
+ dr_increment = vect_get_data_ptr_step (vinfo, dr_info,
+ memory_access_type);
+ dr_bump = vect_get_data_ptr_bump (vinfo, dr_info, aggr_type,
+ memory_access_type);
+ }
poly_uint64 group_elt = 0;
unsigned int inside_cost = 0, prologue_cost = 0;
@@ -11368,8 +11368,8 @@ vectorizable_load (vec_info *vinfo,
dataref_ptr
= vect_create_data_ref_ptr (vinfo, first_stmt_info_for_drptr,
aggr_type, at_loop, offset, &dummy,
- gsi, &ptr_incr, simd_lane_access_p,
- bump);
+ gsi, NULL, simd_lane_access_p,
+ dr_increment);
/* Adjust the pointer by the difference to first_stmt. */
data_reference_p ptrdr
= STMT_VINFO_DATA_REF (first_stmt_info_for_drptr);
@@ -11377,7 +11377,7 @@ vectorizable_load (vec_info *vinfo,
size_binop (MINUS_EXPR,
DR_INIT (first_dr_info->dr),
DR_INIT (ptrdr)));
- dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
+ dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
stmt_info, diff);
if (alignment_support_scheme == dr_explicit_realign)
{
@@ -11393,17 +11393,17 @@ vectorizable_load (vec_info *vinfo,
dataref_ptr
= vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
at_loop,
- offset, &dummy, gsi, &ptr_incr,
- simd_lane_access_p, bump);
+ offset, &dummy, gsi, NULL,
+ simd_lane_access_p, dr_increment);
}
else if (!costing_p)
{
gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
if (dataref_offset)
- dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset, bump);
+ dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset,
dr_bump);
else
- dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
- stmt_info, bump);
+ dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info,
+ dr_bump);
}
auto_vec<tree> dr_chain;
@@ -11429,8 +11429,8 @@ vectorizable_load (vec_info *vinfo,
final_mask, vec_mask, gsi);
if (i > 0)
- dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr,
- gsi, stmt_info, bump);
+ dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info,
+ dr_bump);
}
/* 2. Create the vector-load in the loop. */
@@ -11776,8 +11776,7 @@ vectorizable_load (vec_info *vinfo,
bump = size_binop (MULT_EXPR, vs, TYPE_SIZE_UNIT (elem_type));
bump = size_binop (MINUS_EXPR, bump, size_one_node);
- ptr = bump_vector_ptr (vinfo, dataref_ptr, NULL, gsi, stmt_info,
- bump);
+ ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info, bump);
new_stmt = gimple_build_assign (NULL_TREE, BIT_AND_EXPR, ptr,
build_int_cst (TREE_TYPE (ptr),
-(HOST_WIDE_INT)
align));
@@ -11934,8 +11933,8 @@ vectorizable_load (vec_info *vinfo,
if (tree_int_cst_sgn (vect_dr_behavior (vinfo, dr_info)->step) == -1)
bump_val = -bump_val;
tree bump = wide_int_to_tree (sizetype, bump_val);
- dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
- stmt_info, bump);
+ dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info,
+ bump);
group_elt = 0;
}
}
@@ -11950,8 +11949,7 @@ vectorizable_load (vec_info *vinfo,
if (tree_int_cst_sgn (vect_dr_behavior (vinfo, dr_info)->step) == -1)
bump_val = -bump_val;
tree bump = wide_int_to_tree (sizetype, bump_val);
- dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
- stmt_info, bump);
+ dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info, bump);
}
if (ls.slp_perm)
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 0b646d6b29e..92444ad46b4 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -978,6 +978,10 @@ public:
it can be that no unrolling is needed, and thus this is 1. */
poly_uint64 vectorization_factor;
+ /* Gimple operand for the number of scalar iteration handed per loop
+ iteration, and therefore how much to increment each IV by. */
+ tree iv_increment;
+
/* If this loop is an epilogue loop whose main loop can be skipped,
MAIN_LOOP_EDGE is the edge from the main loop to this loop's
preheader. SKIP_MAIN_LOOP_EDGE is then the edge that skips the
@@ -1302,6 +1306,9 @@ public:
#define LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT(L) (L)-
allow_mutual_alignment
#define LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS(L) (L)-
partial_load_store_bias
#define LOOP_VINFO_VECT_FACTOR(L) (L)->vectorization_factor
+#define LOOP_VINFO_IV_INCREMENT(L) (L)->iv_increment
+#define LOOP_VINFO_IV_INCREMENT_INVARIANT_P(L)\
+ (!(L) || !LOOP_VINFO_USING_SELECT_VL_P (L))
