Various vectorizable_* functions have been updated to return false
to abort the analysis phase of vectorization immediately instead of
setting LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P to false if a partial
vector is required (i.e. the number of subparts in the vector type
may be greater than the number of active lanes for the node) but
not available (e.g., direct_internal_fn_supported_p returned false).
To avoid duplicating the controlling expression, a new helper,
can_use_partial_vectors_p, is used. When analyzing a loop, this
simply returns the result of LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P.
The vect_get_num_copies function used during statement analysis
is updated to use division away from zero for BB SLP. This avoids
an ICE in vect_get_num_vectors when the SLP group size is not an
integral multiple of the number of subparts in the vector type.
The return value of can_use_partial_vectors_p may sometimes appear
to have been assigned unnecessarily. That is to reduce the size of
other patches in this series.
Clear the chosen partial vector style if loop analysis ultimately decides
not to use partial vectors, to avoid producing SSA names for masks or
lengths that will not subsequently be defined.
gcc/ChangeLog:
* tree-vect-loop.cc (vect_analyze_loop_2): Clear
partial_vector_style if the loop does not use partial vectors.
* tree-vect-stmts.cc (can_use_partial_vectors_p): New helper
function to determine whether partial vectors can (still) be
used to vectorize a loop or SLP region. In the case of SLP,
partial vectors can only be used if they might be required
(based on the count of subparts passed by the caller, relative
to the group size).
(vectorizable_call): Call can_use_partial_vectors_p instead of
relying on LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P to determine
whether partial vectors can be used. If so, but no
conditional operation is available, then return false for BB SLP
instead of setting LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P to false.
Pass vec_info and the total number of active lanes across the
SLP vector sequence instead of loop_vec_info when calling
vect_record_len or vect_record_mask.
(vectorizable_simd_clone_call): Disallow SIMD clones for BB SLP
if a partial vector might be required at some possible vector
length.
(vectorizable_operation): Call can_use_partial_vectors_p instead of
relying on LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P to determine
whether partial vectors can be used. If so, but no
conditional operation is available, then return false for BB SLP
instead of setting LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P to false.
Pass vec_info and the total number of active lanes across the
SLP vector sequence instead of loop_vec_info when calling
vect_record_len or vect_record_mask.
(vectorizable_store): During the analysis phase, if doing BB SLP
vectorization, call check_load_store_for_partial_vectors as for
loop vectorization but propagate a return value of false up the
callstack instead of assigning the return value to
LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P.
(vectorizable_load): As above.
When transforming a permuted load, pass the size of the containing
group to vect_get_mask or vect_get_len (as the number of active
lanes), instead of passing the number of lanes in the SLP node.
(vectorizable_condition): Call can_use_partial_vectors_p instead
of relying on LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P to determine
whether partial vectors can be used. If so, but no
conditional operation is available, then return false for BB SLP
instead of setting LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P to false.
Pass vec_info and the total number of active lanes across the
SLP vector sequence instead of loop_vec_info when calling
vect_record_len or vect_record_mask.
* tree-vectorizer.h (vect_get_num_copies): Use division away from
zero for BB SLP to avoid an ICE in vect_get_num_vectors when the
SLP group size is not an integral multiple of the number of subparts
in the vector type.
---
gcc/tree-vect-loop.cc | 6 +
gcc/tree-vect-stmts.cc | 424 +++++++++++++++++++++++++++--------------
gcc/tree-vectorizer.h | 25 ++-
3 files changed, 305 insertions(+), 150 deletions(-)
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index 8d004a951f4..9c47fe63fe5 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -2405,6 +2405,12 @@ start_over:
if (!ok)
return ok;
+ /* Clear the chosen partial vector style if the loop does not use partial
+ vectors. This prevents production of SSA names for lengths or masks that
+ are not subsequently defined. */
+ if (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
+ LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo) = vect_partial_vectors_none;
+
/* If we're vectorizing a loop that uses length "controls" and
can iterate more than once, we apply decrementing IV approach
in loop control. */
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index c897166e8ef..edb76b3bf51 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -1476,7 +1476,7 @@ check_load_store_for_partial_vectors (vec_info *vinfo,
tree vectype,
vect_memory_access_type memory_access_type = ls->memory_access_type;
/* For SLP vectorization, we cannot use partial vectors if the
- group could be wider than the vector type. */
+ group could be wider than the vector type (e.g. 8 lanes of VNx4SI). */
poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
if (!loop_vinfo && maybe_gt (SLP_TREE_LANES (slp_node), nunits))
return false;
@@ -3455,6 +3455,23 @@ simple_integer_narrowing (tree vectype_out, tree
vectype_in,
return true;
}
+/* Return true if the loop or SLP vectorization region represented by
+ VINFO can use partial vectors. SLP_NODE can be null unless VINFO
+ represents an SLP region. NUNITS gives the number of subparts in the
+ vector type that would be used to vectorize stmts. */
+static bool
+can_use_partial_vectors_p (vec_info *vinfo, slp_tree slp_node,
+ poly_uint64 nunits)
+{
+ gcc_assert (!is_a <bb_vec_info> (vinfo) || slp_node);
+ loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
+
+ if (loop_vinfo)
+ return LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo);
+ else
+ return maybe_lt (SLP_TREE_LANES (slp_node), nunits);
+}
+
/* Function vectorizable_call.
Check if STMT_INFO performs a function call that can be vectorized.
@@ -3667,24 +3684,26 @@ vectorizable_call (vec_info *vinfo,
ifn = vectorizable_internal_function (cfn, callee, vectype_out,
vectype_in);
+ bool use_partial_vectors_p
+ = can_use_partial_vectors_p (vinfo, slp_node, nunits_in);
+
/* Check if the operation traps. */
bool could_trap = gimple_could_trap_p (STMT_VINFO_STMT (stmt_info));
- if (could_trap && cost_vec && loop_vinfo)
+ if (could_trap && cost_vec && use_partial_vectors_p)
{
/* If the operation can trap it must be conditional, otherwise fail. */
internal_fn cond_fn = (internal_fn_mask_index (ifn) != -1
- ? ifn : get_conditional_internal_fn (ifn));
+ ? ifn
+ : get_conditional_internal_fn (ifn));
internal_fn cond_len_fn = get_len_internal_fn (cond_fn);
- if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
+ if ((cond_fn == IFN_LAST
+ || !direct_internal_fn_supported_p (cond_fn, vectype_out,
+ OPTIMIZE_FOR_SPEED))
+ && (cond_len_fn == IFN_LAST
+ || !direct_internal_fn_supported_p (cond_len_fn, vectype_out,
+ OPTIMIZE_FOR_SPEED)))
{
- /* We assume that BB SLP fills all lanes, so no inactive lanes can
- cause issues. */
- if ((cond_fn == IFN_LAST
- || !direct_internal_fn_supported_p (cond_fn, vectype_out,
- OPTIMIZE_FOR_SPEED))
- && (cond_len_fn == IFN_LAST
- || !direct_internal_fn_supported_p (cond_len_fn, vectype_out,
- OPTIMIZE_FOR_SPEED)))
+ if (loop_vinfo)
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -3692,6 +3711,14 @@ vectorizable_call (vec_info *vinfo,
" conditional operation is available.\n");
LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
}
+ else
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "SLP can't use a mask or length because no"
+ " conditional operation is available.\n");
+ return false;
+ }
}
}
@@ -3757,9 +3784,8 @@ vectorizable_call (vec_info *vinfo,
DUMP_VECT_SCOPE ("vectorizable_call");
vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
- if (loop_vinfo
- && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
- && (reduc_idx >= 0 || could_trap || mask_opno >= 0))
+ if ((reduc_idx >= 0 || could_trap || mask_opno >= 0)
+ && use_partial_vectors_p)
{
if (reduc_idx >= 0
&& (cond_fn == IFN_LAST
@@ -3769,11 +3795,22 @@ vectorizable_call (vec_info *vinfo,
|| !direct_internal_fn_supported_p (cond_len_fn, vectype_out,
OPTIMIZE_FOR_SPEED)))
{
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "can't use a fully-masked loop because no"
- " conditional operation is available.\n");
- LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ if (loop_vinfo)
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "can't use a fully-masked loop because no"
+ " conditional operation is available.\n");
+ LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ }
+ else
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "SLP can't use a mask or length because no"
+ " conditional operation is available.\n");
+ return false;
+ }
}
else
{
@@ -3783,10 +3820,11 @@ vectorizable_call (vec_info *vinfo,
if (cond_len_fn != IFN_LAST
&& direct_internal_fn_supported_p (cond_len_fn, vectype_out,
OPTIMIZE_FOR_SPEED))
- vect_record_len (loop_vinfo, nvectors, vectype_out, 1);
+ vect_record_len (vinfo, nvectors, vectype_out, 1,
+ SLP_TREE_LANES (slp_node));
else
- vect_record_mask (loop_vinfo, nvectors, vectype_out,
- scalar_mask);
+ vect_record_mask (vinfo, nvectors, vectype_out, scalar_mask,
+ SLP_TREE_LANES (slp_node));
}
}
return true;
@@ -3801,8 +3839,15 @@ vectorizable_call (vec_info *vinfo,
scalar_dest = gimple_call_lhs (stmt);
vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
- bool masked_loop_p = loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
- bool len_loop_p = loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
+ bool masked_loop_p
+ = use_partial_vectors_p
+ && (vinfo->partial_vector_style == vect_partial_vectors_while_ult
+ || vinfo->partial_vector_style == vect_partial_vectors_avx512);
+
+ bool len_loop_p
+ = use_partial_vectors_p
+ && (vinfo->partial_vector_style == vect_partial_vectors_len);
+
unsigned int vect_nargs = nargs;
if (len_loop_p && (reduc_idx >= 0 || could_trap || mask_opno >= 0))
{
@@ -3866,8 +3911,9 @@ vectorizable_call (vec_info *vinfo,
if (masked_loop_p)
{
unsigned int vec_num = vec_oprnds0.length ();
- vargs[varg++] = vect_get_mask (loop_vinfo, gsi, vec_num,
- vectype_out, i);
+ vargs[varg++]
+ = vect_get_mask (vinfo, gsi, vec_num, vectype_out, i,
+ SLP_TREE_LANES (slp_node));
}
else
{
@@ -3926,10 +3972,10 @@ vectorizable_call (vec_info *vinfo,
if (len_opno >= 0 && len_loop_p)
{
unsigned int vec_num = vec_oprnds0.length ();
- tree len = vect_get_len (loop_vinfo, gsi, vec_num,
- vectype_out, i, 1, true);
- signed char biasval
- = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
+ tree len
+ = vect_get_len (vinfo, gsi, vec_num, vectype_out, i, 1,
+ true, SLP_TREE_LANES (slp_node));
+ signed char biasval = vinfo->partial_load_store_bias;
tree bias = build_int_cst (intQI_type_node, biasval);
vargs[len_opno] = len;
vargs[len_opno + 1] = bias;
@@ -3937,10 +3983,11 @@ vectorizable_call (vec_info *vinfo,
else if (mask_opno >= 0 && masked_loop_p)
{
unsigned int vec_num = vec_oprnds0.length ();
- tree mask = vect_get_mask (loop_vinfo, gsi, vec_num,
- vectype_out, i);
+ tree mask
+ = vect_get_mask (vinfo, gsi, vec_num, vectype_out, i,
+ SLP_TREE_LANES (slp_node));
vargs[mask_opno]
- = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
+ = prepare_vec_mask (vinfo, TREE_TYPE (mask), mask,
vargs[mask_opno], gsi);
}
@@ -4166,6 +4213,12 @@ vectorizable_simd_clone_call (vec_info *vinfo,
stmt_vec_info stmt_info,
if (loop_vinfo && nested_in_vect_loop_p (loop, stmt_info))
return false;
+ /* FORNOW Disallow SIMD clones if BB SLP vectorization might need a partial
+ vector at some possible vector length (including for all VLA types). */
+ if (!loop_vinfo
+ && maybe_lt (SLP_TREE_LANES (slp_node), TYPE_VECTOR_SUBPARTS (vectype)))
+ return false;
+
/* Process function arguments. */
nargs = gimple_call_num_args (stmt) - masked_call_offset;
@@ -5731,17 +5784,29 @@ vectorizable_conversion (vec_info *vinfo,
}
if (modifier == WIDEN
- && loop_vinfo
- && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
- && (code1 == VEC_WIDEN_MULT_EVEN_EXPR
- || widening_evenodd_fn_p (code1)))
+ && (code1 == VEC_WIDEN_MULT_EVEN_EXPR || widening_evenodd_fn_p (code1))
+ && can_use_partial_vectors_p (vinfo, slp_node, nunits_in))
{
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "can't use a fully-masked loop because"
- " widening operation on even/odd elements"
- " mixes up lanes.\n");
- LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ if (loop_vinfo)
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "can't use a fully-masked loop because"
+ " widening operation on even/odd elements"
+ " mixes up lanes.\n");
+ LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ }
+ else
+ {
+ gcc_checking_assert (
+ maybe_lt (SLP_TREE_LANES (slp_node), nunits_out));
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "SLP can't use a mask or length because"
+ " widening operation on even/odd elements"
+ " mixes up lanes.\n");
+ return false;
+ }
}
if (cost_vec) /* transformation not required. */
@@ -6869,27 +6934,41 @@ vectorizable_operation (vec_info *vinfo,
bool mask_out_inactive = ((!is_invariant && gimple_could_trap_p (stmt))
|| reduc_idx >= 0);
+ bool use_partial_vectors_p
+ = can_use_partial_vectors_p (vinfo, slp_node, nunits_in);
+
if (cost_vec) /* transformation not required. */
{
- if (loop_vinfo
- && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
- && mask_out_inactive)
+ if (mask_out_inactive && use_partial_vectors_p)
{
if (cond_len_fn != IFN_LAST
&& direct_internal_fn_supported_p (cond_len_fn, vectype,
OPTIMIZE_FOR_SPEED))
- vect_record_len (loop_vinfo, vec_num, vectype, 1);
+ vect_record_len (vinfo, vec_num, vectype, 1,
+ SLP_TREE_LANES (slp_node));
else if (cond_fn != IFN_LAST
&& direct_internal_fn_supported_p (cond_fn, vectype,
OPTIMIZE_FOR_SPEED))
- vect_record_mask (loop_vinfo, vec_num, vectype, NULL_TREE);
+ vect_record_mask (vinfo, vec_num, vectype, NULL_TREE,
+ SLP_TREE_LANES (slp_node));
else
{
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "can't use a fully-masked loop because no"
- " conditional operation is available.\n");
- LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ if (loop_vinfo)
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "can't use a fully-masked loop because no"
+ " conditional operation is available.\n");
+ LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ }
+ else
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "SLP can't use a mask or length because no"
+ " conditional operation is available.\n");
+ return false;
+ }
}
}
@@ -6949,8 +7028,13 @@ vectorizable_operation (vec_info *vinfo,
dump_printf_loc (MSG_NOTE, vect_location,
"transform binary/unary operation.\n");
- bool masked_loop_p = loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
- bool len_loop_p = loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
+ bool masked_loop_p
+ = use_partial_vectors_p
+ && (vinfo->partial_vector_style == vect_partial_vectors_while_ult
+ || vinfo->partial_vector_style == vect_partial_vectors_avx512);
+
+ bool len_loop_p = use_partial_vectors_p
+ && vinfo->partial_vector_style == vect_partial_vectors_len;
/* POINTER_DIFF_EXPR has pointer arguments which are vectorized as
vectors with unsigned elements, but the result is signed. So, we
@@ -7142,7 +7226,8 @@ vectorizable_operation (vec_info *vinfo,
{
tree mask;
if (masked_loop_p)
- mask = vect_get_mask (loop_vinfo, gsi, vec_num, vectype, i);
+ mask = vect_get_mask (vinfo, gsi, vec_num, vectype, i,
+ SLP_TREE_LANES (slp_node));
else
/* Dummy mask. */
mask = build_minus_one_cst (truth_type_for (vectype));
@@ -7168,10 +7253,9 @@ vectorizable_operation (vec_info *vinfo,
}
if (len_loop_p)
{
- tree len
- = vect_get_len (loop_vinfo, gsi, vec_num, vectype, i, 1, true);
- signed char biasval
- = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
+ tree len = vect_get_len (vinfo, gsi, vec_num, vectype, i, 1, true,
+ SLP_TREE_LANES (slp_node));
+ signed char biasval = vinfo->partial_load_store_bias;
tree bias = build_int_cst (intQI_type_node, biasval);
vops.quick_push (len);
vops.quick_push (bias);
@@ -7191,9 +7275,10 @@ vectorizable_operation (vec_info *vinfo,
tree mask = NULL_TREE;
/* When combining two masks check if either of them is elsewhere
combined with a loop mask, if that's the case we can mark that the
- new combined mask doesn't need to be combined with a loop mask. */
- if (masked_loop_p
- && code == BIT_AND_EXPR
+ new combined mask doesn't need to be combined with a loop mask.
+ Not applicable to BB SLP vectorization because two SLP nodes could
+ have the same OP0 but different group sizes or vector types. */
+ if (loop_vinfo && masked_loop_p && code == BIT_AND_EXPR
&& VECTOR_BOOLEAN_TYPE_P (vectype))
{
if (loop_vinfo->scalar_cond_masked_set.contains ({ op0, vec_num
}))
@@ -8351,15 +8436,28 @@ vectorizable_store (vec_info *vinfo,
return false;
}
+ bool use_partial_vectors_p
+ = can_use_partial_vectors_p (vinfo, slp_node, nunits);
+
bool costing_p = cost_vec;
if (costing_p) /* transformation not required. */
{
- if (loop_vinfo
- && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
- && !check_load_store_for_partial_vectors (loop_vinfo, vectype,
- slp_node, vls_type,
- group_size, &ls, mask_node))
- LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ if (use_partial_vectors_p
+ && !check_load_store_for_partial_vectors (vinfo, vectype, slp_node,
+ vls_type, group_size, &ls,
+ mask_node))
+ {
+ if (loop_vinfo)
+ LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ else
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (
+ MSG_MISSED_OPTIMIZATION, vect_location,
+ "SLP store needs but cannot use partial vectors.\n");
+ return false;
+ }
+ }
if (!vect_maybe_update_slp_op_vectype (op_node, vectype)
|| (mask_node
@@ -8735,8 +8833,14 @@ vectorizable_store (vec_info *vinfo,
}
gcc_assert (alignment_support_scheme);
- bool masks_p = loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
- bool lens_p = loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
+
+ bool masks_p
+ = use_partial_vectors_p
+ && (vinfo->partial_vector_style == vect_partial_vectors_while_ult
+ || vinfo->partial_vector_style == vect_partial_vectors_avx512);
+
+ bool lens_p = use_partial_vectors_p
+ && vinfo->partial_vector_style == vect_partial_vectors_len;
/* The vect_transform_stmt and vect_analyze_stmt will go here but there
are some difference here. We cannot enable both the lens and masks
@@ -8890,20 +8994,19 @@ vectorizable_store (vec_info *vinfo,
tree final_len = NULL;
tree bias = NULL;
if (masks_p)
- final_mask
- = vect_get_mask (loop_vinfo, gsi, ncopies, vectype, j);
+ final_mask = vect_get_mask (vinfo, gsi, ncopies, vectype, j,
+ SLP_TREE_LANES (slp_node));
if (vec_mask)
- final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
+ final_mask = prepare_vec_mask (vinfo, mask_vectype, final_mask,
vec_mask, gsi);
if (lanes_ifn == IFN_MASK_LEN_STORE_LANES)
{
if (lens_p)
- final_len = vect_get_len (loop_vinfo, gsi, ncopies, vectype, j,
- 1, true);
+ final_len = vect_get_len (vinfo, gsi, ncopies, vectype, j, 1,
+ true, SLP_TREE_LANES (slp_node));
else final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
- signed char biasval
- = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
+ signed char biasval = vinfo->partial_load_store_bias;
bias = build_int_cst (intQI_type_node, biasval);
if (!final_mask)
{
@@ -9031,11 +9134,11 @@ vectorizable_store (vec_info *vinfo,
if (!costing_p)
{
if (masks_p)
- final_mask
- = vect_get_mask (loop_vinfo, gsi, num_stmts, vectype, j);
+ final_mask = vect_get_mask (vinfo, gsi, num_stmts, vectype, j,
+ SLP_TREE_LANES (slp_node));
if (vec_mask)
- final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
- final_mask, vec_mask, gsi);
+ final_mask = prepare_vec_mask (vinfo, mask_vectype, final_mask,
+ vec_mask, gsi);
}
unsigned align = get_object_alignment (DR_REF (first_dr_info->dr));
@@ -9099,13 +9202,13 @@ vectorizable_store (vec_info *vinfo,
if (ls.gs.ifn == IFN_MASK_LEN_SCATTER_STORE)
{
if (lens_p)
- final_len = vect_get_len (loop_vinfo, gsi, num_stmts,
- vectype, j, 1, true);
+ final_len
+ = vect_get_len (vinfo, gsi, num_stmts, vectype, j, 1,
+ true, SLP_TREE_LANES (slp_node));
else
final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
- signed char biasval
- = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
+ signed char biasval = vinfo->partial_load_store_bias;
bias = build_int_cst (intQI_type_node, biasval);
if (!final_mask)
{
@@ -9447,12 +9550,13 @@ vectorizable_store (vec_info *vinfo,
tree final_len = NULL_TREE;
tree bias = NULL_TREE;
if (masks_p)
- final_mask = vect_get_mask (loop_vinfo, gsi, vec_num, vectype, i);
+ final_mask = vect_get_mask (vinfo, gsi, vec_num, vectype, i,
+ SLP_TREE_LANES (slp_node));
if (vec_mask)
vec_mask = vec_masks[i];
if (vec_mask)
- final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
- vec_mask, gsi);
+ final_mask
+ = prepare_vec_mask (vinfo, mask_vectype, final_mask, vec_mask, gsi);
if (i > 0)
/* Bump the vector pointer. */
@@ -9487,8 +9591,8 @@ vectorizable_store (vec_info *vinfo,
new_vmode = new_ovmode.require ();
unsigned factor
= (new_ovmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vmode);
- final_len
- = vect_get_len (loop_vinfo, gsi, vec_num, vectype, i, factor, true);
+ final_len = vect_get_len (vinfo, gsi, vec_num, vectype, i, factor,
+ true, SLP_TREE_LANES (slp_node));
}
else if (final_mask)
{
@@ -9516,7 +9620,7 @@ vectorizable_store (vec_info *vinfo,
}
if (final_len)
{
- signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
+ signed char biasval = vinfo->partial_load_store_bias;
bias = build_int_cst (intQI_type_node, biasval);
}
@@ -9997,17 +10101,28 @@ vectorizable_load (vec_info *vinfo,
}
}
- bool costing_p = cost_vec;
+ bool use_partial_vectors_p
+ = can_use_partial_vectors_p (vinfo, slp_node, nunits);
+ bool costing_p = cost_vec;
if (costing_p) /* transformation not required. */
{
- if (loop_vinfo
- && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
- && !check_load_store_for_partial_vectors (loop_vinfo, vectype,
- slp_node, VLS_LOAD,
- group_size, &ls, mask_node,
- &ls.elsvals))
- LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ if (use_partial_vectors_p
+ && !check_load_store_for_partial_vectors (vinfo, vectype, slp_node,
+ VLS_LOAD, group_size, &ls,
+ mask_node, &ls.elsvals))
+ {
+ if (loop_vinfo)
+ LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ else
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (
+ MSG_MISSED_OPTIMIZATION, vect_location,
+ "SLP load needs but cannot use partial vectors.\n");
+ return false;
+ }
+ }
/* If the type needs padding we must zero inactive elements.
Check if we can do that with a VEC_COND_EXPR and store the
@@ -10570,6 +10685,8 @@ vectorizable_load (vec_info *vinfo,
&& !ls.ls_type)
grouped_load = false;
+ unsigned int active_lanes = SLP_TREE_LANES (slp_node);
+
if (grouped_load
|| SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
{
@@ -10620,6 +10737,7 @@ vectorizable_load (vec_info *vinfo,
unsigned int const_nunits = nunits.to_constant ();
vec_num = CEIL (group_size * const_vf, const_nunits);
group_gap_adj = vf * group_size - nunits * vec_num;
+ active_lanes = group_size;
}
else
{
@@ -10637,8 +10755,13 @@ vectorizable_load (vec_info *vinfo,
ref_type = reference_alias_ptr_type (DR_REF (first_dr_info->dr));
}
- bool masks_p = loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
- bool lens_p = loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
+ bool masks_p
+ = use_partial_vectors_p
+ && (vinfo->partial_vector_style == vect_partial_vectors_while_ult
+ || vinfo->partial_vector_style == vect_partial_vectors_avx512);
+
+ bool lens_p = use_partial_vectors_p
+ && vinfo->partial_vector_style == vect_partial_vectors_len;
/* The vect_transform_stmt and vect_analyze_stmt will go here but there
are some difference here. We cannot enable both the lens and masks
@@ -10860,20 +10983,19 @@ vectorizable_load (vec_info *vinfo,
tree bias = NULL_TREE;
if (masks_p)
final_mask
- = vect_get_mask (loop_vinfo, gsi, ncopies, vectype, j);
+ = vect_get_mask (vinfo, gsi, ncopies, vectype, j, active_lanes);
if (vec_mask)
- final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
+ final_mask = prepare_vec_mask (vinfo, mask_vectype, final_mask,
vec_mask, gsi);
if (lanes_ifn == IFN_MASK_LEN_LOAD_LANES)
{
if (lens_p)
- final_len
- = vect_get_len (loop_vinfo, gsi, ncopies, vectype, j, 1,
true);
+ final_len = vect_get_len (vinfo, gsi, ncopies, vectype, j, 1,
+ true, active_lanes);
else
final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
- signed char biasval
- = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
+ signed char biasval = vinfo->partial_load_store_bias;
bias = build_int_cst (intQI_type_node, biasval);
if (!final_mask)
{
@@ -11009,11 +11131,11 @@ vectorizable_load (vec_info *vinfo,
if (mask_node)
vec_mask = vec_masks[i];
if (masks_p)
- final_mask
- = vect_get_mask (loop_vinfo, gsi, vec_num, vectype, i);
+ final_mask = vect_get_mask (vinfo, gsi, vec_num, vectype, i,
+ active_lanes);
if (vec_mask)
- final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
- final_mask, vec_mask, gsi);
+ final_mask = prepare_vec_mask (vinfo, mask_vectype, final_mask,
+ vec_mask, gsi);
if (i > 0 && !STMT_VINFO_GATHER_SCATTER_P (stmt_info))
dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
@@ -11081,13 +11203,12 @@ vectorizable_load (vec_info *vinfo,
if (ls.gs.ifn == IFN_MASK_LEN_GATHER_LOAD)
{
if (lens_p)
- final_len = vect_get_len (loop_vinfo, gsi, vec_num, vectype,
- i, 1, true);
+ final_len = vect_get_len (vinfo, gsi, vec_num, vectype, i,
+ 1, true, active_lanes);
else
final_len = build_int_cst (sizetype,
TYPE_VECTOR_SUBPARTS (vectype));
- signed char biasval
- = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
+ signed char biasval = vinfo->partial_load_store_bias;
bias = build_int_cst (intQI_type_node, biasval);
if (!final_mask)
{
@@ -11484,10 +11605,10 @@ vectorizable_load (vec_info *vinfo,
vec_mask = vec_masks[i];
if (masks_p)
final_mask
- = vect_get_mask (loop_vinfo, gsi, vec_num, vectype, i);
+ = vect_get_mask (vinfo, gsi, vec_num, vectype, i, active_lanes);
if (vec_mask)
- final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
- final_mask, vec_mask, gsi);
+ final_mask = prepare_vec_mask (vinfo, mask_vectype, final_mask,
+ vec_mask, gsi);
if (i > 0)
dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info,
@@ -11532,8 +11653,8 @@ vectorizable_load (vec_info *vinfo,
new_vmode = new_ovmode.require ();
unsigned factor
= (new_ovmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vmode);
- final_len = vect_get_len (loop_vinfo, gsi, vec_num, vectype, i,
- factor, true);
+ final_len = vect_get_len (vinfo, gsi, vec_num, vectype, i,
+ factor, true, active_lanes);
}
else if (final_mask)
{
@@ -11562,8 +11683,7 @@ vectorizable_load (vec_info *vinfo,
}
if (final_len)
{
- signed char biasval
- = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
+ signed char biasval = vinfo->partial_load_store_bias;
bias = build_int_cst (intQI_type_node, biasval);
}
@@ -12179,6 +12299,10 @@ vectorizable_condition (vec_info *vinfo,
/* std::swap (then_clause, else_clause); */
}
+ bool use_partial_vectors_p
+ = can_use_partial_vectors_p (vinfo, slp_node,
+ TYPE_VECTOR_SUBPARTS (vectype));
+
if (cost_vec)
{
vect_cost_for_stmt kind = vector_stmt;
@@ -12199,25 +12323,39 @@ vectorizable_condition (vec_info *vinfo,
return false;
}
- if (loop_vinfo && for_reduction
- && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
+ if (for_reduction && use_partial_vectors_p)
{
if (reduction_type == EXTRACT_LAST_REDUCTION)
{
if (direct_internal_fn_supported_p (IFN_LEN_FOLD_EXTRACT_LAST,
vectype, OPTIMIZE_FOR_SPEED))
- vect_record_len (loop_vinfo, vec_num, vectype, 1);
+ vect_record_len (vinfo, vec_num, vectype, 1,
+ SLP_TREE_LANES (slp_node));
else
- vect_record_mask (loop_vinfo, vec_num, vectype, NULL_TREE);
+ vect_record_mask (vinfo, vec_num, vectype, NULL_TREE,
+ SLP_TREE_LANES (slp_node));
}
/* Extra inactive lanes should be safe for vect_nested_cycle. */
else if (!nested_cycle_p)
{
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "conditional reduction prevents the use"
- " of partial vectors.\n");
- LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ if (loop_vinfo)
+ {
+ if (dump_enabled_p ()
+ && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "conditional reduction prevents the use"
+ " of partial vectors.\n");
+ LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ }
+ else
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "SLP can't use a mask or length because"
+ " conditional reduction prevents the use"
+ " of partial vectors.\n");
+ return false;
+ }
}
}
@@ -12240,16 +12378,19 @@ vectorizable_condition (vec_info *vinfo,
bool masks_p = false;
bool lens_p = false;
- if (loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
+ if (use_partial_vectors_p
+ && vinfo->partial_vector_style == vect_partial_vectors_len)
{
if (reduction_type == EXTRACT_LAST_REDUCTION)
lens_p = true;
}
- else if (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
+ else if (use_partial_vectors_p
+ && (vinfo->partial_vector_style == vect_partial_vectors_while_ult
+ || vinfo->partial_vector_style == vect_partial_vectors_avx512))
{
if (reduction_type == EXTRACT_LAST_REDUCTION)
masks_p = true;
- else
+ else if (loop_vinfo)
{
scalar_cond_masked_key cond (cond_expr, 1);
if (loop_vinfo->scalar_cond_masked_set.contains (cond))
@@ -12348,10 +12489,9 @@ vectorizable_condition (vec_info *vinfo,
{
/* ??? Do we really want the adjusted LEN here? Isn't this
based on number of elements? */
- len = vect_get_len (loop_vinfo, gsi, vec_num, vectype, i, 1,
- true);
- signed char biasval
- = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
+ len = vect_get_len (vinfo, gsi, vec_num, vectype, i, 1,
+ true, SLP_TREE_LANES (slp_node));
+ signed char biasval = vinfo->partial_load_store_bias;
bias = build_int_cst (intQI_type_node, biasval);
}
else
@@ -12362,8 +12502,8 @@ vectorizable_condition (vec_info *vinfo,
}
if (masks_p)
{
- tree loop_mask
- = vect_get_mask (loop_vinfo, gsi, vec_num, vectype, i);
+ tree loop_mask = vect_get_mask (vinfo, gsi, vec_num, vectype, i,
+ SLP_TREE_LANES (slp_node));
tree tmp2 = make_ssa_name (vec_cmp_type);
gassign *g
= gimple_build_assign (tmp2, BIT_AND_EXPR, vec_compare,
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 32851a31011..37788e21b57 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -2356,17 +2356,26 @@ vect_get_num_vectors (poly_uint64 nunits, tree vectype)
inline unsigned int
vect_get_num_copies (vec_info *vinfo, slp_tree node)
{
- poly_uint64 vf;
+ tree vectype = SLP_TREE_VECTYPE (node);
+ unsigned int group_size = SLP_TREE_LANES (node);
if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
- vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+ {
+ poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+ return vect_get_num_vectors (vf * group_size, vectype);
+ }
else
- vf = 1;
-
- vf *= SLP_TREE_LANES (node);
- tree vectype = SLP_TREE_VECTYPE (node);
-
- return vect_get_num_vectors (vf, vectype);
+ {
+ /* The group size is not necessarily an integral multiple of the number
+ of subparts in the vector type if doing BB SLP vectorization with
+ predicated tails. */
+ poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
+ unsigned int nvectors;
+ if (can_div_away_from_zero_p (poly_uint64 (group_size), nunits,
+ &nvectors))
+ return nvectors;
+ gcc_unreachable ();
+ }
}
/* Return the vectorization factor that should be used for costing
--
2.43.0
