SLP discovery, load permutation handling and load/store generation calculate
the bounds of a memory group separately. To support source vectors we need
them to share the same calculation before changing its units to element lanes.
For example, this scalar group covers four unsigned elements:
void memory_group (unsigned *restrict out, const unsigned *a)
{
out[0] = a[0] + 1;
out[1] = a[1] + 2;
out[2] = a[2] + 3;
out[3] = a[3] + 4;
}
We introduce vect_slp_memory_layout to hold the canonical data reference,
group lane count, gap and element size. vect_get_slp_memory_layout calculates
these values, giving the following metadata for this example:
dr_info: canonical reference for a[0]
group_lanes: 4
gap: 0
element_size: sizeof(unsigned), 4 bytes on AArch64
covered bytes: 4 * 4 = 16
vect_slp_memory_stmts uses the layout to build a statement list indexed by
lane, with null entries for gaps. vect_load_store_data caches the layout so
transformation uses the bounds checked during analysis.
Alignment and speculative-read checks use vect_first_dr_info to obtain the
same canonical reference for grouped and standalone accesses. The bounds
describe the source memory group even when a permutation selects fewer result
lanes.
The permutation cleanup uses the existing load permutation to recognize a
consecutive subchain, rather than walking the source statement chain again.
For grouped loop loads it uses the shared layout to check the group size and
trailing gap. The separate rule for ungrouped scalar loop loads stays here;
the next patch extends it to complete vector loads.
This patch leaves the scalar metadata and codegen unchanged. The next patch
extends the shared calculation to vector references.
Bootstrapped Regtested on aarch64-none-linux-gnu,
arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
-m32, -m64 and no issues.
Any feedback?
Thanks,
Tamar
gcc/ChangeLog:
PR tree-optimization/119187
* tree-vectorizer.h (vect_slp_memory_layout): New.
(vect_load_store_data): Cache the source memory layout.
(vect_first_dr_info): New.
(dr_target_alignment, dr_safe_speculative_read_required,
dr_set_safe_speculative_read_required): Use vect_first_dr_info.
(vect_get_scalar_dr_size): Clarify the units.
(vect_get_slp_memory_layout): Declare.
* tree-vect-data-refs.cc (vect_slp_analyze_node_alignment): Use
vect_first_dr_info.
* tree-vect-slp.cc (vect_get_slp_memory_layout, vect_slp_memory_stmts):
New.
(vect_build_slp_tree_2, vect_lower_load_permutations): Use the shared
memory layout and statement collection.
(vect_optimize_slp_pass::remove_redundant_permutations): Check load
permutations for consecutive accesses and use the shared memory
layout for grouped loop loads.
(vect_optimize_slp_pass::internal_node_cost,
vect_optimize_slp_pass::start_choosing_layouts,
vect_transform_slp_perm_load_1): Use the shared memory layout.
* tree-vect-stmts.cc (get_load_store_type, vectorizable_load,
vectorizable_store): Use the cached memory layout.
* tree-vect-stmts.cc (vectorizable_operation): Remove the temporary
source element-type adjustment.
---
--
diff --git a/gcc/tree-vect-data-refs.cc b/gcc/tree-vect-data-refs.cc
index
08b94149e7c28d10ee9272a3b18d981cac5efa3a..5af85a7ffc3b5705d2efa1c46b02cc50cbf4360c
100644
--- a/gcc/tree-vect-data-refs.cc
+++ b/gcc/tree-vect-data-refs.cc
@@ -3020,9 +3020,8 @@ static bool
vect_slp_analyze_node_alignment (vec_info *vinfo, slp_tree node)
{
/* Alignment is maintained in the first element of the group. */
- stmt_vec_info first_stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
- first_stmt_info = DR_GROUP_FIRST_ELEMENT (first_stmt_info);
- dr_vec_info *dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
+ dr_vec_info *dr_info
+ = vect_first_dr_info (SLP_TREE_SCALAR_STMTS (node)[0]);
tree vectype = SLP_TREE_VECTYPE (node);
poly_uint64 vector_alignment
= exact_div (targetm.vectorize.preferred_vector_alignment (vectype),
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index
b76400b12f2bcc236682136f4b36efcee9cd580a..2f2ff08ac45db67e01f3820ca1e78bb75ff11513
100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -444,6 +444,51 @@ vect_get_place_in_interleaving_chain (stmt_vec_info
stmt_info,
return -1;
}
+/* Return the source reference, group bounds and element size for NODE.
+ For ungrouped loads, PERM determines how many source elements are used. */
+
+vect_slp_memory_layout
+vect_get_slp_memory_layout (slp_tree node, const load_permutation_t &perm)
+{
+ stmt_vec_info stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
+ dr_vec_info *dr_info = vect_first_dr_info (stmt_info);
+ vect_slp_memory_layout memory
+ = { dr_info, 1, 0, vect_get_scalar_dr_size (dr_info) };
+
+ if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
+ {
+ memory.group_lanes = DR_GROUP_SIZE (dr_info->stmt);
+ memory.gap = DR_GROUP_GAP (dr_info->stmt);
+ }
+ else
+ /* Cover splats and collections of distinct invariant references. */
+ for (unsigned int lane : perm)
+ memory.group_lanes = MAX (memory.group_lanes, lane + 1);
+
+ return memory;
+}
+
+/* Collect the source statements in MEMORY, with one entry per element lane
+ and null entries for gaps. */
+
+static vec<stmt_vec_info>
+vect_slp_memory_stmts (const vect_slp_memory_layout &memory)
+{
+ vec<stmt_vec_info> stmts;
+ stmts.create (memory.group_lanes);
+ stmts.quick_grow_cleared (memory.group_lanes);
+ stmt_vec_info first = memory.dr_info->stmt;
+ unsigned int pos = 0;
+ for (stmt_vec_info si = first; si; si = DR_GROUP_NEXT_ELEMENT (si))
+ {
+ if (si != first)
+ pos += DR_GROUP_GAP (si);
+ stmts[pos] = si;
+ }
+
+ return stmts;
+}
+
/* Check whether it is possible to load COUNT elements of type ELT_TYPE
using the method implemented by duplicate_and_interleave. Return true
if so, returning the number of intermediate vectors in *NVECTORS_OUT
@@ -2298,9 +2343,7 @@ vect_build_slp_tree_2 (vec_info *vinfo, slp_tree node,
int j;
stmt_vec_info load_info;
load_permutation.create (group_size);
- stmt_vec_info first_stmt_info
- = STMT_VINFO_GROUPED_ACCESS (stmt_info)
- ? DR_GROUP_FIRST_ELEMENT (stmt_info) : stmt_info;
+ stmt_vec_info first_stmt_info = vect_first_dr_info (stmt_info)->stmt;
bool any_permute = false;
FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), j, load_info)
{
@@ -2313,18 +2356,22 @@ vect_build_slp_tree_2 (vec_info *vinfo, slp_tree node,
load_place = 0;
}
else if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
- load_place = vect_get_place_in_interleaving_chain
- (load_info, first_stmt_info);
+ load_place
+ = vect_get_place_in_interleaving_chain (load_info,
+ first_stmt_info);
else
/* Recognize the splat case as { 0, 0, ... } but make
sure to use the appropriate refs for collections
of invariant refs. */
load_place = (load_info == stmt_info) ? 0 : j;
+
gcc_assert (load_place != -1);
any_permute |= load_place != j;
load_permutation.quick_push (load_place);
}
+ const vect_slp_memory_layout memory
+ = vect_get_slp_memory_layout (node, load_permutation);
if (gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt))
{
gcc_assert (gimple_call_internal_p (stmt, IFN_MASK_LOAD));
@@ -2352,23 +2399,11 @@ vect_build_slp_tree_2 (vec_info *vinfo, slp_tree node,
the whole group followed by a SLP permute node. */
if (any_permute
|| (STMT_VINFO_GROUPED_ACCESS (stmt_info)
- && DR_GROUP_SIZE (first_stmt_info) != group_size))
+ && memory.group_lanes != group_size))
{
/* Discover the whole unpermuted load. */
- vec<stmt_vec_info> stmts2;
- unsigned dr_group_size = STMT_VINFO_GROUPED_ACCESS (stmt_info)
- ? DR_GROUP_SIZE (first_stmt_info) : 1;
- stmts2.create (dr_group_size);
- stmts2.quick_grow_cleared (dr_group_size);
- unsigned i = 0;
- for (stmt_vec_info si = first_stmt_info;
- si; si = DR_GROUP_NEXT_ELEMENT (si))
- {
- if (si != first_stmt_info)
- for (unsigned k = 1; k < DR_GROUP_GAP (si); ++k)
- stmts2[i++] = NULL;
- stmts2[i++] = si;
- }
+ vec<stmt_vec_info> stmts2 = vect_slp_memory_stmts (memory);
+ unsigned dr_group_size = memory.group_lanes;
match_elt_t *matches2
= XALLOCAVEC (match_elt_t, dr_group_size);
slp_tree unperm_load
@@ -2403,8 +2438,9 @@ vect_build_slp_tree_2 (vec_info *vinfo, slp_tree node,
{
if (!any_permute
&& STMT_VINFO_GROUPED_ACCESS (stmt_info)
- && group_size == DR_GROUP_SIZE (first_stmt_info))
+ && group_size == memory.group_lanes)
load_permutation.release ();
+
SLP_TREE_LOAD_PERMUTATION (node) = load_permutation;
return node;
}
@@ -5687,9 +5723,11 @@ vect_lower_load_permutations (loop_vec_info loop_vinfo,
size in mind which means we cannot actually compute whether we
need three or more vectors for a load permutation yet. So always
lower. */
- stmt_vec_info first
- = DR_GROUP_FIRST_ELEMENT (SLP_TREE_SCALAR_STMTS (loads[0])[0]);
- unsigned group_lanes = DR_GROUP_SIZE (first);
+ const vect_slp_memory_layout memory
+ = vect_get_slp_memory_layout (loads[0],
+ SLP_TREE_LOAD_PERMUTATION (loads[0]));
+ stmt_vec_info first = memory.dr_info->stmt;
+ unsigned group_lanes = memory.group_lanes;
/* Verify if all load permutations can be implemented with a suitably
large element load-lanes operation. */
@@ -5791,17 +5829,7 @@ vect_lower_load_permutations (loop_vec_info loop_vinfo,
/* First build (and possibly re-use) a load node for the
unpermuted group. Gaps in the middle and on the end are
represented with NULL stmts. */
- vec<stmt_vec_info> stmts;
- stmts.create (group_lanes);
- for (stmt_vec_info s = first; s; s = DR_GROUP_NEXT_ELEMENT (s))
- {
- if (s != first)
- for (unsigned i = 1; i < DR_GROUP_GAP (s); ++i)
- stmts.quick_push (NULL);
- stmts.quick_push (s);
- }
- for (unsigned i = 0; i < DR_GROUP_GAP (first); ++i)
- stmts.quick_push (NULL);
+ vec<stmt_vec_info> stmts = vect_slp_memory_stmts (memory);
match_elt_t *matches = XALLOCAVEC (match_elt_t, group_lanes);
unsigned limit = 1;
unsigned tree_size = 0;
@@ -7343,16 +7371,16 @@ vect_optimize_slp_pass::internal_node_cost (slp_tree
node, int in_layout_i,
if (!vect_transform_slp_perm_load_1 (m_vinfo, node, tmp_perm, vNULL,
nullptr, vf, true, false, &n_perms))
{
- auto rep = SLP_TREE_REPRESENTATIVE (node);
if (out_layout_i == 0)
{
/* Use the fallback cost if the load is an N-to-N permutation.
Otherwise assume that the node will be rejected later
and rebuilt from scalars. */
if (STMT_VINFO_GROUPED_ACCESS (rep)
- && (DR_GROUP_SIZE (DR_GROUP_FIRST_ELEMENT (rep))
+ && (vect_get_slp_memory_layout (node, tmp_perm).group_lanes
== SLP_TREE_LANES (node)))
return fallback_cost;
+
return 0;
}
return -1;
@@ -7415,8 +7443,11 @@ vect_optimize_slp_pass::start_choosing_layouts ()
partition.layout = -1;
continue;
}
- dr_stmt = DR_GROUP_FIRST_ELEMENT (dr_stmt);
- imin = DR_GROUP_SIZE (dr_stmt) + 1;
+
+ imin
+ = vect_get_slp_memory_layout (node,
+ SLP_TREE_LOAD_PERMUTATION
+ (node)).group_lanes;
tmp_perm.safe_splice (SLP_TREE_LOAD_PERMUTATION (node));
}
else if (SLP_TREE_PERMUTE_P (node)
@@ -8242,22 +8273,11 @@ vect_optimize_slp_pass::remove_redundant_permutations ()
FORNOW: not in loop SLP because of realignment complications. */
if (is_a <bb_vec_info> (m_vinfo))
{
- bool subchain_p = true;
- stmt_vec_info next_load_info = NULL;
- stmt_vec_info load_info;
- unsigned j;
- FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), j, load_info)
- {
- if (j != 0
- && (next_load_info != load_info
- || ! load_info
- || DR_GROUP_GAP (load_info) != 1))
- {
- subchain_p = false;
- break;
- }
- next_load_info = DR_GROUP_NEXT_ELEMENT (load_info);
- }
+ bool subchain_p = vect_load_perm_consecutive_p (node);
+ for (stmt_vec_info load_info : SLP_TREE_SCALAR_STMTS (node))
+ if (!load_info)
+ subchain_p = false;
+
if (subchain_p)
{
SLP_TREE_LOAD_PERMUTATION (node).release ();
@@ -8278,15 +8298,16 @@ vect_optimize_slp_pass::remove_redundant_permutations ()
SLP_TREE_LOAD_PERMUTATION (node).release ();
continue;
}
- stmt_vec_info first_stmt_info
- = DR_GROUP_FIRST_ELEMENT (SLP_TREE_SCALAR_STMTS (node)[0]);
+ const vect_slp_memory_layout memory
+ = vect_get_slp_memory_layout (node,
+ SLP_TREE_LOAD_PERMUTATION (node));
if (!this_load_permuted
/* The load requires permutation when unrolling exposes
a gap either because the group is larger than the SLP
group-size or because there is a gap between the groups. */
&& (known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), 1U)
- || ((SLP_TREE_LANES (node) == DR_GROUP_SIZE (first_stmt_info))
- && DR_GROUP_GAP (first_stmt_info) == 0)))
+ || (SLP_TREE_LANES (node) == memory.group_lanes
+ && memory.gap == 0)))
{
SLP_TREE_LOAD_PERMUTATION (node).release ();
continue;
@@ -11507,28 +11528,18 @@ vect_transform_slp_perm_load_1 (vec_info *vinfo,
slp_tree node,
unsigned *n_perms, unsigned int *n_loads,
bool dce_chain)
{
- stmt_vec_info stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
int vec_index = 0;
tree vectype = SLP_TREE_VECTYPE (node);
unsigned int group_size = SLP_TREE_LANES (node);
unsigned int mask_element;
- unsigned dr_group_size;
machine_mode mode;
- if (!STMT_VINFO_GROUPED_ACCESS (stmt_info))
- {
- /* We have both splats of the same non-grouped load and groups
- of distinct invariant loads entering here. */
- unsigned max_idx = 0;
- for (auto idx : perm)
- max_idx = idx > max_idx ? idx : max_idx;
- dr_group_size = max_idx + 1;
- }
- else
- {
- stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
- dr_group_size = DR_GROUP_SIZE (stmt_info);
- }
+ const vect_slp_memory_layout memory
+ = node->data
+ ? static_cast <vect_load_store_data *> (node->data)->memory
+ : vect_get_slp_memory_layout (node, perm);
+ stmt_vec_info stmt_info = memory.dr_info->stmt;
+ unsigned int dr_group_size = memory.group_lanes;
mode = TYPE_MODE (vectype);
poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index
5b5e0c7d2227cc805d15667f138925c778bd51dd..696a7bb60bfecf0d34c17f0b54a4351810c4116e
100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -2099,10 +2099,6 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info
stmt_info,
loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
class loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
- stmt_vec_info first_stmt_info;
- unsigned int group_size;
- unsigned HOST_WIDE_INT gap;
- bool single_element_p;
poly_int64 neg_ldst_offset = 0;
*misalignment = DR_MISALIGNMENT_UNKNOWN;
@@ -2112,6 +2108,17 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info
stmt_info,
*n_perms = -1U;
*n_loads = -1U;
ls->subchain_p = false;
+ ls->memory
+ = vect_get_slp_memory_layout (slp_node,
+ SLP_TREE_LOAD_PERMUTATION (slp_node));
+
+ stmt_vec_info first_stmt_info = ls->memory.dr_info->stmt;
+ dr_vec_info *first_dr_info = ls->memory.dr_info;
+ unsigned int group_size = ls->memory.group_lanes;
+ unsigned HOST_WIDE_INT gap = ls->memory.gap;
+ bool single_element_p
+ = (stmt_info == first_stmt_info
+ && !DR_GROUP_NEXT_ELEMENT (first_stmt_info));
bool perm_ok = true;
poly_int64 vf = loop_vinfo ? LOOP_VINFO_VECT_FACTOR (loop_vinfo) : 1;
@@ -2120,23 +2127,6 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info
stmt_info,
perm_ok = vect_transform_slp_perm_load (vinfo, slp_node, vNULL, NULL,
vf, true, n_perms, n_loads);
- if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
- {
- first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
- group_size = DR_GROUP_SIZE (first_stmt_info);
- gap = DR_GROUP_GAP (first_stmt_info);
- single_element_p = (stmt_info == first_stmt_info
- && !DR_GROUP_NEXT_ELEMENT (stmt_info));
- }
- else
- {
- first_stmt_info = stmt_info;
- group_size = 1;
- gap = 0;
- single_element_p = true;
- }
- dr_vec_info *first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
-
/* There can only be a gap at the end of the group if the stride is
known at compile time. */
gcc_assert (!STMT_VINFO_STRIDED_P (first_stmt_info) || gap == 0);
@@ -2324,7 +2314,7 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info
stmt_info,
return false;
}
- unsigned HOST_WIDE_INT dr_size = vect_get_scalar_dr_size (first_dr_info);
+ unsigned HOST_WIDE_INT dr_size = ls->memory.element_size;
poly_int64 off = 0;
if (*memory_access_type == VMAT_CONTIGUOUS_REVERSE)
off = (TYPE_VECTOR_SUBPARTS (vectype) - 1) * -dr_size;
@@ -2614,8 +2604,7 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info
stmt_info,
poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
poly_uint64 read_amount
= vf * TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
- if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
- read_amount *= group_size;
+ read_amount *= group_size;
auto target_alignment
= DR_TARGET_ALIGNMENT (STMT_VINFO_DR_INFO (stmt_info));
@@ -2731,8 +2720,6 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info
stmt_info,
{
/* In BB vectorization we may not actually use a loaded vector
accessing elements in excess of DR_GROUP_SIZE. */
- stmt_vec_info group_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
- group_info = DR_GROUP_FIRST_ELEMENT (group_info);
unsigned HOST_WIDE_INT nunits;
unsigned j, k, maxk = 0;
FOR_EACH_VEC_ELT (SLP_TREE_LOAD_PERMUTATION (slp_node), j, k)
@@ -2740,7 +2727,7 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info
stmt_info,
maxk = k;
tree vectype = SLP_TREE_VECTYPE (slp_node);
if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nunits)
- || maxk >= (DR_GROUP_SIZE (group_info) & ~(nunits - 1)))
+ || maxk >= (ls->memory.group_lanes & ~(nunits - 1)))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -6722,9 +6709,6 @@ vectorizable_operation (vec_info *vinfo,
scalar_dest = gimple_assign_lhs (stmt);
tree scalar_type = TREE_TYPE (scalar_dest);
- if (VECTOR_TYPE_P (scalar_type))
- scalar_type = TREE_TYPE (scalar_type);
-
vectype_out = SLP_TREE_VECTYPE (slp_node);
/* Most operations cannot handle bit-precision types without extra
@@ -8383,18 +8367,8 @@ vectorizable_store (vec_info *vinfo,
dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info), *first_dr_info = NULL;
grouped_store = (STMT_VINFO_GROUPED_ACCESS (stmt_info)
&& !mat_gather_scatter_p (memory_access_type));
- if (grouped_store)
- {
- first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
- first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
- group_size = DR_GROUP_SIZE (first_stmt_info);
- }
- else
- {
- first_stmt_info = stmt_info;
- first_dr_info = dr_info;
- group_size = 1;
- }
+ group_size
+ = mat_gather_scatter_p (memory_access_type) ? 1 : ls.memory.group_lanes;
if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) > 1 && cost_vec)
{
@@ -9967,9 +9941,6 @@ vectorizable_load (vec_info *vinfo,
gcc_assert (!nested_in_vect_loop);
gcc_assert (!STMT_VINFO_GATHER_SCATTER_P (stmt_info));
- first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
- group_size = DR_GROUP_SIZE (first_stmt_info);
-
/* Invalidate assumptions made by dependence analysis when vectorization
on the unrolled body effectively re-orders stmts. */
if (STMT_VINFO_MIN_NEG_DIST (stmt_info) != 0
@@ -9983,8 +9954,6 @@ vectorizable_load (vec_info *vinfo,
return false;
}
}
- else
- group_size = 1;
vect_load_store_data _ls_data{};
vect_load_store_data &ls = slp_node->get_data (_ls_data);
@@ -10000,6 +9969,7 @@ vectorizable_load (vec_info *vinfo,
const int misalignment = ls.misalignment;
const poly_int64 poffset = ls.poffset;
const vec<int> &elsvals = ls.elsvals;
+ group_size = ls.memory.group_lanes;
int maskload_elsval = 0;
bool need_zeroing = false;
@@ -10326,7 +10296,7 @@ vectorizable_load (vec_info *vinfo,
/* If we elided a consecutive load permutation, adjust
the group size here. */
if (!ls.subchain_p)
- group_size = DR_GROUP_SIZE (first_stmt_info);
+ group_size = ls.memory.group_lanes;
else
group_size = SLP_TREE_LANES (slp_node);
}
@@ -10643,16 +10613,8 @@ vectorizable_load (vec_info *vinfo,
if (grouped_load
|| SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
{
- if (grouped_load)
- {
- first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
- group_size = DR_GROUP_SIZE (first_stmt_info);
- }
- else
- {
- first_stmt_info = stmt_info;
- group_size = 1;
- }
+ first_stmt_info = ls.memory.dr_info->stmt;
+
/* For SLP vectorization we directly vectorize a subchain
without permutation. */
if (! SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
@@ -10702,7 +10664,7 @@ vectorizable_load (vec_info *vinfo,
{
first_stmt_info = stmt_info;
first_dr_info = dr_info;
- group_size = 1;
+ group_size = ls.memory.group_lanes;
group_gap_adj = 0;
ref_type = reference_alias_ptr_type (DR_REF (first_dr_info->dr));
}
@@ -11715,9 +11677,8 @@ vectorizable_load (vec_info *vinfo,
{
tree ltype = vectype;
tree new_vtype = NULL_TREE;
- unsigned HOST_WIDE_INT gap = DR_GROUP_GAP (first_stmt_info);
- unsigned HOST_WIDE_INT dr_size
- = vect_get_scalar_dr_size (first_dr_info);
+ unsigned HOST_WIDE_INT gap = ls.memory.gap;
+ unsigned HOST_WIDE_INT dr_size = ls.memory.element_size;
poly_int64 off = 0;
if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
off = (TYPE_VECTOR_SUBPARTS (vectype) - 1) * -dr_size;
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index
eeeee6e1fc63a2e6d8808f765add38b7d66b8e69..fcfc14c1a114d4a007d3c52c81b5b5051c34ca7c
100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -255,6 +255,20 @@ typedef auto_vec<std::pair<unsigned, unsigned>, 16>
auto_lane_permutation_t;
typedef vec<unsigned> load_permutation_t;
typedef auto_vec<unsigned, 16> auto_load_permutation_t;
+/* The source memory layout of an SLP load or store. Unlike DR_GROUP_SIZE
+ and DR_GROUP_GAP, all counts are in scalar element lanes, including when
+ the source statements access whole vectors. This describes the inputs
+ to a load permutation, rather than the lanes produced by the SLP node. */
+struct vect_slp_memory_layout {
+ /* First reference of the source group, or the ungrouped reference. */
+ class dr_vec_info *dr_info;
+ unsigned int group_lanes;
+ unsigned int gap;
+
+ /* Size of one scalar element in bytes. */
+ unsigned HOST_WIDE_INT element_size;
+};
+
struct vect_data {
virtual ~vect_data () = default;
};
@@ -283,6 +297,7 @@ struct vect_load_store_data : vect_data {
vect_load_store_data () = default;
virtual ~vect_load_store_data () = default;
+ vect_slp_memory_layout memory;
vect_memory_access_type memory_access_type;
dr_alignment_support alignment_support_scheme;
int misalignment;
@@ -2190,13 +2205,22 @@ extern int dr_misalignment (dr_vec_info *dr_info, tree
vectype,
#define SET_DR_MISALIGNMENT(DR, VAL) set_dr_misalignment (DR, VAL)
+/* Return the first reference of STMT_INFO's group, or its own reference
+ when it is ungrouped. */
+inline dr_vec_info *
+vect_first_dr_info (stmt_vec_info stmt_info)
+{
+ if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
+ stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
+
+ return STMT_VINFO_DR_INFO (stmt_info);
+}
+
/* Only defined once DR_MISALIGNMENT is defined. */
inline const poly_uint64
dr_target_alignment (dr_vec_info *dr_info)
{
- if (STMT_VINFO_GROUPED_ACCESS (dr_info->stmt))
- dr_info = STMT_VINFO_DR_INFO (DR_GROUP_FIRST_ELEMENT (dr_info->stmt));
- return dr_info->target_alignment;
+ return vect_first_dr_info (dr_info->stmt)->target_alignment;
}
#define DR_TARGET_ALIGNMENT(DR) dr_target_alignment (DR)
#define DR_SCALAR_KNOWN_BOUNDS(DR) (DR)->scalar_access_known_in_bounds
@@ -2205,13 +2229,7 @@ dr_target_alignment (dr_vec_info *dr_info)
inline bool
dr_safe_speculative_read_required (stmt_vec_info stmt_info)
{
- dr_vec_info *dr_info;
- if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
- dr_info = STMT_VINFO_DR_INFO (DR_GROUP_FIRST_ELEMENT (stmt_info));
- else
- dr_info = STMT_VINFO_DR_INFO (stmt_info);
-
- return dr_info->safe_speculative_read_required;
+ return vect_first_dr_info (stmt_info)->safe_speculative_read_required;
}
/* Set the safe_speculative_read_required for the stmt_vec_info, if group
@@ -2220,13 +2238,8 @@ inline void
dr_set_safe_speculative_read_required (stmt_vec_info stmt_info,
bool requires_alignment)
{
- dr_vec_info *dr_info;
- if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
- dr_info = STMT_VINFO_DR_INFO (DR_GROUP_FIRST_ELEMENT (stmt_info));
- else
- dr_info = STMT_VINFO_DR_INFO (stmt_info);
-
- dr_info->safe_speculative_read_required = requires_alignment;
+ vect_first_dr_info (stmt_info)->safe_speculative_read_required
+ = requires_alignment;
}
inline void
@@ -2408,11 +2421,8 @@ vect_max_vf (loop_vec_info loop_vinfo)
}
/* Return the size of the value accessed by unvectorized data reference
- DR_INFO. This is only valid once STMT_VINFO_VECTYPE has been calculated
- for the associated gimple statement, since that guarantees that DR_INFO
- accesses either a scalar or a scalar equivalent. ("Scalar equivalent"
- here includes things like V1SI, which can be vectorized in the same way
- as a plain SI.) */
+ DR_INFO. For a source vector this is the size of the whole vector,
+ rather than its scalar elements. */
inline unsigned int
vect_get_scalar_dr_size (dr_vec_info *dr_info)
@@ -2795,6 +2805,8 @@ extern bool can_duplicate_and_interleave_p (vec_info *,
unsigned int, tree,
extern void duplicate_and_interleave (vec_info *, gimple_seq *, tree,
const vec<tree> &, unsigned int,
vec<tree> &);
extern int vect_get_place_in_interleaving_chain (stmt_vec_info, stmt_vec_info);
+extern vect_slp_memory_layout vect_get_slp_memory_layout
+ (slp_tree, const load_permutation_t &);
extern slp_tree vect_create_new_slp_node (unsigned, tree_code);
extern void vect_free_slp_tree (slp_tree);
extern bool compatible_calls_p (gcall *, gcall *, bool);
