To analyze vector loads in SLP we must express their memory groups in element
lanes. Data-reference groups count source accesses, but each vector access
contains several elements.
e.g.
typedef unsigned V8 __attribute__((vector_size(32)));
void bb_load (V8 *out, const V8 *a, const V8 *b)
{
*out = *a + *b;
}
The SLP tree output for data ref analysis now shows:
=== vect_analyze_data_refs ===
got vectype for stmt: _1 = *a_5(D);
vector(4) unsigned int
...
=== vect_analyze_data_ref_accesses ===
Allowing vector memory access in BB
The source reference still has type V8 and vector(4) is the target type
selected for SLP.
For the load from a, group_lanes becomes 8 and element_size is 4, covering the
original 32 bytes. In general a group of K accesses to N-element vectors has K
* N lanes. We scale gaps by N and map element J of access K to lane K * N + J
in the load permutation.
With the store-root support from patch 5, the SLP load node for this example
looks as follows, with names shortened:
target type: vector(4) unsigned int
stmt 0..7: load = *a; // the same source statement
load permutation { 0 1 2 3 4 5 6 7 }
On Adv. SIMD it produces two native loads:
a_lo = MEM <vector(4) unsigned int> [a];
a_hi = MEM <vector(4) unsigned int> [a + 16B];
The source group still covers eight lanes if a later extract selects only four
of them. Keeping the source bounds separate from the result width preserves
the original access size and stride. The element size is used when splitting
the load.
Only linear vector loads are accepted for SLP at this time. BB SLP can split
one standalone vector load, but cannot combine distinct ungrouped loads into
one load node. Gather/scatter calls are not supported here. Neither are VLA
vectors because data-ref cannot currently model these.
Vector lowering produces the same two loads for this example, so no effect on
codegen yet, however SLP can now analyze their loads.
Bootstrapped Regtested on aarch64-none-linux-gnu,
arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
-m32, -m64 and no issues.
Any feedback?
Thanks,
Tamar
gcc/ChangeLog:
PR tree-optimization/119187
* tree-vect-data-refs.cc (vect_analyze_group_access_1): Handle
fixed-length vector references.
(vect_analyze_data_ref_access): Allow vector references in BB SLP while
preserving data-reference groups.
* tree-vect-slp.cc (vect_get_slp_memory_layout, vect_slp_memory_stmts):
Count source vector elements as lanes.
(vect_build_slp_tree_3): Handle vector loads and share the ungrouped
load condition.
(vect_build_slp_tree_2): Map grouped source accesses to element lanes.
(vect_optimize_slp_pass::remove_redundant_permutations): Apply the
shared-layout check to complete ungrouped vector loads.
* tree-vect-stmts.cc (get_load_store_type): Account for complete vector
accesses in gap and speculative-read checks.
(vectorizable_load): Check padding using the source element type.
* tree-vect-stmts.cc (vectorizable_operation): Use the source element
type.
gcc/testsuite/ChangeLog:
PR tree-optimization/119187
* gcc.dg/vect/vect-revectorize-strided-reverse.c: New test.
* gcc.dg/vect/vect-revectorize-bb-loads.c: New test.
---
--
diff --git a/gcc/testsuite/gcc.dg/vect/vect-revectorize-bb-loads.c
b/gcc/testsuite/gcc.dg/vect/vect-revectorize-bb-loads.c
new file mode 100644
index
0000000000000000000000000000000000000000..07c684673b1fd957c13b03db05ea79308501044a
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-revectorize-bb-loads.c
@@ -0,0 +1,47 @@
+/* { dg-do run } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-O3 -fno-vect-cost-model -Wno-psabi" } */
+
+/* Ungrouped loads may be nonadjacent or in different basic blocks. */
+#include "tree-vect.h"
+
+#define TEST(SIZE) \
+typedef unsigned V##SIZE __attribute__ ((vector_size (SIZE))); \
+__attribute__ ((noipa)) void \
+copy##SIZE (V##SIZE *out, const V##SIZE *a, const V##SIZE *b, int skip) \
+{ \
+ V##SIZE x = *a; \
+ if (skip) \
+ return; \
+ \
+ V##SIZE y = *b; \
+ out[0] = x + (V##SIZE) { 1 }; \
+ out[1] = y + (V##SIZE) { 1 }; \
+} \
+__attribute__ ((optimize ("no-tree-vectorize"))) void \
+test##SIZE (void) \
+{ \
+ const unsigned input_a[] = { 10, 11, 12, 13, 14, 15, 16, 17 }; \
+ const unsigned input_b[] = { 200, 201, 202, 203, 204, 205, 206, 207 }; \
+ const unsigned expected_a[] = { 11, 11, 12, 13, 14, 15, 16, 17 }; \
+ const unsigned expected_b[] = { 201, 201, 202, 203, 204, 205, 206, 207 }; \
+ V##SIZE a, b, out[2]; \
+ __builtin_memcpy (&a, input_a, sizeof (a)); \
+ __builtin_memcpy (&b, input_b, sizeof (b)); \
+ copy##SIZE (out, &a, &b, 0); \
+ if (__builtin_memcmp (&out[0], expected_a, sizeof (a)) \
+ || __builtin_memcmp (&out[1], expected_b, sizeof (b))) \
+ __builtin_abort (); \
+}
+
+TEST (16)
+TEST (32)
+
+int
+main (void)
+{
+ check_vect ();
+ test16 ();
+ test32 ();
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/vect/vect-revectorize-strided-reverse.c
b/gcc/testsuite/gcc.dg/vect/vect-revectorize-strided-reverse.c
new file mode 100644
index
0000000000000000000000000000000000000000..279285cb5f872d95929dee87cc0f680dbca638d6
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-revectorize-strided-reverse.c
@@ -0,0 +1,47 @@
+/* { dg-do run } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-O3 -fno-vect-cost-model" } */
+
+#include "tree-vect.h"
+
+__attribute__ ((noipa)) void
+f (unsigned *restrict dst, const unsigned *restrict src, unsigned n)
+{
+ for (unsigned i = 0; i < n; ++i)
+ {
+ dst[i] = *src;
+ src -= 2;
+ }
+}
+
+static const unsigned a[] = {
+ 3, 14, 25, 36, 47, 58, 69, 80, 91, 102, 113, 124, 135, 146, 157, 168, 179,
+ 190, 201, 212, 223, 234, 245, 256, 267, 278, 289, 300, 311, 322, 333, 344,
+ 355, 366, 377, 388, 399, 410, 421, 432, 443, 454, 465, 476, 487, 498, 509,
+ 520, 531, 542, 553, 564, 575, 586, 597, 608, 619, 630, 641, 652, 663, 674,
+ 685, 696, 707, 718, 729, 740, 751, 762, 773, 784, 795, 806, 817, 828, 839,
+ 850, 861, 872, 883, 894, 905, 916, 927, 938, 949, 960, 971, 982, 993,
+ 1004, 1015, 1026, 1037, 1048, 1059, 1070, 1081, 1092, 1103, 1114, 1125,
+ 1136, 1147, 1158, 1169, 1180, 1191, 1202, 1213, 1224, 1235, 1246, 1257,
+ 1268, 1279, 1290, 1301, 1312, 1323, 1334, 1345, 1356, 1367, 1378, 1389,
+ 1400, 1411, 1422
+};
+static const unsigned expected[] = {
+ 1411, 1389, 1367, 1345, 1323, 1301, 1279, 1257, 1235, 1213, 1191, 1169,
+ 1147, 1125, 1103, 1081, 1059, 1037, 1015, 993, 971, 949, 927, 905, 883,
+ 861, 839, 817, 795, 773, 751, 729, 707, 685, 663, 641, 619, 597, 575, 553,
+ 531, 509, 487, 465, 443, 421, 399, 377, 355, 333, 311, 289, 267, 245, 223,
+ 201, 179, 157, 135, 113, 91, 69, 47, 25
+};
+
+int
+main (void)
+{
+ check_vect ();
+ unsigned out[64];
+ f (out, a + 128, 64);
+ if (__builtin_memcmp (out, expected, sizeof (out)))
+ __builtin_abort ();
+
+ return 0;
+}
diff --git a/gcc/tree-vect-data-refs.cc b/gcc/tree-vect-data-refs.cc
index
5af85a7ffc3b5705d2efa1c46b02cc50cbf4360c..2a4c7936a4d5a393419a900d784df38e5118dfa1
100644
--- a/gcc/tree-vect-data-refs.cc
+++ b/gcc/tree-vect-data-refs.cc
@@ -3086,6 +3086,18 @@ vect_analyze_group_access_1 (vec_info *vinfo,
dr_vec_info *dr_info)
data_reference *dr = dr_info->dr;
tree step = DR_STEP (dr);
tree scalar_type = TREE_TYPE (DR_REF (dr));
+ unsigned HOST_WIDE_INT nunits = 0;
+ if (VECTOR_TYPE_P (scalar_type)
+ && !TYPE_VECTOR_SUBPARTS (scalar_type).is_constant (&nunits))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "variable-length vector access: %T\n",
+ scalar_type);
+
+ return false;
+ }
+
HOST_WIDE_INT type_size = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (scalar_type));
stmt_vec_info stmt_info = dr_info->stmt;
loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
@@ -3117,6 +3129,8 @@ vect_analyze_group_access_1 (vec_info *vinfo, dr_vec_info
*dr_info)
}
groupsize = absu_hwi (dr_step) / type_size;
}
+ else if (bb_vinfo && nunits > 0)
+ groupsize = nunits;
else
groupsize = 0;
@@ -3373,6 +3387,20 @@ vect_analyze_data_ref_access (vec_info *vinfo,
dr_vec_info *dr_info)
}
}
+ /* A non-grouped fixed-length vector reference can be split into element
+ accesses during BB vectorization. Keep groups so that adjacent source
+ vectors can also be combined. */
+ if (!loop_vinfo && integer_zerop (step)
+ && !STMT_VINFO_GROUPED_ACCESS (stmt_info)
+ && vect_can_revectorize_type_p (scalar_type))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "Allowing vector memory access in BB\n");
+
+ return true;
+ }
+
if (loop && nested_in_vect_loop_p (loop, stmt_info))
{
/* Interleaved accesses are not yet supported within outer-loop
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index
2f2ff08ac45db67e01f3820ca1e78bb75ff11513..fbf13d3faa2b9ea812fabec4407a7e8c966eab1a
100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -444,24 +444,35 @@ vect_get_place_in_interleaving_chain (stmt_vec_info
stmt_info,
return -1;
}
-/* Return the source reference, group bounds and element size for NODE.
- For ungrouped loads, PERM determines how many source elements are used. */
+/* Describe the memory covered by NODE in scalar elements. For example,
+ two V4SI accesses have group_lanes == 8 and element_size == 4, although
+ DR_GROUP_SIZE is 2. Scale any trailing gap by the same factor.
+
+ For ungrouped loads, PERM identifies the source elements that are used;
+ it can describe a splat or several distinct invariant references. */
vect_slp_memory_layout
vect_get_slp_memory_layout (slp_tree node, const load_permutation_t &perm)
{
stmt_vec_info stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
dr_vec_info *dr_info = vect_first_dr_info (stmt_info);
+ tree type = TREE_TYPE (DR_REF (dr_info->dr));
+ unsigned int source_lanes
+ = VECTOR_TYPE_P (type) ? TYPE_VECTOR_SUBPARTS (type).to_constant () : 1;
vect_slp_memory_layout memory
- = { dr_info, 1, 0, vect_get_scalar_dr_size (dr_info) };
+ = { dr_info, source_lanes, 0,
+ vect_get_scalar_dr_size (dr_info) / source_lanes };
if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
{
- memory.group_lanes = DR_GROUP_SIZE (dr_info->stmt);
- memory.gap = DR_GROUP_GAP (dr_info->stmt);
+ memory.group_lanes *= DR_GROUP_SIZE (dr_info->stmt);
+ memory.gap = DR_GROUP_GAP (dr_info->stmt) * source_lanes;
}
else
- /* Cover splats and collections of distinct invariant references. */
+ /* Non-grouped loads include both splats of one reference and collections
+ of distinct invariant references. The permutation already uses lane
+ indices. Retain the full source access even if only a prefix is
+ used. */
for (unsigned int lane : perm)
memory.group_lanes = MAX (memory.group_lanes, lane + 1);
@@ -477,13 +488,17 @@ vect_slp_memory_stmts (const vect_slp_memory_layout
&memory)
vec<stmt_vec_info> stmts;
stmts.create (memory.group_lanes);
stmts.quick_grow_cleared (memory.group_lanes);
+ unsigned int source_lanes
+ = vect_get_scalar_dr_size (memory.dr_info) / memory.element_size;
stmt_vec_info first = memory.dr_info->stmt;
unsigned int pos = 0;
for (stmt_vec_info si = first; si; si = DR_GROUP_NEXT_ELEMENT (si))
{
if (si != first)
- pos += DR_GROUP_GAP (si);
- stmts[pos] = si;
+ pos += DR_GROUP_GAP (si) * source_lanes;
+
+ for (unsigned int i = 0; i < source_lanes; ++i)
+ stmts[pos + i] = vect_stmt_to_vectorize (si);
}
return stmts;
@@ -1368,20 +1383,21 @@ vect_build_slp_tree_3 (vec_info *vinfo,
vec<stmt_vec_info> stmts,
return false;
}
- /* Non-grouped store or load. */
- if (ldst_p
- && !STMT_VINFO_GROUPED_ACCESS (stmt_info)
- && DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info))
- && rhs_code != CFN_GATHER_LOAD
- && rhs_code != CFN_MASK_GATHER_LOAD
- && rhs_code != CFN_MASK_LEN_GATHER_LOAD
- && rhs_code != CFN_SCATTER_STORE
- && rhs_code != CFN_MASK_SCATTER_STORE
- && rhs_code != CFN_MASK_LEN_SCATTER_STORE
- && !STMT_VINFO_GATHER_SCATTER_P (stmt_info)
- /* Not grouped loads are handled as externals for BB
- vectorization. Treat them as not vectorizable. */
- && is_a <bb_vec_info> (vinfo))
+ /* Gather/scatter calls do not necessarily have an ordinary data ref. */
+ bool ungrouped_load_p
+ = (ldst_p
+ && !STMT_VINFO_GROUPED_ACCESS (stmt_info)
+ && !(call_stmt && gimple_call_internal_p (call_stmt)
+ && internal_gather_scatter_fn_p
+ (gimple_call_internal_fn (call_stmt)))
+ && !STMT_VINFO_GATHER_SCATTER_P (stmt_info)
+ && DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)));
+
+ /* BB SLP can split one vector load, but leaves scalar ungrouped
+ loads as external operands. */
+ if (ungrouped_load_p
+ && is_a <bb_vec_info> (vinfo)
+ && (!lhs || !vect_can_revectorize_type_p (TREE_TYPE (lhs))))
{
/* Not grouped load. */
if (dump_enabled_p ())
@@ -1665,31 +1681,22 @@ vect_build_slp_tree_3 (vec_info *vinfo,
vec<stmt_vec_info> stmts,
/* Non-grouped store or load. */
else if (ldst_p)
{
- if (DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info))
- && rhs_code != CFN_GATHER_LOAD
- && rhs_code != CFN_MASK_GATHER_LOAD
- && rhs_code != CFN_MASK_LEN_GATHER_LOAD
- && rhs_code != CFN_SCATTER_STORE
- && rhs_code != CFN_MASK_SCATTER_STORE
- && rhs_code != CFN_MASK_LEN_SCATTER_STORE
- && !STMT_VINFO_GATHER_SCATTER_P (stmt_info))
- {
- gcc_assert (!is_a <bb_vec_info> (vinfo));
- /* For loop vectorization we can handle splats the same we
- handle single element interleaving. Likewise we can handle
- a collection of invariant refs. */
- if (stmt_info != first_stmt_info
- && !(integer_zerop (DR_STEP (STMT_VINFO_DATA_REF (stmt_info)))
- && integer_zerop (DR_STEP (STMT_VINFO_DATA_REF
- (first_stmt_info)))))
- {
- /* Not grouped load. */
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "Build SLP failed: not grouped load %G",
- stmt);
- continue;
- }
+ /* BB SLP must stay within one source vector. Loop SLP also allows
+ distinct invariant loads, or repeated uses of the same load. */
+ if (ungrouped_load_p
+ && stmt_info != first_stmt_info
+ && (is_a <bb_vec_info> (vinfo)
+ || !integer_zerop
+ (DR_STEP (STMT_VINFO_DATA_REF (stmt_info)))
+ || !integer_zerop
+ (DR_STEP (STMT_VINFO_DATA_REF (first_stmt_info)))))
+ {
+ /* Not grouped load. */
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "Build SLP failed: not grouped load %G", stmt);
+
+ continue;
}
}
/* Not memory operation. */
@@ -2344,17 +2351,39 @@ vect_build_slp_tree_2 (vec_info *vinfo, slp_tree node,
stmt_vec_info load_info;
load_permutation.create (group_size);
stmt_vec_info first_stmt_info = vect_first_dr_info (stmt_info)->stmt;
+ tree lhs = gimple_get_lhs (stmt_info->stmt);
+ bool split_vector_load
+ = lhs && vect_can_revectorize_type_p (TREE_TYPE (lhs));
bool any_permute = false;
FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), j, load_info)
{
int load_place;
+ tree type;
if (! load_info)
{
if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
load_place = j;
+ else if (split_vector_load)
+ load_place = j;
else
load_place = 0;
}
+
+ /* For V4SI, source access 1 denotes element lanes 4..7.
+ J % NELTS selects the element within that source access. */
+ else if ((type = TREE_TYPE (gimple_get_lhs (load_info->stmt)))
+ && VECTOR_TYPE_P (type)
+ && TYPE_VECTOR_SUBPARTS (type).is_constant ())
+ {
+ unsigned nelts = TYPE_VECTOR_SUBPARTS (type).to_constant ();
+ int place
+ = STMT_VINFO_GROUPED_ACCESS (load_info)
+ ? vect_get_place_in_interleaving_chain (load_info,
+ first_stmt_info)
+ : 0;
+ gcc_assert (place >= 0);
+ load_place = place * nelts + j % nelts;
+ }
else if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
load_place
= vect_get_place_in_interleaving_chain (load_info,
@@ -8288,16 +8317,6 @@ vect_optimize_slp_pass::remove_redundant_permutations ()
{
loop_vec_info loop_vinfo = as_a<loop_vec_info> (m_vinfo);
bool this_load_permuted = !vect_load_perm_consecutive_p (node, 0);
- /* When this isn't a grouped access we know it's single element
- and contiguous. */
- if (!STMT_VINFO_GROUPED_ACCESS (SLP_TREE_SCALAR_STMTS (node)[0]))
- {
- if (!this_load_permuted
- && (known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), 1U)
- || SLP_TREE_LANES (node) == 1))
- SLP_TREE_LOAD_PERMUTATION (node).release ();
- continue;
- }
const vect_slp_memory_layout memory
= vect_get_slp_memory_layout (node,
SLP_TREE_LOAD_PERMUTATION (node));
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index
696a7bb60bfecf0d34c17f0b54a4351810c4116e..f0862fc9dd7723dfa4b3cd78abc091bfbe7b5a35
100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -2271,13 +2271,14 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info
stmt_info,
&& (*memory_access_type != VMAT_STRIDED_SLP
|| cmp > 0));
- /* If this is single-element interleaving with an element
- distance that leaves unused vector loads around fall back
+ /* If this is single-element interleaving with a trailing gap
+ that leaves unused vector loads around fall back
to elementwise access if possible - we otherwise least
create very sub-optimal code in that case (and
blow up memory, see PR65518). */
if (loop_vinfo
&& single_element_p
+ && gap != 0
&& (*memory_access_type == VMAT_CONTIGUOUS
|| *memory_access_type == VMAT_CONTIGUOUS_REVERSE)
&& maybe_gt (group_size, TYPE_VECTOR_SUBPARTS (vectype)))
@@ -2604,6 +2605,8 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info
stmt_info,
poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
poly_uint64 read_amount
= vf * TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
+
+ /* GROUP_SIZE counts elements even for an ungrouped vector access. */
read_amount *= group_size;
auto target_alignment
@@ -6709,6 +6712,9 @@ vectorizable_operation (vec_info *vinfo,
scalar_dest = gimple_assign_lhs (stmt);
tree scalar_type = TREE_TYPE (scalar_dest);
+ if (VECTOR_TYPE_P (scalar_type))
+ scalar_type = TREE_TYPE (scalar_type);
+
vectype_out = SLP_TREE_VECTYPE (slp_node);
/* Most operations cannot handle bit-precision types without extra
@@ -9978,6 +9984,9 @@ vectorizable_load (vec_info *vinfo,
padding bits in the type that might leak otherwise.
Refer to PR115336. */
tree scalar_type = TREE_TYPE (scalar_dest);
+ if (VECTOR_TYPE_P (scalar_type))
+ scalar_type = TREE_TYPE (scalar_type);
+
bool type_mode_padding_p
= TYPE_PRECISION (scalar_type) < GET_MODE_PRECISION (GET_MODE_INNER
(mode));
