To handle vector permutations, extracts, inserts and constructors in SLP we
need to map each result element to an element of a source operand. The
operands can have different lane counts, so they cannot all be described using
the result width.
e.g.
typedef unsigned V8 __attribute__((vector_size(32)));
void insert_lane (V8 *out, const V8 *a, unsigned value)
{
V8 x = *a;
x[5] = value;
*out = x;
}
slp_oprnds records the lane count of each operand and decodes constant
permutations once per statement. SLP builds VEC_PERM_EXPR nodes from
those mappings. For the BIT_INSERT_EXPR at bit 160 above, the map is:
lane permutation { 0[0] 0[1] 0[2] 0[3] 0[4] 1[0] 0[6] 0[7] }
With -O3 -fno-vect-cost-model -march=armv8-a we generated before:
ldp x4, x5, [x1]
ldp x1, x3, [x1, 16]
stp x4, x5, [x0]
bfi x1, x2, 32, 32
stp x1, x3, [x0, 16]
ret
After these patches we get:
ldp q30, q31, [x1]
ins v31.s[1], w2
stp q30, q31, [x0]
ret
Matching and failure handling use the child lane positions. This also lets e.g.
a 4 lane extract discover the eight-lane source without changing the
source's 32-byte memory stride. Constructor roots use the same operand mapping
when building the SLP tree so are handled similarly.
If SLP declines an insert into an emulated vector, lower_vec_insert provides
the generic lowering needed to expand the remaining operation.
Bootstrapped Regtested on aarch64-none-linux-gnu,
arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
-m32, -m64 and no issues.
Any feedback?
Thanks,
Tamar
Assisted-by: Codex GPT 6.1 Sol
AI was used to generate test cases for various types and inputs.
gcc/ChangeLog:
PR tree-optimization/119187
* tree-vect-slp.cc (vect_get_operand_map, slp_oprnds,
vect_get_and_check_slp_defs): Map vector operands and their lane counts.
(slp_oprnds::get_op_nunits, slp_oprnds::get_permutation): New.
(vect_build_slp_tree_3, vect_build_slp_tree_2): Build lane-routing
operations as ordinary SLP permutation nodes.
(vect_slp_check_for_roots): Recognize vector-valued constructors.
* tree-vect-generic.cc (lower_vec_insert): New.
(expand_vector_operations_1): Lower inserts into emulated vectors.
* tree-vect-slp.cc (slp_oprnds::get_op_for_slp_child): Share constructor
operand mapping.
(vect_slp_is_lane_insert): Use the common constructor operand count.
gcc/testsuite/ChangeLog:
PR tree-optimization/119187
* gcc.dg/vect/vect-revectorize-permute.c: New test.
* g++.target/aarch64/vect-revectorize-permute.C: New test.
* gcc.dg/vect/vect-revectorize-constructor.c: New test.
* gcc.dg/vect/vect-revectorize-subvectors.c: New test.
* gcc.dg/vect/vect-revectorize-insert-lower.c: New test.
* gcc.dg/vect/vect-revectorize-memory-lanes.c: New test.
* gcc.target/aarch64/vect-revectorize-memory-lanes.c: New test.
* gcc.target/aarch64/vect-revectorize-invariant-cost.c: New test.
* gcc.target/aarch64/sve/vect-revectorize-perm-const-2048-run.c: New
XFAIL test.
* gcc.dg/vect/vect-revectorize-constructor-loop.c: New test.
* gcc.dg/vect/vect-revectorize-memory-lanes-2.c: New test.
* gcc.dg/vect/vect-revectorize-memory-lanes-3.c: New test.
* gcc.dg/vect/vect-revectorize-memory-lanes.h: New test support.
* gcc.target/aarch64/vect-revectorize-memory-lanes-2.c: New test.
* gcc.target/aarch64/vect-revectorize-memory-lanes-3.c: New test.
---
--
diff --git a/gcc/testsuite/g++.target/aarch64/vect-revectorize-permute.C
b/gcc/testsuite/g++.target/aarch64/vect-revectorize-permute.C
new file mode 100644
index
0000000000000000000000000000000000000000..69de9244c9a93bbc56c37975b5256a75462713d6
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/vect-revectorize-permute.C
@@ -0,0 +1,33 @@
+/* { dg-do run } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-options "-O3 -march=armv8-a -fdump-tree-slp-details -save-temps" } */
+
+#include "../../gcc.dg/vect/tree-vect.h"
+
+typedef unsigned V __attribute__ ((vector_size (32)));
+
+__attribute__ ((noipa)) void
+shuffle (V &x)
+{
+ x ^= 1;
+ x = (V) { x[4], x[0], x[5], x[1], x[6], x[2], x[7], x[3] };
+}
+
+__attribute__ ((optimize ("no-tree-vectorize")))
+int
+main ()
+{
+ check_vect ();
+ V x = { 0, 101, 202, 303, 404, 505, 606, 707 };
+ const V expected = { 405, 1, 504, 100, 607, 203, 706, 302 };
+ shuffle (x);
+ if (__builtin_memcmp (&x, &expected, sizeof (x)))
+ __builtin_abort ();
+ return 0;
+}
+
+/* Retain the cost model: the original unsupported shuffle is lowered
+ elementwise, whereas SLP needs two native permutations. */
+/* { dg-final { scan-tree-dump "basic block part vectorized using 16 byte
vectors" "slp2" } } */
+/* { dg-final { scan-assembler {zip1\tv[0-9]+\.4s} } } */
+/* { dg-final { scan-assembler {zip2\tv[0-9]+\.4s} } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-revectorize-constructor-loop.c
b/gcc/testsuite/gcc.dg/vect/vect-revectorize-constructor-loop.c
new file mode 100644
index
0000000000000000000000000000000000000000..9c86696831e33c99a451c599be4d996f531e00ea
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-revectorize-constructor-loop.c
@@ -0,0 +1,75 @@
+/* { dg-do run } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-O3 -fgimple -fno-vect-cost-model -Wno-psabi" } */
+/* { dg-additional-options "-fdump-tree-slp-details -fdump-tree-vect-details"
} */
+
+#include "tree-vect.h"
+
+typedef unsigned V4 __attribute__ ((vector_size (16)));
+typedef unsigned V8 __attribute__ ((vector_size (32)));
+
+__attribute__ ((always_inline)) static inline V8 __GIMPLE ()
+concat (V4 a, V4 b)
+{
+ V8 result;
+ result = _Literal (V8) { a, b };
+ return result;
+}
+
+/* Nested constructors, including an invariant operand. */
+__attribute__ ((noipa)) void
+nested (V8 *restrict out, const V4 *restrict in, V4 bias, unsigned n)
+{
+ for (unsigned i = 0; i < n; ++i)
+ {
+ V4 a = in[2 * i] + (V4) { 1, 3, 5, 7 };
+ V4 b = in[2 * i + 1] ^ (V4) { 9, 11, 13, 15 };
+ out[2 * i] = concat (a, b) + 1;
+ out[2 * i + 1] = concat (b, bias) ^ 17;
+ }
+}
+
+__attribute__ ((noipa)) void
+scalars (V4 *restrict out, const unsigned *restrict in, unsigned n)
+{
+ for (unsigned i = 0; i < n; ++i)
+ out[i] = (V4) { in[4 * i] + 1, in[4 * i + 1] + 3,
+ in[4 * i + 2] + 5, in[4 * i + 3] + 7 } ^ 17;
+}
+
+__attribute__ ((optimize ("no-tree-vectorize"))) int
+main (void)
+{
+ check_vect ();
+ V4 in[34], scalar_out[34], expected_scalars[17];
+ V8 out[34], expected_nested[34];
+ const V4 bias = { 31, 37, 41, 43 };
+ for (unsigned i = 0; i < 34; ++i)
+ in[i] = (V4) { 0, 7, 14, 21 } + 101 * i;
+
+ /* Scalar references keep the two halves and the invariant lanes separate.
*/
+ for (unsigned k = 0; k < 17 * 4; ++k)
+ {
+ unsigned i = k / 4, j = k % 4;
+ expected_nested[2 * i][j] = 202 * i + 9 * j + 2;
+ expected_nested[2 * i][j + 4]
+ = ((202 * i + 101 + 7 * j) ^ (9 + 2 * j)) + 1;
+ expected_nested[2 * i + 1][j]
+ = (202 * i + 101 + 7 * j) ^ (9 + 2 * j) ^ 17;
+ expected_nested[2 * i + 1][j + 4] = bias[j] ^ 17;
+ expected_scalars[i][j] = (101 * i + 9 * j + 1) ^ 17;
+ }
+
+ for (unsigned n = 0; n <= 17; ++n)
+ {
+ nested (out, in, bias, n);
+ scalars (scalar_out, (const unsigned *) in, n);
+ if (__builtin_memcmp (out, expected_nested, 2 * n * sizeof (V8))
+ || __builtin_memcmp (scalar_out, expected_scalars, n * sizeof (V4)))
+ __builtin_abort ();
+ }
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops in function" 2 "vect"
{ target aarch64*-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-revectorize-constructor.c
b/gcc/testsuite/gcc.dg/vect/vect-revectorize-constructor.c
new file mode 100644
index
0000000000000000000000000000000000000000..37071c6d350fe35d9f9a8433753e504c67a90a80
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-revectorize-constructor.c
@@ -0,0 +1,110 @@
+/* { dg-do run } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-O3 -fgimple -fno-vect-cost-model -Wno-psabi" } */
+/* { dg-additional-options "-fdump-tree-slp-details -fdump-tree-vect-details"
} */
+
+#include "tree-vect.h"
+
+typedef unsigned V2 __attribute__ ((vector_size (8)));
+typedef unsigned V4 __attribute__ ((vector_size (16)));
+typedef unsigned V8 __attribute__ ((vector_size (32)));
+
+/* Start at SLP to test constructor roots without intervening lowering. */
+__attribute__ ((noipa)) V4 __GIMPLE (ssa, startwith ("slp"))
+root (const V2 *p)
+{
+ V2 a;
+ V2 b;
+ V2 low;
+ V2 high;
+ V4 result;
+
+ __BB(2):
+ a_2 = __MEM <V2> (p_1(D));
+ b_3 = __MEM <V2> (p_1(D) + _Literal (const V2 *) 8);
+ low_4 = a_2 + _Literal (V2) { 1u, 3u };
+ high_5 = b_3 + _Literal (V2) { 5u, 7u };
+ result_6 = _Literal (V4) { low_4, high_5 };
+ return result_6;
+}
+
+__attribute__ ((noipa)) V8 __GIMPLE (ssa, startwith ("slp"))
+wide_root (const V2 *p)
+{
+ V2 a;
+ V2 b;
+ V2 c;
+ V2 d;
+ V2 low;
+ V2 high;
+ V2 low2;
+ V2 high2;
+ V8 result;
+
+ __BB(2):
+ a_2 = __MEM <V2> (p_1(D));
+ b_3 = __MEM <V2> (p_1(D) + _Literal (const V2 *) 8);
+ c_4 = __MEM <V2> (p_1(D) + _Literal (const V2 *) 16);
+ d_5 = __MEM <V2> (p_1(D) + _Literal (const V2 *) 24);
+ low_6 = a_2 + _Literal (V2) { 1u, 3u };
+ high_7 = b_3 + _Literal (V2) { 5u, 7u };
+ low2_8 = c_4 + _Literal (V2) { 9u, 11u };
+ high2_9 = d_5 + _Literal (V2) { 13u, 15u };
+ result_10 = _Literal (V8) { low_6, high_7, low2_8, high2_9 };
+ return result_10;
+}
+
+/* Keep the ordinary lowering for constructors with implicit zero lanes. */
+__attribute__ ((noipa)) V4 __GIMPLE (ssa, startwith ("slp"))
+partial (unsigned x)
+{
+ unsigned low;
+ unsigned high;
+ V4 result;
+
+ __BB(2):
+ low_2 = x_1(D) + 1u;
+ high_3 = x_1(D) + 3u;
+ result_4 = _Literal (V4) { low_2, high_3 };
+ return result_4;
+}
+
+__attribute__ ((noipa)) V4 __GIMPLE (ssa, startwith ("slp"))
+empty (void)
+{
+ V4 result;
+
+ __BB(2):
+ result_1 = _Literal (V4) {};
+ return result_1;
+}
+
+__attribute__ ((optimize ("no-tree-vectorize"))) int
+main (void)
+{
+ check_vect ();
+ V4 in[] = { { 0, 7, 14, 21 }, { 101, 108, 115, 122 } };
+ V4 expected = { 1, 10, 19, 28 };
+ V8 expected_wide = { 1, 10, 19, 28, 110, 119, 128, 137 };
+ V4 r = root ((const V2 *) in);
+ if (__builtin_memcmp (&r, &expected, sizeof (r)))
+ __builtin_abort ();
+
+ V8 wide = wide_root ((const V2 *) in);
+ if (__builtin_memcmp (&wide, &expected_wide, sizeof (wide)))
+ __builtin_abort ();
+
+ V4 p = partial (123);
+ V4 expected_p = { 124, 126, 0, 0 };
+ V4 zero = empty ();
+ for (unsigned j = 0; j < 4; ++j)
+ if (p[j] != expected_p[j] || zero[j] != 0)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* Check root discovery and vectorization of both kinds of constructor. */
+/* { dg-final { scan-tree-dump "Analyzing vectorizable constructor: result_6
=" "slp2" { target aarch64*-*-* } } } */
+/* { dg-final { scan-tree-dump "Analyzing vectorizable constructor: result_10
=" "slp2" { target aarch64*-*-* } } } */
+/* { dg-final { scan-tree-dump-times {basic block part vectorized using
[^\n]*vectors} 2 "slp2" { target aarch64*-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-revectorize-insert-lower.c
b/gcc/testsuite/gcc.dg/vect/vect-revectorize-insert-lower.c
new file mode 100644
index
0000000000000000000000000000000000000000..36370473a75929601381ba739927ec3d368585db
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-revectorize-insert-lower.c
@@ -0,0 +1,40 @@
+/* Inserts into unsupported vectors must also lower without vectorization. */
+/* { dg-do run } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-O2 -fgimple -fno-tree-vectorize -Wno-psabi" } */
+
+#include "tree-vect.h"
+
+typedef unsigned V8 __attribute__ ((vector_size (32)));
+typedef unsigned V2 __attribute__ ((vector_size (8)));
+__attribute__ ((noipa)) V8 __GIMPLE
+insert_pair (V8 x, V2 y)
+{
+ V8 res;
+ res = __BIT_INSERT (x, y, 64);
+ return res;
+}
+
+__attribute__ ((noipa)) V8 __GIMPLE
+insert_scalar (V8 x, unsigned y)
+{
+ V8 res;
+ res = __BIT_INSERT (x, y, 160);
+ return res;
+}
+
+int
+main (void)
+{
+ check_vect ();
+ V8 x = { 0, 1, 2, 3, 4, 5, 6, 7 };
+ V2 pair = { 99, 101 };
+ V8 expected_a = { 0, 1, 99, 101, 4, 5, 6, 7 };
+ V8 expected_b = { 0, 1, 2, 3, 4, 23, 6, 7 };
+ V8 a = insert_pair (x, pair), b = insert_scalar (x, 23);
+ if (__builtin_memcmp (&a, &expected_a, sizeof (a))
+ || __builtin_memcmp (&b, &expected_b, sizeof (b)))
+ __builtin_abort ();
+
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/vect/vect-revectorize-memory-lanes-2.c
b/gcc/testsuite/gcc.dg/vect/vect-revectorize-memory-lanes-2.c
new file mode 100644
index
0000000000000000000000000000000000000000..9fb407030156ef617fd1fc071f0ae994e9a9bf0e
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-revectorize-memory-lanes-2.c
@@ -0,0 +1,7 @@
+/* { dg-do run } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-O3 -fno-vect-cost-model -Wno-psabi" } */
+/* { dg-xfail-run-if "SVE 2048-bit two-input permutation" { aarch64_sve } {
"-msve-vector-bits=2048" } } */
+
+typedef unsigned V __attribute__ ((vector_size (32)));
+#include "vect-revectorize-memory-lanes.h"
diff --git a/gcc/testsuite/gcc.dg/vect/vect-revectorize-memory-lanes-3.c
b/gcc/testsuite/gcc.dg/vect/vect-revectorize-memory-lanes-3.c
new file mode 100644
index
0000000000000000000000000000000000000000..b59c04e283d8715d56a63bb1addd9a5341c0d2db
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-revectorize-memory-lanes-3.c
@@ -0,0 +1,6 @@
+/* { dg-do run } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-O3 -fno-vect-cost-model -Wno-psabi" } */
+
+typedef unsigned V __attribute__ ((vector_size (64)));
+#include "vect-revectorize-memory-lanes.h"
diff --git a/gcc/testsuite/gcc.dg/vect/vect-revectorize-memory-lanes.c
b/gcc/testsuite/gcc.dg/vect/vect-revectorize-memory-lanes.c
new file mode 100644
index
0000000000000000000000000000000000000000..59346ecb4b56bc751ecf2a27f09d0a4ee1983d5f
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-revectorize-memory-lanes.c
@@ -0,0 +1,7 @@
+/* { dg-do run } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-O3 -fno-vect-cost-model -Wno-psabi" } */
+/* { dg-xfail-run-if "SVE 2048-bit two-input permutation" { aarch64_sve } {
"-msve-vector-bits=2048" } } */
+
+typedef unsigned V __attribute__ ((vector_size (16)));
+#include "vect-revectorize-memory-lanes.h"
diff --git a/gcc/testsuite/gcc.dg/vect/vect-revectorize-memory-lanes.h
b/gcc/testsuite/gcc.dg/vect/vect-revectorize-memory-lanes.h
new file mode 100644
index
0000000000000000000000000000000000000000..c1d8c4e848847814e8fe464b76db2d86b25d0688
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-revectorize-memory-lanes.h
@@ -0,0 +1,77 @@
+/* Source loads have more lanes than the prefix or repeated-lane results. */
+#include "tree-vect.h"
+
+typedef unsigned V2 __attribute__ ((vector_size (8)));
+typedef unsigned V4 __attribute__ ((vector_size (16)));
+
+__attribute__ ((noipa)) void
+prefix_load (V2 *restrict out, const V *restrict in, unsigned n)
+{
+ for (unsigned i = 0; i < n; ++i)
+ out[i] = __builtin_shufflevector (in[i], in[i], 0, 1);
+}
+
+__attribute__ ((noipa)) void
+repeat_load (V4 *restrict out, const V *restrict in, unsigned n)
+{
+ for (unsigned i = 0; i < n; ++i)
+ out[i] = __builtin_shufflevector (in[i], in[i], 1, 0, 1, 0);
+}
+
+__attribute__ ((noipa)) void
+group_load (V4 *restrict out, const V *restrict in, unsigned n)
+{
+ for (unsigned i = 0; i < n; ++i)
+ out[i]
+ = __builtin_shufflevector (in[2 * i], in[2 * i + 1], 1, 0,
+ sizeof (V) / sizeof (unsigned) + 1,
+ sizeof (V) / sizeof (unsigned));
+}
+
+__attribute__ ((optimize ("no-tree-vectorize"))) int
+main (void)
+{
+ check_vect ();
+ V in[130];
+ unsigned input[130][2];
+ V4 expected_repeat[65], expected_group[65];
+ V2 prefix[66];
+ V4 repeated[66], grouped[66];
+ const V2 guard2 = { -1, -1 };
+ const V4 guard4 = { -1, -1, -1, -1 };
+
+ /* Only two source lanes are used. Distinct rows check source strides. */
+ for (unsigned i = 0; i < 130; ++i)
+ {
+ input[i][0] = 37 * i;
+ input[i][1] = 37 * i + 11;
+ in[i] = (V) { input[i][0], input[i][1] };
+ }
+
+ for (unsigned i = 0; i < 65; ++i)
+ {
+ expected_repeat[i] = (V4) { 37 * i + 11, 37 * i,
+ 37 * i + 11, 37 * i };
+ expected_group[i] = (V4) { 74 * i + 11, 74 * i,
+ 74 * i + 48, 74 * i + 37 };
+ }
+
+ /* Cross vector boundaries even at the largest SVE vector length. */
+ for (unsigned n = 0; n <= 65; ++n)
+ {
+ prefix[n] = guard2;
+ repeated[n] = grouped[n] = guard4;
+ prefix_load (prefix, in, n);
+ repeat_load (repeated, in, n);
+ group_load (grouped, in, n);
+ if (__builtin_memcmp (prefix, input, n * sizeof (V2))
+ || __builtin_memcmp (repeated, expected_repeat, n * sizeof (V4))
+ || __builtin_memcmp (grouped, expected_group, n * sizeof (V4))
+ || __builtin_memcmp (&prefix[n], &guard2, sizeof (V2))
+ || __builtin_memcmp (&repeated[n], &guard4, sizeof (V4))
+ || __builtin_memcmp (&grouped[n], &guard4, sizeof (V4)))
+ __builtin_abort ();
+ }
+
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/vect/vect-revectorize-permute.c
b/gcc/testsuite/gcc.dg/vect/vect-revectorize-permute.c
new file mode 100644
index
0000000000000000000000000000000000000000..add6aa413ef8d4ecd84663b81e82885c9b4dae8f
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-revectorize-permute.c
@@ -0,0 +1,54 @@
+/* { dg-do run } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-O3 -fno-vect-cost-model" } */
+
+#include "tree-vect.h"
+
+typedef unsigned V __attribute__ ((vector_size (32)));
+
+__attribute__ ((noipa)) void
+permute (V *out, const V *a, const V *b)
+{
+ out[0] = __builtin_shuffle (*a, *b, (V) { 7, 8, 2, 13, 4, 11, 6, 9 });
+ out[1] = __builtin_shuffle (*a, *b,
+ (V) { -1, -8, 18, 29, 36, 43, 54, 57 });
+ out[2] = __builtin_shuffle (*a, (V) { 7, 6, 5, 4, 3, 2, 1, 0 });
+}
+
+__attribute__ ((noipa)) void
+permute_variable (V *out, const V *a, const V *b, const V *mask)
+{
+ V t = __builtin_shuffle (*a, *b, *mask);
+ out[0] = __builtin_shuffle (t, (V) { 7, 6, 5, 4, 3, 2, 1, 0 });
+ out[1] = __builtin_shuffle (*a, *mask);
+ out[2] = t + *b;
+}
+
+int
+main (void)
+{
+ check_vect ();
+ V a = { 0, 9, 18, 27, 36, 45, 54, 63 };
+ V b = { 61, 58, 55, 52, 49, 46, 43, 40 };
+ V expected[] = {
+ { 63, 61, 18, 46, 36, 52, 54, 58 },
+ { 40, 61, 18, 46, 36, 52, 54, 58 },
+ { 63, 54, 45, 36, 27, 18, 9, 0 }
+ };
+ V out[3];
+ permute (out, &a, &b);
+ if (__builtin_memcmp (out, expected, sizeof (out)))
+ __builtin_abort ();
+
+ V variable_mask = { -1, -8, 18, 29, 36, 43, 54, 57 };
+ V expected_variable[] = {
+ { 58, 54, 52, 36, 46, 18, 61, 40 },
+ { 63, 0, 18, 45, 36, 27, 54, 9 },
+ { 101, 119, 73, 98, 85, 98, 97, 98 }
+ };
+ permute_variable (out, &a, &b, &variable_mask);
+ if (__builtin_memcmp (out, expected_variable, sizeof (out)))
+ __builtin_abort ();
+
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/vect/vect-revectorize-subvectors.c
b/gcc/testsuite/gcc.dg/vect/vect-revectorize-subvectors.c
new file mode 100644
index
0000000000000000000000000000000000000000..da2306298f297c94bbe0b36825b630f31e6f2097
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-revectorize-subvectors.c
@@ -0,0 +1,133 @@
+/* { dg-do run } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-O3 -fgimple -fno-vect-cost-model -Wno-psabi
-fdump-tree-slp-details -fdump-tree-vect-details" } */
+
+#include "tree-vect.h"
+
+typedef unsigned V2 __attribute__ ((vector_size (8)));
+typedef unsigned V4 __attribute__ ((vector_size (16)));
+typedef unsigned V8 __attribute__ ((vector_size (32)));
+
+/* Preserve shared subvector extracts until SLP. */
+__attribute__ ((noipa)) void __GIMPLE (ssa, startwith ("slp"))
+extracts (V4 *out, const V8 *in)
+{
+ V8 x;
+ V8 y;
+ V2 low;
+ V2 high;
+ V2 a;
+ V2 b;
+ V4 result;
+
+ __BB(2):
+ x_3 = __MEM <V8> (in_2(D));
+ y_4 = x_3 ^ _Literal (V8) { 17u, 17u, 17u, 17u, 17u, 17u, 17u, 17u };
+ low_5 = __BIT_FIELD_REF <V2> (y_4, 64, 64);
+ high_6 = __BIT_FIELD_REF <V2> (y_4, 64, 192);
+ a_7 = low_5 + _Literal (V2) { 1u, 3u };
+ b_8 = high_6 + _Literal (V2) { 5u, 7u };
+ result_9 = _Literal (V4) { a_7, b_8 };
+ __MEM <V4> (out_1(D)) = result_9;
+ return;
+}
+
+/* Chained subvector and scalar inserts at nonzero offsets. */
+__attribute__ ((noipa)) void __GIMPLE (ssa, startwith ("slp"))
+inserts (V8 *out, const V8 *in, const V2 *pair, unsigned value)
+{
+ V8 x;
+ V8 y;
+ V2 p;
+ V2 q;
+ unsigned v;
+ V8 tmp;
+ V8 result;
+
+ __BB(2):
+ x_5 = __MEM <V8> (in_2(D));
+ p_6 = __MEM <V2> (pair_3(D));
+ y_7 = x_5 ^ _Literal (V8) { 17u, 17u, 17u, 17u, 17u, 17u, 17u, 17u };
+ q_8 = p_6 + _Literal (V2) { 1u, 3u };
+ v_9 = value_4(D) + 1u;
+ tmp_10 = __BIT_INSERT (y_7, q_8, 64);
+ result_11 = __BIT_INSERT (tmp_10, v_9, 160);
+ __MEM <V8> (out_1(D)) = result_11;
+ return;
+}
+
+__attribute__ ((always_inline)) static inline V8 __GIMPLE ()
+update (V8 x)
+{
+ V2 p;
+ V2 q;
+ V8 result;
+ p = __BIT_FIELD_REF <V2> (x, 64, 64);
+ q = p + _Literal (V2) { 1u, 3u };
+ result = __BIT_INSERT (x, q, 64);
+ return result;
+}
+
+__attribute__ ((noipa)) void
+updates (V8 *p, unsigned n)
+{
+ for (unsigned i = 0; i < n; ++i)
+ p[i] = update (p[i]);
+}
+
+__attribute__ ((noipa)) V8
+recurrence (V8 x, unsigned n)
+{
+ for (unsigned i = 0; i < n; ++i)
+ x = update (x);
+ return x;
+}
+
+__attribute__ ((optimize ("no-tree-vectorize"))) int
+main (void)
+{
+ check_vect ();
+ for (unsigned n = 0; n < 17; ++n)
+ {
+ V8 in = (V8) { 0, 7, 14, 21, 28, 35, 42, 49 } + 101 * n;
+ V8 b, data[17], want_recurrence = in, want_update = in;
+ V2 pair = { 3 * n, 7 * n };
+ V4 a;
+
+ /* Compute the references by scalar lane, independently of the
+ subvector extracts, inserts and constructors under test. */
+ want_recurrence[2] += n;
+ want_recurrence[3] += 3 * n;
+ want_update[2] += 1;
+ want_update[3] += 3;
+ V4 want_extract = { (in[2] ^ 17) + 1, (in[3] ^ 17) + 3,
+ (in[6] ^ 17) + 5, (in[7] ^ 17) + 7 };
+ V8 want_insert = in ^ 17;
+ want_insert[2] = 3 * n + 1;
+ want_insert[3] = 7 * n + 3;
+ want_insert[5] = 11 * n + 1;
+
+ for (unsigned i = 0; i < 17; ++i)
+ data[i] = in;
+ updates (data, n);
+ V8 repeated = recurrence (in, n);
+ if (__builtin_memcmp (&repeated, &want_recurrence, sizeof (V8)))
+ __builtin_abort ();
+
+ for (unsigned i = 0; i < 17; ++i)
+ if (__builtin_memcmp (&data[i], i < n ? &want_update : &in,
+ sizeof (V8)))
+ __builtin_abort ();
+
+ extracts (&a, &in);
+ inserts (&b, &in, &pair, 11 * n);
+ if (__builtin_memcmp (&a, &want_extract, sizeof (V4))
+ || __builtin_memcmp (&b, &want_insert, sizeof (V8)))
+ __builtin_abort ();
+ }
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump "basic block part vectorized" "slp2" { target
aarch64*-*-* } } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops in function" 2 "vect"
{ target aarch64*-*-* } } } */
diff --git
a/gcc/testsuite/gcc.target/aarch64/sve/vect-revectorize-perm-const-2048-run.c
b/gcc/testsuite/gcc.target/aarch64/sve/vect-revectorize-perm-const-2048-run.c
new file mode 100644
index
0000000000000000000000000000000000000000..5ef039e9a10f46f71ff9c47a2b049f13a91dab57
--- /dev/null
+++
b/gcc/testsuite/gcc.target/aarch64/sve/vect-revectorize-perm-const-2048-run.c
@@ -0,0 +1,56 @@
+/* { dg-do run { target aarch64_sve2048_hw } } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-options "-O3 -march=armv8-a+sve -msve-vector-bits=2048
-fno-tree-vectorize" } */
+/* { dg-xfail-run-if "SVE 2048-bit two-input permutation" { *-*-* } } */
+
+/* Check selection from both inputs without involving the vectorizer. */
+#include "../../../gcc.dg/vect/tree-vect.h"
+
+typedef unsigned V __attribute__ ((vector_size (256)));
+
+__attribute__ ((noipa)) void
+perm (V *out, const V *a, const V *b)
+{
+ *out
+ = __builtin_shufflevector (*a, *b, 1, 0, 65, 64, 5, 4, 69, 68, 9, 8, 73,
+ 72, 13, 12, 77, 76, 17, 16, 81, 80, 21, 20, 85,
+ 84, 25, 24, 89, 88, 29, 28, 93, 92, 33, 32, 97,
+ 96, 37, 36, 101, 100, 41, 40, 105, 104, 45, 44,
+ 109, 108, 49, 48, 113, 112, 53, 52, 117, 116,
+ 57, 56, 121, 120, 61, 60, 125, 124);
+}
+
+static const V inputs[] = {
+ { 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011,
+ 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023,
+ 1024, 1025, 1026, 1027, 1028, 1029, 1030, 1031, 1032, 1033, 1034, 1035,
+ 1036, 1037, 1038, 1039, 1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047,
+ 1048, 1049, 1050, 1051, 1052, 1053, 1054, 1055, 1056, 1057, 1058, 1059,
+ 1060, 1061, 1062, 1063 },
+ { 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011,
+ 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023,
+ 2024, 2025, 2026, 2027, 2028, 2029, 2030, 2031, 2032, 2033, 2034, 2035,
+ 2036, 2037, 2038, 2039, 2040, 2041, 2042, 2043, 2044, 2045, 2046, 2047,
+ 2048, 2049, 2050, 2051, 2052, 2053, 2054, 2055, 2056, 2057, 2058, 2059,
+ 2060, 2061, 2062, 2063 }
+};
+static const V expected[] = {
+ { 1001, 1000, 2001, 2000, 1005, 1004, 2005, 2004, 1009, 1008, 2009, 2008,
+ 1013, 1012, 2013, 2012, 1017, 1016, 2017, 2016, 1021, 1020, 2021, 2020,
+ 1025, 1024, 2025, 2024, 1029, 1028, 2029, 2028, 1033, 1032, 2033, 2032,
+ 1037, 1036, 2037, 2036, 1041, 1040, 2041, 2040, 1045, 1044, 2045, 2044,
+ 1049, 1048, 2049, 2048, 1053, 1052, 2053, 2052, 1057, 1056, 2057, 2056,
+ 1061, 1060, 2061, 2060 }
+};
+
+int
+main (void)
+{
+ check_vect ();
+ V out;
+ perm (&out, &inputs[0], &inputs[1]);
+ if (__builtin_memcmp (&out, &expected[0], sizeof (out)))
+ __builtin_abort ();
+
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-revectorize-invariant-cost.c
b/gcc/testsuite/gcc.target/aarch64/vect-revectorize-invariant-cost.c
new file mode 100644
index
0000000000000000000000000000000000000000..08efc70da403edeeabcb720bac09fc3e2a641d2b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/vect-revectorize-invariant-cost.c
@@ -0,0 +1,38 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-require-effective-target vect_float } */
+/* { dg-options "-O3 -march=armv8-a+sve -Wno-psabi -fdump-tree-slp-details
-save-temps" } */
+
+#include "../../gcc.dg/vect/tree-vect.h"
+
+typedef float V4 __attribute__ ((vector_size (16)));
+typedef float V8 __attribute__ ((vector_size (32)));
+
+__attribute__ ((noipa)) V8
+swap_halves (V8 v)
+{
+ V4 d[2];
+ __builtin_memcpy (d, &v, sizeof v);
+ V4 t = d[0];
+ d[0] = d[1];
+ d[1] = t;
+ __builtin_memcpy (&v, d, sizeof v);
+ return v;
+}
+
+__attribute__ ((optimize ("no-tree-vectorize")))
+int
+main (void)
+{
+ check_vect ();
+ V8 x = { 0, 101, 202, 303, 404, 505, 606, 707 };
+ const V8 expected = { 404, 505, 606, 707, 0, 101, 202, 303 };
+ V8 y = swap_halves (x);
+ if (__builtin_memcmp (&y, &expected, sizeof (y)))
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* Existing native vector pieces require no construction in the prologue. */
+/* { dg-final { scan-tree-dump "basic block part vectorized using 16 byte
vectors" "slp2" } } */
+/* { dg-final { scan-tree-dump-not "times scalar_to_vec" "slp2" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-revectorize-memory-lanes-2.c
b/gcc/testsuite/gcc.target/aarch64/vect-revectorize-memory-lanes-2.c
new file mode 100644
index
0000000000000000000000000000000000000000..9a313c8f61509f176691649529d4d37b134c4cd0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/vect-revectorize-memory-lanes-2.c
@@ -0,0 +1,8 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-options "-O3 -march=armv8-a+sve -fno-vect-cost-model -Wno-psabi
-fdump-tree-vect-details" } */
+/* { dg-xfail-run-if "SVE 2048-bit two-input permutation" { *-*-* } {
"-msve-vector-bits=2048" } } */
+
+#include "../../gcc.dg/vect/vect-revectorize-memory-lanes-2.c"
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops in function" 3 "vect"
} } */
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-revectorize-memory-lanes-3.c
b/gcc/testsuite/gcc.target/aarch64/vect-revectorize-memory-lanes-3.c
new file mode 100644
index
0000000000000000000000000000000000000000..40fdadcb009191c8ffcf8aa4ae2173aeda5dd74c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/vect-revectorize-memory-lanes-3.c
@@ -0,0 +1,7 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-options "-O3 -march=armv8-a+sve -fno-vect-cost-model -Wno-psabi
-fdump-tree-vect-details" } */
+
+#include "../../gcc.dg/vect/vect-revectorize-memory-lanes-3.c"
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops in function" 3 "vect"
} } */
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-revectorize-memory-lanes.c
b/gcc/testsuite/gcc.target/aarch64/vect-revectorize-memory-lanes.c
new file mode 100644
index
0000000000000000000000000000000000000000..4eda51c04b0ddd1e979a7da63598aad1969769b4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/vect-revectorize-memory-lanes.c
@@ -0,0 +1,8 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-options "-O3 -march=armv8-a+sve -fno-vect-cost-model -Wno-psabi
-fdump-tree-vect-details" } */
+/* { dg-xfail-run-if "SVE 2048-bit two-input permutation" { *-*-* } {
"-msve-vector-bits=2048" } } */
+
+#include "../../gcc.dg/vect/vect-revectorize-memory-lanes.c"
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops in function" 3 "vect"
} } */
diff --git a/gcc/tree-vect-generic.cc b/gcc/tree-vect-generic.cc
index
f9cd17db8b49a85a8b770b402faa32f101fed881..8df7ffc148ebbe4272139de8ad8c538556e54a45
100644
--- a/gcc/tree-vect-generic.cc
+++ b/gcc/tree-vect-generic.cc
@@ -2167,6 +2167,37 @@ expand_vector_conversion (gimple_stmt_iterator *gsi)
gsi_replace (gsi, g, false);
}
+/* Lower an insert into an emulated vector using its source subvector size.
+ This also provides the fallback for whole-vector values exposed by the
+ vectorizer when SLP does not replace the insert. */
+
+static void
+lower_vec_insert (gimple_stmt_iterator *gsi)
+{
+ gassign *stmt = as_a<gassign *> (gsi_stmt (*gsi));
+ tree type = TREE_TYPE (gimple_assign_lhs (stmt));
+ tree old = gimple_assign_rhs1 (stmt);
+ tree value = gimple_assign_rhs2 (stmt);
+ tree size = TYPE_SIZE (TREE_TYPE (value));
+ unsigned HOST_WIDE_INT bits = tree_to_uhwi (size);
+ unsigned HOST_WIDE_INT first = tree_to_uhwi (gimple_assign_rhs3 (stmt));
+ unsigned HOST_WIDE_INT total = tree_to_uhwi (TYPE_SIZE (type));
+ vec<constructor_elt, va_gc> *elts = nullptr;
+ for (unsigned HOST_WIDE_INT pos = 0; pos < total; pos += bits)
+ {
+ tree part = value;
+ if (pos != first)
+ part
+ = gimplify_build3 (gsi, BIT_FIELD_REF, TREE_TYPE (value), old, size,
+ bitsize_int (pos));
+
+ CONSTRUCTOR_APPEND_ELT (elts, NULL_TREE, part);
+ }
+
+ gimple_assign_set_rhs_from_tree (gsi, build_constructor (type, elts));
+ update_stmt (gsi_stmt (*gsi));
+}
+
/* Process one statement. If we identify a vector operation, expand it. */
static void
@@ -2191,6 +2222,14 @@ expand_vector_operations_1 (gimple_stmt_iterator *gsi)
rhs_class = get_gimple_rhs_class (code);
lhs = gimple_assign_lhs (stmt);
+ if (code == BIT_INSERT_EXPR
+ && VECTOR_TYPE_P (TREE_TYPE (lhs))
+ && TYPE_MODE (TREE_TYPE (lhs)) == BLKmode)
+ {
+ lower_vec_insert (gsi);
+ return;
+ }
+
if (code == VEC_PERM_EXPR)
{
lower_vec_perm (gsi);
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index
109cac7142f94f22dd35546e21882dc84be87ed3..c6e94bd422b4d4b811c20cac4407c4aaa09fc318
100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -618,6 +618,7 @@ vect_get_operand_map (const gimple *stmt, bool
gather_scatter_p,
{
static const int no_arg_map[] = { 0 };
static const int arg0_map[] = { 1, 0 };
+ static const int arg0_arg1_map[] = { 2, 0, 1 };
static const int arg2_map[] = { 1, 2 };
static const int arg2_arg3_map[] = { 2, 2, 3 };
static const int arg2_arg4_map[] = { 2, 2, 4 };
@@ -656,8 +657,17 @@ vect_get_operand_map (const gimple *stmt, bool
gather_scatter_p,
|| commutative_tree_code (code))
&& swap)
return op1_op0_map;
- else if (code == VIEW_CONVERT_EXPR)
+ else if (code == VIEW_CONVERT_EXPR
+ || (code == BIT_FIELD_REF
+ && vect_can_revectorize_type_p
+ (TREE_TYPE (gimple_assign_lhs (assign)))))
return op00_map;
+ else if ((code == BIT_INSERT_EXPR
+ || (code == VEC_PERM_EXPR
+ && TREE_CODE (gimple_assign_rhs3 (assign)) == VECTOR_CST))
+ && vect_can_revectorize_type_p
+ (TREE_TYPE (gimple_assign_lhs (assign))))
+ return arg0_arg1_map;
else if (gather_scatter_p)
return (TREE_CODE (gimple_assign_lhs (assign)) != SSA_NAME
? off_op0_map : off_map);
@@ -738,16 +748,37 @@ vect_slp_child_index_for_operand (const stmt_vec_info
stmt, int op)
vect_get_operand_map. Adjust all consumers. */
struct slp_oprnds {
- slp_oprnds (stmt_vec_info);
+ slp_oprnds (stmt_vec_info, unsigned char = 0);
tree get_op_for_slp_child (stmt_vec_info, unsigned);
+ unsigned get_op_nunits (tree) const;
+ bool get_permutation (stmt_vec_info, unsigned, auto_lane_permutation_t &);
const int *opmap;
const unsigned int num_slp_children;
+
+ /* Output lanes per statement for a permutation, or zero otherwise. */
+ unsigned int num_lanes = 0;
};
-slp_oprnds::slp_oprnds (stmt_vec_info stmt_info)
- : opmap (vect_get_operand_map (stmt_info)),
- num_slp_children (opmap ? opmap[0] : gimple_num_args (stmt_info->stmt))
+slp_oprnds::slp_oprnds (stmt_vec_info stmt_info, unsigned char swap)
+ : opmap (vect_get_operand_map (stmt_info, swap)),
+ num_slp_children (opmap ? opmap[0]
+ : (is_gimple_assign (stmt_info->stmt)
+ && gimple_assign_rhs_code (stmt_info->stmt)
+ == CONSTRUCTOR)
+ ? CONSTRUCTOR_NELTS (gimple_assign_rhs1 (stmt_info->stmt))
+ : gimple_num_args (stmt_info->stmt))
{
+ if (gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt))
+ {
+ tree type = TREE_TYPE (gimple_assign_lhs (stmt));
+ tree_code code = gimple_assign_rhs_code (stmt);
+ if (vect_can_revectorize_type_p (type)
+ && (code == CONSTRUCTOR || code == BIT_FIELD_REF
+ || code == BIT_INSERT_EXPR
+ || (code == VEC_PERM_EXPR
+ && TREE_CODE (gimple_assign_rhs3 (stmt)) == VECTOR_CST)))
+ num_lanes = TYPE_VECTOR_SUBPARTS (type).to_constant ();
+ }
}
/* For SLP child number N get the corresponding tree operand from GIMPLE
@@ -757,6 +788,10 @@ tree
slp_oprnds::get_op_for_slp_child (stmt_vec_info stmt_info, unsigned n)
{
gcc_assert (n < num_slp_children);
+ if (is_gimple_assign (stmt_info->stmt)
+ && gimple_assign_rhs_code (stmt_info->stmt) == CONSTRUCTOR)
+ return CONSTRUCTOR_ELT (gimple_assign_rhs1 (stmt_info->stmt), n)->value;
+
int opno = opmap ? opmap[n + 1] : (int) n;
if (opno == GATHER_SCATTER_OFFSET)
gcc_unreachable (); // TODO
@@ -766,6 +801,101 @@ slp_oprnds::get_op_for_slp_child (stmt_vec_info
stmt_info, unsigned n)
return gimple_arg (stmt_info->stmt, opno);
}
+/* Return the number of lanes contributed by OP to its SLP child. */
+
+unsigned
+slp_oprnds::get_op_nunits (tree op) const
+{
+ if (!num_lanes || !VECTOR_TYPE_P (TREE_TYPE (op)))
+ return 1;
+
+ unsigned HOST_WIDE_INT count;
+ return TYPE_VECTOR_SUBPARTS (TREE_TYPE (op)).is_constant (&count) ? count :
0;
+}
+
+/* Append the permutation for statement STMT_INFO at position INDEX. */
+
+bool
+slp_oprnds::get_permutation (stmt_vec_info stmt_info, unsigned index,
+ auto_lane_permutation_t &perm)
+{
+ if (!num_slp_children)
+ return false;
+
+ auto_vec<unsigned, 4> op_lanes (num_slp_children);
+ for (unsigned i = 0; i < num_slp_children; ++i)
+ {
+ unsigned count = get_op_nunits (get_op_for_slp_child (stmt_info, i));
+ if (!count)
+ return false;
+
+ op_lanes.quick_push (count);
+ }
+
+ gassign *stmt = as_a <gassign *> (stmt_info->stmt);
+ tree_code code = gimple_assign_rhs_code (stmt);
+ vec_perm_indices indices;
+ unsigned HOST_WIDE_INT first = 0;
+ if (code == CONSTRUCTOR)
+ {
+ if (num_lanes != num_slp_children * op_lanes[0])
+ return false;
+ }
+ else if (code == VEC_PERM_EXPR)
+ {
+ if (!tree_to_vec_perm_indices (&indices, gimple_assign_rhs1 (stmt),
+ gimple_assign_rhs2 (stmt),
+ gimple_assign_rhs3 (stmt)))
+ return false;
+ }
+ else
+ {
+ tree type = TREE_TYPE (gimple_assign_lhs (stmt));
+ auto offset
+ = code == BIT_FIELD_REF
+ ? bit_field_offset (gimple_assign_rhs1 (stmt))
+ : tree_to_poly_uint64 (gimple_assign_rhs3 (stmt));
+ if (!constant_multiple_p (offset,
+ tree_to_poly_uint64
+ (TYPE_SIZE (TREE_TYPE (type))), &first)
+ || first >= op_lanes[0])
+ return false;
+ }
+
+ for (unsigned i = 0; i < num_lanes; ++i)
+ {
+ unsigned input = 0, lane = i;
+ if (code == CONSTRUCTOR)
+ {
+ input = i / op_lanes[0];
+ lane = i % op_lanes[0];
+ }
+ else if (code == VEC_PERM_EXPR)
+ {
+ HOST_WIDE_INT idx;
+ if (!indices[i].is_constant (&idx))
+ return false;
+
+ input = idx / op_lanes[0];
+ lane = idx % op_lanes[0];
+ }
+ else if (code == BIT_INSERT_EXPR)
+ {
+ input = i >= first && i - first < op_lanes[1];
+ lane = input ? i - first : i;
+ }
+ else
+ lane += first;
+
+ if (lane >= op_lanes[input])
+ return false;
+
+ perm.safe_push (std::make_pair (input, index * op_lanes[input] + lane));
+ }
+
+ return true;
+}
+
/* Get the defs for the rhs of STMT (collect them in OPRNDS_INFO), check that
they are of a valid type and that they match the defs of the first stmt of
the SLP group (stored in OPRNDS_INFO). This function tries to match stmts
@@ -809,10 +939,11 @@ vect_get_and_check_slp_defs (vec_info *vinfo, tree
vectype, unsigned char swap,
&& !is_a<gphi *> (stmt_info->stmt))
return -1;
- number_of_oprnds = gimple_num_args (stmt_info->stmt);
- const int *map = vect_get_operand_map (stmt_info, swap);
- if (map)
- number_of_oprnds = *map++;
+ slp_oprnds child_ops (stmt_info, swap);
+ number_of_oprnds = child_ops.num_slp_children;
+ if (number_of_oprnds != oprnds_info->length ())
+ return 1;
+
if (gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt))
{
if (gimple_call_internal_p (stmt))
@@ -842,7 +973,7 @@ vect_get_and_check_slp_defs (vec_info *vinfo, tree vectype,
unsigned char swap,
for (i = 0; i < number_of_oprnds; i++)
{
oprnd_info = (*oprnds_info)[i];
- int opno = map ? map[i] : int (i);
+ int opno = child_ops.opmap ? child_ops.opmap[i + 1] : int (i);
if (opno == GATHER_SCATTER_OFFSET)
{
gcc_assert (STMT_VINFO_GATHER_SCATTER_P (stmt_info));
@@ -864,11 +995,9 @@ vect_get_and_check_slp_defs (vec_info *vinfo, tree
vectype, unsigned char swap,
oprnd = gs_info.offset;
}
}
- else if (opno < 0)
- oprnd = TREE_OPERAND (gimple_arg (stmt_info->stmt, 0), -1 - opno);
else
{
- oprnd = gimple_arg (stmt_info->stmt, opno);
+ oprnd = child_ops.get_op_for_slp_child (stmt_info, i);
if (gphi *stmt = dyn_cast <gphi *> (stmt_info->stmt))
{
edge e = gimple_phi_arg_edge (stmt, opno);
@@ -898,9 +1027,7 @@ vect_get_and_check_slp_defs (vec_info *vinfo, tree
vectype, unsigned char swap,
continue;
}
- oprnd_info->def_stmts.quick_push (def_stmt_info);
- oprnd_info->ops.quick_push (oprnd);
-
+ tree op = oprnd;
if (def_stmt_info
&& is_pattern_stmt_p (def_stmt_info))
{
@@ -909,8 +1036,14 @@ vect_get_and_check_slp_defs (vec_info *vinfo, tree
vectype, unsigned char swap,
oprnd_info->any_pattern = true;
else
/* If we promote this to external use the original stmt def. */
- oprnd_info->ops.last ()
- = gimple_get_lhs (vect_orig_stmt (def_stmt_info)->stmt);
+ op = gimple_get_lhs (vect_orig_stmt (def_stmt_info)->stmt);
+ }
+
+ unsigned count = child_ops.get_op_nunits (oprnd);
+ for (unsigned lane = 0; lane < count; ++lane)
+ {
+ oprnd_info->def_stmts.safe_push (def_stmt_info);
+ oprnd_info->ops.safe_push (op);
}
/* If there's a extern def on a backedge make sure we can
@@ -986,8 +1119,11 @@ vect_get_and_check_slp_defs (vec_info *vinfo, tree
vectype, unsigned char swap,
oprnd_info = (*oprnds_info)[i];
dt = dts[i];
- stmt_vec_info def_stmt_info = oprnd_info->def_stmts[stmt_num];
- oprnd = oprnd_info->ops[stmt_num];
+
+ /* Child operands can contribute a different number of lanes from
+ the output statement, so STMT_NUM is not an operand index. */
+ stmt_vec_info def_stmt_info = oprnd_info->def_stmts.last ();
+ oprnd = oprnd_info->ops.last ();
tree type = TREE_TYPE (oprnd);
if (!types_compatible_p (oprnd_info->first_op_type, type))
@@ -1059,10 +1195,11 @@ vect_get_and_check_slp_defs (vec_info *vinfo, tree
vectype, unsigned char swap,
dump_printf_loc (MSG_NOTE, vect_location,
"trying swapped operands\n");
std::swap (dts[i], dts[i+1]);
- std::swap ((*oprnds_info)[i]->def_stmts[stmt_num],
- (*oprnds_info)[i+1]->def_stmts[stmt_num]);
- std::swap ((*oprnds_info)[i]->ops[stmt_num],
- (*oprnds_info)[i+1]->ops[stmt_num]);
+ std::swap ((*oprnds_info)[i]->def_stmts.last (),
+ (*oprnds_info)[i+1]->def_stmts.last ());
+ std::swap ((*oprnds_info)[i]->ops.last (),
+ (*oprnds_info)[i+1]->ops.last ());
+
/* After swapping some operands we lost track whether an
operand has any pattern defs so be conservative here. */
if ((*oprnds_info)[i]->any_pattern
@@ -1109,8 +1246,8 @@ vect_get_and_check_slp_defs (vec_info *vinfo, tree
vectype, unsigned char swap,
&& (REDUC_GROUP_FIRST_ELEMENT (def_stmt_info)
== REDUC_GROUP_FIRST_ELEMENT (stmt_info)))
{
- oprnd_info->def_stmts[stmt_num] = oprnd_info->def_stmts[0];
- oprnd_info->ops[stmt_num] = oprnd_info->ops[0];
+ oprnd_info->def_stmts.last () = oprnd_info->def_stmts[0];
+ oprnd_info->ops.last () = oprnd_info->ops[0];
}
++i;
@@ -1370,6 +1507,8 @@ vect_build_slp_tree_3 (vec_info *vinfo,
vec<stmt_vec_info> stmts,
&& rhs_code != VIEW_CONVERT_EXPR
&& rhs_code != CALL_EXPR
&& rhs_code != BIT_FIELD_REF
+ && !(rhs_code == CONSTRUCTOR
+ && vect_can_revectorize_type_p (TREE_TYPE (lhs)))
&& rhs_code != SSA_NAME)
{
if (dump_enabled_p ())
@@ -1410,17 +1549,30 @@ vect_build_slp_tree_3 (vec_info *vinfo,
vec<stmt_vec_info> stmts,
if (!ldst_p && rhs_code == BIT_FIELD_REF)
{
- tree vec = TREE_OPERAND (gimple_assign_rhs1 (stmt), 0);
- if (!is_a <bb_vec_info> (vinfo)
+ tree ref = gimple_assign_rhs1 (stmt);
+ tree vec = TREE_OPERAND (ref, 0);
+ tree source_type = TREE_TYPE (vec);
+ bool same_elements_p
+ = (VECTOR_TYPE_P (source_type)
+ && types_compatible_p (TREE_TYPE (vectype),
+ TREE_TYPE (source_type)));
+ bool supported_p;
+ if (VECTOR_TYPE_P (TREE_TYPE (lhs)))
+ /* Vector slices keep the source element type and lane order. */
+ supported_p
+ = (vect_can_revectorize_type_p (source_type) && same_elements_p);
+ else
+ /* BB SLP also allows scalar extracts after reinterpreting a
+ source of the same size as VECTYPE. */
+ supported_p
+ = (is_a <bb_vec_info> (vinfo)
+ && (same_elements_p
+ || operand_equal_p (TYPE_SIZE (vectype),
+ TYPE_SIZE (source_type))));
+
+ if (!supported_p
|| TREE_CODE (vec) != SSA_NAME
- || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (vec)
- /* When the element types are not compatible we pun the
- source to the target vectype which requires equal size. */
- || ((!VECTOR_TYPE_P (TREE_TYPE (vec))
- || !types_compatible_p (TREE_TYPE (vectype),
- TREE_TYPE (TREE_TYPE (vec))))
- && !operand_equal_p (TYPE_SIZE (vectype),
- TYPE_SIZE (TREE_TYPE (vec)))))
+ || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (vec))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -1561,6 +1713,7 @@ vect_build_slp_tree_3 (vec_info *vinfo,
vec<stmt_vec_info> stmts,
if (!ldst_p
&& first_stmt_code == BIT_FIELD_REF
+ && !vect_can_revectorize_type_p (TREE_TYPE (lhs))
&& (TREE_OPERAND (gimple_assign_rhs1 (first_stmt_info->stmt), 0)
!= TREE_OPERAND (gimple_assign_rhs1 (stmt_info->stmt), 0)))
{
@@ -2247,9 +2400,7 @@ vect_build_slp_tree_2 (vec_info *vinfo, slp_tree node,
}
}
- nops = gimple_num_args (stmt_info->stmt);
- if (const int *map = vect_get_operand_map (stmt_info))
- nops = map[0];
+ nops = slp_oprnds (stmt_info).num_slp_children;
/* If the SLP node is a PHI (induction or reduction), terminate
the recursion. */
@@ -2478,7 +2629,8 @@ vect_build_slp_tree_2 (vec_info *vinfo, slp_tree node,
}
else if (gimple_assign_single_p (stmt_info->stmt)
&& !gimple_vuse (stmt_info->stmt)
- && gimple_assign_rhs_code (stmt_info->stmt) == BIT_FIELD_REF)
+ && gimple_assign_rhs_code (stmt_info->stmt) == BIT_FIELD_REF
+ && !vect_can_revectorize_type_p (type))
{
if (soft_fail)
{
@@ -2927,14 +3079,28 @@ out:
}
/* Get at the operands, verifying they are compatible. */
+ slp_oprnds operands (stmt_info);
+ unsigned stmt_lanes = MAX (1, operands.num_lanes);
+ auto_lane_permutation_t lane_perm;
+ if (operands.num_lanes)
+ for (i = 0; i < group_size; i += stmt_lanes)
+ {
+ slp_oprnds ops (stmts[i]);
+ if (ops.num_slp_children != nops
+ || !ops.get_permutation (stmts[i], i / stmt_lanes, lane_perm))
+ {
+ matches[0] = -1;
+ return NULL;
+ }
+ }
+
vec<slp_oprnd_info> oprnds_info = vect_create_oprnd_info (nops, group_size);
slp_oprnd_info oprnd_info;
- FOR_EACH_VEC_ELT (stmts, i, stmt_info)
+ for (i = 0; i < group_size; i += stmt_lanes)
{
- int res = vect_get_and_check_slp_defs (vinfo, vectype,
- swap[i], skip_args,
- stmts, i, &oprnds_info,
- soft_fail);
+ int res
+ = vect_get_and_check_slp_defs (vinfo, vectype, swap[i], skip_args,
+ stmts, i, &oprnds_info, soft_fail);
if (res != 0)
/* ??? This puts -1 back into matches[] and the cache. */
matches[(res == -1) ? 0 : i] = -1;
@@ -3066,9 +3232,22 @@ out:
int gs_scale = 0;
tree gs_base = NULL_TREE;
+ match_elt_t *parent_matches = matches;
+
/* Create SLP_TREE nodes for the definition node/s. */
FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
{
+ unsigned group_size = oprnd_info->ops.length ();
+ auto_vec<match_elt_t> child_matches;
+ match_elt_t *matches = parent_matches;
+
+ /* Child matches use the child's lane coordinates. */
+ if (group_size != stmts.length ())
+ {
+ child_matches.safe_grow (group_size);
+ matches = child_matches.address ();
+ }
+
slp_tree child = nullptr;
unsigned int j;
unsigned old_swap_distance;
@@ -3240,6 +3419,7 @@ out:
}
can_swap = (i == 0
+ && group_size == stmts.length ()
&& (nops == 2 || nops == 3)
&& oprnds_info.length () > 1
&& oprnds_info[1]->first_dt == vect_internal_def
@@ -3402,6 +3582,9 @@ fail:
}
gcc_assert (child == NULL);
+ if (group_size != stmts.length ())
+ parent_matches[0] = -1;
+
FOR_EACH_VEC_ELT (children, j, child)
if (child)
vect_free_slp_tree (child);
@@ -3544,6 +3727,14 @@ fail:
node = vect_create_new_slp_node (node, stmts, nops);
SLP_TREE_VECTYPE (node) = vectype;
+
+ if (!lane_perm.is_empty ())
+ {
+ SLP_TREE_CODE (node) = VEC_PERM_EXPR;
+ SLP_TREE_LANE_PERMUTATION (node) = lane_perm.copy ();
+ SLP_TREE_REPRESENTATIVE (node) = NULL;
+ }
+
SLP_TREE_CHILDREN (node).splice (children);
SLP_TREE_GS_SCALE (node) = gs_scale;
SLP_TREE_GS_BASE (node) = gs_base;
@@ -10493,8 +10684,7 @@ vect_slp_is_lane_insert (gimple *use_stmt, tree vec,
unsigned *this_lane)
return true;
}
-/* Find any vectorizable constructors and add them to the grouped_store
- array. */
+/* Find vectorizable constructors and other SLP roots. */
static void
vect_slp_check_for_roots (bb_vec_info bb_vinfo)
@@ -10516,28 +10706,38 @@ vect_slp_check_for_roots (bb_vec_info bb_vinfo)
if (code == CONSTRUCTOR)
{
if (!VECTOR_TYPE_P (TREE_TYPE (rhs))
- || maybe_ne (TYPE_VECTOR_SUBPARTS (TREE_TYPE (rhs)),
- CONSTRUCTOR_NELTS (rhs))
- || VECTOR_TYPE_P (TREE_TYPE (CONSTRUCTOR_ELT (rhs, 0)->value))
+ || !TYPE_VECTOR_SUBPARTS (TREE_TYPE (rhs)).is_constant ()
|| uniform_vector_p (rhs))
continue;
+ slp_oprnds operands (bb_vinfo->lookup_stmt (assign));
+ vec<stmt_vec_info> stmts = vNULL;
unsigned j;
- tree val;
- FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (rhs), j, val)
- if (TREE_CODE (val) != SSA_NAME
- || !bb_vinfo->lookup_def (val))
- break;
- if (j != CONSTRUCTOR_NELTS (rhs))
- continue;
+ for (j = 0; j < operands.num_slp_children; ++j)
+ {
+ tree val
+ = operands.get_op_for_slp_child (bb_vinfo->lookup_stmt (assign),
+ j);
+ stmt_vec_info def = bb_vinfo->lookup_def (val);
+ tree type = TREE_TYPE (val);
+ if (!def || (VECTOR_TYPE_P (type)
+ && !vect_can_revectorize_type_p (type)))
+ break;
+
+ unsigned nelts = operands.get_op_nunits (val);
+ for (unsigned k = 0; k < nelts; ++k)
+ stmts.safe_push (vect_stmt_to_vectorize (def));
+ }
+
+ if (j != operands.num_slp_children
+ || stmts.length () != operands.num_lanes)
+ {
+ stmts.release ();
+ continue;
+ }
vec<stmt_vec_info> roots = vNULL;
roots.safe_push (bb_vinfo->lookup_stmt (assign));
- vec<stmt_vec_info> stmts;
- stmts.create (CONSTRUCTOR_NELTS (rhs));
- FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (rhs), j, val)
- stmts.quick_push
- (vect_stmt_to_vectorize (bb_vinfo->lookup_def (val)));
bb_vinfo->roots.safe_push (slp_root (slp_inst_kind_ctor,
stmts, roots));
}
