The patterns merging
c = VEC_PERM_EXPR <a, b, VCST0>;
d = VEC_PERM_EXPR <x, c, VCST1>; (or <c, x, VCST1>)
only fire when d uses elements of just one of a and b, since the merged
permute can only take x and one other input. When x is itself a or b,
d's elements all come from a and b however they are distributed, so it
can always be rewritten as VEC_PERM_EXPR <a, b, NEW_VCST>.
Handle that in the same two patterns: when x is a or b and c has no
other use, take the elements of c that come from x from x itself. d
then reads at most one input besides x, so the existing code builds the
merged permute, with the same check against creating one the target
can't handle. The restriction to a single use of c is because the new
mask may be harder to do than d's.
Building a vector element by element, as in
v4su d;
d[0] = a[2]; d[1] = a[3]; d[2] = b[2]; d[3] = b[3];
gives a chain of VEC_PERM_EXPRs each of which has one of these shapes,
the first one reading the uninitialized d:
d_3 = VEC_PERM_EXPR <a, d_2(D), { 2, 5, 6, 7 }>;
d_4 = VEC_PERM_EXPR <d_3, a, { 0, 7, 2, 3 }>;
d_6 = VEC_PERM_EXPR <d_4, b, { 0, 1, 6, 3 }>;
d_7 = VEC_PERM_EXPR <d_6, b, { 0, 1, 2, 7 }>;
The chain now folds to
d_7 = VEC_PERM_EXPR <a, b, { 2, 3, 6, 7 }>;
(one shufps on x86-64 instead of eleven instructions). Since d_2(D) is
then unused, this also removes the bogus "'d' is used uninitialized"
warning that the first permute used to give; a d that is only partly
built still warns.
Assisted-by: Claude Opus 5.5
gcc/ChangeLog:
* match.pd (vec_perm (vec_perm) merges): When the outer operand
is one of the inner permute's inputs, take the inner permute's
elements of it from the outer operand.
gcc/testsuite/ChangeLog:
* gcc.dg/torture/vec-perm-merge-two-inputs-1.c: New test.
* gcc.dg/tree-ssa/vec-perm-merge-two-inputs.c: New test.
* gcc.dg/tree-ssa/vec-perm-merge-two-inputs-2.c: New test.
Signed-off-by: Igor Shevlyakov <[email protected]>
---
Bootstrapped and regtested on x86_64-pc-linux-gnu (-m32/-m64), no new
regressions.
gcc/match.pd | 45 +++--
.../torture/vec-perm-merge-two-inputs-1.c | 164 ++++++++++++++++++
.../tree-ssa/vec-perm-merge-two-inputs-2.c | 27 +++
.../tree-ssa/vec-perm-merge-two-inputs.c | 32 ++++
4 files changed, 257 insertions(+), 11 deletions(-)
create mode 100644 gcc/testsuite/gcc.dg/torture/vec-perm-merge-two-inputs-1.c
create mode 100644 gcc/testsuite/gcc.dg/tree-ssa/vec-perm-merge-two-inputs-2.c
create mode 100644 gcc/testsuite/gcc.dg/tree-ssa/vec-perm-merge-two-inputs.c
diff --git a/gcc/match.pd b/gcc/match.pd
index b936c23f2ea..a44f9b8d325 100644
--- a/gcc/match.pd
+++ b/gcc/match.pd
@@ -12974,7 +12974,12 @@ and,
to
d = VEC_PERM_EXPR <x, {a,b}, NEW_VCST>;
when all elements from a or b are replaced by the later
- permutation. */
+ permutation.
+
+ If x is a, the elements c takes from a can be read from x instead,
+ so only b remains and d becomes VEC_PERM_EXPR <a, b, NEW_VCST>
+ whatever mix of a and b it uses. Likewise if x is b. Do this only
+ when c has no other use, as the new mask may be more costly. */
(simplify
(vec_perm @5 (vec_perm@0 @1 @2 VECTOR_CST@3) VECTOR_CST@4)
@@ -12996,6 +13001,9 @@ and,
(with
{
bool use_1 = false, use_2 = false;
+ /* Whether to read the elements c takes from a or b from x. */
+ bool x_is_1 = single_use (@0) && operand_equal_p (@5, @1, 0);
+ bool x_is_2 = single_use (@0) && operand_equal_p (@5, @2, 0);
vec_perm_builder builder2 (nelts, nelts, 2);
for (int i = 0; i < nelts; i++)
@@ -13006,18 +13014,23 @@ and,
{
poly_uint64 j = sel0[(sel1[i] - sel1.nelts_per_input ())
.to_constant ()];
+ bool from_x;
if (known_lt (j, sel0.nelts_per_input ()))
- use_1 = true;
+ {
+ from_x = x_is_1;
+ use_1 |= !from_x;
+ }
else
{
- use_2 = true;
+ from_x = x_is_2;
+ use_2 |= !from_x;
j -= sel0.nelts_per_input ();
}
- builder2.quick_push (j + sel1.nelts_per_input ());
+ builder2.quick_push (from_x ? j : j + sel1.nelts_per_input ());
}
}
}
- (if (use_1 ^ use_2)
+ (if (x_is_1 || x_is_2 || (use_1 ^ use_2))
(with
{
vec_perm_indices sel2 (builder2, 2, sel1.nelts_per_input ());
@@ -13044,7 +13057,8 @@ and,
(if (use_1)
(vec_perm @5 @1 { op0; }))
(if (use_2)
- (vec_perm @5 @2 { op0; })))))))))))
+ (vec_perm @5 @2 { op0; }))
+ (vec_perm @5 @5 { op0; }))))))))))
/* And the case with swapped outer permute sources. */
@@ -13068,6 +13082,9 @@ and,
(with
{
bool use_1 = false, use_2 = false;
+ /* Whether to read the elements c takes from a or b from x. */
+ bool x_is_1 = single_use (@0) && operand_equal_p (@5, @1, 0);
+ bool x_is_2 = single_use (@0) && operand_equal_p (@5, @2, 0);
vec_perm_builder builder2 (nelts, nelts, 2);
for (int i = 0; i < nelts; i++)
@@ -13077,18 +13094,23 @@ and,
else
{
poly_uint64 j = sel0[sel1[i].to_constant ()];
+ bool from_x;
if (known_lt (j, sel0.nelts_per_input ()))
- use_1 = true;
+ {
+ from_x = x_is_1;
+ use_1 |= !from_x;
+ }
else
{
- use_2 = true;
+ from_x = x_is_2;
+ use_2 |= !from_x;
j -= sel0.nelts_per_input ();
}
- builder2.quick_push (j);
+ builder2.quick_push (from_x ? j + sel1.nelts_per_input () : j);
}
}
}
- (if (use_1 ^ use_2)
+ (if (x_is_1 || x_is_2 || (use_1 ^ use_2))
(with
{
vec_perm_indices sel2 (builder2, 2, sel1.nelts_per_input ());
@@ -13115,7 +13137,8 @@ and,
(if (use_1)
(vec_perm @1 @5 { op0; }))
(if (use_2)
- (vec_perm @2 @5 { op0; })))))))))))
+ (vec_perm @2 @5 { op0; }))
+ (vec_perm @5 @5 { op0; }))))))))))
#if GIMPLE
diff --git a/gcc/testsuite/gcc.dg/torture/vec-perm-merge-two-inputs-1.c
b/gcc/testsuite/gcc.dg/torture/vec-perm-merge-two-inputs-1.c
new file mode 100644
index 00000000000..2ca18d90a6b
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/vec-perm-merge-two-inputs-1.c
@@ -0,0 +1,164 @@
+/* Merging VEC_PERM_EXPRs whose combined inputs are only a and b, and
+ permutes reading lanes of an uninitialized vector. */
+/* { dg-do run } */
+/* { dg-additional-options "-fgimple -Wno-psabi" } */
+
+typedef short v4hi __attribute__((vector_size(8)));
+typedef short v8hi __attribute__((vector_size(16)));
+
+v4hi a4, b4, r4;
+v8hi a8, b8, r8;
+
+/* d = <a, c>, c = <a, b>: d uses both a and b. */
+void __attribute__((noipa)) __GIMPLE() f1 ()
+{
+ v4hi a_; v4hi b_; v4hi c_; v4hi d_;
+ a_ = a4; b_ = b4;
+ c_ = __VEC_PERM (a_, b_, _Literal (v4hi) { 7, 0, 5, 2 });
+ d_ = __VEC_PERM (a_, c_, _Literal (v4hi) { 4, 3, 5, 1 });
+ r4 = d_;
+}
+
+/* d = <b, c>. */
+void __attribute__((noipa)) __GIMPLE() f2 ()
+{
+ v4hi a_; v4hi b_; v4hi c_; v4hi d_;
+ a_ = a4; b_ = b4;
+ c_ = __VEC_PERM (a_, b_, _Literal (v4hi) { 7, 0, 5, 2 });
+ d_ = __VEC_PERM (b_, c_, _Literal (v4hi) { 4, 3, 5, 1 });
+ r4 = d_;
+}
+
+/* d = <c, a>. */
+void __attribute__((noipa)) __GIMPLE() f3 ()
+{
+ v8hi a_; v8hi b_; v8hi c_; v8hi d_;
+ a_ = a8; b_ = b8;
+ c_ = __VEC_PERM (a_, b_, _Literal (v8hi) { 15, 0, 9, 3, 12, 6, 8, 1 });
+ d_ = __VEC_PERM (c_, a_, _Literal (v8hi) { 0, 13, 2, 8, 4, 11, 6, 1 });
+ r8 = d_;
+}
+
+/* d = <c, b>. */
+void __attribute__((noipa)) __GIMPLE() f4 ()
+{
+ v8hi a_; v8hi b_; v8hi c_; v8hi d_;
+ a_ = a8; b_ = b8;
+ c_ = __VEC_PERM (a_, b_, _Literal (v8hi) { 15, 0, 9, 3, 12, 6, 8, 1 });
+ d_ = __VEC_PERM (c_, b_, _Literal (v8hi) { 0, 13, 2, 8, 4, 11, 6, 1 });
+ r8 = d_;
+}
+
+/* Building a vector up from an uninitialized one. c takes
+ lanes from the uninitialized u that later permutes overwrite. */
+void __attribute__((noipa)) __GIMPLE() f5 ()
+{
+ v4hi a_; v4hi b_; v4hi u_; v4hi c_; v4hi d_; v4hi e_; v4hi f_;
+ a_ = a4; b_ = b4;
+ c_ = __VEC_PERM (a_, u_, _Literal (v4hi) { 2, 5, 6, 7 });
+ d_ = __VEC_PERM (c_, a_, _Literal (v4hi) { 0, 7, 2, 3 });
+ e_ = __VEC_PERM (d_, b_, _Literal (v4hi) { 0, 1, 6, 3 });
+ f_ = __VEC_PERM (e_, b_, _Literal (v4hi) { 0, 1, 2, 7 });
+ r4 = f_;
+}
+
+/* As f5 with the uninitialized vector first. */
+void __attribute__((noipa)) __GIMPLE() f6 ()
+{
+ v4hi a_; v4hi b_; v4hi u_; v4hi c_; v4hi d_; v4hi e_; v4hi f_;
+ a_ = a4; b_ = b4;
+ c_ = __VEC_PERM (u_, b_, _Literal (v4hi) { 0, 5, 2, 3 });
+ d_ = __VEC_PERM (c_, b_, _Literal (v4hi) { 7, 1, 2, 3 });
+ e_ = __VEC_PERM (d_, a_, _Literal (v4hi) { 0, 1, 4, 3 });
+ f_ = __VEC_PERM (e_, a_, _Literal (v4hi) { 0, 1, 2, 6 });
+ r4 = f_;
+}
+
+/* d = <c, c>: x is c itself rather than one of a and b. */
+void __attribute__((noipa)) __GIMPLE() f7 ()
+{
+ v4hi a_; v4hi b_; v4hi c_; v4hi d_;
+ a_ = a4; b_ = b4;
+ c_ = __VEC_PERM (a_, b_, _Literal (v4hi) { 6, 2, 2, 1 });
+ d_ = __VEC_PERM (c_, c_, _Literal (v4hi) { 4, 3, 5, 1 });
+ r4 = d_;
+}
+
+/* Reference VEC_PERM_EXPR: OUT[i] = concat (X, Y)[SEL[i] % (2 * N)]. */
+static void
+perm (short *out, const short *x, const short *y, int n, const int *sel)
+{
+ short tmp[8];
+ for (int i = 0; i < n; i++)
+ {
+ int j = sel[i] % (2 * n);
+ tmp[i] = j < n ? x[j] : y[j - n];
+ }
+ for (int i = 0; i < n; i++)
+ out[i] = tmp[i];
+}
+
+static void
+check (const short *got, const short *want, int n)
+{
+ for (int i = 0; i < n; i++)
+ if (got[i] != want[i])
+ __builtin_abort ();
+}
+
+int
+main ()
+{
+ short c[8], d[8];
+ for (int i = 0; i < 4; i++)
+ {
+ a4[i] = i + 1;
+ b4[i] = i + 101;
+ }
+ for (int i = 0; i < 8; i++)
+ {
+ a8[i] = i + 1;
+ b8[i] = i + 101;
+ }
+
+ {
+ static const int s0[4] = { 7, 0, 5, 2 }, s1[4] = { 4, 3, 5, 1 };
+ perm (c, (short *) &a4, (short *) &b4, 4, s0);
+ perm (d, (short *) &a4, c, 4, s1);
+ f1 ();
+ check ((short *) &r4, d, 4);
+ perm (d, (short *) &b4, c, 4, s1);
+ f2 ();
+ check ((short *) &r4, d, 4);
+ }
+ {
+ static const int s0[4] = { 6, 2, 2, 1 }, s1[4] = { 4, 3, 5, 1 };
+ perm (c, (short *) &a4, (short *) &b4, 4, s0);
+ perm (d, c, c, 4, s1);
+ f7 ();
+ check ((short *) &r4, d, 4);
+ }
+ {
+ static const int s0[8] = { 15, 0, 9, 3, 12, 6, 8, 1 };
+ static const int s1[8] = { 0, 13, 2, 8, 4, 11, 6, 1 };
+ perm (c, (short *) &a8, (short *) &b8, 8, s0);
+ perm (d, c, (short *) &a8, 8, s1);
+ f3 ();
+ check ((short *) &r8, d, 8);
+ perm (d, c, (short *) &b8, 8, s1);
+ f4 ();
+ check ((short *) &r8, d, 8);
+ }
+
+ /* f5: { a2, a3, b2, b3 }. */
+ f5 ();
+ if (r4[0] != 3 || r4[1] != 4 || r4[2] != 103 || r4[3] != 104)
+ __builtin_abort ();
+
+ /* f6: { b3, b1, a0, a2 }. */
+ f6 ();
+ if (r4[0] != 104 || r4[1] != 102 || r4[2] != 1 || r4[3] != 3)
+ __builtin_abort ();
+
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/vec-perm-merge-two-inputs-2.c
b/gcc/testsuite/gcc.dg/tree-ssa/vec-perm-merge-two-inputs-2.c
new file mode 100644
index 00000000000..2ba56ee16ba
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/vec-perm-merge-two-inputs-2.c
@@ -0,0 +1,27 @@
+/* Once the permutes building d fold to one that reads only a and b,
+ the uninitialized d is no longer used and there is nothing to warn
+ about. Building only part of d still warns. */
+/* { dg-do compile { target { aarch64*-*-* i?86-*-* x86_64-*-* } } } */
+/* { dg-options "-O2 -Wuninitialized" } */
+/* { dg-additional-options "-msse2" { target i?86-*-* x86_64-*-* } } */
+
+typedef unsigned int v4su __attribute__((vector_size(16)));
+
+v4su
+f1 (v4su a, v4su b)
+{
+ v4su d;
+ d[0] = a[2]; /* { dg-bogus "uninitialized" } */
+ d[1] = a[3];
+ d[2] = b[2];
+ d[3] = b[3];
+ return d;
+}
+
+v4su
+f2 (v4su a)
+{
+ v4su d;
+ d[0] = a[2]; d[1] = a[3]; /* { dg-warning "used uninitialized" } */
+ return d;
+}
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/vec-perm-merge-two-inputs.c
b/gcc/testsuite/gcc.dg/tree-ssa/vec-perm-merge-two-inputs.c
new file mode 100644
index 00000000000..05b1db4ea2b
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/vec-perm-merge-two-inputs.c
@@ -0,0 +1,32 @@
+/* Permute chains reading only a and b, including one built up from an
+ uninitialized vector, fold to a single VEC_PERM_EXPR. */
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wno-psabi -fdump-tree-optimized" } */
+/* { dg-additional-options "-msse2" { target i?86-*-* x86_64-*-* } } */
+
+typedef unsigned int v4su __attribute__((vector_size(16)));
+
+v4su
+f1 (v4su a, v4su b)
+{
+ v4su d;
+ d[0] = a[2];
+ d[1] = a[3];
+ d[2] = b[2];
+ d[3] = b[3];
+ return d;
+}
+
+v4su
+f2 (v4su a, v4su b)
+{
+ v4su d = a;
+ d[0] = a[2];
+ d[1] = a[3];
+ d[2] = b[2];
+ d[3] = b[3];
+ return d;
+}
+
+/* { dg-final { scan-tree-dump-times "VEC_PERM_EXPR <a_\[0-9\]+\\(D\\),
b_\[0-9\]+\\(D\\), \{ 2, 3, 6, 7 \}>" 2 "optimized" { target { aarch64*-*-*
i?86-*-* x86_64-*-* } } } } */
+/* { dg-final { scan-tree-dump-times "VEC_PERM_EXPR" 2 "optimized" { target {
aarch64*-*-* i?86-*-* x86_64-*-* } } } } */
--
2.55.0
