https://gcc.gnu.org/g:bb6efe4d690515e49c5da9167d9ca4836d1ab546
commit r17-5345-gbb6efe4d690515e49c5da9167d9ca4836d1ab546 Author: Igor Shevlyakov <[email protected]> Date: Wed Oct 14 10:44:39 2026 -0700 middle-end: Fix VEC_PERM_EXPR merges of different-length vectors [PR127862] Since GCC 14 a VEC_PERM_EXPR with a constant selector may have inputs with a different number of elements than its result. The match.pd patterns that merge a VEC_PERM_EXPR into a VEC_PERM_EXPR using it, and forwprop's identity check for two consecutive permutes, still assume that every permute and input involved has the same number of elements. On aarch64 this happens for plain SVE intrinsics: svld1rq folds to a permute from a 128-bit vector to the SVE vector, and with -msve-vector-bits=256 a following svrev, svzip1, svuzp1 or svtrn2 of it is merged into an invalid permute. That fails GIMPLE verification, and without checking it is miscompiled or ICEs in expand_vec_perm_const. - Merging c = <a, b> into d = <c, c> builds the new selector with d's element count as the number of elements per input, and emits the new permute with a's type but d's selector length. That is invalid GIMPLE: the verifier rejects it, and without checking it is miscompiled or ICEs during expansion. - Merging c = <a, b> into d = <x, c> or <c, x> builds the new selector with d's element count as the number of elements per input too, so when d is narrower than c, lanes of c are taken from x: silent wrong code. When c and a differ in length, the new permute mixes inputs x and a of different widths. - is_combined_permutation_identity folds the outer mask through the inner one with the inner mask's type and reduces the result by its length, so mixed widths ICE in fold_vec_perm or wrongly look like an identity (a V2HI permute of a V2HI permute of two V4HI vectors gets replaced by one of the V4HI inputs). Build each selector with the element count of its own inputs. In the <c, c> merge, emit the new permute as a vector of a's element type with d's length, which is d's type unless c is view-converted to a different element type, and query it with that type's mode. In the <x, c> and <c, x> merges, require a and b to have x's type and query the inner permute with its own result mode. In forwprop, leave permutes of different widths to match.pd. Assisted-by: Claude Opus 5.5 PR middle-end/127862 gcc/ChangeLog: * match.pd (vec_perm (vec_perm) merges): Handle inputs and results with different numbers of elements. * tree-ssa-forwprop.cc (simplify_permutation): Don't check for an identity when the permutes have different widths. gcc/testsuite/ChangeLog: * gcc.dg/torture/vec-perm-merge-wide-input-1.c: New test. * gcc.dg/torture/vec-perm-merge-wide-input-2.c: New test. * gcc.dg/torture/vec-perm-merge-wide-input-3.c: New test. * gcc.target/aarch64/sve/pr127862.c: New test. * gcc.target/aarch64/sve/pr127862_run.c: New test. Signed-off-by: Igor Shevlyakov <[email protected]> Diff: --- gcc/match.pd | 48 ++- .../gcc.dg/torture/vec-perm-merge-wide-input-1.c | 193 +++++++++++ .../gcc.dg/torture/vec-perm-merge-wide-input-2.c | 64 ++++ .../gcc.dg/torture/vec-perm-merge-wide-input-3.c | 382 +++++++++++++++++++++ gcc/testsuite/gcc.target/aarch64/sve/pr127862.c | 53 +++ .../gcc.target/aarch64/sve/pr127862_run.c | 46 +++ gcc/tree-ssa-forwprop.cc | 9 + 7 files changed, 782 insertions(+), 13 deletions(-) diff --git a/gcc/match.pd b/gcc/match.pd index 2adb45c50dea..f3d9e55f1390 100644 --- a/gcc/match.pd +++ b/gcc/match.pd @@ -12893,11 +12893,16 @@ and, to result = VIEW_CONVERT_EXPR <VEC_PERM_EXPR <op2, op3, new_sel>>; The VIEW_CONVERT_EXPR may change the element type but not the element - count. */ + count. op2 and op3 have the same number of elements, but inner may have + a different number than op2 and op3, and result a different number + than inner, so index each selector by the element count of its own + inputs. */ (simplify (vec_perm (view_convert?@0 (vec_perm@1 @2 @3 VECTOR_CST@4)) @0 VECTOR_CST@5) - (if (TYPE_VECTOR_SUBPARTS (type).is_constant ()) + (if (TYPE_VECTOR_SUBPARTS (type).is_constant () + && TYPE_VECTOR_SUBPARTS (TREE_TYPE (@0)).is_constant () + && TYPE_VECTOR_SUBPARTS (TREE_TYPE (@2)).is_constant ()) (with { machine_mode result_mode = TYPE_MODE (type); @@ -12917,16 +12922,23 @@ and, for (int i = 0; i < nelts; i++) new_builder.quick_push (sel4[sel5[i].to_constant ()]); - vec_perm_indices new_sel (new_builder, 2, nelts); + vec_perm_indices new_sel (new_builder, 2, sel4.nelts_per_input ()); + + /* The new permute has the element count of result and the element + type of op2, so it has the type of result unless the + VIEW_CONVERT_EXPR changes the element type. */ + tree perm_type = type; + if (!types_match (TREE_TYPE (type), TREE_TYPE (TREE_TYPE (@2)))) + perm_type = build_vector_type (TREE_TYPE (TREE_TYPE (@2)), nelts); + machine_mode perm_mode = TYPE_MODE (perm_type); tree new_vcst = NULL_TREE; /* If the new VEC_PERM_EXPR can't be handled but both original VEC_PERM_EXPRs can, punt. If one or both of the original VEC_PERM_EXPRs can't be handled and the new one can't be either, don't increase - number of VEC_PERM_EXPRs that can't be handled. - The new VEC_PERM_EXPR has the type of op2. */ - if (can_vec_perm_const_p (op2_mode, op2_mode, new_sel, false) + number of VEC_PERM_EXPRs that can't be handled. */ + if (can_vec_perm_const_p (perm_mode, op2_mode, new_sel, false) || (single_use (@0) ? (!can_vec_perm_const_p (inner_mode, op2_mode, sel4, true) || !can_vec_perm_const_p (result_mode, vc_mode, sel5, true)) @@ -12934,7 +12946,7 @@ and, new_vcst = vec_perm_indices_to_tree (TREE_TYPE (@5), new_sel); } (if (new_vcst) - (view_convert (vec_perm @2 @3 { new_vcst; })))))))) + (view_convert (vec_perm:perm_type @2 @3 { new_vcst; })))))))) /* Merge c = VEC_PERM_EXPR <a, b, VCST0>; @@ -12946,11 +12958,16 @@ and, (simplify (vec_perm @5 (vec_perm@0 @1 @2 VECTOR_CST@3) VECTOR_CST@4) - (if (TYPE_VECTOR_SUBPARTS (type).is_constant ()) + /* The new permute replaces c by a or b, which therefore must have the + type of x. c may have more elements than d. */ + (if (TYPE_VECTOR_SUBPARTS (type).is_constant () + && TYPE_VECTOR_SUBPARTS (TREE_TYPE (@0)).is_constant () + && types_match (@1, @5)) (with { machine_mode result_mode = TYPE_MODE (type); machine_mode op_mode = TYPE_MODE (TREE_TYPE (@1)); + machine_mode c_mode = TYPE_MODE (TREE_TYPE (@0)); int nelts = TYPE_VECTOR_SUBPARTS (type).to_constant (); vec_perm_indices sel0, sel1; } @@ -12983,7 +13000,7 @@ and, (if (use_1 ^ use_2) (with { - vec_perm_indices sel2 (builder2, 2, nelts); + vec_perm_indices sel2 (builder2, 2, sel1.nelts_per_input ()); tree op0 = NULL_TREE; /* If the new VEC_PERM_EXPR can't be handled but both original VEC_PERM_EXPRs can, punt. @@ -12992,7 +13009,7 @@ and, number of VEC_PERM_EXPRs that can't be handled. */ if (can_vec_perm_const_p (result_mode, op_mode, sel2, false) || (single_use (@0) - ? (!can_vec_perm_const_p (result_mode, op_mode, sel0, true) + ? (!can_vec_perm_const_p (c_mode, op_mode, sel0, true) || !can_vec_perm_const_p (result_mode, op_mode, sel1, true)) : !can_vec_perm_const_p (result_mode, op_mode, sel1, true))) op0 = vec_perm_indices_to_tree (TREE_TYPE (@4), sel2); @@ -13008,11 +13025,16 @@ and, (simplify (vec_perm (vec_perm@0 @1 @2 VECTOR_CST@3) @5 VECTOR_CST@4) - (if (TYPE_VECTOR_SUBPARTS (type).is_constant ()) + /* The new permute replaces c by a or b, which therefore must have the + type of x. c may have more elements than d. */ + (if (TYPE_VECTOR_SUBPARTS (type).is_constant () + && TYPE_VECTOR_SUBPARTS (TREE_TYPE (@0)).is_constant () + && types_match (@1, @5)) (with { machine_mode result_mode = TYPE_MODE (type); machine_mode op_mode = TYPE_MODE (TREE_TYPE (@1)); + machine_mode c_mode = TYPE_MODE (TREE_TYPE (@0)); int nelts = TYPE_VECTOR_SUBPARTS (type).to_constant (); vec_perm_indices sel0, sel1; } @@ -13044,7 +13066,7 @@ and, (if (use_1 ^ use_2) (with { - vec_perm_indices sel2 (builder2, 2, nelts); + vec_perm_indices sel2 (builder2, 2, sel1.nelts_per_input ()); tree op0 = NULL_TREE; /* If the new VEC_PERM_EXPR can't be handled but both original VEC_PERM_EXPRs can, punt. @@ -13053,7 +13075,7 @@ and, number of VEC_PERM_EXPRs that can't be handled. */ if (can_vec_perm_const_p (result_mode, op_mode, sel2, false) || (single_use (@0) - ? (!can_vec_perm_const_p (result_mode, op_mode, sel0, true) + ? (!can_vec_perm_const_p (c_mode, op_mode, sel0, true) || !can_vec_perm_const_p (result_mode, op_mode, sel1, true)) : !can_vec_perm_const_p (result_mode, op_mode, sel1, true))) op0 = vec_perm_indices_to_tree (TREE_TYPE (@4), sel2); diff --git a/gcc/testsuite/gcc.dg/torture/vec-perm-merge-wide-input-1.c b/gcc/testsuite/gcc.dg/torture/vec-perm-merge-wide-input-1.c new file mode 100644 index 000000000000..b047c05bc9ae --- /dev/null +++ b/gcc/testsuite/gcc.dg/torture/vec-perm-merge-wide-input-1.c @@ -0,0 +1,193 @@ +/* VEC_PERM_EXPR merges with inputs wider than the result. */ +/* { dg-do run } */ +/* { dg-additional-options "-fgimple -Wno-psabi" } */ + +typedef int v4si __attribute__((vector_size(16))); +typedef int v8si __attribute__((vector_size(32))); +typedef int v16si __attribute__((vector_size(64))); +typedef float v4sf __attribute__((vector_size(16))); +typedef float v8sf __attribute__((vector_size(32))); + +v8si a8, b8, x8; +v16si a16, b16; +v4si r4; +v8si r8; +v4sf f4; + +/* c = <a, b> v8si; d = <c, c> v4si. */ +void __attribute__((noipa)) __GIMPLE() fa1 () +{ + v8si a_; v8si b_; v8si c_; v4si d_; + a_ = a8; b_ = b8; + c_ = __VEC_PERM (a_, b_, _Literal (v8si) { 8, 0, 9, 1, 10, 2, 11, 3 }); + d_ = __VEC_PERM (c_, c_, _Literal (v4si) { 1, 2, 3, 4 }); + r4 = d_; +} + +/* c = <a, b> v8si from v16si; d = <c, c> v8si. */ +void __attribute__((noipa)) __GIMPLE() fa2 () +{ + v16si a_; v16si b_; v8si c_; v8si d_; + a_ = a16; b_ = b16; + c_ = __VEC_PERM (a_, b_, _Literal (v8si) { 31, 0, 17, 14, 16, 15, 1, 30 }); + d_ = __VEC_PERM (c_, c_, _Literal (v8si) { 7, 6, 5, 4, 3, 2, 1, 0 }); + r8 = d_; +} + +/* As fa1, through a VIEW_CONVERT_EXPR. */ +void __attribute__((noipa)) __GIMPLE() fa3 () +{ + v8si a_; v8si b_; v8si c_; v8sf e_; v4sf d_; + a_ = a8; b_ = b8; + c_ = __VEC_PERM (a_, b_, _Literal (v8si) { 8, 0, 9, 1, 10, 2, 11, 3 }); + e_ = __VIEW_CONVERT <v8sf> (c_); + d_ = __VEC_PERM (e_, e_, _Literal (v4si) { 7, 0, 5, 2 }); + f4 = d_; +} + +/* c = <a, b> v8si; d = <x, c> v4si; only a used from c. */ +void __attribute__((noipa)) __GIMPLE() fb1 () +{ + v8si a_; v8si b_; v8si x_; v8si c_; v4si d_; + a_ = a8; b_ = b8; x_ = x8; + c_ = __VEC_PERM (a_, b_, _Literal (v8si) { 7, 6, 5, 4, 8, 9, 10, 11 }); + d_ = __VEC_PERM (x_, c_, _Literal (v4si) { 0, 9, 7, 11 }); + r4 = d_; +} + +/* c = <a, b> v8si from v16si; d = <x, c> v8si; only b used from c. */ +void __attribute__((noipa)) __GIMPLE() fb2 () +{ + v16si a_; v16si b_; v8si x_; v8si c_; v8si d_; + a_ = a16; b_ = b16; x_ = x8; + c_ = __VEC_PERM (a_, b_, _Literal (v8si) { 31, 1, 17, 2, 16, 3, 20, 4 }); + d_ = __VEC_PERM (x_, c_, _Literal (v8si) { 0, 8, 1, 10, 2, 12, 3, 14 }); + r8 = d_; +} + +/* c = <a, b> v8si; d = <c, x> v4si; only b used from c. */ +void __attribute__((noipa)) __GIMPLE() fc1 () +{ + v8si a_; v8si b_; v8si x_; v8si c_; v4si d_; + a_ = a8; b_ = b8; x_ = x8; + c_ = __VEC_PERM (a_, b_, _Literal (v8si) { 0, 1, 2, 3, 15, 14, 13, 12 }); + d_ = __VEC_PERM (c_, x_, _Literal (v4si) { 4, 9, 6, 15 }); + r4 = d_; +} + +/* c = <a, b> v8si from v16si; d = <c, x> v8si; only a used from c. */ +void __attribute__((noipa)) __GIMPLE() fc2 () +{ + v16si a_; v16si b_; v8si x_; v8si c_; v8si d_; + a_ = a16; b_ = b16; x_ = x8; + c_ = __VEC_PERM (a_, b_, _Literal (v8si) { 15, 0, 13, 2, 11, 4, 9, 6 }); + d_ = __VEC_PERM (c_, x_, _Literal (v8si) { 0, 8, 2, 9, 4, 10, 6, 11 }); + r8 = d_; +} + +/* Reference VEC_PERM_EXPR: OUT[i] = concat (X, Y)[SEL[i] % (2 * NX)]. */ +static void +perm (int *out, const int *x, const int *y, int nx, const int *sel, int n) +{ + int tmp[16]; + for (int i = 0; i < n; i++) + { + int j = sel[i] % (2 * nx); + tmp[i] = j < nx ? x[j] : y[j - nx]; + } + for (int i = 0; i < n; i++) + out[i] = tmp[i]; +} + +static int fails; + +static void +check (const char *name, const int *got, const int *want, int n) +{ + for (int i = 0; i < n; i++) + if (got[i] != want[i]) + { + __builtin_printf ("%s: lane %d got %d want %d\n", name, i, got[i], + want[i]); + fails++; + return; + } +} + +int +main () +{ + int c[8], d[8]; + for (int i = 0; i < 8; i++) + { + a8[i] = i + 1; + b8[i] = i + 101; + x8[i] = i + 201; + } + for (int i = 0; i < 16; i++) + { + a16[i] = i + 301; + b16[i] = i + 401; + } + + { + static const int s0[8] = { 8, 0, 9, 1, 10, 2, 11, 3 }; + static const int s1[4] = { 1, 2, 3, 4 }; + perm (c, (int *) &a8, (int *) &b8, 8, s0, 8); + perm (d, c, c, 8, s1, 4); + fa1 (); + check ("fa1", (int *) &r4, d, 4); + } + { + static const int s0[8] = { 31, 0, 17, 14, 16, 15, 1, 30 }; + static const int s1[8] = { 7, 6, 5, 4, 3, 2, 1, 0 }; + perm (c, (int *) &a16, (int *) &b16, 16, s0, 8); + perm (d, c, c, 8, s1, 8); + fa2 (); + check ("fa2", (int *) &r8, d, 8); + } + { + static const int s0[8] = { 8, 0, 9, 1, 10, 2, 11, 3 }; + static const int s1[4] = { 7, 0, 5, 2 }; + perm (c, (int *) &a8, (int *) &b8, 8, s0, 8); + perm (d, c, c, 8, s1, 4); + fa3 (); + check ("fa3", (int *) &f4, d, 4); + } + { + static const int s0[8] = { 7, 6, 5, 4, 8, 9, 10, 11 }; + static const int s1[4] = { 0, 9, 7, 11 }; + perm (c, (int *) &a8, (int *) &b8, 8, s0, 8); + perm (d, (int *) &x8, c, 8, s1, 4); + fb1 (); + check ("fb1", (int *) &r4, d, 4); + } + { + static const int s0[8] = { 31, 1, 17, 2, 16, 3, 20, 4 }; + static const int s1[8] = { 0, 8, 1, 10, 2, 12, 3, 14 }; + perm (c, (int *) &a16, (int *) &b16, 16, s0, 8); + perm (d, (int *) &x8, c, 8, s1, 8); + fb2 (); + check ("fb2", (int *) &r8, d, 8); + } + { + static const int s0[8] = { 0, 1, 2, 3, 15, 14, 13, 12 }; + static const int s1[4] = { 4, 9, 6, 15 }; + perm (c, (int *) &a8, (int *) &b8, 8, s0, 8); + perm (d, c, (int *) &x8, 8, s1, 4); + fc1 (); + check ("fc1", (int *) &r4, d, 4); + } + { + static const int s0[8] = { 15, 0, 13, 2, 11, 4, 9, 6 }; + static const int s1[8] = { 0, 8, 2, 9, 4, 10, 6, 11 }; + perm (c, (int *) &a16, (int *) &b16, 16, s0, 8); + perm (d, c, (int *) &x8, 8, s1, 8); + fc2 (); + check ("fc2", (int *) &r8, d, 8); + } + + if (fails) + __builtin_abort (); + return 0; +} diff --git a/gcc/testsuite/gcc.dg/torture/vec-perm-merge-wide-input-2.c b/gcc/testsuite/gcc.dg/torture/vec-perm-merge-wide-input-2.c new file mode 100644 index 000000000000..6df62d3347d8 --- /dev/null +++ b/gcc/testsuite/gcc.dg/torture/vec-perm-merge-wide-input-2.c @@ -0,0 +1,64 @@ +/* forwprop must not treat permutes of different widths as an identity. */ +/* { dg-do run } */ +/* { dg-additional-options "-fgimple -Wno-psabi" } */ + +typedef float v8sf __attribute__((vector_size(32))); +typedef float v4sf __attribute__((vector_size(16))); +typedef float v2sf __attribute__((vector_size(8))); +typedef int v4si __attribute__((vector_size(16))); +typedef int v2si __attribute__((vector_size(8))); +typedef short v4hi __attribute__((vector_size(8))); +typedef short v2hi __attribute__((vector_size(4))); + +v8sf fa, fb; +v2sf fr; +v4hi ha, hb; +v2hi hr; + +/* The composed mask { 3, 4 } has fewer elements than the inner one. */ +void __attribute__((noipa)) __GIMPLE() f1 () +{ + v8sf a_; v8sf b_; v4sf c_; v2sf d_; + a_ = fa; b_ = fb; + c_ = __VEC_PERM (a_, b_, _Literal (v4si) { 4, 6, 12, 3 }); + d_ = __VEC_PERM (c_, c_, _Literal (v2si) { 6, 0 }); + fr = d_; +} + +/* { 1, 0 } of { 7, 6 } looks like an identity on b if the indices are + reduced by the length of c rather than of b. */ +void __attribute__((noipa)) __GIMPLE() f2 () +{ + v4hi a_; v4hi b_; v2hi c_; v2hi d_; + a_ = ha; b_ = hb; + c_ = __VEC_PERM (a_, b_, _Literal (v2hi) { 7, 6 }); + d_ = __VEC_PERM (c_, c_, _Literal (v2hi) { 1, 2 }); + hr = d_; +} + +int +main () +{ + for (int i = 0; i < 8; i++) + { + fa[i] = i + 1; + fb[i] = i + 101; + } + for (int i = 0; i < 4; i++) + { + ha[i] = i + 1; + hb[i] = i + 101; + } + + f1 (); + /* c = { a4, a6, b4, a3 }, d = { c2, c0 }. */ + if (fr[0] != 105 || fr[1] != 5) + __builtin_abort (); + + f2 (); + /* c = { b3, b2 }, d = { c1, c0 }. */ + if (hr[0] != 103 || hr[1] != 104) + __builtin_abort (); + + return 0; +} diff --git a/gcc/testsuite/gcc.dg/torture/vec-perm-merge-wide-input-3.c b/gcc/testsuite/gcc.dg/torture/vec-perm-merge-wide-input-3.c new file mode 100644 index 000000000000..337328136ecd --- /dev/null +++ b/gcc/testsuite/gcc.dg/torture/vec-perm-merge-wide-input-3.c @@ -0,0 +1,382 @@ +/* VEC_PERM_EXPR merges of c = <a, b> into d = <c, c> with c wider than, as + wide as and narrower than a, and d wider than, as wide as and narrower + than c. */ +/* { dg-do run } */ +/* { dg-additional-options "-fgimple -Wno-psabi" } */ + +typedef int v2si __attribute__((vector_size(8))); +typedef int v4si __attribute__((vector_size(16))); +typedef int v8si __attribute__((vector_size(32))); +typedef int v16si __attribute__((vector_size(64))); +typedef int v32si __attribute__((vector_size(128))); + +v8si a, b; +v2si r2; +v4si r4; +v8si r8; +v16si r16; +v32si r32; + +/* c wider than a (v8si -> v16si), d wider than c (-> v32si). */ + +void __attribute__((noipa)) __GIMPLE() f1 () +{ + v8si a_; v8si b_; v16si c_; v32si d_; + a_ = a; b_ = b; + c_ = __VEC_PERM (a_, b_, _Literal (v16si) { + 14, 14, 14, 6, 5, 15, 5, 3, 14, 9, 4, 2, 1, 12, 14, 5 }); + d_ = __VEC_PERM (c_, c_, _Literal (v32si) { + 0, 4, 3, 2, 12, 15, 1, 29, 20, 28, 12, 14, 18, 31, 0, 5, 29, 17, 26, 5, + 16, 20, 14, 18, 1, 4, 6, 25, 6, 18, 24, 4 }); + r32 = d_; +} + +void __attribute__((noipa)) __GIMPLE() f2 () +{ + v8si a_; v8si b_; v16si c_; v32si d_; + a_ = a; b_ = b; + c_ = __VEC_PERM (a_, b_, _Literal (v16si) { + 0, 0, 6, 6, 1, 15, 12, 12, 13, 2, 6, 8, 10, 2, 9, 10 }); + d_ = __VEC_PERM (c_, c_, _Literal (v32si) { + 0, 26, 7, 8, 15, 6, 0, 3, 29, 31, 11, 12, 28, 12, 8, 26, 24, 7, 25, 26, + 13, 0, 17, 19, 1, 13, 11, 25, 6, 2, 9, 13 }); + r32 = d_; +} + +/* c wider than a (v8si -> v16si), d as wide as c (-> v16si). */ + +void __attribute__((noipa)) __GIMPLE() f3 () +{ + v8si a_; v8si b_; v16si c_; v16si d_; + a_ = a; b_ = b; + c_ = __VEC_PERM (a_, b_, _Literal (v16si) { + 14, 8, 0, 10, 9, 12, 2, 2, 2, 6, 7, 0, 11, 11, 14, 4 }); + d_ = __VEC_PERM (c_, c_, _Literal (v16si) { + 30, 8, 24, 11, 9, 19, 14, 15, 12, 10, 12, 24, 30, 5, 26, 3 }); + r16 = d_; +} + +void __attribute__((noipa)) __GIMPLE() f4 () +{ + v8si a_; v8si b_; v16si c_; v16si d_; + a_ = a; b_ = b; + c_ = __VEC_PERM (a_, b_, _Literal (v16si) { + 3, 3, 1, 8, 7, 12, 8, 13, 15, 9, 5, 2, 4, 7, 15, 2 }); + d_ = __VEC_PERM (c_, c_, _Literal (v16si) { + 17, 13, 13, 1, 4, 17, 26, 28, 15, 3, 2, 11, 18, 23, 8, 5 }); + r16 = d_; +} + +/* c wider than a (v8si -> v16si), d narrower than c (-> v8si). */ + +void __attribute__((noipa)) __GIMPLE() f5 () +{ + v8si a_; v8si b_; v16si c_; v8si d_; + a_ = a; b_ = b; + c_ = __VEC_PERM (a_, b_, _Literal (v16si) { + 11, 4, 14, 10, 4, 1, 0, 15, 11, 9, 1, 0, 2, 15, 2, 9 }); + d_ = __VEC_PERM (c_, c_, _Literal (v8si) { 20, 8, 4, 4, 28, 23, 2, 8 }); + r8 = d_; +} + +void __attribute__((noipa)) __GIMPLE() f6 () +{ + v8si a_; v8si b_; v16si c_; v8si d_; + a_ = a; b_ = b; + c_ = __VEC_PERM (a_, b_, _Literal (v16si) { + 10, 11, 2, 15, 2, 13, 0, 15, 0, 12, 12, 0, 2, 2, 2, 3 }); + d_ = __VEC_PERM (c_, c_, _Literal (v8si) { 16, 26, 21, 24, 29, 28, 29, 5 }); + r8 = d_; +} + +/* c as wide as a (v8si -> v8si), d wider than c (-> v16si). */ + +void __attribute__((noipa)) __GIMPLE() f7 () +{ + v8si a_; v8si b_; v8si c_; v16si d_; + a_ = a; b_ = b; + c_ = __VEC_PERM (a_, b_, _Literal (v8si) { 0, 9, 2, 15, 0, 7, 3, 15 }); + d_ = __VEC_PERM (c_, c_, _Literal (v16si) { + 15, 8, 0, 11, 9, 4, 6, 5, 10, 14, 15, 7, 10, 12, 8, 6 }); + r16 = d_; +} + +void __attribute__((noipa)) __GIMPLE() f8 () +{ + v8si a_; v8si b_; v8si c_; v16si d_; + a_ = a; b_ = b; + c_ = __VEC_PERM (a_, b_, _Literal (v8si) { 13, 6, 6, 12, 7, 10, 6, 4 }); + d_ = __VEC_PERM (c_, c_, _Literal (v16si) { + 4, 15, 11, 1, 2, 8, 5, 3, 14, 15, 8, 6, 13, 12, 15, 10 }); + r16 = d_; +} + +/* c as wide as a (v8si -> v8si), d as wide as c (-> v8si). */ + +void __attribute__((noipa)) __GIMPLE() f9 () +{ + v8si a_; v8si b_; v8si c_; v8si d_; + a_ = a; b_ = b; + c_ = __VEC_PERM (a_, b_, _Literal (v8si) { 14, 10, 2, 1, 8, 1, 8, 11 }); + d_ = __VEC_PERM (c_, c_, _Literal (v8si) { 9, 0, 4, 12, 14, 6, 0, 8 }); + r8 = d_; +} + +void __attribute__((noipa)) __GIMPLE() f10 () +{ + v8si a_; v8si b_; v8si c_; v8si d_; + a_ = a; b_ = b; + c_ = __VEC_PERM (a_, b_, _Literal (v8si) { 7, 4, 1, 3, 14, 3, 11, 2 }); + d_ = __VEC_PERM (c_, c_, _Literal (v8si) { 6, 6, 15, 8, 5, 0, 15, 1 }); + r8 = d_; +} + +/* c as wide as a (v8si -> v8si), d narrower than c (-> v4si). */ + +void __attribute__((noipa)) __GIMPLE() f11 () +{ + v8si a_; v8si b_; v8si c_; v4si d_; + a_ = a; b_ = b; + c_ = __VEC_PERM (a_, b_, _Literal (v8si) { 5, 7, 8, 11, 5, 12, 7, 2 }); + d_ = __VEC_PERM (c_, c_, _Literal (v4si) { 13, 12, 4, 14 }); + r4 = d_; +} + +void __attribute__((noipa)) __GIMPLE() f12 () +{ + v8si a_; v8si b_; v8si c_; v4si d_; + a_ = a; b_ = b; + c_ = __VEC_PERM (a_, b_, _Literal (v8si) { 14, 6, 0, 12, 10, 14, 10, 6 }); + d_ = __VEC_PERM (c_, c_, _Literal (v4si) { 3, 3, 6, 7 }); + r4 = d_; +} + +/* c narrower than a (v8si -> v4si), d wider than c (-> v8si). */ + +void __attribute__((noipa)) __GIMPLE() f13 () +{ + v8si a_; v8si b_; v4si c_; v8si d_; + a_ = a; b_ = b; + c_ = __VEC_PERM (a_, b_, _Literal (v4si) { 12, 2, 9, 10 }); + d_ = __VEC_PERM (c_, c_, _Literal (v8si) { 4, 0, 5, 1, 0, 7, 5, 6 }); + r8 = d_; +} + +void __attribute__((noipa)) __GIMPLE() f14 () +{ + v8si a_; v8si b_; v4si c_; v8si d_; + a_ = a; b_ = b; + c_ = __VEC_PERM (a_, b_, _Literal (v4si) { 8, 15, 0, 6 }); + d_ = __VEC_PERM (c_, c_, _Literal (v8si) { 1, 6, 0, 2, 5, 2, 7, 2 }); + r8 = d_; +} + +/* c narrower than a (v8si -> v4si), d as wide as c (-> v4si). */ + +void __attribute__((noipa)) __GIMPLE() f15 () +{ + v8si a_; v8si b_; v4si c_; v4si d_; + a_ = a; b_ = b; + c_ = __VEC_PERM (a_, b_, _Literal (v4si) { 14, 15, 2, 7 }); + d_ = __VEC_PERM (c_, c_, _Literal (v4si) { 7, 4, 2, 4 }); + r4 = d_; +} + +void __attribute__((noipa)) __GIMPLE() f16 () +{ + v8si a_; v8si b_; v4si c_; v4si d_; + a_ = a; b_ = b; + c_ = __VEC_PERM (a_, b_, _Literal (v4si) { 9, 12, 6, 9 }); + d_ = __VEC_PERM (c_, c_, _Literal (v4si) { 2, 4, 7, 3 }); + r4 = d_; +} + +/* c narrower than a (v8si -> v4si), d narrower than c (-> v2si). */ + +void __attribute__((noipa)) __GIMPLE() f17 () +{ + v8si a_; v8si b_; v4si c_; v2si d_; + a_ = a; b_ = b; + c_ = __VEC_PERM (a_, b_, _Literal (v4si) { 13, 3, 0, 12 }); + d_ = __VEC_PERM (c_, c_, _Literal (v2si) { 0, 0 }); + r2 = d_; +} + +void __attribute__((noipa)) __GIMPLE() f18 () +{ + v8si a_; v8si b_; v4si c_; v2si d_; + a_ = a; b_ = b; + c_ = __VEC_PERM (a_, b_, _Literal (v4si) { 12, 3, 15, 2 }); + d_ = __VEC_PERM (c_, c_, _Literal (v2si) { 2, 1 }); + r2 = d_; +} + +/* Reference VEC_PERM_EXPR: OUT[i] = concat (X, Y)[SEL[i] % (2 * NX)]. */ +static void +perm (int *out, const int *x, const int *y, int nx, const int *sel, int n) +{ + int tmp[32]; + for (int i = 0; i < n; i++) + { + int j = sel[i] % (2 * nx); + tmp[i] = j < nx ? x[j] : y[j - nx]; + } + for (int i = 0; i < n; i++) + out[i] = tmp[i]; +} + +static int fails; + +static void +check (const char *name, const int *got, const int *sel0, int nc, + const int *sel1, int nd) +{ + int c[32], want[32]; + perm (c, (int *) &a, (int *) &b, 8, sel0, nc); + perm (want, c, c, nc, sel1, nd); + for (int i = 0; i < nd; i++) + if (got[i] != want[i]) + { + __builtin_printf ("%s: lane %d got %d want %d\n", name, i, got[i], + want[i]); + fails++; + return; + } +} + +int +main () +{ + for (int i = 0; i < 8; i++) + { + a[i] = i + 1; + b[i] = i + 101; + } + + { + static const int s0[] = { + 14, 14, 14, 6, 5, 15, 5, 3, 14, 9, 4, 2, 1, 12, 14, 5 }; + static const int s1[] = { + 0, 4, 3, 2, 12, 15, 1, 29, 20, 28, 12, 14, 18, 31, 0, 5, 29, 17, 26, + 5, 16, 20, 14, 18, 1, 4, 6, 25, 6, 18, 24, 4 }; + f1 (); + check ("f1", (int *) &r32, s0, 16, s1, 32); + } + { + static const int s0[] = { + 0, 0, 6, 6, 1, 15, 12, 12, 13, 2, 6, 8, 10, 2, 9, 10 }; + static const int s1[] = { + 0, 26, 7, 8, 15, 6, 0, 3, 29, 31, 11, 12, 28, 12, 8, 26, 24, 7, 25, + 26, 13, 0, 17, 19, 1, 13, 11, 25, 6, 2, 9, 13 }; + f2 (); + check ("f2", (int *) &r32, s0, 16, s1, 32); + } + { + static const int s0[] = { + 14, 8, 0, 10, 9, 12, 2, 2, 2, 6, 7, 0, 11, 11, 14, 4 }; + static const int s1[] = { + 30, 8, 24, 11, 9, 19, 14, 15, 12, 10, 12, 24, 30, 5, 26, 3 }; + f3 (); + check ("f3", (int *) &r16, s0, 16, s1, 16); + } + { + static const int s0[] = { + 3, 3, 1, 8, 7, 12, 8, 13, 15, 9, 5, 2, 4, 7, 15, 2 }; + static const int s1[] = { + 17, 13, 13, 1, 4, 17, 26, 28, 15, 3, 2, 11, 18, 23, 8, 5 }; + f4 (); + check ("f4", (int *) &r16, s0, 16, s1, 16); + } + { + static const int s0[] = { + 11, 4, 14, 10, 4, 1, 0, 15, 11, 9, 1, 0, 2, 15, 2, 9 }; + static const int s1[] = { 20, 8, 4, 4, 28, 23, 2, 8 }; + f5 (); + check ("f5", (int *) &r8, s0, 16, s1, 8); + } + { + static const int s0[] = { + 10, 11, 2, 15, 2, 13, 0, 15, 0, 12, 12, 0, 2, 2, 2, 3 }; + static const int s1[] = { 16, 26, 21, 24, 29, 28, 29, 5 }; + f6 (); + check ("f6", (int *) &r8, s0, 16, s1, 8); + } + { + static const int s0[] = { 0, 9, 2, 15, 0, 7, 3, 15 }; + static const int s1[] = { + 15, 8, 0, 11, 9, 4, 6, 5, 10, 14, 15, 7, 10, 12, 8, 6 }; + f7 (); + check ("f7", (int *) &r16, s0, 8, s1, 16); + } + { + static const int s0[] = { 13, 6, 6, 12, 7, 10, 6, 4 }; + static const int s1[] = { + 4, 15, 11, 1, 2, 8, 5, 3, 14, 15, 8, 6, 13, 12, 15, 10 }; + f8 (); + check ("f8", (int *) &r16, s0, 8, s1, 16); + } + { + static const int s0[] = { 14, 10, 2, 1, 8, 1, 8, 11 }; + static const int s1[] = { 9, 0, 4, 12, 14, 6, 0, 8 }; + f9 (); + check ("f9", (int *) &r8, s0, 8, s1, 8); + } + { + static const int s0[] = { 7, 4, 1, 3, 14, 3, 11, 2 }; + static const int s1[] = { 6, 6, 15, 8, 5, 0, 15, 1 }; + f10 (); + check ("f10", (int *) &r8, s0, 8, s1, 8); + } + { + static const int s0[] = { 5, 7, 8, 11, 5, 12, 7, 2 }; + static const int s1[] = { 13, 12, 4, 14 }; + f11 (); + check ("f11", (int *) &r4, s0, 8, s1, 4); + } + { + static const int s0[] = { 14, 6, 0, 12, 10, 14, 10, 6 }; + static const int s1[] = { 3, 3, 6, 7 }; + f12 (); + check ("f12", (int *) &r4, s0, 8, s1, 4); + } + { + static const int s0[] = { 12, 2, 9, 10 }; + static const int s1[] = { 4, 0, 5, 1, 0, 7, 5, 6 }; + f13 (); + check ("f13", (int *) &r8, s0, 4, s1, 8); + } + { + static const int s0[] = { 8, 15, 0, 6 }; + static const int s1[] = { 1, 6, 0, 2, 5, 2, 7, 2 }; + f14 (); + check ("f14", (int *) &r8, s0, 4, s1, 8); + } + { + static const int s0[] = { 14, 15, 2, 7 }; + static const int s1[] = { 7, 4, 2, 4 }; + f15 (); + check ("f15", (int *) &r4, s0, 4, s1, 4); + } + { + static const int s0[] = { 9, 12, 6, 9 }; + static const int s1[] = { 2, 4, 7, 3 }; + f16 (); + check ("f16", (int *) &r4, s0, 4, s1, 4); + } + { + static const int s0[] = { 13, 3, 0, 12 }; + static const int s1[] = { 0, 0 }; + f17 (); + check ("f17", (int *) &r2, s0, 4, s1, 2); + } + { + static const int s0[] = { 12, 3, 15, 2 }; + static const int s1[] = { 2, 1 }; + f18 (); + check ("f18", (int *) &r2, s0, 4, s1, 2); + } + + if (fails) + __builtin_abort (); + return 0; +} diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr127862.c b/gcc/testsuite/gcc.target/aarch64/sve/pr127862.c new file mode 100644 index 000000000000..4b088391fa71 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/pr127862.c @@ -0,0 +1,53 @@ +/* VEC_PERM_EXPR merges with a result wider than the inputs. */ +/* { dg-do compile } */ +/* { dg-options "-O2 -msve-vector-bits=256" } */ + +#include <arm_sve.h> + +/* svld1rq is c = VEC_PERM <a, a> from a 128-bit vector; d = <c, c>. */ + +svint32_t +f1 (int32_t *p) +{ + svint32_t c = svld1rq_s32 (svptrue_b8 (), p); + return svrev_s32 (c); +} + +svint32_t +f2 (int32_t *p) +{ + svint32_t c = svld1rq_s32 (svptrue_b8 (), p); + return svzip1_s32 (c, c); +} + +svint32_t +f3 (int32_t *p) +{ + svint32_t c = svld1rq_s32 (svptrue_b8 (), p); + return svuzp2_s32 (c, c); +} + +/* d = <x, c>. */ + +svint32_t +f4 (int32_t *p, svint32_t x) +{ + svint32_t c = svld1rq_s32 (svptrue_b8 (), p); + return svzip1_s32 (x, c); +} + +svint32_t +f5 (int32_t *p, svint32_t x) +{ + svint32_t c = svld1rq_s32 (svptrue_b8 (), p); + return svuzp1_s32 (x, c); +} + +/* d = <c, x>. */ + +svint32_t +f6 (int32_t *p, svint32_t x) +{ + svint32_t c = svld1rq_s32 (svptrue_b8 (), p); + return svtrn2_s32 (c, x); +} diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr127862_run.c b/gcc/testsuite/gcc.target/aarch64/sve/pr127862_run.c new file mode 100644 index 000000000000..abbec284cb85 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/pr127862_run.c @@ -0,0 +1,46 @@ +/* VEC_PERM_EXPR merges with a result wider than the inputs. */ +/* { dg-do run { target aarch64_sve256_hw } } */ +/* { dg-options "-O2 -msve-vector-bits=256" } */ + +#include <arm_sve.h> + +int32_t p[4] = { 1, 2, 3, 4 }; +int32_t xs[8] = { 11, 12, 13, 14, 15, 16, 17, 18 }; +int32_t r[8]; + +/* c = VEC_PERM <a, a>, d = VEC_PERM <c, c>. */ +void __attribute__((noipa)) +f1 (int32_t *p) +{ + svint32_t c = svld1rq_s32 (svptrue_b8 (), p); + svst1_s32 (svptrue_b8 (), r, svrev_s32 (c)); +} + +/* c = VEC_PERM <a, a>, d = VEC_PERM <c, x>. */ +void __attribute__((noipa)) +f2 (int32_t *p, svint32_t x) +{ + svint32_t c = svld1rq_s32 (svptrue_b8 (), p); + svst1_s32 (svptrue_b8 (), r, svtrn2_s32 (c, x)); +} + +static void +check (const int32_t *want) +{ + for (int i = 0; i < 8; i++) + if (r[i] != want[i]) + __builtin_abort (); +} + +int +main (void) +{ + static const int32_t want1[8] = { 4, 3, 2, 1, 4, 3, 2, 1 }; + static const int32_t want2[8] = { 2, 12, 4, 14, 2, 16, 4, 18 }; + + f1 (p); + check (want1); + f2 (p, svld1_s32 (svptrue_b8 (), xs)); + check (want2); + return 0; +} diff --git a/gcc/tree-ssa-forwprop.cc b/gcc/tree-ssa-forwprop.cc index 3ef7f4311f2b..d840a68a00c5 100644 --- a/gcc/tree-ssa-forwprop.cc +++ b/gcc/tree-ssa-forwprop.cc @@ -5032,6 +5032,15 @@ simplify_permutation (gimple_stmt_iterator *gsi) op3 = gimple_assign_rhs3 (def_stmt); if (TREE_CODE (op3) != VECTOR_CST) return false; + /* is_combined_permutation_identity assumes that both permutes and + their inputs have the same number of elements. Mixed widths are + left to the match.pd VEC_PERM merge patterns. */ + if (maybe_ne (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op2)), + TYPE_VECTOR_SUBPARTS (TREE_TYPE (op3))) + || maybe_ne (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op0)), + TYPE_VECTOR_SUBPARTS + (TREE_TYPE (gimple_assign_rhs1 (def_stmt))))) + return false; ident = is_combined_permutation_identity (op3, op2); if (!ident) return false;
