https://gcc.gnu.org/g:88e5b890064857b44d67f204be0566ad8550955f
commit r17-6102-g88e5b890064857b44d67f204be0566ad8550955f Author: Tamar Christina <[email protected]> Date: Tue Nov 10 11:16:59 2026 +0000 AArch64: add general LUTI EVPC support This patch adds LUTI2, LUTI4 and LUTI6 to constant vector permutation expansion. It covers byte and halfword permutations for Advanced SIMD and SVE, including the two-register table forms and repeating permutations with a scalable vector length. LUTI instructions look up elements in a table held in vector registers. The number in the name gives the number of bits used for each index: 2, 4 or 6. The indices are packed together in another register, so LUTI2 can hold four indices in each byte and LUTI4 can hold two. Both byte and halfword table elements are supported. For LUTI2 and LUTI4, the patch only uses the instruction when the packed indices can be built with an immediate move. This avoids loading a full permutation vector from the constant pool. For example, compile this with: With -O2 -march=armv9-a+lut and the following testcase typedef unsigned char v16qi __attribute__((vector_size (16))); v16qi shuffle (v16qi x) { return __builtin_shufflevector (x, x, 2, 0, 3, 1, 2, 0, 3, 1, 2, 0, 3, 1, 2, 0, 3, 1); } We generate before: shuffle: adrp x0, .LANCHOR0 ldr q31, [x0, #:lo12:.LANCHOR0] tbl v0.16b, {v0.16b}, v31.16b ret .section .rodata .align 4 .set .LANCHOR0,. + 0 .LC0: .byte 2 .byte 0 .byte 3 .byte 1 .byte 2 .byte 0 .byte 3 .byte 1 .byte 2 .byte 0 .byte 3 .byte 1 .byte 2 .byte 0 .byte 3 .byte 1 This also needs a 16-byte constant containing the permutation indices. After: shuffle: movi v31.8b, 0x72 luti2 v0.16b, {v0.16b}, v31[0] ret Here 0x72 packs the indices {2, 0, 3, 1}, starting at the low bits. Repeating that byte gives the complete permutation, so the constant-pool load goes away. LUTI6 can handle arbitrary two-input permutations with 32 result elements. The packed indices can still need a constant-pool load, but a single LUTI6 replaces the two TBL instructions and the operations used to combine them. The options for the new matches are: * LUTI2/4: -march=armv9-a+lut enables both the Advanced SIMD and SVE forms. These matches require little-endian code. SVE also works with -msve-vector-bits=scalable for supported repeating permutations; a fixed vector length is not required. * LUTI6: -march=armv9-a+sve2p3, in non-streaming mode. The current matches require -msve-vector-bits=256 for 32 byte elements, or -msve-vector-bits=512 for 32 halfword elements. Note that this conflicts are bit with Ezra's LUTI6 intrinsics patches and I'll rebase and merge the patterns when that gets done. The example uses __builtin_shufflevector; the same expansion is available for constant permutations produced by the vectorizer. gcc/ChangeLog: * config/aarch64/aarch64-sve2.md (@aarch64_sve_luti6<mode>): New pattern. * config/aarch64/aarch64.cc (aarch64_evpc_luti_indices): New function. (aarch64_evpc_luti): Likewise. (aarch64_expand_vec_perm_const_1): Try LUTI before falling back to TBL. * config/aarch64/iterators.md (UNSPEC_SVE_LUTI6): New unspec. * doc/sourcebuild.texi (aarch64_lut_hw): Document effective target. gcc/testsuite/ChangeLog: * lib/target-supports.exp (check_effective_target_aarch64_lut_hw): New effective-target check. * gcc.target/aarch64/perm-luti-2-run.c: New test. * gcc.target/aarch64/perm-luti-2.c: New test. * gcc.target/aarch64/perm-luti-fallback.c: New test. * gcc.target/aarch64/perm-luti-nolut.c: New test. * gcc.target/aarch64/perm-luti-run.c: New test. * gcc.target/aarch64/perm-luti.c: New test. * gcc.target/aarch64/sve/perm_luti_1.c: New test. * gcc.target/aarch64/sve/perm_luti_1_run.c: New test. * gcc.target/aarch64/sve/perm_luti_2.c: New test. * gcc.target/aarch64/sve/perm_luti_2_run.c: New test. * gcc.target/aarch64/sve/perm_luti_3.c: New test. * gcc.target/aarch64/sve/perm_luti_3_run.c: New test. * gcc.target/aarch64/sve/perm_luti_4.c: New test. * gcc.target/aarch64/sve/perm_luti_4_run.c: New test. * gcc.target/aarch64/sve/perm_luti_5.c: New test. * gcc.target/aarch64/sve/perm_luti_6.c: New test. * gcc.target/aarch64/sve/perm_luti_6_run.c: New test. * gcc.target/aarch64/sve/perm_luti_7.c: New test. * gcc.target/aarch64/sve/perm_luti_7_run.c: New test. * gcc.target/aarch64/sve/perm_luti_8.c: New test. Diff: --- gcc/config/aarch64/aarch64-sve2.md | 17 +++ gcc/config/aarch64/aarch64.cc | 150 +++++++++++++++++++++ gcc/config/aarch64/iterators.md | 1 + gcc/doc/sourcebuild.texi | 2 + gcc/testsuite/gcc.target/aarch64/perm-luti-2-run.c | 23 ++++ gcc/testsuite/gcc.target/aarch64/perm-luti-2.c | 26 ++++ .../gcc.target/aarch64/perm-luti-fallback.c | 15 +++ gcc/testsuite/gcc.target/aarch64/perm-luti-nolut.c | 7 + gcc/testsuite/gcc.target/aarch64/perm-luti-run.c | 23 ++++ gcc/testsuite/gcc.target/aarch64/perm-luti.c | 31 +++++ gcc/testsuite/gcc.target/aarch64/sve/perm_luti_1.c | 38 ++++++ .../gcc.target/aarch64/sve/perm_luti_1_run.c | 28 ++++ gcc/testsuite/gcc.target/aarch64/sve/perm_luti_2.c | 15 +++ .../gcc.target/aarch64/sve/perm_luti_2_run.c | 24 ++++ gcc/testsuite/gcc.target/aarch64/sve/perm_luti_3.c | 20 +++ .../gcc.target/aarch64/sve/perm_luti_3_run.c | 21 +++ gcc/testsuite/gcc.target/aarch64/sve/perm_luti_4.c | 22 +++ .../gcc.target/aarch64/sve/perm_luti_4_run.c | 25 ++++ gcc/testsuite/gcc.target/aarch64/sve/perm_luti_5.c | 22 +++ gcc/testsuite/gcc.target/aarch64/sve/perm_luti_6.c | 22 +++ .../gcc.target/aarch64/sve/perm_luti_6_run.c | 27 ++++ gcc/testsuite/gcc.target/aarch64/sve/perm_luti_7.c | 46 +++++++ .../gcc.target/aarch64/sve/perm_luti_7_run.c | 52 +++++++ gcc/testsuite/gcc.target/aarch64/sve/perm_luti_8.c | 24 ++++ gcc/testsuite/lib/target-supports.exp | 18 +++ 25 files changed, 699 insertions(+) diff --git a/gcc/config/aarch64/aarch64-sve2.md b/gcc/config/aarch64/aarch64-sve2.md index 1e575fa50fd9..a60ff46d2bf6 100644 --- a/gcc/config/aarch64/aarch64-sve2.md +++ b/gcc/config/aarch64/aarch64-sve2.md @@ -4858,8 +4858,25 @@ ;; Includes: ;; - LUTI2 ;; - LUTI4 +;; - LUTI6 ;; ------------------------------------------------------------------------- +(define_insn "@aarch64_sve_luti6<mode>" + [(set (match_operand:SVE_FULL_BH 0 "register_operand" "=w") + (unspec:SVE_FULL_BH + [(match_operand:<VDOUBLE> 1 "register_operand" "w") + (match_operand:VNx16QI 2 "register_operand" "w")] + UNSPEC_SVE_LUTI6))] + "TARGET_SVE2p3 && TARGET_NON_STREAMING + && known_ge (BYTES_PER_SVE_VECTOR, 32 * GET_MODE_UNIT_SIZE (<MODE>mode))" +{ + if (<MODE>mode == VNx16QImode) + return "luti6\t%0.b, %1, %2"; + return "luti6\t%0.h, %1, %2[0]"; +} + [(set_attr "sve_type" "sve_int_general")] +) + (define_insn "@aarch64_sve_luti<LUTI_BITS><mode>" [(set (match_operand:SVE_FULL_BH 0 "register_operand" "=w") (unspec:SVE_FULL_BH diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc index 730a1fee8359..1ef3afc9a80a 100644 --- a/gcc/config/aarch64/aarch64.cc +++ b/gcc/config/aarch64/aarch64.cc @@ -28817,6 +28817,152 @@ aarch64_evpc_hvla (struct expand_vec_perm_d *d) return try_zip () || try_uzp () || try_extq () || try_dupq (); } +/* Pack D's constant indices into BITS-bit fields. For LUTI2/4, repeat the + packed pattern into the unused part of SEL_MODE too, to give immediate + materialization more freedom. Only use those forms when the selector + needs no constant-pool load. For LUTI6, pack all the indices and leave + the unused high bytes zero. Stepped scalable selectors need another + expansion. */ + +static rtx +aarch64_evpc_luti_indices (struct expand_vec_perm_d *d, unsigned int bits, + machine_mode sel_mode) +{ + poly_int64 nelt = d->perm.length (); + unsigned int npatterns, packed_npatterns; + if (bits == 6) + { + npatterns = nelt.to_constant (); + packed_npatterns = GET_MODE_NUNITS (sel_mode).to_constant (); + } + else + { + const auto &encoding = d->perm.encoding (); + if (encoding.nelts_per_pattern () != 1) + return NULL_RTX; + npatterns = encoding.npatterns (); + packed_npatterns = CEIL (npatterns * bits, BITS_PER_UNIT); + } + rtx_vector_builder builder (sel_mode, packed_npatterns, 1); + /* Two-input LUTI4 reads eight halfwords from each register, + even when the input vectors are wider than that. */ + bool remap_inputs = bits == 4 && !d->one_vector_p; + unsigned int value = 0, nbits = 0, next = 0; + unsigned int nbytes = CEIL (npatterns * bits, BITS_PER_UNIT); + for (unsigned int i = 0; i < nbytes; ++i) + { + /* Append indices until we have a complete byte. LUTI6 fields can + straddle bytes, so keep any unused bits for the next byte. */ + while (nbits < BITS_PER_UNIT) + { + poly_int64 elt = d->perm[next % npatterns]; + bool second = remap_inputs && known_ge (elt, nelt); + if (second) + elt -= nelt; + HOST_WIDE_INT index; + if (!elt.is_constant (&index) + || !IN_RANGE (index, 0, remap_inputs ? 7 : (1U << bits) - 1)) + return NULL_RTX; + value |= (index + (second ? 8 : 0)) << nbits; + nbits += bits; + ++next; + } + builder.quick_push (gen_int_mode (value, QImode)); + value >>= BITS_PER_UNIT; + nbits -= BITS_PER_UNIT; + } + /* LUTI6 leaves the rest of the selector zero. */ + for (unsigned int i = nbytes; i < packed_npatterns; ++i) + builder.quick_push (const0_rtx); + rtx sel = builder.build (); + if (bits != 6 && !aarch64_simd_valid_mov_imm (sel)) + return NULL_RTX; + return sel; +} + +/* Try to implement D using LUTI2, LUTI4 or LUTI6, as selected by BITS. + Prefer LUTI2/4 when their packed selectors can be loaded as an immediate. + LUTI6 can implement any two-input permutation with 32 elements, avoiding + the two TBLs and their combining operations. */ + +static bool +aarch64_evpc_luti (struct expand_vec_perm_d *d, unsigned int bits) +{ + machine_mode mode = d->vmode; + bool sve_p = d->vec_flags == VEC_SVE_DATA; + /* LUTI only supports .B and .H. */ + if (GET_MODE_UNIT_SIZE (mode) > 2 + || (!sve_p && (d->vec_flags != VEC_ADVSIMD + || !known_eq (GET_MODE_SIZE (mode), 16)))) + return false; + + bool two_tables; + if (bits == 6) + { + if (!sve_p || !TARGET_SVE2p3 || !TARGET_NON_STREAMING + || d->one_vector_p || !known_eq (GET_MODE_NUNITS (mode), 32)) + return false; + two_tables = true; + } + else + { + if (!TARGET_LUT || BYTES_BIG_ENDIAN + || (sve_p && !TARGET_SVE2_OR_SME2)) + return false; + + /* Advanced SIMD halfword LUTI4 always needs two table registers. + SVE needs two below VL == 256 or when using both inputs. */ + two_tables = (bits == 4 && GET_MODE_UNIT_SIZE (mode) == 2 + && (!d->one_vector_p + || !known_ge (GET_MODE_SIZE (mode), 32))); + if (!d->one_vector_p && !two_tables) + return false; + + /* Duplicating the table does not make its upper half accessible, but + there's an open question whether a DUP or EXT like rotation could be + used to get access. For now just reject. */ + if (two_tables && d->one_vector_p + && !d->perm.all_in_range_p (0, 8)) + return false; + } + + machine_mode sel_mode = sve_p ? VNx16QImode : V8QImode; + rtx sel = aarch64_evpc_luti_indices (d, bits, sel_mode); + if (!sel) + return false; + if (d->testing_p) + return true; + + machine_mode table_mode = mode; + rtx table = d->op0; + if (two_tables) + { + table_mode = aarch64_array_mode (mode, 2).require (); + table = gen_reg_rtx (table_mode); + /* Emit clobber to start live region for the entire table. */ + emit_clobber (table); + emit_move_insn (gen_lowpart (mode, table), d->op0); + emit_move_insn (simplify_gen_subreg (mode, table, table_mode, + GET_MODE_SIZE (mode)), d->op1); + } + else if (!sve_p && GET_MODE_UNIT_SIZE (mode) == 2) + { + /* LUTI2 uses only the low four halfwords of the input. */ + table_mode = mode_for_vector (GET_MODE_INNER (mode), 4).require (); + table = gen_lowpart (table_mode, table); + } + sel = force_reg (sel_mode, sel); + if (bits == 6) + emit_insn (gen_aarch64_sve_luti6 (mode, d->target, table, sel)); + else if (sve_p) + emit_insn (gen_aarch64_sve_luti (bits, table_mode, d->target, table, + sel, const0_rtx)); + else + emit_insn (gen_aarch64_lut (table_mode, sel_mode, d->target, table, + sel, const0_rtx, GEN_INT (bits))); + return true; +} + static bool aarch64_evpc_tbl (struct expand_vec_perm_d *d) { @@ -29149,6 +29295,10 @@ aarch64_expand_vec_perm_const_1 (struct expand_vec_perm_d *d) return true; else if (aarch64_evpc_reencode (d)) return true; + else if (aarch64_evpc_luti (d, 2) + || aarch64_evpc_luti (d, 4) + || aarch64_evpc_luti (d, 6)) + return true; if (d->vec_flags == VEC_SVE_DATA) return aarch64_evpc_sve_tbl (d); diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md index b18094d9088d..d3ca1c09e396 100644 --- a/gcc/config/aarch64/iterators.md +++ b/gcc/config/aarch64/iterators.md @@ -1411,6 +1411,7 @@ UNSPEC_ZIPQ1 UNSPEC_ZIPQ2 UNSPEC_SVE_LUTI + UNSPEC_SVE_LUTI6 ;; All used in aarch64-sme.md UNSPEC_SME_ADD diff --git a/gcc/doc/sourcebuild.texi b/gcc/doc/sourcebuild.texi index 20c3c2d10d6b..f24c6f825d20 100644 --- a/gcc/doc/sourcebuild.texi +++ b/gcc/doc/sourcebuild.texi @@ -2404,6 +2404,8 @@ for AArch64 small memory model. AArch64 target that generates instructions for SME. @item aarch64_sme2 AArch64 target that generates instructions for SME2. +@item aarch64_lut_hw +AArch64 target that is able to generate and execute LUT instructions. @item aarch64_sve_hw AArch64 target that is able to generate and execute SVE code (regardless of whether it does so by default). diff --git a/gcc/testsuite/gcc.target/aarch64/perm-luti-2-run.c b/gcc/testsuite/gcc.target/aarch64/perm-luti-2-run.c new file mode 100644 index 000000000000..a9c756d06941 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/perm-luti-2-run.c @@ -0,0 +1,23 @@ +/* { dg-do run { target { aarch64_lut_hw && le } } } */ +/* { dg-options "-O2 -march=armv9-a+lut" } */ + +#include "perm-luti-2.c" + +int +main (void) +{ + v8hi x = { 0, 1001, 2002, 3003, 4004, 5005, 6006, 7007 }; + v8hi y = { 8008, 9009, 10010, 11011, 12012, 13013, 14014, 15015 }; + v8hi a = luti4_h (x); + v8hi b = luti4_h2 (x, y); + v8hi c = (v8hi) luti4_hf2 ((v8hf) x, (v8hf) y); + v8hi d = (v8hi) luti4_bf2 ((v8bf) x, (v8bf) y); + for (unsigned int i = 0; i < 8; ++i) + { + unsigned short expected = i % 2 ? y[1] : x[2]; + if (a[i] != x[i % 2 ? 5 : 6] + || b[i] != expected || c[i] != expected || d[i] != expected) + __builtin_abort (); + } + return 0; +} diff --git a/gcc/testsuite/gcc.target/aarch64/perm-luti-2.c b/gcc/testsuite/gcc.target/aarch64/perm-luti-2.c new file mode 100644 index 000000000000..ebdd4ef5c3d3 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/perm-luti-2.c @@ -0,0 +1,26 @@ +/* { dg-do compile { target le } } */ +/* { dg-options "-O2 -march=armv9-a+lut" } */ + +typedef unsigned short v8hi __attribute__((vector_size (16))); +typedef __fp16 v8hf __attribute__((vector_size (16))); +typedef __bf16 v8bf __attribute__((vector_size (16))); + +v8hi __attribute__((noipa)) +luti4_h (v8hi x) +{ + return __builtin_shufflevector (x, x, 6, 5, 6, 5, 6, 5, 6, 5); +} + +#define TWO_TABLES(NAME, TYPE) \ +TYPE __attribute__((noipa)) \ +NAME (TYPE x, TYPE y) \ +{ \ + return __builtin_shufflevector (x, y, 2, 9, 2, 9, 2, 9, 2, 9); \ +} + +TWO_TABLES (luti4_h2, v8hi) +TWO_TABLES (luti4_hf2, v8hf) +TWO_TABLES (luti4_bf2, v8bf) + +/* { dg-final { scan-assembler-times {\tluti4\tv[0-9]+\.8h, \{v[0-9]+\.8h, v[0-9]+\.8h\}, v[0-9]+\[0\]} 4 } } */ +/* { dg-final { scan-assembler-not {\ttbl\t|\tldr\t} } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/perm-luti-fallback.c b/gcc/testsuite/gcc.target/aarch64/perm-luti-fallback.c new file mode 100644 index 000000000000..8fad6d41c137 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/perm-luti-fallback.c @@ -0,0 +1,15 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=armv9-a+lut" } */ + +typedef unsigned char v16qi __attribute__((vector_size (16))); + +v16qi +non_immediate_indices (v16qi x) +{ + return __builtin_shufflevector (x, x, 2, 0, 3, 1, 1, 3, 0, 2, + 3, 2, 0, 1, 1, 0, 2, 3); +} + +/* Packing the indices would still need a load. Keep the ordinary TBL. */ +/* { dg-final { scan-assembler-not {\tluti[24]\t} } } */ +/* { dg-final { scan-assembler-times {\ttbl\t} 1 } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/perm-luti-nolut.c b/gcc/testsuite/gcc.target/aarch64/perm-luti-nolut.c new file mode 100644 index 000000000000..3ed5efb15a02 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/perm-luti-nolut.c @@ -0,0 +1,7 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=armv8-a" } */ + +#include "perm-luti.c" + +/* { dg-final { scan-assembler-not {\tluti[246]\t} } } */ +/* { dg-final { scan-assembler-times {\ttbl\t} 3 } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/perm-luti-run.c b/gcc/testsuite/gcc.target/aarch64/perm-luti-run.c new file mode 100644 index 000000000000..893338785f20 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/perm-luti-run.c @@ -0,0 +1,23 @@ +/* { dg-do run { target { aarch64_lut_hw && le } } } */ +/* { dg-options "-O2 -march=armv9-a+lut" } */ + +#include "perm-luti.c" + +int +main (void) +{ + v16qi x = { 0, 17, 34, 51, 68, 85, 102, 119, + 136, 153, 170, 187, 204, 221, 238, 255 }; + v8hi y = { 0, 1001, 2002, 3003, 4004, 5005, 6006, 7007 }; + v16qi a = luti2_b (x); + v16qi b = luti4_b (x); + v8hi c = luti2_h (y); + const unsigned char indices[4] = { 2, 0, 3, 1 }; + for (unsigned int i = 0; i < 16; ++i) + if (a[i] != x[indices[i % 4]] || b[i] != x[i % 2 ? 9 : 10]) + __builtin_abort (); + for (unsigned int i = 0; i < 8; ++i) + if (c[i] != y[indices[i % 4]]) + __builtin_abort (); + return 0; +} diff --git a/gcc/testsuite/gcc.target/aarch64/perm-luti.c b/gcc/testsuite/gcc.target/aarch64/perm-luti.c new file mode 100644 index 000000000000..803b9e8244b4 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/perm-luti.c @@ -0,0 +1,31 @@ +/* { dg-do compile { target { le } } } */ +/* { dg-options "-O2 -march=armv9-a+lut" } */ + +typedef unsigned char v16qi __attribute__((vector_size (16))); +typedef unsigned short v8hi __attribute__((vector_size (16))); + +v16qi __attribute__((noipa)) +luti2_b (v16qi x) +{ + return __builtin_shufflevector (x, x, 2, 0, 3, 1, 2, 0, 3, 1, + 2, 0, 3, 1, 2, 0, 3, 1); +} + +v16qi __attribute__((noipa)) +luti4_b (v16qi x) +{ + return __builtin_shufflevector (x, x, 10, 9, 10, 9, 10, 9, 10, 9, + 10, 9, 10, 9, 10, 9, 10, 9); +} + +v8hi __attribute__((noipa)) +luti2_h (v8hi x) +{ + return __builtin_shufflevector (x, x, 2, 0, 3, 1, 2, 0, 3, 1); +} + +/* The packed indices can all be materialized with MOVI. */ +/* { dg-final { scan-assembler-times {\tluti2\t} 2 } } */ +/* { dg-final { scan-assembler-times {\tluti4\t} 1 } } */ +/* { dg-final { scan-assembler-not {\ttbl\t} } } */ +/* { dg-final { scan-assembler-not {\tldr\t} } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_1.c b/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_1.c new file mode 100644 index 000000000000..5c5047c33912 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_1.c @@ -0,0 +1,38 @@ +/* { dg-do compile { target { le } } } */ +/* { dg-options "-O2 -march=armv9-a+lut -msve-vector-bits=256" } */ + +typedef __SVUint8_t v32qi __attribute__((arm_sve_vector_bits (256))); +typedef __SVUint16_t v16hi __attribute__((arm_sve_vector_bits (256))); + +#define FOUR 2, 0, 3, 1 +#define EIGHT 10, 9, 10, 9, 10, 9, 10, 9 + +v32qi __attribute__((noipa)) +luti2_b (v32qi x) +{ + return __builtin_shufflevector (x, x, FOUR, FOUR, FOUR, FOUR, + FOUR, FOUR, FOUR, FOUR); +} + +v32qi __attribute__((noipa)) +luti4_b (v32qi x) +{ + return __builtin_shufflevector (x, x, EIGHT, EIGHT, EIGHT, EIGHT); +} + +v16hi __attribute__((noipa)) +luti2_h (v16hi x) +{ + return __builtin_shufflevector (x, x, FOUR, FOUR, FOUR, FOUR); +} + +v16hi __attribute__((noipa)) +luti4_h (v16hi x) +{ + return __builtin_shufflevector (x, x, EIGHT, EIGHT); +} + +/* { dg-final { scan-assembler-times {\tluti2\t} 2 } } */ +/* { dg-final { scan-assembler-times {\tluti4\t} 2 } } */ +/* { dg-final { scan-assembler-not {\ttbl\t} } } */ +/* { dg-final { scan-assembler-not {\tld1[rqbhwd]*\t|\tldr\t} } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_1_run.c new file mode 100644 index 000000000000..3902369547e4 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_1_run.c @@ -0,0 +1,28 @@ +/* { dg-do run { target { { aarch64_sve256_hw && aarch64_lut_hw } && le } } } */ +/* { dg-options "-O2 -march=armv9-a+lut -msve-vector-bits=256" } */ + +#include "perm_luti_1.c" + +int +main (void) +{ + v32qi x; + v16hi y; + for (unsigned int i = 0; i < 32; ++i) + x[i] = i * 7 + 3; + for (unsigned int i = 0; i < 16; ++i) + y[i] = i * 1001 + 123; + + v32qi a = luti2_b (x); + v32qi b = luti4_b (x); + v16hi c = luti2_h (y); + v16hi d = luti4_h (y); + const unsigned char indices[4] = { FOUR }; + for (unsigned int i = 0; i < 32; ++i) + if (a[i] != x[indices[i % 4]] || b[i] != x[i % 2 ? 9 : 10]) + __builtin_abort (); + for (unsigned int i = 0; i < 16; ++i) + if (c[i] != y[indices[i % 4]] || d[i] != y[i % 2 ? 9 : 10]) + __builtin_abort (); + return 0; +} diff --git a/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_2.c b/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_2.c new file mode 100644 index 000000000000..0dfa0571cf6c --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_2.c @@ -0,0 +1,15 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=armv9-a+sve2p3 -msve-vector-bits=256" } */ + +typedef __SVUint8_t v32qi __attribute__((arm_sve_vector_bits (256))); + +v32qi __attribute__((noipa)) +luti6_b (v32qi x, v32qi y) +{ + return __builtin_shufflevector (x, y, + 0, 35, 2, 39, 4, 43, 6, 47, 8, 51, 10, 55, 12, 59, 14, 63, + 16, 33, 18, 37, 20, 41, 22, 45, 24, 49, 26, 53, 28, 57, 30, 61); +} + +/* { dg-final { scan-assembler-times {\tluti6\t} 1 } } */ +/* { dg-final { scan-assembler-not {\ttbl\t|\torr\t|\tsub\t} } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_2_run.c b/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_2_run.c new file mode 100644 index 000000000000..cfc7a68e6d84 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_2_run.c @@ -0,0 +1,24 @@ +/* { dg-do run { target { aarch64_sve256_hw && aarch64_sve2p3_hw } } } */ +/* { dg-options "-O2 -march=armv9-a+sve2p3 -msve-vector-bits=256" } */ + +#include "perm_luti_2.c" + +int +main (void) +{ + v32qi x, y; + for (unsigned int i = 0; i < 32; ++i) + { + x[i] = i; + y[i] = i + 100; + } + v32qi z = luti6_b (x, y); + for (unsigned int i = 0; i < 32; ++i) + { + unsigned int index = (i % 2 == 0 ? i + : (2 * i + (i < 16 ? 1 : -1)) % 32); + if (z[i] != (i % 2 == 0 ? x[index] : y[index])) + __builtin_abort (); + } + return 0; +} diff --git a/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_3.c b/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_3.c new file mode 100644 index 000000000000..18ef6bae0058 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_3.c @@ -0,0 +1,20 @@ +/* { dg-do compile { target le } } */ +/* { dg-options "-O2 -march=armv9-a+lut -msve-vector-bits=128" } */ + +typedef __SVUint16_t v8hi __attribute__((arm_sve_vector_bits (128))); + +v8hi __attribute__((noipa)) +small_halfword_table (v8hi x) +{ + return __builtin_shufflevector (x, x, 7, 6, 7, 6, 7, 6, 7, 6); +} + +v8hi __attribute__((noipa)) +two_halfword_tables (v8hi x, v8hi y) +{ + return __builtin_shufflevector (x, y, 2, 9, 2, 9, 2, 9, 2, 9); +} + +/* At VL == 128, use the two-register form for both cases. */ +/* { dg-final { scan-assembler-times {\tluti4\tz[0-9]+\.h, \{z[0-9]+\.h - z[0-9]+\.h\}, z[0-9]+\[0\]} 2 } } */ +/* { dg-final { scan-assembler-not {\ttbl\t} } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_3_run.c b/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_3_run.c new file mode 100644 index 000000000000..c5b7269efb61 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_3_run.c @@ -0,0 +1,21 @@ +/* { dg-do run { target { { aarch64_sve128_hw && aarch64_lut_hw } && le } } } */ +/* { dg-options "-O2 -march=armv9-a+lut -msve-vector-bits=128" } */ + +#include "perm_luti_3.c" + +int +main (void) +{ + v8hi x, y; + for (unsigned int i = 0; i < 8; ++i) + { + x[i] = i * 1001 + 123; + y[i] = i * 2001 + 456; + } + v8hi a = small_halfword_table (x); + v8hi b = two_halfword_tables (x, y); + for (unsigned int i = 0; i < 8; ++i) + if (a[i] != x[i % 2 ? 6 : 7] || b[i] != (i % 2 ? y[1] : x[2])) + __builtin_abort (); + return 0; +} diff --git a/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_4.c b/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_4.c new file mode 100644 index 000000000000..69256b255db8 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_4.c @@ -0,0 +1,22 @@ +/* { dg-do compile { target le } } */ +/* { dg-options "-O2 -march=armv9-a+lut -msve-vector-bits=256" } */ + +typedef __SVUint16_t v16hi __attribute__((arm_sve_vector_bits (256))); +typedef __SVFloat16_t v16hf __attribute__((arm_sve_vector_bits (256))); +typedef __SVBfloat16_t v16bf __attribute__((arm_sve_vector_bits (256))); + +#define TWO_TABLES(NAME, TYPE) \ +TYPE __attribute__((noipa)) \ +NAME (TYPE x, TYPE y) \ +{ \ + return __builtin_shufflevector (x, y, 2, 17, 2, 17, 2, 17, 2, 17, \ + 2, 17, 2, 17, 2, 17, 2, 17); \ +} + +TWO_TABLES (luti4_h2, v16hi) +TWO_TABLES (luti4_hf2, v16hf) +TWO_TABLES (luti4_bf2, v16bf) + +/* Indices into Y start at 16, but LUTI4 needs them to start at 8. */ +/* { dg-final { scan-assembler-times {\tluti4\tz[0-9]+\.h, \{z[0-9]+\.h - z[0-9]+\.h\}, z[0-9]+\[0\]} 3 } } */ +/* { dg-final { scan-assembler-not {\ttbl\t|\tld1[rqbhwd]*\t|\tldr\t} } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_4_run.c b/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_4_run.c new file mode 100644 index 000000000000..17a2411cc7c7 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_4_run.c @@ -0,0 +1,25 @@ +/* { dg-do run { target { { aarch64_sve256_hw && aarch64_lut_hw } && le } } } */ +/* { dg-options "-O2 -march=armv9-a+lut -msve-vector-bits=256" } */ + +#include "perm_luti_4.c" + +int +main (void) +{ + v16hi x, y; + for (unsigned int i = 0; i < 16; ++i) + { + x[i] = i * 1001 + 123; + y[i] = i * 2001 + 456; + } + v16hi a = luti4_h2 (x, y); + v16hi b = (v16hi) luti4_hf2 ((v16hf) x, (v16hf) y); + v16hi c = (v16hi) luti4_bf2 ((v16bf) x, (v16bf) y); + for (unsigned int i = 0; i < 16; ++i) + { + unsigned short expected = i % 2 ? y[1] : x[2]; + if (a[i] != expected || b[i] != expected || c[i] != expected) + __builtin_abort (); + } + return 0; +} diff --git a/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_5.c b/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_5.c new file mode 100644 index 000000000000..de3c7b543d64 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_5.c @@ -0,0 +1,22 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=armv9-a+lut -msve-vector-bits=256" } */ + +typedef __SVUint16_t v16hi __attribute__((arm_sve_vector_bits (256))); + +v16hi +first_table_upper_half (v16hi x, v16hi y) +{ + return __builtin_shufflevector (x, y, 10, 17, 10, 17, 10, 17, 10, 17, + 10, 17, 10, 17, 10, 17, 10, 17); +} + +v16hi +second_table_upper_half (v16hi x, v16hi y) +{ + return __builtin_shufflevector (x, y, 2, 25, 2, 25, 2, 25, 2, 25, + 2, 25, 2, 25, 2, 25, 2, 25); +} + +/* The two-register form cannot read beyond the low eight halfwords of + either table, even though each input vector has sixteen halfwords. */ +/* { dg-final { scan-assembler-not {\tluti[24]\t} } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_6.c b/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_6.c new file mode 100644 index 000000000000..5885efa995c2 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_6.c @@ -0,0 +1,22 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=armv9-a+sve2p3 -msve-vector-bits=512" } */ + +typedef __SVUint16_t v32hi __attribute__((arm_sve_vector_bits (512))); +typedef __SVFloat16_t v32hf __attribute__((arm_sve_vector_bits (512))); +typedef __SVBfloat16_t v32bf __attribute__((arm_sve_vector_bits (512))); + +#define TWO_TABLES(NAME, TYPE) \ +TYPE __attribute__((noipa)) \ +NAME (TYPE x, TYPE y) \ +{ \ + return __builtin_shufflevector (x, y, \ + 0, 35, 2, 39, 4, 43, 6, 47, 8, 51, 10, 55, 12, 59, 14, 63, \ + 16, 33, 18, 37, 20, 41, 22, 45, 24, 49, 26, 53, 28, 57, 30, 61); \ +} + +TWO_TABLES (luti6_h, v32hi) +TWO_TABLES (luti6_hf, v32hf) +TWO_TABLES (luti6_bf, v32bf) + +/* { dg-final { scan-assembler-times {\tluti6\tz[0-9]+\.h, \{z[0-9]+\.h - z[0-9]+\.h\}, z[0-9]+\[0\]} 3 } } */ +/* { dg-final { scan-assembler-not {\ttbl\t|\torr\t|\tsub\t} } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_6_run.c b/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_6_run.c new file mode 100644 index 000000000000..f11d6ccac01e --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_6_run.c @@ -0,0 +1,27 @@ +/* { dg-do run { target { aarch64_sve512_hw && aarch64_sve2p3_hw } } } */ +/* { dg-options "-O2 -march=armv9-a+sve2p3 -msve-vector-bits=512" } */ + +#include "perm_luti_6.c" + +int +main (void) +{ + v32hi x, y; + for (unsigned int i = 0; i < 32; ++i) + { + x[i] = i * 1001 + 123; + y[i] = i * 2001 + 456; + } + v32hi a = luti6_h (x, y); + v32hi b = (v32hi) luti6_hf ((v32hf) x, (v32hf) y); + v32hi c = (v32hi) luti6_bf ((v32bf) x, (v32bf) y); + for (unsigned int i = 0; i < 32; ++i) + { + unsigned int index = (i % 2 == 0 ? i + : (2 * i + (i < 16 ? 1 : -1)) % 32); + unsigned short expected = i % 2 == 0 ? x[index] : y[index]; + if (a[i] != expected || b[i] != expected || c[i] != expected) + __builtin_abort (); + } + return 0; +} diff --git a/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_7.c b/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_7.c new file mode 100644 index 000000000000..28158fd9b8ea --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_7.c @@ -0,0 +1,46 @@ +/* { dg-do compile { target le } } */ +/* { dg-options "-O2 -march=armv9-a+lut -msve-vector-bits=scalable" } */ + +#include <arm_sve.h> + +/* Repeat the indices across the full scalable result. */ +svuint8_t __attribute__((noipa)) +luti2_b (svuint8_t x) +{ + return __builtin_shuffle (x, svdupq_n_u8 (2, 0, 3, 1, 2, 0, 3, 1, + 2, 0, 3, 1, 2, 0, 3, 1)); +} + +svuint8_t __attribute__((noipa)) +luti4_b (svuint8_t x) +{ + return __builtin_shuffle (x, svdupq_n_u8 (10, 9, 10, 9, 10, 9, 10, 9, + 10, 9, 10, 9, 10, 9, 10, 9)); +} + +svuint16_t __attribute__((noipa)) +luti2_h (svuint16_t x) +{ + return __builtin_shuffle (x, svdupq_n_u16 (2, 0, 3, 1, 2, 0, 3, 1)); +} + +/* VL can be 128, so use two copies of X for the halfword LUTI4 table. */ +svuint16_t __attribute__((noipa)) +luti4_h (svuint16_t x) +{ + return __builtin_shuffle (x, svdupq_n_u16 (7, 6, 7, 6, 7, 6, 7, 6)); +} + +/* The permutation index of Y[1] depends on VL, but its LUTI4 index is 9. */ +svuint16_t __attribute__((noipa)) +luti4_h2 (svuint16_t x, svuint16_t y) +{ + unsigned short n = svcnth () + 1; + return __builtin_shuffle (x, y, svdupq_n_u16 (2, n, 2, n, 2, n, 2, n)); +} + +/* { dg-final { scan-assembler-times {\tluti2\tz[0-9]+\.b,} 1 } } */ +/* { dg-final { scan-assembler-times {\tluti2\tz[0-9]+\.h,} 1 } } */ +/* { dg-final { scan-assembler-times {\tluti4\tz[0-9]+\.b,} 1 } } */ +/* { dg-final { scan-assembler-times {\tluti4\tz[0-9]+\.h, \{z[0-9]+\.h - z[0-9]+\.h\}, z[0-9]+\[0\]} 2 } } */ +/* { dg-final { scan-assembler-not {\ttbl\t|\tld1[rqbhwd]*\t|\tldr\t} } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_7_run.c b/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_7_run.c new file mode 100644 index 000000000000..7e601afd2ad4 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_7_run.c @@ -0,0 +1,52 @@ +/* { dg-do run { target { { aarch64_sve_hw && aarch64_lut_hw } && le } } } */ +/* { dg-options "-O2 -march=armv9-a+lut -msve-vector-bits=scalable" } */ + +#include "perm_luti_7.c" + +int +main (void) +{ + unsigned int nb = svcntb (); + unsigned int nh = svcnth (); + uint8_t bytes[nb], byte_result[nb]; + uint16_t x[nh], y[nh], halfword_result[nh]; + const unsigned char indices[4] = { 2, 0, 3, 1 }; + for (unsigned int i = 0; i < nb; ++i) + bytes[i] = i * 7 + 3; + for (unsigned int i = 0; i < nh; ++i) + { + x[i] = i * 101 + 123; + y[i] = i * 201 + 456; + } + + svuint8_t b = svld1_u8 (svptrue_b8 (), bytes); + svuint16_t h = svld1_u16 (svptrue_b16 (), x); + svuint16_t h2 = svld1_u16 (svptrue_b16 (), y); + + svst1_u8 (svptrue_b8 (), byte_result, luti2_b (b)); + for (unsigned int i = 0; i < nb; ++i) + if (byte_result[i] != bytes[indices[i % 4]]) + __builtin_abort (); + + svst1_u8 (svptrue_b8 (), byte_result, luti4_b (b)); + for (unsigned int i = 0; i < nb; ++i) + if (byte_result[i] != bytes[i % 2 ? 9 : 10]) + __builtin_abort (); + + svst1_u16 (svptrue_b16 (), halfword_result, luti2_h (h)); + for (unsigned int i = 0; i < nh; ++i) + if (halfword_result[i] != x[indices[i % 4]]) + __builtin_abort (); + + svst1_u16 (svptrue_b16 (), halfword_result, luti4_h (h)); + for (unsigned int i = 0; i < nh; ++i) + if (halfword_result[i] != x[i % 2 ? 6 : 7]) + __builtin_abort (); + + svst1_u16 (svptrue_b16 (), halfword_result, luti4_h2 (h, h2)); + for (unsigned int i = 0; i < nh; ++i) + if (halfword_result[i] != (i % 2 ? y[1] : x[2])) + __builtin_abort (); + + return 0; +} diff --git a/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_8.c b/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_8.c new file mode 100644 index 000000000000..4f46e96dcb73 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/perm_luti_8.c @@ -0,0 +1,24 @@ +/* { dg-do compile { target le } } */ +/* { dg-options "-O2 -march=armv9-a+lut -msve-vector-bits=scalable" } */ + +#include <arm_sve.h> + +/* At VL == 128 these indices wrap to X[2] and X[1]. At larger VLs they + select X[10] and X[9], which a duplicated two-register table cannot read. + The single-register halfword LUTI4 requires VL >= 256. */ +svuint16_t __attribute__((noipa)) +upper_halfwords (svuint16_t x) +{ + return __builtin_shuffle (x, svdupq_n_u16 (10, 9, 10, 9, 10, 9, 10, 9)); +} + +/* Byte LUTI4 cannot read beyond the low sixteen bytes of its table. */ +svuint8_t __attribute__((noipa)) +upper_bytes (svuint8_t x) +{ + return __builtin_shuffle (x, svdupq_n_u8 (18, 17, 18, 17, 18, 17, 18, 17, + 18, 17, 18, 17, 18, 17, 18, 17)); +} + +/* { dg-final { scan-assembler-not {\tluti[246]\t} } } */ +/* { dg-final { scan-assembler-times {\ttbl\t} 2 } } */ diff --git a/gcc/testsuite/lib/target-supports.exp b/gcc/testsuite/lib/target-supports.exp index 5a102323c01d..c5ef4cbb333d 100644 --- a/gcc/testsuite/lib/target-supports.exp +++ b/gcc/testsuite/lib/target-supports.exp @@ -6990,6 +6990,24 @@ proc check_effective_target_aarch64_sve_hw { } { } [add_options_for_aarch64_sve ""]] } +# Return true if this is an AArch64 target that can run LUT instructions. + +proc check_effective_target_aarch64_lut_hw { } { + if { ![istarget aarch64*-*-*] } { + return 0 + } + return [check_runtime aarch64_lut_hw_available { + #pragma GCC target "+lut" + int + main (void) + { + asm volatile ("movi v0.16b, #0\n\t" + "luti2 v0.16b, {v0.16b}, v0[0]" ::: "v0"); + return 0; + } + }] +} + # Return true if this is an AArch64 target that can run SVE2 code. proc check_effective_target_aarch64_sve2_hw { } {
