https://gcc.gnu.org/g:8c0a15694343ace40a2965c47783589cb5057491
commit r17-5815-g8c0a15694343ace40a2965c47783589cb5057491 Author: Stefan Schulze Frielinghaus <[email protected]> Date: Mon Nov 2 13:54:52 2026 +0100 s390: For halfwords a byte-swap is a rotate by 8 bits [PR100867] Instead of a vperm, which needs a permutation vector loaded from the constant pool, rotate each halfword by 8 bits, which is equivalent. This also applies to constant permutations dealt with via TARGET_VECTORIZE_VEC_PERM_CONST where we previously bailed out early without VXE2. However, the halfword case is folded to a rotate which does not require VXE2, therefore, don't bail out early. The implementation is more invasive than expected. A new named insn bswapv8hi_vxe2 is needed that must not carry a permutation vector, unlike the existing *bswap<mode> split pattern. If the existing insn were reused, during RA the reg-reg alternative would still push the constant into the constant pool. This would make r13 the pool pointer, which may result in spills, and a vector register is allocated to load the constant into although it is unused and removed in the end. To circumvent this, *vec_splats_bswap_vec<mode>, *vec_insert_and_zero_bswap<mode>, and *vec_set_bswap_vec<mode> are duplicated for V8HI, the only difference being that the permutation vector is left out. For V4SI we could basically do the same by emitting two rotates instead of a load+vperm. However, a priori it is not clear whether this is really worthwhile. In the best case the load of the constant is hoisted out of the hot path which means only a cheap vperm remains. Only in case the load cannot be hoisted and the cache line for the load is evicted, two rotates are beneficial. PR target/100867 gcc/ChangeLog: * config/s390/s390.cc (expand_perm_as_a_vlbr_vstbr_candidate): Handle the halfword byte-swap permutation also without VXE2. * config/s390/vector.md (VT_HW_SDT): New mode iterator. (V_HW_SD): Ditto. (bswap<mode>): Deal with V8HI directly. (bswapv8hi_vxe2): New insn. (*bswap<mode>): Remove V8HI mode handling which is done in bswap<mode>, now. (*vec_splats_bswap_vecv8hi): New insn for V8HI and VXE2 which doesn't make use of a permutation vector. (*vstbr<mode>): Use mode iterator VI_HW_SDT instead of VI_HW_HSDT. * config/s390/vx-builtins.md (*vec_insert_and_zero_bswapv8hi): New insn for V8HI and VXE2 which doesn't make use of a permutation vector. (*vec_set_bswap_vecv8hi): Ditto. gcc/testsuite/ChangeLog: * gcc.target/s390/vector/vec-perm-halfword-swap-1.c: New test. * gcc.target/s390/zvector/vec-revb-halfword-1.c: New test. Diff: --- gcc/config/s390/s390.cc | 7 +- gcc/config/s390/vector.md | 78 ++++++++++++++++------ gcc/config/s390/vx-builtins.md | 48 ++++++++++--- .../s390/vector/vec-perm-halfword-swap-1.c | 45 +++++++++++++ .../gcc.target/s390/zvector/vec-revb-halfword-1.c | 77 +++++++++++++++++++++ 5 files changed, 224 insertions(+), 31 deletions(-) diff --git a/gcc/config/s390/s390.cc b/gcc/config/s390/s390.cc index 61ca55161f9a..577b0d0c8a20 100644 --- a/gcc/config/s390/s390.cc +++ b/gcc/config/s390/s390.cc @@ -18167,9 +18167,11 @@ expand_perm_as_a_vlbr_vstbr_candidate (const struct expand_vec_perm_d &d) { 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8 }, { 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 } }; - if (!TARGET_VXE2 || d.vmode != V16QImode || d.op0 != d.op1) + if (d.vmode != V16QImode || d.op0 != d.op1) return false; + /* For halfwords a byte-swap is a rotate by 8 bits (verllh), which does not + require VXE2. */ if (memcmp (d.perm, perm[0], MAX_VECT_LEN) == 0) { if (!d.testing_p) @@ -18181,6 +18183,9 @@ expand_perm_as_a_vlbr_vstbr_candidate (const struct expand_vec_perm_d &d) return true; } + if (!TARGET_VXE2) + return false; + if (memcmp (d.perm, perm[1], MAX_VECT_LEN) == 0) { if (!d.testing_p) diff --git a/gcc/config/s390/vector.md b/gcc/config/s390/vector.md index 3f5abca1e6ef..500cb0b59615 100644 --- a/gcc/config/s390/vector.md +++ b/gcc/config/s390/vector.md @@ -41,7 +41,9 @@ (define_mode_iterator V_HW3 [V16QI V8HI V4SI V2DI V1TI TI V4SF V2DF V1TF TF]) (define_mode_iterator VT_HW_HSDT [V8HI V4SI V4SF V2DI V2DF V1TI V1TF TI TF]) +(define_mode_iterator VT_HW_SDT [V4SI V4SF V2DI V2DF V1TI V1TF TI TF]) (define_mode_iterator V_HW_HSD [V8HI V4SI (V4SF "TARGET_VXE") V2DI V2DF]) +(define_mode_iterator V_HW_SD [V4SI (V4SF "TARGET_VXE") V2DI V2DF]) ; Including TI for instructions that support it (va, vn, ...) (define_mode_iterator VT_HW [V16QI V8HI V4SI V2DI V2DF V1TI TI (V4SF "TARGET_VXE") (V1TF "TARGET_VXE")]) @@ -815,16 +817,27 @@ #" [(set_attr "op_type" "VRX,VRI,VRI,*")]) -; vlbrreph, vlbrrepf, vlbrrepg +; vlbrrepf, vlbrrepg (define_insn "*vec_splats_bswap_vec<mode>" - [(set (match_operand:V_HW_HSD 0 "register_operand" "=v") - (bswap:V_HW_HSD - (vec_duplicate:V_HW_HSD (match_operand:<non_vec> 1 "memory_operand" "R")))) - (use (match_operand:V16QI 2 "permute_pattern_operand" "X"))] + [(set (match_operand:V_HW_SD 0 "register_operand" "=v") + (bswap:V_HW_SD + (vec_duplicate:V_HW_SD (match_operand:<non_vec> 1 "memory_operand" "R")))) + (use (match_operand:V16QI 2 "permute_pattern_operand" "X"))] "TARGET_VXE2" "vlbrrep<bhfgq>\t%v0,%1" [(set_attr "op_type" "VRX")]) +; bswapv8hi does not emit a permutation vector, so this variant has +; no (use ...) operand. +; vlbrreph +(define_insn "*vec_splats_bswap_vecv8hi" + [(set (match_operand:V8HI 0 "register_operand" "=v") + (bswap:V8HI + (vec_duplicate:V8HI (match_operand:HI 1 "memory_operand" "R"))))] + "TARGET_VXE2" + "vlbrreph\t%v0,%1" + [(set_attr "op_type" "VRX")]) + ; Why do we need both? Shouldn't there be a canonical form? ; vlbrreph, vlbrrepf, vlbrrepg (define_insn "*vec_splats_bswap_elem<mode>" @@ -3514,9 +3527,25 @@ (use (match_dup 2))])] "TARGET_VX" { - static const char p[4][16] = - { { 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14 }, /* H */ - { 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12 }, /* S */ + /* For halfwords a byte-swap is a rotate by 8 bits. */ + if (<MODE>mode == V8HImode) + { + /* With VXE2, the memory instructions vlbrh and vstbrh are available + (the reg-reg case uses verllh). */ + if (TARGET_VXE2) + emit_insn (gen_bswapv8hi_vxe2 (operands[0], operands[1])); + else + { + rtx in = force_reg (V8HImode, operands[1]); + rtx out = gen_reg_rtx (V8HImode); + emit_insn (gen_rotlv8hi3 (out, in, GEN_INT (8))); + emit_move_insn (operands[0], out); + } + DONE; + } + + static const char p[3][16] = + { { 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12 }, /* S */ { 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8 }, /* D */ { 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 } }; /* T */ const char *perm; @@ -3524,10 +3553,9 @@ switch (GET_MODE_SIZE (GET_MODE_INNER (<MODE>mode))) { - case 2: perm = p[0]; break; - case 4: perm = p[1]; break; - case 8: perm = p[2]; break; - case 16: perm = p[3]; break; + case 4: perm = p[0]; break; + case 8: perm = p[1]; break; + case 16: perm = p[2]; break; default: gcc_unreachable (); } for (int i = 0; i < 16; i++) @@ -3551,6 +3579,17 @@ } }) +; verllh, vlbrh, vstbrh +(define_insn "bswapv8hi_vxe2" + [(set (match_operand:V8HI 0 "nonimmediate_operand" "=v,v,R") + (bswap:V8HI (match_operand:V8HI 1 "nonimmediate_operand" "v,R,v")))] + "TARGET_VXE2" + "@ + verllh\t%v0,%v1,8 + vlbrh\t%v0,%1 + vstbrh\t%v1,%0" + [(set_attr "op_type" "VRS,VRX,VRX")]) + ; Switching late to the reg-reg variant requires the vector permute ; pattern to be pushed into literal pool and allocating a vector ; register to load it into. We rely on both being provided by LRA @@ -3563,11 +3602,11 @@ ; ^R: Prevent these alternatives from being chosen if it would require ; pushing the operand into memory first -; vlbrh, vlbrf, vlbrg, vlbrq, vstbrh, vstbrf, vstbrg, vstbrq +; vlbrf, vlbrg, vlbrq, vstbrf, vstbrg, vstbrq (define_insn_and_split "*bswap<mode>" - [(set (match_operand:VT_HW_HSDT 0 "nonimmediate_operand" "=v, v,^R") - (bswap:VT_HW_HSDT (match_operand:VT_HW_HSDT 1 "nonimmediate_operand" "v,^R, v"))) - (use (match_operand:V16QI 2 "permute_pattern_operand" "v, X, X"))] + [(set (match_operand:VT_HW_SDT 0 "nonimmediate_operand" "=v, v,^R") + (bswap:VT_HW_SDT (match_operand:VT_HW_SDT 1 "nonimmediate_operand" "v,^R, v"))) + (use (match_operand:V16QI 2 "permute_pattern_operand" "v, X, X"))] "TARGET_VXE2" "@ # @@ -3577,7 +3616,7 @@ && !memory_operand (operands[0], <MODE>mode) && !memory_operand (operands[1], <MODE>mode)" [(set (match_dup 0) - (subreg:VT_HW_HSDT + (subreg:VT_HW_SDT (unspec:V16QI [(subreg:V16QI (match_dup 1) 0) (subreg:V16QI (match_dup 1) 0) (match_dup 2)] @@ -3585,9 +3624,10 @@ "" [(set_attr "op_type" "*,VRX,VRX")]) +; The halfword variant is handled by bswapv8hi_vxe2. (define_insn "*vstbr<mode>" - [(set (match_operand:VI_HW_HSDT 0 "memory_operand" "=R") - (bswap:VI_HW_HSDT (match_operand:VI_HW_HSDT 1 "register_operand" "v")))] + [(set (match_operand:VI_HW_SDT 0 "memory_operand" "=R") + (bswap:VI_HW_SDT (match_operand:VI_HW_SDT 1 "register_operand" "v")))] "TARGET_VXE2" "vstbr<bhfgq>\t%v1,%0" [(set_attr "op_type" "VRX")]) diff --git a/gcc/config/s390/vx-builtins.md b/gcc/config/s390/vx-builtins.md index 6d06f1a7915a..4837290af013 100644 --- a/gcc/config/s390/vx-builtins.md +++ b/gcc/config/s390/vx-builtins.md @@ -181,10 +181,10 @@ [(set_attr "op_type" "VRX")]) ; vec_revb (vec_insert_and_zero(x)) bswap-and-replicate-1.c -; vllebrzh, vllebrzf, vllebrzg +; vllebrzf, vllebrzg (define_insn "*vec_insert_and_zero_bswap<mode>" - [(set (match_operand:V_HW_HSD 0 "register_operand" "=v") - (bswap:V_HW_HSD (unspec:V_HW_HSD + [(set (match_operand:V_HW_SD 0 "register_operand" "=v") + (bswap:V_HW_SD (unspec:V_HW_SD [(match_operand:<non_vec> 1 "memory_operand" "R")] UNSPEC_VEC_INSERT_AND_ZERO))) (use (match_operand:V16QI 2 "permute_pattern_operand" "X"))] @@ -192,6 +192,18 @@ "vllebrz<bhfgq>\t%v0,%1" [(set_attr "op_type" "VRX")]) +; bswapv8hi does not emit a permutation vector, so this variant has +; no (use ...) operand. +; vllebrzh +(define_insn "*vec_insert_and_zero_bswapv8hi" + [(set (match_operand:V8HI 0 "register_operand" "=v") + (bswap:V8HI (unspec:V8HI + [(match_operand:HI 1 "memory_operand" "R")] + UNSPEC_VEC_INSERT_AND_ZERO)))] + "TARGET_VXE2" + "vllebrzh\t%v0,%1" + [(set_attr "op_type" "VRX")]) + (define_insn "vlbb" [(set (match_operand:V16QI 0 "register_operand" "=v") @@ -2122,19 +2134,33 @@ [(set_attr "op_type" "VRX")]) ; vec_revb (vec_insert (*a, vec_revb (b), 1)) set-element-bswap-1.c -; vlebrh, vlebrf, vlebrg +; vlebrf, vlebrg (define_insn "*vec_set_bswap_vec<mode>" - [(set (match_operand:V_HW_HSD 0 "register_operand" "=v") - (bswap:V_HW_HSD - (unspec:V_HW_HSD [(match_operand:<non_vec> 1 "memory_operand" "R") - (match_operand:SI 2 "const_int_operand" "C") - (bswap:V_HW_HSD (match_operand:V_HW_HSD 3 "register_operand" "0"))] + [(set (match_operand:V_HW_SD 0 "register_operand" "=v") + (bswap:V_HW_SD + (unspec:V_HW_SD [(match_operand:<non_vec> 1 "memory_operand" "R") + (match_operand:SI 2 "const_int_operand" "C") + (bswap:V_HW_SD (match_operand:V_HW_SD 3 "register_operand" "0"))] UNSPEC_VEC_SET))) - (use (match_operand:V16QI 4 "permute_pattern_operand" "X"))] - "TARGET_VXE2 && UINTVAL (operands[2]) < GET_MODE_NUNITS (<V_HW_HSD:MODE>mode)" + (use (match_operand:V16QI 4 "permute_pattern_operand" "X"))] + "TARGET_VXE2 && UINTVAL (operands[2]) < GET_MODE_NUNITS (<V_HW_SD:MODE>mode)" "vlebr<bhfgq>\t%v0,%1,%2" [(set_attr "op_type" "VRX")]) +; bswapv8hi does not emit a permutation vector, so this variant has +; no (use ...) operand. +; vlebrh +(define_insn "*vec_set_bswap_vecv8hi" + [(set (match_operand:V8HI 0 "register_operand" "=v") + (bswap:V8HI + (unspec:V8HI [(match_operand:HI 1 "memory_operand" "R") + (match_operand:SI 2 "const_int_operand" "C") + (bswap:V8HI (match_operand:V8HI 3 "register_operand" "0"))] + UNSPEC_VEC_SET)))] + "TARGET_VXE2 && UINTVAL (operands[2]) < 8" + "vlebrh\t%v0,%1,%2" + [(set_attr "op_type" "VRX")]) + ; *a = vec_extract (vec_revb (b), 1); get-element-bswap-3.c ; *a = vec_revb (b)[1]; get-element-bswap-4.c ; vstebrh, vstebrf, vstebrg diff --git a/gcc/testsuite/gcc.target/s390/vector/vec-perm-halfword-swap-1.c b/gcc/testsuite/gcc.target/s390/vector/vec-perm-halfword-swap-1.c new file mode 100644 index 000000000000..54f8d3bf1e7f --- /dev/null +++ b/gcc/testsuite/gcc.target/s390/vector/vec-perm-halfword-swap-1.c @@ -0,0 +1,45 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=z13" } */ +/* { dg-final { check-function-bodies "**" "" "" } } */ + +/* For halfwords a byte-swap is a rotate by 8. */ + +typedef unsigned char v16qi __attribute__ ((vector_size (16))); + +/* +** test_reg_reg: +** verllh %v24,%v24,8 +** br %r14 +*/ +v16qi +test_reg_reg (v16qi x) +{ + return __builtin_shuffle (x, (v16qi) { 1, 0, 3, 2, 5, 4, 7, 6, + 9, 8, 11, 10, 13, 12, 15, 14 }); +} + +/* +** test_reg_mem: +** vl %v24,0\(%r2\),3 +** verllh %v24,%v24,8 +** br %r14 +*/ +v16qi +test_reg_mem (v16qi *x) +{ + return __builtin_shuffle (*x, (v16qi) { 1, 0, 3, 2, 5, 4, 7, 6, + 9, 8, 11, 10, 13, 12, 15, 14 }); +} + +/* +** test_mem_reg: +** verllh %v24,%v24,8 +** vst %v24,0\(%r2\),3 +** br %r14 +*/ +void +test_mem_reg (v16qi *x, v16qi y) +{ + *x = __builtin_shuffle (y, (v16qi) { 1, 0, 3, 2, 5, 4, 7, 6, + 9, 8, 11, 10, 13, 12, 15, 14 }); +} diff --git a/gcc/testsuite/gcc.target/s390/zvector/vec-revb-halfword-1.c b/gcc/testsuite/gcc.target/s390/zvector/vec-revb-halfword-1.c new file mode 100644 index 000000000000..c0931ac51f6f --- /dev/null +++ b/gcc/testsuite/gcc.target/s390/zvector/vec-revb-halfword-1.c @@ -0,0 +1,77 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=z13 -mzvector" } */ +/* { dg-final { check-function-bodies "**" "" "" } } */ +/* { dg-final { scan-assembler-not {\tvperm\t} } } */ + +#include <vecintrin.h> + +/* +** test_z13_rr: +** verllh %v24,%v24,8 +** br %r14 +*/ +vector unsigned short +test_z13_rr (vector unsigned short a) +{ + return vec_revb (a); +} + +/* +** test_z15_rr: +** verllh %v24,%v24,8 +** br %r14 +*/ +__attribute__ ((target ("arch=z15"))) +vector unsigned short +test_z15_rr (vector unsigned short a) +{ + return vec_revb (a); +} + +/* +** test_z13_rm: +** vl %v24,0\(%r2\),3 +** verllh %v24,%v24,8 +** br %r14 +*/ +vector unsigned short +test_z13_rm (vector unsigned short *a) +{ + return vec_revb (*a); +} + +/* +** test_z15_rm: +** vlbrh %v24,0\(%r2\) +** br %r14 +*/ +__attribute__ ((target ("arch=z15"))) +vector unsigned short +test_z15_rm (vector unsigned short *a) +{ + return vec_revb (*a); +} + +/* +** test_z13_mr: +** verllh %v24,%v24,8 +** vst %v24,0\(%r2\),3 +** br %r14 +*/ +void +test_z13_mr (vector unsigned short *dst, vector unsigned short a) +{ + *dst = vec_revb (a); +} + +/* +** test_z15_mr: +** vstbrh %v24,0\(%r2\) +** br %r14 +*/ +__attribute__ ((target ("arch=z15"))) +void +test_z15_mr (vector unsigned short *dst, vector unsigned short a) +{ + *dst = vec_revb (a); +}
