https://gcc.gnu.org/g:8e356a38b8a28d838230914ccb8a7a010464ae1d
commit r17-5046-g8e356a38b8a28d838230914ccb8a7a010464ae1d Author: Stefan Schulze Frielinghaus <[email protected]> Date: Sat Oct 10 09:51:30 2026 +0200 s390: Rework clz/ctz For ctzsi2 the input is zero extended and ctzdi2 is utilized which means for zero we end up with 64. Since some optimizations may kick in when ctz(0) evaluates to the precision of the mode, change ctzsi2 by setting bit 31 of the input value to one so we are guaranteed to end up with a value in [0,32]. The set manifests in an OR IMMEDIATE or INSERT IMMEDIATE which group nicely back-to-back and the overhead of a further instruction outweighs a potential compare and branch. This also aligns with the vector instruction vctz where for zero the elements precision is computed. This is analogue to clzsi2 where the input is zero extended, too, passed to the 64-bit variant and afterwards 32 is subtracted which also aligns with vector instruction vclz. Deal with vector modes in target hooks C[LT]Z_DEFINED_VALUE_AT_ZERO. Insn c[lt]zti2 are implemented through vector patterns since the underlying instructions are vc[lt]zq. This in turn forces the results to be of TI mode in vector registers which is cumbersome to deal with. For scalar TI mode expect the result to be of SI mode which will be consumed in GPRs. gcc/ChangeLog: * config/s390/s390.h (CLZ_DEFINED_VALUE_AT_ZERO): Deal with vector modes. (CTZ_DEFINED_VALUE_AT_ZERO): Ditto. * config/s390/s390.md (ctzsi2): Compute 32 for zero value. * config/s390/vector.md (VIT_VXE3_NO_TI): New mode iterator. (clzti2): New expander. (ctzti2): Ditto. gcc/testsuite/ChangeLog: * gcc.dg/tree-ssa/pr123311-2.c: Pattern ctzsi2 becomes available with z17. * gcc.target/s390/clz-ctz-1.c: New test. Diff: --- gcc/config/s390/s390.h | 6 +- gcc/config/s390/s390.md | 3 + gcc/config/s390/vector.md | 27 +- gcc/testsuite/gcc.dg/tree-ssa/pr123311-2.c | 1 + gcc/testsuite/gcc.target/s390/clz-ctz-1.c | 481 +++++++++++++++++++++++++++++ 5 files changed, 512 insertions(+), 6 deletions(-) diff --git a/gcc/config/s390/s390.h b/gcc/config/s390/s390.h index 948fe506a9c1..ba839c8bee7b 100644 --- a/gcc/config/s390/s390.h +++ b/gcc/config/s390/s390.h @@ -917,10 +917,12 @@ do { \ #define FUNCTION_MODE QImode /* Specify the value which is used when clz operand is zero. */ -#define CLZ_DEFINED_VALUE_AT_ZERO(MODE, VALUE) ((VALUE) = GET_MODE_PRECISION (MODE), 2) +#define CLZ_DEFINED_VALUE_AT_ZERO(MODE, VALUE) \ + ((VALUE) = GET_MODE_UNIT_PRECISION (MODE), 2) /* Specify the value which is used when ctz operand is zero. */ -#define CTZ_DEFINED_VALUE_AT_ZERO(MODE, VALUE) ((VALUE) = 64, 2) +#define CTZ_DEFINED_VALUE_AT_ZERO(MODE, VALUE) \ + ((VALUE) = GET_MODE_UNIT_PRECISION (MODE), 2) /* Machine-specific symbol_ref flags. */ #define SYMBOL_FLAG_ALIGN_SHIFT SYMBOL_FLAG_MACH_DEP_SHIFT diff --git a/gcc/config/s390/s390.md b/gcc/config/s390/s390.md index 9ca5ad06cfb5..6d59795ea2e0 100644 --- a/gcc/config/s390/s390.md +++ b/gcc/config/s390/s390.md @@ -9013,6 +9013,9 @@ (define_expand "ctzsi2" [(set (match_dup 2) (zero_extend:DI (match_operand:SI 1 "register_operand" ""))) + ; Set bit 31 to one which ensures that for zero we compute 32. + (set (zero_extract:DI (match_dup 2) (const_int 32) (const_int 0)) + (const_int 1)) (set (match_dup 3) (ctz:DI (match_dup 2))) (set (match_operand:SI 0 "register_operand" "") (subreg:SI (match_dup 3) 4))] "TARGET_Z17" diff --git a/gcc/config/s390/vector.md b/gcc/config/s390/vector.md index f3b9636e3988..3f5abca1e6ef 100644 --- a/gcc/config/s390/vector.md +++ b/gcc/config/s390/vector.md @@ -72,6 +72,7 @@ V1SI V2SI V4SI (V1DI "TARGET_VXE3") (V2DI "TARGET_VXE3") (V1TI "TARGET_VXE3") (TI "TARGET_VXE3")]) +(define_mode_iterator VIT_VXE3_NO_TI [V1QI V2QI V4QI V8QI V16QI V1HI V2HI V4HI V8HI V1SI V2SI V4SI V1DI V2DI (V1TI "TARGET_VXE3")]) (define_mode_iterator VI [V1QI V2QI V4QI V8QI V16QI V1HI V2HI V4HI V8HI V1SI V2SI V4SI V1DI V2DI]) (define_mode_iterator VI_VXE3 [V1QI V2QI V4QI V8QI V16QI V1HI V2HI V4HI V8HI V1SI V2SI V4SI V1DI (V2DI "TARGET_VXE3")]) (define_mode_iterator VI_QHS [V1QI V2QI V4QI V8QI V16QI V1HI V2HI V4HI V8HI V1SI V2SI V4SI]) @@ -1620,21 +1621,39 @@ ; Count leading zeros ; vclzb, vclzh, vclzf, vclzg, vclzq (define_insn "clz<mode>2" - [(set (match_operand:VT_VXE3 0 "register_operand" "=v") - (clz:VT_VXE3 (match_operand:VT_VXE3 1 "register_operand" "v")))] + [(set (match_operand:VIT_VXE3_NO_TI 0 "register_operand" "=v") + (clz:VIT_VXE3_NO_TI (match_operand:VIT_VXE3_NO_TI 1 "register_operand" "v")))] "TARGET_VX" "vclz<bhfgq>\t%v0,%v1" [(set_attr "op_type" "VRR")]) +(define_expand "clzti2" + [(set (match_dup 2) + (clz:V1TI (subreg:V1TI (match_operand:TI 1 "register_operand") 0))) + (set (match_operand:SI 0 "register_operand") + (vec_select:SI (subreg:V4SI (match_dup 2) 0) + (parallel [(const_int 3)])))] + "TARGET_VXE3" + "operands[2] = gen_reg_rtx (V1TImode);") + ; Count trailing zeros ; vctzb, vctzh, vctzf, vctzg, vctzq (define_insn "ctz<mode>2" - [(set (match_operand:VT_VXE3 0 "register_operand" "=v") - (ctz:VT_VXE3 (match_operand:VT_VXE3 1 "register_operand" "v")))] + [(set (match_operand:VIT_VXE3_NO_TI 0 "register_operand" "=v") + (ctz:VIT_VXE3_NO_TI (match_operand:VIT_VXE3_NO_TI 1 "register_operand" "v")))] "TARGET_VX" "vctz<bhfgq>\t%v0,%v1" [(set_attr "op_type" "VRR")]) +(define_expand "ctzti2" + [(set (match_dup 2) + (ctz:V1TI (subreg:V1TI (match_operand:TI 1 "register_operand") 0))) + (set (match_operand:SI 0 "register_operand") + (vec_select:SI (subreg:V4SI (match_dup 2) 0) + (parallel [(const_int 3)])))] + "TARGET_VXE3" + "operands[2] = gen_reg_rtx (V1TImode);") + ; Each vector element rotated by the corresponding vector element diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr123311-2.c b/gcc/testsuite/gcc.dg/tree-ssa/pr123311-2.c index 0f55c32a1bbd..72f52853c188 100644 --- a/gcc/testsuite/gcc.dg/tree-ssa/pr123311-2.c +++ b/gcc/testsuite/gcc.dg/tree-ssa/pr123311-2.c @@ -1,6 +1,7 @@ /* { dg-do compile } */ /* { dg-options "-O2 -fdump-tree-optimized" } */ /* { dg-additional-options "-mbmi -mlzcnt" { target i?86-*-* x86_64-*-* } } */ +/* { dg-additional-options "-march=z17" { target s390x-*-* } } */ /* { dg-require-effective-target clz } */ /* { dg-require-effective-target ctz } */ diff --git a/gcc/testsuite/gcc.target/s390/clz-ctz-1.c b/gcc/testsuite/gcc.target/s390/clz-ctz-1.c new file mode 100644 index 000000000000..523c089755c4 --- /dev/null +++ b/gcc/testsuite/gcc.target/s390/clz-ctz-1.c @@ -0,0 +1,481 @@ +/* { dg-do compile } */ +/* { dg-options "-O2" } */ +/* { dg-final { check-function-bodies "**" "" "" } } */ +/* { dg-final { scan-assembler-times {\n\tvzero\t} 9 } } */ + +typedef unsigned char __attribute__ ((vector_size (16))) v16qi; +typedef unsigned short __attribute__ ((vector_size (16))) v8hi; +typedef unsigned int __attribute__ ((vector_size (16))) v4si; +typedef unsigned long __attribute__ ((vector_size (16))) v2di; +typedef unsigned __int128 __attribute__ ((vector_size (16))) v1ti; + +/* +** clzsi_z900: +** ... +** brasl %r14,__clzdi2@PLT +** ... +*/ + +__attribute__ ((target ("arch=z900"))) +int clzsi_z900 (unsigned int x) +{ + return x ? __builtin_clz (x) : 32; +} + +/* +** clzdi_z900: +** ... +** brasl %r14,__clzdi2@PLT +** ... +*/ + +__attribute__ ((target ("arch=z900"))) +int clzdi_z900 (unsigned long x) +{ + return x ? __builtin_clzl (x) : 64; +} + +/* +** clzsi_z9_109: +** flogr %r2,%r2 +** ahi %r2,-32 +** lgfr %r2,%r2 +** br %r14 +*/ + +__attribute__ ((target ("arch=z9-109"))) +int clzsi_z9_109 (unsigned int x) +{ + return x ? __builtin_clz (x) : 32; +} + +/* +** clzdi_z9_109: +** flogr %r2,%r2 +** lgfr %r2,%r2 +** br %r14 +*/ + +__attribute__ ((target ("arch=z9-109"))) +int clzdi_z9_109 (unsigned long x) +{ + return x ? __builtin_clzl (x) : 64; +} + +/* +** clzsi_z17: +** clzg %r2,%r2 +** ahi %r2,-32 +** lgfr %r2,%r2 +** br %r14 +*/ + +__attribute__ ((target ("arch=z17"))) +int clzsi_z17 (unsigned int x) +{ + return x ? __builtin_clz (x) : 32; +} + +/* +** clzdi_z17: +** clzg %r2,%r2 +** br %r14 +*/ + +__attribute__ ((target ("arch=z17"))) +int clzdi_z17 (unsigned long x) +{ + return x ? __builtin_clzl (x) : 64; +} + +/* +** clzti_z17: +** vl (%v[0-9]+),0\(%r2\),3 +** vclzq (%v[0-9]+),\1 +** vlgvf (%r[0-9]+),\2,3 +** lgfr %r2,\3 +** br %r14 +*/ + +__attribute__ ((target ("arch=z17"))) +int clzti_z17 (unsigned __int128 x) +{ + return x ? __builtin_clzg (x) : 128; +} + +/* +** clzsi_z13_autovec: +** ... +** vclzf .* +** ... +** vst .* +** ... +*/ + +__attribute__ ((target ("arch=z13"))) +void clzsi_z13_autovec (unsigned int *x) +{ + for (int i = 0; i < 64; ++i) + x[i] = x[i] ? __builtin_clz (x[i]) : 32; +} + +/* +** clzdi_z13_autovec: +** ... +** vclzg .* +** ... +** vpkg .* +** ... +** vst .* +** ... +*/ + +__attribute__ ((target ("arch=z13"))) +void clzdi_z13_autovec (int *x, unsigned long *y) +{ + for (int i = 0; i < 64; ++i) + x[i] = y[i] ? __builtin_clzl (y[i]) : 64; +} + +/* +** clzti_z17_autovec: +** ... +** vclzq .* +** ... +** vstef .* +** ... +*/ + +__attribute__ ((target ("arch=z17"))) +void clzti_z17_autovec (int *x, unsigned __int128 *y) +{ + for (int i = 0; i < 64; ++i) + x[i] = y[i] ? __builtin_clzg (y[i]) : 128; +} + +/* For __builtin_clz arguments are promoted to unsigned int which is why we are + faced with an extra addend .CLZ(x, 8) + 24. + +** clzv16qi_z13: +** ... +** vzero .* +** ... +** vclzb %v[0-9]+,%v[0-9]+ +** ... +*/ + +__attribute__ ((target ("arch=z13"))) +v16qi clzv16qi_z13 (v16qi x) +{ + for (int i = 0; i < 16; ++i) + x[i] = x[i] ? __builtin_clz (x[i]) : 8; + return x; +} + +/* TODO: fold zero test. + +** clzgv16qi_z13: +** vzero (%v[0-9]+) +** vceqb (%v[0-9]+),%v24,\1 +** vgm (%v[0-9]+),4,4,0 +** vclzb %v24,%v24 +** vsel %v24,\3,%v24,\2 +** br %r14 +*/ + +__attribute__ ((target ("arch=z13"))) +v16qi clzgv16qi_z13 (v16qi x) +{ + for (int i = 0; i < 16; ++i) + x[i] = x[i] ? __builtin_clzg (x[i]) : 8; + return x; +} + +/* For __builtin_clz arguments are promoted to unsigned int which is why we are + faced with an extra addend .CLZ(x, 16) + 16. + +** clzv8hi_z13: +** ... +** vzero .* +** ... +** vclzh %v[0-9]+,%v[0-9]+ +** ... +*/ + +__attribute__ ((target ("arch=z13"))) +v8hi clzv8hi_z13 (v8hi x) +{ + for (int i = 0; i < 8; ++i) + x[i] = x[i] ? __builtin_clz (x[i]) : 16; + return x; +} + +/* TODO: fold zero test. + +** clzgv8hi_z13: +** vzero (%v[0-9]+) +** vceqh (%v[0-9]+),%v24,\1 +** vgm (%v[0-9]+),11,11,1 +** vclzh %v24,%v24 +** vsel %v24,\3,%v24,\2 +** br %r14 +*/ + +__attribute__ ((target ("arch=z13"))) +v8hi clzgv8hi_z13 (v8hi x) +{ + for (int i = 0; i < 8; ++i) + x[i] = x[i] ? __builtin_clzg (x[i]) : 16; + return x; +} + +/* +** clzv4si_z13: +** vclzf %v24,%v24 +** br %r14 +*/ + +__attribute__ ((target ("arch=z13"))) +v4si clzv4si_z13 (v4si x) +{ + for (int i = 0; i < 4; ++i) + x[i] = x[i] ? __builtin_clz (x[i]) : 32; + return x; +} + + + +/* +** ctzsi_z900: +** ... +** brasl %r14,__ctzdi2@PLT +** ... +*/ + +__attribute__ ((target ("arch=z900"))) +int ctzsi_z900 (unsigned int x) +{ + return x ? __builtin_ctz (x) : 32; +} + +/* +** ctzdi_z900: +** ... +** brasl %r14,__ctzdi2@PLT +** ... +*/ + +__attribute__ ((target ("arch=z900"))) +int ctzdi_z900 (unsigned long x) +{ + return x ? __builtin_ctzl (x) : 64; +} + +/* +** ctzsi_z9_109: +** ... +** flogr %r[0-9]+,%r[0-9]+ +** ... +*/ + +__attribute__ ((target ("arch=z9-109"))) +int ctzsi_z9_109 (unsigned int x) +{ + return x ? __builtin_ctz (x) : 32; +} + +/* +** ctzdi_z9_109: +** ... +** flogr %r[0-9]+,%r[0-9]+ +** ... +*/ + +__attribute__ ((target ("arch=z9-109"))) +int ctzdi_z9_109 (unsigned long x) +{ + return x ? __builtin_ctzl (x) : 64; +} + +/* +** ctzsi_z17: +** oihl %r2,1 +** ctzg %r2,%r2 +** br %r14 +*/ + +__attribute__ ((target ("arch=z17"))) +int ctzsi_z17 (unsigned int x) +{ + return x ? __builtin_ctz (x) : 32; +} + +/* +** ctzdi_z17: +** ctzg %r2,%r2 +** br %r14 +*/ + +__attribute__ ((target ("arch=z17"))) +int ctzdi_z17 (unsigned long x) +{ + return x ? __builtin_ctzl (x) : 64; +} + +/* +** ctzti_z17: +** vl (%v[0-9]+),0\(%r2\),3 +** vctzq (%v[0-9]+),\1 +** vlgvf (%r[0-9]+),\2,3 +** lgfr %r2,\3 +** br %r14 +*/ + +__attribute__ ((target ("arch=z17"))) +int ctzti_z17 (unsigned __int128 x) +{ + return x ? __builtin_ctzg (x) : 128; +} + +/* +** ctzsi_z13_autovec: +** ... +** vctzf .* +** ... +** vst .* +** ... +*/ + +__attribute__ ((target ("arch=z13"))) +void ctzsi_z13_autovec (unsigned int *x) +{ + for (int i = 0; i < 64; ++i) + x[i] = x[i] ? __builtin_ctz (x[i]) : 32; +} + +/* TODO: fold zero test. + +** ctzdi_z13_autovec: +** ... +** vzero .* +** ... +** vctzg .* +** ... +** vpkg .* +** ... +** vst .* +** ... +*/ + +__attribute__ ((target ("arch=z13"))) +void ctzdi_z13_autovec (int *x, unsigned long *y) +{ + for (int i = 0; i < 64; ++i) + x[i] = y[i] ? __builtin_ctzl (y[i]) : 64; +} + +/* +** ctzdi_z17_autovec: +** ... +** vctzg .* +** ... +** vpkg .* +** ... +** vst .* +** ... +*/ + +__attribute__ ((target ("arch=z17"))) +void ctzdi_z17_autovec (int *x, unsigned long *y) +{ + for (int i = 0; i < 64; ++i) + x[i] = y[i] ? __builtin_ctzl (y[i]) : 64; +} + +/* +** ctzti_z17_autovec: +** ... +** vctzq .* +** ... +** vstef .* +** ... +*/ + +__attribute__ ((target ("arch=z17"))) +void ctzti_z17_autovec (int *x, unsigned __int128 *y) +{ + for (int i = 0; i < 64; ++i) + x[i] = y[i] ? __builtin_ctzg (y[i]) : 128; +} + + +/* TODO: fold zero test. + +** ctzv16qi_z13: +** vzero (%v[0-9]+) +** vceqb (%v[0-9]+),%v24,\1 +** vgm (%v[0-9]+),4,4,0 +** vctzb %v24,%v24 +** vsel %v24,\3,%v24,\2 +** br %r14 +*/ + +__attribute__ ((target ("arch=z13"))) +v16qi ctzv16qi_z13 (v16qi x) +{ + for (int i = 0; i < 16; ++i) + x[i] = x[i] ? __builtin_ctz (x[i]) : 8; + return x; +} + +/* TODO: fold zero test. + +** ctzv8hi_z13: +** vzero (%v[0-9]+) +** vceqh (%v[0-9]+),%v24,\1 +** vgm (%v[0-9]+),11,11,1 +** vctzh %v24,%v24 +** vsel %v24,\3,%v24,\2 +** br %r14 +*/ + +__attribute__ ((target ("arch=z13"))) +v8hi ctzv8hi_z13 (v8hi x) +{ + for (int i = 0; i < 8; ++i) + x[i] = x[i] ? __builtin_ctz (x[i]) : 16; + return x; +} + +/* TODO: fold zero test. + +** ctzv4si_z13: +** vzero (%v[0-9]+) +** vceqf (%v[0-9]+),%v24,\1 +** vgm (%v[0-9]+),26,26,2 +** vctzf %v24,%v24 +** vsel %v24,\3,%v24,\2 +** br %r14 +*/ + +__attribute__ ((target ("arch=z13"))) +v4si ctzv4si_z13 (v4si x) +{ + for (int i = 0; i < 4; ++i) + x[i] = x[i] ? __builtin_ctz (x[i]) : 32; + return x; +} + +/* +** ctzv4si_z17: +** vctzf %v24,%v24 +** br %r14 +*/ + +__attribute__ ((target ("arch=z17"))) +v4si ctzv4si_z17 (v4si x) +{ + for (int i = 0; i < 4; ++i) + x[i] = x[i] ? __builtin_ctz (x[i]) : 32; + return x; +}
