From: Stefan Schulze Frielinghaus <[email protected]>
For ctzsi2 the input is zero extended and ctzdi2 is utilized which means
for zero we end up with 64. Since some optimizations may kick in when
ctz(0) evaluates to the precision of the mode, change ctzsi2 by setting
bit 31 of the input value to one so we are guaranteed to end up with a
value in [0,32]. The set manifests in an OR IMMEDIATE or INSERT
IMMEDIATE which group nicely back-to-back and the overhead of a further
instruction outweighs a potential compare and branch. This also aligns
with the vector instruction vctz where for zero the elements precision
is computed. This is analogue to clzsi2 where the input is zero
extended, too, passed to the 64-bit variant and afterwards 32 is
subtracted which also aligns with vector instruction vclz.
Deal with vector modes in target hooks C[LT]Z_DEFINED_VALUE_AT_ZERO.
gcc/ChangeLog:
* config/s390/s390.h (CLZ_DEFINED_VALUE_AT_ZERO): Deal with
vector modes.
(CTZ_DEFINED_VALUE_AT_ZERO): Ditto.
* config/s390/s390.md (ctzsi2): Compute 32 for zero value.
gcc/testsuite/ChangeLog:
* gcc.dg/tree-ssa/pr123311-2.c: Pattern ctzsi2 becomes available
with z17.
* gcc.target/s390/clz-ctz-1.c: New test.
---
gcc/config/s390/s390.h | 6 +-
gcc/config/s390/s390.md | 3 +
gcc/testsuite/gcc.dg/tree-ssa/pr123311-2.c | 1 +
gcc/testsuite/gcc.target/s390/clz-ctz-1.c | 320 +++++++++++++++++++++
4 files changed, 328 insertions(+), 2 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/s390/clz-ctz-1.c
diff --git a/gcc/config/s390/s390.h b/gcc/config/s390/s390.h
index 948fe506a9c..ba839c8bee7 100644
--- a/gcc/config/s390/s390.h
+++ b/gcc/config/s390/s390.h
@@ -917,10 +917,12 @@ do {
\
#define FUNCTION_MODE QImode
/* Specify the value which is used when clz operand is zero. */
-#define CLZ_DEFINED_VALUE_AT_ZERO(MODE, VALUE) ((VALUE) = GET_MODE_PRECISION
(MODE), 2)
+#define CLZ_DEFINED_VALUE_AT_ZERO(MODE, VALUE) \
+ ((VALUE) = GET_MODE_UNIT_PRECISION (MODE), 2)
/* Specify the value which is used when ctz operand is zero. */
-#define CTZ_DEFINED_VALUE_AT_ZERO(MODE, VALUE) ((VALUE) = 64, 2)
+#define CTZ_DEFINED_VALUE_AT_ZERO(MODE, VALUE) \
+ ((VALUE) = GET_MODE_UNIT_PRECISION (MODE), 2)
/* Machine-specific symbol_ref flags. */
#define SYMBOL_FLAG_ALIGN_SHIFT SYMBOL_FLAG_MACH_DEP_SHIFT
diff --git a/gcc/config/s390/s390.md b/gcc/config/s390/s390.md
index aa9b011278d..a7ab4c9b548 100644
--- a/gcc/config/s390/s390.md
+++ b/gcc/config/s390/s390.md
@@ -9013,6 +9013,9 @@
(define_expand "ctzsi2"
[(set (match_dup 2)
(zero_extend:DI (match_operand:SI 1 "register_operand" "")))
+ ; Set bit 31 to one which ensures that for zero we compute 32.
+ (set (zero_extract:DI (match_dup 2) (const_int 32) (const_int 0))
+ (const_int 1))
(set (match_dup 3) (ctz:DI (match_dup 2)))
(set (match_operand:SI 0 "register_operand" "") (subreg:SI (match_dup 3)
4))]
"TARGET_Z17"
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr123311-2.c
b/gcc/testsuite/gcc.dg/tree-ssa/pr123311-2.c
index 0f55c32a1bb..72f52853c18 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr123311-2.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr123311-2.c
@@ -1,6 +1,7 @@
/* { dg-do compile } */
/* { dg-options "-O2 -fdump-tree-optimized" } */
/* { dg-additional-options "-mbmi -mlzcnt" { target i?86-*-* x86_64-*-* } } */
+/* { dg-additional-options "-march=z17" { target s390x-*-* } } */
/* { dg-require-effective-target clz } */
/* { dg-require-effective-target ctz } */
diff --git a/gcc/testsuite/gcc.target/s390/clz-ctz-1.c
b/gcc/testsuite/gcc.target/s390/clz-ctz-1.c
new file mode 100644
index 00000000000..4d3fc148a0a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/clz-ctz-1.c
@@ -0,0 +1,320 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-optimized" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+/* { dg-final { scan-tree-dump-times "\.CLZ \\(\[^,\]+, 32\\)" 3 "optimized" }
} */
+/* { dg-final { scan-tree-dump-times "\.CTZ \\(\[^,\]+, 64\\)" 3 "optimized" }
} */
+
+typedef unsigned char __attribute__ ((vector_size (16))) v16qi;
+typedef unsigned short __attribute__ ((vector_size (16))) v8hi;
+typedef unsigned int __attribute__ ((vector_size (16))) v4si;
+typedef unsigned long __attribute__ ((vector_size (16))) v2di;
+
+/*
+** clzsi_z900:
+** ...
+** brasl %r14,__clzdi2@PLT
+** ...
+*/
+
+__attribute__ ((target ("arch=z900")))
+int clzsi_z900 (unsigned int x)
+{
+ return x ? __builtin_clz (x) : 32;
+}
+
+/*
+** clzdi_z900:
+** ...
+** brasl %r14,__clzdi2@PLT
+** ...
+*/
+
+__attribute__ ((target ("arch=z900")))
+int clzdi_z900 (unsigned long x)
+{
+ return x ? __builtin_clzl (x) : 64;
+}
+
+/*
+** clzsi_z9_109:
+** flogr %r2,%r2
+** ahi %r2,-32
+** lgfr %r2,%r2
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z9-109")))
+int clzsi_z9_109 (unsigned int x)
+{
+ return x ? __builtin_clz (x) : 32;
+}
+
+/*
+** clzdi_z9_109:
+** flogr %r2,%r2
+** lgfr %r2,%r2
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z9-109")))
+int clzdi_z9_109 (unsigned long x)
+{
+ return x ? __builtin_clzl (x) : 64;
+}
+
+/*
+** clzsi_z17:
+** clzg %r2,%r2
+** ahi %r2,-32
+** lgfr %r2,%r2
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z17")))
+int clzsi_z17 (unsigned int x)
+{
+ return x ? __builtin_clz (x) : 32;
+}
+
+/*
+** clzdi_z17:
+** clzg %r2,%r2
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z17")))
+int clzdi_z17 (unsigned long x)
+{
+ return x ? __builtin_clzl (x) : 64;
+}
+
+/* The promoted and unpromoted precision does not coincide which is why the
+ zero test is not folded.
+
+** clzv16qi_z13:
+** ...
+** vclzb %v[0-9]+,%v[0-9]+
+** ...
+*/
+
+__attribute__ ((target ("arch=z13")))
+v16qi clzv16qi_z13 (v16qi x)
+{
+ for (int i = 0; i < 16; ++i)
+ x[i] = x[i] ? __builtin_clz (x[i]) : 8;
+ return x;
+}
+
+/* The promoted and unpromoted precision does not coincide which is why the
+ zero test is not folded.
+
+** clzv8hi_z13:
+** ...
+** vclzh %v[0-9]+,%v[0-9]+
+** ...
+*/
+
+__attribute__ ((target ("arch=z13")))
+v8hi clzv8hi_z13 (v8hi x)
+{
+ for (int i = 0; i < 8; ++i)
+ x[i] = x[i] ? __builtin_clz (x[i]) : 16;
+ return x;
+}
+
+/*
+** clzv4si_z13:
+** vclzf %v24,%v24
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z13")))
+v4si clzv4si_z13 (v4si x)
+{
+ for (int i = 0; i < 4; ++i)
+ x[i] = x[i] ? __builtin_clz (x[i]) : 32;
+ return x;
+}
+
+/* For V2DI use the auto vectorizer since if we directly operate on vector
+ registers the loop is unrolled and SLP vectorization fails.
+
+** clzv2di_z13:
+** ...
+** vclzg .*
+** ...
+** vclzg .*
+** ...
+*/
+
+__attribute__ ((target ("arch=z13")))
+void clzv2di_z13 (unsigned long *x)
+{
+ for (int i = 0; i < 4; ++i)
+ x[i] = x[i] ? __builtin_clzl (x[i]) : 64;
+}
+
+/*
+** ctzsi_z900:
+** ...
+** brasl %r14,__ctzdi2@PLT
+** ...
+*/
+
+__attribute__ ((target ("arch=z900")))
+int ctzsi_z900 (unsigned int x)
+{
+ return x ? __builtin_ctz (x) : 32;
+}
+
+/*
+** ctzdi_z900:
+** ...
+** brasl %r14,__ctzdi2@PLT
+** ...
+*/
+
+__attribute__ ((target ("arch=z900")))
+int ctzdi_z900 (unsigned long x)
+{
+ return x ? __builtin_ctzl (x) : 64;
+}
+
+/*
+** ctzsi_z9_109:
+** ...
+** flogr %r[0-9]+,%r[0-9]+
+** ...
+*/
+
+__attribute__ ((target ("arch=z9-109")))
+int ctzsi_z9_109 (unsigned int x)
+{
+ return x ? __builtin_ctz (x) : 32;
+}
+
+/*
+** ctzdi_z9_109:
+** ...
+** flogr %r[0-9]+,%r[0-9]+
+** ...
+*/
+
+__attribute__ ((target ("arch=z9-109")))
+int ctzdi_z9_109 (unsigned long x)
+{
+ return x ? __builtin_ctzl (x) : 64;
+}
+
+/*
+** ctzsi_z17:
+** oihl %r2,1
+** ctzg %r2,%r2
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z17")))
+int ctzsi_z17 (unsigned int x)
+{
+ return x ? __builtin_ctz (x) : 32;
+}
+
+/*
+** ctzdi_z17:
+** ctzg %r2,%r2
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z17")))
+int ctzdi_z17 (unsigned long x)
+{
+ return x ? __builtin_ctzl (x) : 64;
+}
+
+
+/* The promoted and unpromoted precision does not coincide which is why the
+ zero test is not folded.
+
+** ctzv16qi_z13:
+** ...
+** vctzb %v[0-9]+,%v[0-9]+
+** ...
+*/
+
+__attribute__ ((target ("arch=z13")))
+v16qi ctzv16qi_z13 (v16qi x)
+{
+ for (int i = 0; i < 16; ++i)
+ x[i] = x[i] ? __builtin_ctz (x[i]) : 8;
+ return x;
+}
+
+/* The promoted and unpromoted precision does not coincide which is why the
+ zero test is not folded.
+
+** ctzv8hi_z13:
+** ...
+** vctzh %v[0-9]+,%v[0-9]+
+** ...
+*/
+
+__attribute__ ((target ("arch=z13")))
+v8hi ctzv8hi_z13 (v8hi x)
+{
+ for (int i = 0; i < 8; ++i)
+ x[i] = x[i] ? __builtin_ctz (x[i]) : 16;
+ return x;
+}
+
+/* Prior z17 we do not have a defined value for scalar zero. Although, value
+ at zero is defined for vector modes, this isn't fixed up by vectorizer.
+
+** ctzv4si_z13:
+** vzero (%v[0-9]+)
+** vceqf (%v[0-9]+),%v24,\1
+** vgm (%v[0-9]+),26,26,2
+** vctzf %v24,%v24
+** vsel %v24,\3,%v24,\2
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z13")))
+v4si ctzv4si_z13 (v4si x)
+{
+ for (int i = 0; i < 4; ++i)
+ x[i] = x[i] ? __builtin_ctz (x[i]) : 32;
+ return x;
+}
+
+/*
+** ctzv4si_z17:
+** vctzf %v24,%v24
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z17")))
+v4si ctzv4si_z17 (v4si x)
+{
+ for (int i = 0; i < 4; ++i)
+ x[i] = x[i] ? __builtin_ctz (x[i]) : 32;
+ return x;
+}
+
+/* For V2DI use the auto vectorizer since if we directly operate on vector
+ registers the loop is unrolled and SLP vectorization fails.
+
+** ctzv2di_z13:
+** ...
+** vctzg .*
+** ...
+** vctzg .*
+** ...
+*/
+
+__attribute__ ((target ("arch=z13")))
+void ctzv2di_z13 (unsigned long *x)
+{
+ for (int i = 0; i < 4; ++i)
+ x[i] = x[i] ? __builtin_ctzl (x[i]) : 64;
+}
--
2.55.0
