From: Stefan Schulze Frielinghaus <[email protected]>
Pattern clz{si,di}2 which return for zero the bit precision are limited
to TARGET_EXTIMM. Reflect this in CLZ_DEFINED_VALUE_AT_ZERO. Likewise,
pattern ctz{si,di}2 are limited to TARGET_Z17 which must be reflected in
CTZ_DEFINED_VALUE_AT_ZERO.
So far for ctzsi2 the input is zero extended and ctzdi2 is utilized
which means for zero we end up with 64. Since some optimizations may
kick in when ctz(0) evaluates to the precision of the mode, change
ctzsi2 by setting bit 31 of the input value to one so we are guaranteed
to end up with a value in [0,32]. The set manifests in an OR IMMEDIATE
or INSERT IMMEDIATE which group nicely back-to-back and the overhead of
a further instruction outweighs a potential compare and branch. This
also aligns with the vector instruction vctz where for zero the elements
precision is computed.
In total, for ctz this would mean we have a defined value at zero if
TARGET_Z17 || (TARGET_Z13 && VECTOR_MODE_P (MODE) holds. However, this
makes gcc.dg/vect/pr109011-2.c fail on s390 where we miss a ctz
vectorization. In tree-vect-patterns.cc:1889 we have
if (ifn == IFN_FFS
&& direct_internal_fn_supported_p (IFN_CTZ, vec_rhs_type,
OPTIMIZE_FOR_SPEED))
{
ifnnew = IFN_CTZ;
defined_at_zero_new
= CTZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (rhs_type),
val_new) == 2;
}
The IFN_CTZ is checked for availability w.r.t. vector types but the
defined value at zero for scalar types which seems inconsistent to me.
gccint does not limit C[LT]Z_DEFINED_VALUE_AT_ZERO to scalars.
Furthermore, targets so far do not differentiate between vector and
scalar mode. Therefore, I'm wondering whether it would be acceptable to
change test for vector mode here (and of course all the other places,
too)? Of course targets would need to be prepared to being asked for
vector modes. For s390 I did a quick test while replacing
GET_MODE_PRECISION by GET_MODE_UNIT_PRECISION, though, this needs
further inspection but I wanted to clarify the initial question first.
---
gcc/config/s390/s390.h | 4 +-
gcc/config/s390/s390.md | 2 +
gcc/testsuite/gcc.dg/tree-ssa/pr123311-2.c | 1 +
gcc/testsuite/gcc.target/s390/clz-ctz-1.c | 236 +++++++++++++++++++++
gcc/tree-vect-patterns.cc | 4 +-
5 files changed, 243 insertions(+), 4 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/s390/clz-ctz-1.c
diff --git a/gcc/config/s390/s390.h b/gcc/config/s390/s390.h
index 948fe506a9c..c449623e5ea 100644
--- a/gcc/config/s390/s390.h
+++ b/gcc/config/s390/s390.h
@@ -917,10 +917,10 @@ do {
\
#define FUNCTION_MODE QImode
/* Specify the value which is used when clz operand is zero. */
-#define CLZ_DEFINED_VALUE_AT_ZERO(MODE, VALUE) ((VALUE) = GET_MODE_PRECISION
(MODE), 2)
+#define CLZ_DEFINED_VALUE_AT_ZERO(MODE, VALUE) (TARGET_EXTIMM ? ((VALUE) =
GET_MODE_UNIT_PRECISION (MODE), 2) : 0)
/* Specify the value which is used when ctz operand is zero. */
-#define CTZ_DEFINED_VALUE_AT_ZERO(MODE, VALUE) ((VALUE) = 64, 2)
+#define CTZ_DEFINED_VALUE_AT_ZERO(MODE, VALUE) (TARGET_Z17 || (TARGET_Z13 &&
VECTOR_MODE_P (MODE)) ? ((VALUE) = GET_MODE_UNIT_PRECISION (MODE), 2) : 0)
/* Machine-specific symbol_ref flags. */
#define SYMBOL_FLAG_ALIGN_SHIFT SYMBOL_FLAG_MACH_DEP_SHIFT
diff --git a/gcc/config/s390/s390.md b/gcc/config/s390/s390.md
index aa9b011278d..6e30943319c 100644
--- a/gcc/config/s390/s390.md
+++ b/gcc/config/s390/s390.md
@@ -9013,6 +9013,8 @@
(define_expand "ctzsi2"
[(set (match_dup 2)
(zero_extend:DI (match_operand:SI 1 "register_operand" "")))
+ (set (zero_extract:DI (match_dup 2) (const_int 32) (const_int 0))
+ (const_int 1))
(set (match_dup 3) (ctz:DI (match_dup 2)))
(set (match_operand:SI 0 "register_operand" "") (subreg:SI (match_dup 3)
4))]
"TARGET_Z17"
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr123311-2.c
b/gcc/testsuite/gcc.dg/tree-ssa/pr123311-2.c
index 0f55c32a1bb..72f52853c18 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr123311-2.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr123311-2.c
@@ -1,6 +1,7 @@
/* { dg-do compile } */
/* { dg-options "-O2 -fdump-tree-optimized" } */
/* { dg-additional-options "-mbmi -mlzcnt" { target i?86-*-* x86_64-*-* } } */
+/* { dg-additional-options "-march=z17" { target s390x-*-* } } */
/* { dg-require-effective-target clz } */
/* { dg-require-effective-target ctz } */
diff --git a/gcc/testsuite/gcc.target/s390/clz-ctz-1.c
b/gcc/testsuite/gcc.target/s390/clz-ctz-1.c
new file mode 100644
index 00000000000..9c59cfd22f1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/clz-ctz-1.c
@@ -0,0 +1,236 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-optimized" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+/* { dg-final { scan-tree-dump-times "\.CLZ \\(\[^,\]+, 32\\)" 3 "optimized" }
} */
+/* { dg-final { scan-tree-dump-times "\.CTZ \\(\[^,\]+, 64\\)" 1 "optimized" }
} */
+
+typedef unsigned int __attribute__ ((vector_size (16))) v4si;
+typedef unsigned long __attribute__ ((vector_size (16))) v2di;
+
+/*
+** clzsi_z900:
+** ...
+** brasl %r14,__clzdi2@PLT
+** ...
+*/
+
+__attribute__ ((target ("arch=z900")))
+int clzsi_z900 (unsigned int x)
+{
+ return x ? __builtin_clz (x) : 32;
+}
+
+/*
+** clzdi_z900:
+** ...
+** brasl %r14,__clzdi2@PLT
+** ...
+*/
+
+__attribute__ ((target ("arch=z900")))
+int clzdi_z900 (unsigned long x)
+{
+ return x ? __builtin_clzl (x) : 64;
+}
+
+/*
+** clzsi_z10:
+** flogr %r2,%r2
+** ahi %r2,-32
+** lgfr %r2,%r2
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z10")))
+int clzsi_z10 (unsigned int x)
+{
+ return x ? __builtin_clz (x) : 32;
+}
+
+/*
+** clzdi_z10:
+** flogr %r2,%r2
+** lgfr %r2,%r2
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z10")))
+int clzdi_z10 (unsigned long x)
+{
+ return x ? __builtin_clzl (x) : 64;
+}
+
+/*
+** clzsi_z17:
+** clzg %r2,%r2
+** ahi %r2,-32
+** lgfr %r2,%r2
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z17")))
+int clzsi_z17 (unsigned int x)
+{
+ return x ? __builtin_clz (x) : 32;
+}
+
+/*
+** clzdi_z17:
+** clzg %r2,%r2
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z17")))
+int clzdi_z17 (unsigned long x)
+{
+ return x ? __builtin_clzl (x) : 64;
+}
+
+/*
+** clzv4si_z13:
+** vclzf %v24,%v24
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z13")))
+v4si clzv4si_z13 (v4si x)
+{
+ for (int i = 0; i < 4; ++i)
+ x[i] = x[i] ? __builtin_clz (x[i]) : 32;
+ return x;
+}
+
+/* TODO: This isn't vectorized but scalarized.
+
+__attribute__ ((target ("arch=z13")))
+v2di clzv2di_z13 (v2di x)
+{
+ for (int i = 0; i < 2; ++i)
+ x[i] = x[i] ? __builtin_clzl (x[i]) : 64;
+ return x;
+}
+*/
+
+/*
+** ctzsi_z900:
+** ...
+** brasl %r14,__ctzdi2@PLT
+** ...
+*/
+
+__attribute__ ((target ("arch=z900")))
+int ctzsi_z900 (unsigned int x)
+{
+ return x ? __builtin_ctz (x) : 32;
+}
+
+/*
+** ctzdi_z900:
+** ...
+** brasl %r14,__ctzdi2@PLT
+** ...
+*/
+
+__attribute__ ((target ("arch=z900")))
+int ctzdi_z900 (unsigned long x)
+{
+ return x ? __builtin_ctzl (x) : 64;
+}
+
+/*
+** ctzsi_z10:
+** ...
+** flogr %r[0-9]+,%r[0-9]+
+** ...
+*/
+
+__attribute__ ((target ("arch=z10")))
+int ctzsi_z10 (unsigned int x)
+{
+ return x ? __builtin_ctz (x) : 32;
+}
+
+/*
+** ctzdi_z10:
+** ...
+** flogr %r[0-9]+,%r[0-9]+
+** ...
+*/
+
+__attribute__ ((target ("arch=z10")))
+int ctzdi_z10 (unsigned long x)
+{
+ return x ? __builtin_ctzl (x) : 64;
+}
+
+/*
+** ctzsi_z17:
+** oihl %r2,1
+** ctzg %r2,%r2
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z17")))
+int ctzsi_z17 (unsigned int x)
+{
+ return x ? __builtin_ctz (x) : 32;
+}
+
+/*
+** ctzdi_z17:
+** ctzg %r2,%r2
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z17")))
+int ctzdi_z17 (unsigned long x)
+{
+ return x ? __builtin_ctzl (x) : 64;
+}
+
+/* Prior z17 we do not have a defined value for scalar zero. Although, value
+ at zero is defined for vector modes, this isn't fixed up by vectorizer. */
+
+/*
+** ctzv4si_z13:
+** vzero (%v[0-9]+)
+** vceqf (%v[0-9]+),%v24,\1
+** vgm (%v[0-9]+),26,26,2
+** vctzf %v24,%v24
+** vsel %v24,\3,%v24,\2
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z13")))
+v4si ctzv4si_z13 (v4si x)
+{
+ for (int i = 0; i < 4; ++i)
+ x[i] = x[i] ? __builtin_ctz (x[i]) : 32;
+ return x;
+}
+
+/*
+** ctzv4si_z17:
+** vctzf %v24,%v24
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z17")))
+v4si ctzv4si_z17 (v4si x)
+{
+ for (int i = 0; i < 4; ++i)
+ x[i] = x[i] ? __builtin_ctz (x[i]) : 32;
+ return x;
+}
+
+/* TODO: This isn't vectorized but scalarized.
+
+__attribute__ ((target ("arch=z17")))
+v2di ctzv2di_z17 (v2di x)
+{
+ for (int i = 0; i < 2; ++i)
+ x[i] = x[i] ? __builtin_ctzl (x[i]) : 64;
+ return x;
+}
+*/
diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc
index 406f81b95c8..9ddfd74b914 100644
--- a/gcc/tree-vect-patterns.cc
+++ b/gcc/tree-vect-patterns.cc
@@ -1892,7 +1892,7 @@ vect_recog_ctz_ffs_pattern (vec_info *vinfo,
stmt_vec_info stmt_vinfo,
{
ifnnew = IFN_CTZ;
defined_at_zero_new
- = CTZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (rhs_type),
+ = CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_rhs_type),
val_new) == 2;
}
else if (direct_internal_fn_supported_p (IFN_CLZ, vec_rhs_type,
@@ -1900,7 +1900,7 @@ vect_recog_ctz_ffs_pattern (vec_info *vinfo,
stmt_vec_info stmt_vinfo,
{
ifnnew = IFN_CLZ;
defined_at_zero_new
- = CLZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (rhs_type),
+ = CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_rhs_type),
val_new) == 2;
}
if ((ifnnew == IFN_LAST
--
2.55.0
