From: Stefan Schulze Frielinghaus <[email protected]>
For ctzsi2 the input is zero extended and ctzdi2 is utilized which means
for zero we end up with 64. Since some optimizations may kick in when
ctz(0) evaluates to the precision of the mode, change ctzsi2 by setting
bit 31 of the input value to one so we are guaranteed to end up with a
value in [0,32]. The set manifests in an OR IMMEDIATE or INSERT
IMMEDIATE which group nicely back-to-back and the overhead of a further
instruction outweighs a potential compare and branch. This also aligns
with the vector instruction vctz where for zero the elements precision
is computed. This is analogue to clzsi2 where the input is zero
extended, too, passed to the 64-bit variant and afterwards 32 is
subtracted which also aligns with vector instruction vclz.
Deal with vector modes in target hooks C[LT]Z_DEFINED_VALUE_AT_ZERO.
Insn c[lt]zti2 are implemented through vector patterns since the
underlying instructions are vc[lt]zq. This in turn forces the results
to be of TI mode in vector registers which is cumbersome to deal with.
For scalar TI mode expect the result to be of SI mode which will be
consumed in GPRs.
gcc/ChangeLog:
* config/s390/s390.h (CLZ_DEFINED_VALUE_AT_ZERO): Deal with
vector modes.
(CTZ_DEFINED_VALUE_AT_ZERO): Ditto.
* config/s390/s390.md (ctzsi2): Compute 32 for zero value.
* config/s390/vector.md (VIT_VXE3_NO_TI): New mode iterator.
(clzti2): New expander.
(ctzti2): Ditto.
gcc/testsuite/ChangeLog:
* gcc.dg/tree-ssa/pr123311-2.c: Pattern ctzsi2 becomes available
with z17.
* gcc.target/s390/clz-ctz-1.c: New test.
---
gcc/config/s390/s390.h | 6 +-
gcc/config/s390/s390.md | 3 +
gcc/config/s390/vector.md | 27 +-
gcc/testsuite/gcc.dg/tree-ssa/pr123311-2.c | 1 +
gcc/testsuite/gcc.target/s390/clz-ctz-1.c | 481 +++++++++++++++++++++
5 files changed, 512 insertions(+), 6 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/s390/clz-ctz-1.c
diff --git a/gcc/config/s390/s390.h b/gcc/config/s390/s390.h
index 948fe506a9c..ba839c8bee7 100644
--- a/gcc/config/s390/s390.h
+++ b/gcc/config/s390/s390.h
@@ -917,10 +917,12 @@ do {
\
#define FUNCTION_MODE QImode
/* Specify the value which is used when clz operand is zero. */
-#define CLZ_DEFINED_VALUE_AT_ZERO(MODE, VALUE) ((VALUE) = GET_MODE_PRECISION
(MODE), 2)
+#define CLZ_DEFINED_VALUE_AT_ZERO(MODE, VALUE) \
+ ((VALUE) = GET_MODE_UNIT_PRECISION (MODE), 2)
/* Specify the value which is used when ctz operand is zero. */
-#define CTZ_DEFINED_VALUE_AT_ZERO(MODE, VALUE) ((VALUE) = 64, 2)
+#define CTZ_DEFINED_VALUE_AT_ZERO(MODE, VALUE) \
+ ((VALUE) = GET_MODE_UNIT_PRECISION (MODE), 2)
/* Machine-specific symbol_ref flags. */
#define SYMBOL_FLAG_ALIGN_SHIFT SYMBOL_FLAG_MACH_DEP_SHIFT
diff --git a/gcc/config/s390/s390.md b/gcc/config/s390/s390.md
index aa9b011278d..a7ab4c9b548 100644
--- a/gcc/config/s390/s390.md
+++ b/gcc/config/s390/s390.md
@@ -9013,6 +9013,9 @@
(define_expand "ctzsi2"
[(set (match_dup 2)
(zero_extend:DI (match_operand:SI 1 "register_operand" "")))
+ ; Set bit 31 to one which ensures that for zero we compute 32.
+ (set (zero_extract:DI (match_dup 2) (const_int 32) (const_int 0))
+ (const_int 1))
(set (match_dup 3) (ctz:DI (match_dup 2)))
(set (match_operand:SI 0 "register_operand" "") (subreg:SI (match_dup 3)
4))]
"TARGET_Z17"
diff --git a/gcc/config/s390/vector.md b/gcc/config/s390/vector.md
index acca6c4ea34..32c404cc4ce 100644
--- a/gcc/config/s390/vector.md
+++ b/gcc/config/s390/vector.md
@@ -67,6 +67,7 @@
; All integer vector modes supported in a vector register + TImode
(define_mode_iterator VIT [V1QI V2QI V4QI V8QI V16QI V1HI V2HI V4HI V8HI V1SI
V2SI V4SI V1DI V2DI V1TI TI])
(define_mode_iterator VIT_VXE3 [V1QI V2QI V4QI V8QI V16QI V1HI V2HI V4HI V8HI
V1SI V2SI V4SI V1DI V2DI (V1TI "TARGET_VXE3") (TI "TARGET_VXE3")])
+(define_mode_iterator VIT_VXE3_NO_TI [V1QI V2QI V4QI V8QI V16QI V1HI V2HI V4HI
V8HI V1SI V2SI V4SI V1DI V2DI (V1TI "TARGET_VXE3")])
(define_mode_iterator VI [V1QI V2QI V4QI V8QI V16QI V1HI V2HI V4HI V8HI V1SI
V2SI V4SI V1DI V2DI])
(define_mode_iterator VI_VXE3 [V1QI V2QI V4QI V8QI V16QI V1HI V2HI V4HI V8HI
V1SI V2SI V4SI V1DI (V2DI "TARGET_VXE3")])
(define_mode_iterator VI_QHS [V1QI V2QI V4QI V8QI V16QI V1HI V2HI V4HI V8HI
V1SI V2SI V4SI])
@@ -1615,21 +1616,39 @@
; Count leading zeros
; vclzb, vclzh, vclzf, vclzg, vclzq
(define_insn "clz<mode>2"
- [(set (match_operand:VT_VXE3 0 "register_operand" "=v")
- (clz:VT_VXE3 (match_operand:VT_VXE3 1 "register_operand" "v")))]
+ [(set (match_operand:VIT_VXE3_NO_TI 0 "register_operand"
"=v")
+ (clz:VIT_VXE3_NO_TI (match_operand:VIT_VXE3_NO_TI 1 "register_operand"
"v")))]
"TARGET_VX"
"vclz<bhfgq>\t%v0,%v1"
[(set_attr "op_type" "VRR")])
+(define_expand "clzti2"
+ [(set (match_dup 2)
+ (clz:V1TI (subreg:V1TI (match_operand:TI 1 "register_operand") 0)))
+ (set (match_operand:SI 0 "register_operand")
+ (vec_select:SI (subreg:V4SI (match_dup 2) 0)
+ (parallel [(const_int 3)])))]
+ "TARGET_VXE3"
+ "operands[2] = gen_reg_rtx (V1TImode);")
+
; Count trailing zeros
; vctzb, vctzh, vctzf, vctzg, vctzq
(define_insn "ctz<mode>2"
- [(set (match_operand:VT_VXE3 0 "register_operand" "=v")
- (ctz:VT_VXE3 (match_operand:VT_VXE3 1 "register_operand" "v")))]
+ [(set (match_operand:VIT_VXE3_NO_TI 0 "register_operand"
"=v")
+ (ctz:VIT_VXE3_NO_TI (match_operand:VIT_VXE3_NO_TI 1 "register_operand"
"v")))]
"TARGET_VX"
"vctz<bhfgq>\t%v0,%v1"
[(set_attr "op_type" "VRR")])
+(define_expand "ctzti2"
+ [(set (match_dup 2)
+ (ctz:V1TI (subreg:V1TI (match_operand:TI 1 "register_operand") 0)))
+ (set (match_operand:SI 0 "register_operand")
+ (vec_select:SI (subreg:V4SI (match_dup 2) 0)
+ (parallel [(const_int 3)])))]
+ "TARGET_VXE3"
+ "operands[2] = gen_reg_rtx (V1TImode);")
+
; Each vector element rotated by the corresponding vector element
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr123311-2.c
b/gcc/testsuite/gcc.dg/tree-ssa/pr123311-2.c
index 0f55c32a1bb..72f52853c18 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr123311-2.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr123311-2.c
@@ -1,6 +1,7 @@
/* { dg-do compile } */
/* { dg-options "-O2 -fdump-tree-optimized" } */
/* { dg-additional-options "-mbmi -mlzcnt" { target i?86-*-* x86_64-*-* } } */
+/* { dg-additional-options "-march=z17" { target s390x-*-* } } */
/* { dg-require-effective-target clz } */
/* { dg-require-effective-target ctz } */
diff --git a/gcc/testsuite/gcc.target/s390/clz-ctz-1.c
b/gcc/testsuite/gcc.target/s390/clz-ctz-1.c
new file mode 100644
index 00000000000..523c089755c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/clz-ctz-1.c
@@ -0,0 +1,481 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+/* { dg-final { scan-assembler-times {\n\tvzero\t} 9 } } */
+
+typedef unsigned char __attribute__ ((vector_size (16))) v16qi;
+typedef unsigned short __attribute__ ((vector_size (16))) v8hi;
+typedef unsigned int __attribute__ ((vector_size (16))) v4si;
+typedef unsigned long __attribute__ ((vector_size (16))) v2di;
+typedef unsigned __int128 __attribute__ ((vector_size (16))) v1ti;
+
+/*
+** clzsi_z900:
+** ...
+** brasl %r14,__clzdi2@PLT
+** ...
+*/
+
+__attribute__ ((target ("arch=z900")))
+int clzsi_z900 (unsigned int x)
+{
+ return x ? __builtin_clz (x) : 32;
+}
+
+/*
+** clzdi_z900:
+** ...
+** brasl %r14,__clzdi2@PLT
+** ...
+*/
+
+__attribute__ ((target ("arch=z900")))
+int clzdi_z900 (unsigned long x)
+{
+ return x ? __builtin_clzl (x) : 64;
+}
+
+/*
+** clzsi_z9_109:
+** flogr %r2,%r2
+** ahi %r2,-32
+** lgfr %r2,%r2
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z9-109")))
+int clzsi_z9_109 (unsigned int x)
+{
+ return x ? __builtin_clz (x) : 32;
+}
+
+/*
+** clzdi_z9_109:
+** flogr %r2,%r2
+** lgfr %r2,%r2
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z9-109")))
+int clzdi_z9_109 (unsigned long x)
+{
+ return x ? __builtin_clzl (x) : 64;
+}
+
+/*
+** clzsi_z17:
+** clzg %r2,%r2
+** ahi %r2,-32
+** lgfr %r2,%r2
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z17")))
+int clzsi_z17 (unsigned int x)
+{
+ return x ? __builtin_clz (x) : 32;
+}
+
+/*
+** clzdi_z17:
+** clzg %r2,%r2
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z17")))
+int clzdi_z17 (unsigned long x)
+{
+ return x ? __builtin_clzl (x) : 64;
+}
+
+/*
+** clzti_z17:
+** vl (%v[0-9]+),0\(%r2\),3
+** vclzq (%v[0-9]+),\1
+** vlgvf (%r[0-9]+),\2,3
+** lgfr %r2,\3
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z17")))
+int clzti_z17 (unsigned __int128 x)
+{
+ return x ? __builtin_clzg (x) : 128;
+}
+
+/*
+** clzsi_z13_autovec:
+** ...
+** vclzf .*
+** ...
+** vst .*
+** ...
+*/
+
+__attribute__ ((target ("arch=z13")))
+void clzsi_z13_autovec (unsigned int *x)
+{
+ for (int i = 0; i < 64; ++i)
+ x[i] = x[i] ? __builtin_clz (x[i]) : 32;
+}
+
+/*
+** clzdi_z13_autovec:
+** ...
+** vclzg .*
+** ...
+** vpkg .*
+** ...
+** vst .*
+** ...
+*/
+
+__attribute__ ((target ("arch=z13")))
+void clzdi_z13_autovec (int *x, unsigned long *y)
+{
+ for (int i = 0; i < 64; ++i)
+ x[i] = y[i] ? __builtin_clzl (y[i]) : 64;
+}
+
+/*
+** clzti_z17_autovec:
+** ...
+** vclzq .*
+** ...
+** vstef .*
+** ...
+*/
+
+__attribute__ ((target ("arch=z17")))
+void clzti_z17_autovec (int *x, unsigned __int128 *y)
+{
+ for (int i = 0; i < 64; ++i)
+ x[i] = y[i] ? __builtin_clzg (y[i]) : 128;
+}
+
+/* For __builtin_clz arguments are promoted to unsigned int which is why we are
+ faced with an extra addend .CLZ(x, 8) + 24.
+
+** clzv16qi_z13:
+** ...
+** vzero .*
+** ...
+** vclzb %v[0-9]+,%v[0-9]+
+** ...
+*/
+
+__attribute__ ((target ("arch=z13")))
+v16qi clzv16qi_z13 (v16qi x)
+{
+ for (int i = 0; i < 16; ++i)
+ x[i] = x[i] ? __builtin_clz (x[i]) : 8;
+ return x;
+}
+
+/* TODO: fold zero test.
+
+** clzgv16qi_z13:
+** vzero (%v[0-9]+)
+** vceqb (%v[0-9]+),%v24,\1
+** vgm (%v[0-9]+),4,4,0
+** vclzb %v24,%v24
+** vsel %v24,\3,%v24,\2
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z13")))
+v16qi clzgv16qi_z13 (v16qi x)
+{
+ for (int i = 0; i < 16; ++i)
+ x[i] = x[i] ? __builtin_clzg (x[i]) : 8;
+ return x;
+}
+
+/* For __builtin_clz arguments are promoted to unsigned int which is why we are
+ faced with an extra addend .CLZ(x, 16) + 16.
+
+** clzv8hi_z13:
+** ...
+** vzero .*
+** ...
+** vclzh %v[0-9]+,%v[0-9]+
+** ...
+*/
+
+__attribute__ ((target ("arch=z13")))
+v8hi clzv8hi_z13 (v8hi x)
+{
+ for (int i = 0; i < 8; ++i)
+ x[i] = x[i] ? __builtin_clz (x[i]) : 16;
+ return x;
+}
+
+/* TODO: fold zero test.
+
+** clzgv8hi_z13:
+** vzero (%v[0-9]+)
+** vceqh (%v[0-9]+),%v24,\1
+** vgm (%v[0-9]+),11,11,1
+** vclzh %v24,%v24
+** vsel %v24,\3,%v24,\2
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z13")))
+v8hi clzgv8hi_z13 (v8hi x)
+{
+ for (int i = 0; i < 8; ++i)
+ x[i] = x[i] ? __builtin_clzg (x[i]) : 16;
+ return x;
+}
+
+/*
+** clzv4si_z13:
+** vclzf %v24,%v24
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z13")))
+v4si clzv4si_z13 (v4si x)
+{
+ for (int i = 0; i < 4; ++i)
+ x[i] = x[i] ? __builtin_clz (x[i]) : 32;
+ return x;
+}
+
+
+
+/*
+** ctzsi_z900:
+** ...
+** brasl %r14,__ctzdi2@PLT
+** ...
+*/
+
+__attribute__ ((target ("arch=z900")))
+int ctzsi_z900 (unsigned int x)
+{
+ return x ? __builtin_ctz (x) : 32;
+}
+
+/*
+** ctzdi_z900:
+** ...
+** brasl %r14,__ctzdi2@PLT
+** ...
+*/
+
+__attribute__ ((target ("arch=z900")))
+int ctzdi_z900 (unsigned long x)
+{
+ return x ? __builtin_ctzl (x) : 64;
+}
+
+/*
+** ctzsi_z9_109:
+** ...
+** flogr %r[0-9]+,%r[0-9]+
+** ...
+*/
+
+__attribute__ ((target ("arch=z9-109")))
+int ctzsi_z9_109 (unsigned int x)
+{
+ return x ? __builtin_ctz (x) : 32;
+}
+
+/*
+** ctzdi_z9_109:
+** ...
+** flogr %r[0-9]+,%r[0-9]+
+** ...
+*/
+
+__attribute__ ((target ("arch=z9-109")))
+int ctzdi_z9_109 (unsigned long x)
+{
+ return x ? __builtin_ctzl (x) : 64;
+}
+
+/*
+** ctzsi_z17:
+** oihl %r2,1
+** ctzg %r2,%r2
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z17")))
+int ctzsi_z17 (unsigned int x)
+{
+ return x ? __builtin_ctz (x) : 32;
+}
+
+/*
+** ctzdi_z17:
+** ctzg %r2,%r2
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z17")))
+int ctzdi_z17 (unsigned long x)
+{
+ return x ? __builtin_ctzl (x) : 64;
+}
+
+/*
+** ctzti_z17:
+** vl (%v[0-9]+),0\(%r2\),3
+** vctzq (%v[0-9]+),\1
+** vlgvf (%r[0-9]+),\2,3
+** lgfr %r2,\3
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z17")))
+int ctzti_z17 (unsigned __int128 x)
+{
+ return x ? __builtin_ctzg (x) : 128;
+}
+
+/*
+** ctzsi_z13_autovec:
+** ...
+** vctzf .*
+** ...
+** vst .*
+** ...
+*/
+
+__attribute__ ((target ("arch=z13")))
+void ctzsi_z13_autovec (unsigned int *x)
+{
+ for (int i = 0; i < 64; ++i)
+ x[i] = x[i] ? __builtin_ctz (x[i]) : 32;
+}
+
+/* TODO: fold zero test.
+
+** ctzdi_z13_autovec:
+** ...
+** vzero .*
+** ...
+** vctzg .*
+** ...
+** vpkg .*
+** ...
+** vst .*
+** ...
+*/
+
+__attribute__ ((target ("arch=z13")))
+void ctzdi_z13_autovec (int *x, unsigned long *y)
+{
+ for (int i = 0; i < 64; ++i)
+ x[i] = y[i] ? __builtin_ctzl (y[i]) : 64;
+}
+
+/*
+** ctzdi_z17_autovec:
+** ...
+** vctzg .*
+** ...
+** vpkg .*
+** ...
+** vst .*
+** ...
+*/
+
+__attribute__ ((target ("arch=z17")))
+void ctzdi_z17_autovec (int *x, unsigned long *y)
+{
+ for (int i = 0; i < 64; ++i)
+ x[i] = y[i] ? __builtin_ctzl (y[i]) : 64;
+}
+
+/*
+** ctzti_z17_autovec:
+** ...
+** vctzq .*
+** ...
+** vstef .*
+** ...
+*/
+
+__attribute__ ((target ("arch=z17")))
+void ctzti_z17_autovec (int *x, unsigned __int128 *y)
+{
+ for (int i = 0; i < 64; ++i)
+ x[i] = y[i] ? __builtin_ctzg (y[i]) : 128;
+}
+
+
+/* TODO: fold zero test.
+
+** ctzv16qi_z13:
+** vzero (%v[0-9]+)
+** vceqb (%v[0-9]+),%v24,\1
+** vgm (%v[0-9]+),4,4,0
+** vctzb %v24,%v24
+** vsel %v24,\3,%v24,\2
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z13")))
+v16qi ctzv16qi_z13 (v16qi x)
+{
+ for (int i = 0; i < 16; ++i)
+ x[i] = x[i] ? __builtin_ctz (x[i]) : 8;
+ return x;
+}
+
+/* TODO: fold zero test.
+
+** ctzv8hi_z13:
+** vzero (%v[0-9]+)
+** vceqh (%v[0-9]+),%v24,\1
+** vgm (%v[0-9]+),11,11,1
+** vctzh %v24,%v24
+** vsel %v24,\3,%v24,\2
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z13")))
+v8hi ctzv8hi_z13 (v8hi x)
+{
+ for (int i = 0; i < 8; ++i)
+ x[i] = x[i] ? __builtin_ctz (x[i]) : 16;
+ return x;
+}
+
+/* TODO: fold zero test.
+
+** ctzv4si_z13:
+** vzero (%v[0-9]+)
+** vceqf (%v[0-9]+),%v24,\1
+** vgm (%v[0-9]+),26,26,2
+** vctzf %v24,%v24
+** vsel %v24,\3,%v24,\2
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z13")))
+v4si ctzv4si_z13 (v4si x)
+{
+ for (int i = 0; i < 4; ++i)
+ x[i] = x[i] ? __builtin_ctz (x[i]) : 32;
+ return x;
+}
+
+/*
+** ctzv4si_z17:
+** vctzf %v24,%v24
+** br %r14
+*/
+
+__attribute__ ((target ("arch=z17")))
+v4si ctzv4si_z17 (v4si x)
+{
+ for (int i = 0; i < 4; ++i)
+ x[i] = x[i] ? __builtin_ctz (x[i]) : 32;
+ return x;
+}
--
2.55.0
