From: Stefan Schulze Frielinghaus <[email protected]>
For clz/ctz, scalar and vectorized defined-at-zero values may differ.
Statements of the form
B = (A != 0) ? OP : N
may still be vectorized without a zero test in case the defined-at-zero
value for vector modes equals N.
gcc/ChangeLog:
* tree-vect-patterns.cc
(vect_recog_popcount_clz_ctz_ffs_pattern): Fold zero test for
clz/ctz and vector modes.
gcc/testsuite/ChangeLog:
* gcc.target/s390/clz-ctz-1.c: Remove zero test for ctzv4si_z13.
---
gcc/testsuite/gcc.target/s390/clz-ctz-1.c | 8 +--
gcc/tree-vect-patterns.cc | 67 +++++++++++++++++++++++
2 files changed, 69 insertions(+), 6 deletions(-)
diff --git a/gcc/testsuite/gcc.target/s390/clz-ctz-1.c
b/gcc/testsuite/gcc.target/s390/clz-ctz-1.c
index 4d3fc148a0a..e97772143b0 100644
--- a/gcc/testsuite/gcc.target/s390/clz-ctz-1.c
+++ b/gcc/testsuite/gcc.target/s390/clz-ctz-1.c
@@ -267,15 +267,11 @@ v8hi ctzv8hi_z13 (v8hi x)
return x;
}
-/* Prior z17 we do not have a defined value for scalar zero. Although, value
- at zero is defined for vector modes, this isn't fixed up by vectorizer.
+/* Prior z17 we do not have a defined zero value for scalars. Verify that
+ vectorizer still recognizes that vctz has a defined zero value.
** ctzv4si_z13:
-** vzero (%v[0-9]+)
-** vceqf (%v[0-9]+),%v24,\1
-** vgm (%v[0-9]+),26,26,2
** vctzf %v24,%v24
-** vsel %v24,\3,%v24,\2
** br %r14
*/
diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc
index 38f65c0dafa..41d63de0b68 100644
--- a/gcc/tree-vect-patterns.cc
+++ b/gcc/tree-vect-patterns.cc
@@ -2151,6 +2151,50 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo,
if (!last_stmt)
return NULL;
tree_code code = gimple_assign_rhs_code (last_stmt);
+
+ /* For clz/ctz, scalar and vectorized defined-at-zero values may differ. For
+ scalars we might end up here with LAST_STMT of the form
+ B = (A != 0) ? op_result : N
+ if there is no defined-at-zero value or the value differs from N. Still
+ we might vectorize here without a zero test, if the defined-at-zero value
+ equals N for the vector mode.
+
+ Here we only look through the COND_EXPR. The test whether the vectorized
+ defined-at-zero value equals N is done later on. */
+ gassign *outer_cond_stmt = NULL;
+ tree outer_cond_ne_op0 = NULL;
+ if (code == COND_EXPR)
+ {
+ tree cond = gimple_assign_rhs1 (last_stmt);
+ tree true_val = gimple_assign_rhs2 (last_stmt);
+ tree false_val = gimple_assign_rhs3 (last_stmt);
+
+ if (TREE_CODE (false_val) != INTEGER_CST)
+ return NULL;
+
+ if (TREE_CODE (cond) != SSA_NAME)
+ return NULL;
+ gimple *cond_def = SSA_NAME_DEF_STMT (cond);
+ if (!is_gimple_assign (cond_def)
+ || gimple_assign_rhs_code (cond_def) != NE_EXPR)
+ return NULL;
+ outer_cond_ne_op0 = gimple_assign_rhs1 (cond_def);
+ tree outer_cond_ne_op1 = gimple_assign_rhs2 (cond_def);
+ if (TREE_CODE (outer_cond_ne_op0) != SSA_NAME
+ || !integer_zerop (outer_cond_ne_op1))
+ return NULL;
+
+ if (TREE_CODE (true_val) != SSA_NAME || !has_single_use (true_val))
+ return NULL;
+ gassign *inner = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (true_val));
+ if (!inner)
+ return NULL;
+
+ outer_cond_stmt = last_stmt;
+ last_stmt = inner;
+ code = gimple_assign_rhs_code (last_stmt);
+ }
+
if (!CONVERT_EXPR_CODE_P (code))
return NULL;
@@ -2172,6 +2216,10 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo,
{
int val;
CASE_CFN_POPCOUNT:
+ /* Bail out if there is an outer statement which means we are looking for
+ clz/ctz pattern. */
+ if (outer_cond_stmt)
+ return NULL;
ifn = IFN_POPCOUNT;
break;
CASE_CFN_CLZ:
@@ -2197,6 +2245,10 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo,
return NULL;
break;
CASE_CFN_FFS:
+ /* Bail out if there is an outer statement which means we are looking for
+ clz/ctz pattern. */
+ if (outer_cond_stmt)
+ return NULL;
ifn = IFN_FFS;
break;
default:
@@ -2215,6 +2267,12 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo,
if (!rhs_origin)
return NULL;
+ /* Verify that for
+ B = (A != 0) ? .CLZ (C) : N
+ A equals C. */
+ if (outer_cond_stmt && outer_cond_ne_op0 != rhs_oprnd)
+ return NULL;
+
/* Input and output of .POPCOUNT should be same-precision integer. */
if (TYPE_PRECISION (unprom_diff.type) != TYPE_PRECISION (lhs_type))
return NULL;
@@ -2339,6 +2397,15 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo,
&& CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type),
val) == 2)
arg2 = build_int_cst (integer_type_node, val);
+
+ /* For pattern clz/ctz, reaching this point we have
+ B = (A != 0) ? .CLZ (A) : N
+ Now verify that the defined-at-zero value equals N. */
+ if (outer_cond_stmt
+ && (arg2 == NULL
+ || !tree_int_cst_equal (gimple_assign_rhs3 (outer_cond_stmt), arg2)))
+ return NULL;
+
if (arg2)
pattern_stmt = gimple_build_call_internal (ifn, 2, unprom_diff.op, arg2);
else
--
2.55.0
