On Tue, Sep 29, 2026 at 1:32 PM Stefan Schulze Frielinghaus
<[email protected]> wrote:
>
> From: Stefan Schulze Frielinghaus <[email protected]>
>
> For clz/ctz, scalar and vectorized defined-at-zero values may differ.
> Statements of the form
> B = (A != 0) ? OP : N
> may still be vectorized without a zero test in case the defined-at-zero
> value for vector modes equals N.
In light of the previous comment this doesn't seem correct iff there
would be different answers for different vector types. So I fear
this needs to be handled after vectorization as generic folding
pattern on GIMPLE (forgoing the eventual vectorization cost benefit)
Richard.
> gcc/ChangeLog:
>
> * tree-vect-patterns.cc
> (vect_recog_popcount_clz_ctz_ffs_pattern): Fold zero test for
> clz/ctz and vector modes.
>
> gcc/testsuite/ChangeLog:
>
> * gcc.target/s390/clz-ctz-1.c: Remove zero test for ctzv4si_z13.
> ---
> gcc/testsuite/gcc.target/s390/clz-ctz-1.c | 8 +--
> gcc/tree-vect-patterns.cc | 67 +++++++++++++++++++++++
> 2 files changed, 69 insertions(+), 6 deletions(-)
>
> diff --git a/gcc/testsuite/gcc.target/s390/clz-ctz-1.c
> b/gcc/testsuite/gcc.target/s390/clz-ctz-1.c
> index 4d3fc148a0a..e97772143b0 100644
> --- a/gcc/testsuite/gcc.target/s390/clz-ctz-1.c
> +++ b/gcc/testsuite/gcc.target/s390/clz-ctz-1.c
> @@ -267,15 +267,11 @@ v8hi ctzv8hi_z13 (v8hi x)
> return x;
> }
>
> -/* Prior z17 we do not have a defined value for scalar zero. Although, value
> - at zero is defined for vector modes, this isn't fixed up by vectorizer.
> +/* Prior z17 we do not have a defined zero value for scalars. Verify that
> + vectorizer still recognizes that vctz has a defined zero value.
>
> ** ctzv4si_z13:
> -** vzero (%v[0-9]+)
> -** vceqf (%v[0-9]+),%v24,\1
> -** vgm (%v[0-9]+),26,26,2
> ** vctzf %v24,%v24
> -** vsel %v24,\3,%v24,\2
> ** br %r14
> */
>
> diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc
> index 38f65c0dafa..41d63de0b68 100644
> --- a/gcc/tree-vect-patterns.cc
> +++ b/gcc/tree-vect-patterns.cc
> @@ -2151,6 +2151,50 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info
> *vinfo,
> if (!last_stmt)
> return NULL;
> tree_code code = gimple_assign_rhs_code (last_stmt);
> +
> + /* For clz/ctz, scalar and vectorized defined-at-zero values may differ.
> For
> + scalars we might end up here with LAST_STMT of the form
> + B = (A != 0) ? op_result : N
> + if there is no defined-at-zero value or the value differs from N. Still
> + we might vectorize here without a zero test, if the defined-at-zero
> value
> + equals N for the vector mode.
> +
> + Here we only look through the COND_EXPR. The test whether the
> vectorized
> + defined-at-zero value equals N is done later on. */
> + gassign *outer_cond_stmt = NULL;
> + tree outer_cond_ne_op0 = NULL;
> + if (code == COND_EXPR)
> + {
> + tree cond = gimple_assign_rhs1 (last_stmt);
> + tree true_val = gimple_assign_rhs2 (last_stmt);
> + tree false_val = gimple_assign_rhs3 (last_stmt);
> +
> + if (TREE_CODE (false_val) != INTEGER_CST)
> + return NULL;
> +
> + if (TREE_CODE (cond) != SSA_NAME)
> + return NULL;
> + gimple *cond_def = SSA_NAME_DEF_STMT (cond);
> + if (!is_gimple_assign (cond_def)
> + || gimple_assign_rhs_code (cond_def) != NE_EXPR)
> + return NULL;
> + outer_cond_ne_op0 = gimple_assign_rhs1 (cond_def);
> + tree outer_cond_ne_op1 = gimple_assign_rhs2 (cond_def);
> + if (TREE_CODE (outer_cond_ne_op0) != SSA_NAME
> + || !integer_zerop (outer_cond_ne_op1))
> + return NULL;
> +
> + if (TREE_CODE (true_val) != SSA_NAME || !has_single_use (true_val))
> + return NULL;
> + gassign *inner = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (true_val));
> + if (!inner)
> + return NULL;
> +
> + outer_cond_stmt = last_stmt;
> + last_stmt = inner;
> + code = gimple_assign_rhs_code (last_stmt);
> + }
> +
> if (!CONVERT_EXPR_CODE_P (code))
> return NULL;
>
> @@ -2172,6 +2216,10 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info
> *vinfo,
> {
> int val;
> CASE_CFN_POPCOUNT:
> + /* Bail out if there is an outer statement which means we are looking
> for
> + clz/ctz pattern. */
> + if (outer_cond_stmt)
> + return NULL;
> ifn = IFN_POPCOUNT;
> break;
> CASE_CFN_CLZ:
> @@ -2197,6 +2245,10 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info
> *vinfo,
> return NULL;
> break;
> CASE_CFN_FFS:
> + /* Bail out if there is an outer statement which means we are looking
> for
> + clz/ctz pattern. */
> + if (outer_cond_stmt)
> + return NULL;
> ifn = IFN_FFS;
> break;
> default:
> @@ -2215,6 +2267,12 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info
> *vinfo,
> if (!rhs_origin)
> return NULL;
>
> + /* Verify that for
> + B = (A != 0) ? .CLZ (C) : N
> + A equals C. */
> + if (outer_cond_stmt && outer_cond_ne_op0 != rhs_oprnd)
> + return NULL;
> +
> /* Input and output of .POPCOUNT should be same-precision integer. */
> if (TYPE_PRECISION (unprom_diff.type) != TYPE_PRECISION (lhs_type))
> return NULL;
> @@ -2339,6 +2397,15 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info
> *vinfo,
> && CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type),
> val) == 2)
> arg2 = build_int_cst (integer_type_node, val);
> +
> + /* For pattern clz/ctz, reaching this point we have
> + B = (A != 0) ? .CLZ (A) : N
> + Now verify that the defined-at-zero value equals N. */
> + if (outer_cond_stmt
> + && (arg2 == NULL
> + || !tree_int_cst_equal (gimple_assign_rhs3 (outer_cond_stmt),
> arg2)))
> + return NULL;
> +
> if (arg2)
> pattern_stmt = gimple_build_call_internal (ifn, 2, unprom_diff.op, arg2);
> else
> --
> 2.55.0
>
