From: Kyrylo Tkachov <[email protected]>
The zstd sources in SPEC2026 compute 64-bit counts with the split idiom
on top of 32-bit table based counts:
unsigned
ZSTD_countTrailingZeros64 (U64 val)
{
U32 msw = val >> 32, lsw = val;
if (lsw == 0)
return 32 + ZSTD_countTrailingZeros32 (msw);
return ZSTD_countTrailingZeros32 (lsw);
}
forwprop turns each table lookup into a count that is defined at zero:
a two-argument .CTZ or .CLZ, followed by a mask or a select when the
table value for zero differs from the value of the target. If the
target does not define a value at zero, the select is on a one-argument
count, after the fix for PR127853. On aarch64 phiopt sees
_1 = val_5(D) >> 32;
msw_6 = (U32) _1;
lsw_7 = (U32) val_5(D);
if (lsw_7 == 0) goto <bb 3>; else goto <bb 4>;
<bb 3>:
_9 = .CTZ (msw_6, 32);
_11 = _9 & 31;
_13 = (unsigned int) _11;
_12 = _13 + 32;
goto <bb 5>;
<bb 4>:
_14 = .CTZ (lsw_7, 32);
_16 = (unsigned int) _14;
<bb 5>:
# _3 = PHI <_12(3), _16(4)>
Accept these forms of the counts. The select can also test X >> W,
which is what match.pd makes of a test of the high half. For a zero X
the idiom gives W plus the value at zero of the count in the second arm,
so keep that value with a select:
_15 = val_5(D) != 0;
_4 = .CTZ (val_5(D));
_10 = _15 ? _4 : 32;
_18 = (unsigned int) _10;
match.pd turns the select into a two-argument count when the target
defines that value at zero. Early phiopt also accepts the select. In
callers that test X, as zstd does, the select folds away.
On aarch64 ZSTD_countTrailingZeros64 becomes rbit, clz, cmp, mov and
csel instead of nine instructions with two counts. This gives ~3% on
the zstd benchmark in SPEC2026 on my aarch64 system.
Bootstrapped and tested on aarch64-none-linux-gnu and x86_64-linux.
Cc: Andrea Pinski <[email protected]>
gcc/ChangeLog:
PR tree-optimization/126035
* gimple-match-head.cc (gimple_count_value_at_zero): New
function.
* match.pd (ctz_nonzero, clz_nonzero): Also match counts that
are defined at zero, possibly masked or selected.
(LO != 0 ? ctz (LO) : W + ctz (HI) -> ctz (X)): Keep the value
at zero if it is defined.
(HI != 0 ? clz (HI) : W + clz (LO) -> clz (X)): Likewise.
* tree-ssa-phiopt.cc (phiopt_early_allow): Also allow a selected
count.
gcc/testsuite/ChangeLog:
PR tree-optimization/126035
* gcc.dg/torture/pr126035-3.c: New test.
* gcc.dg/tree-ssa/pr126035-5.c: New test.
* gcc.dg/tree-ssa/pr126035-6.c: New test.
* gcc.target/aarch64/pr126035-2.c: New test.
* gcc.target/aarch64/pr126035-3.c: New test.
Signed-off-by: Kyrylo Tkachov <[email protected]>
---
gcc/gimple-match-head.cc | 31 +++++
gcc/match.pd | 52 ++++++-
gcc/testsuite/gcc.dg/torture/pr126035-3.c | 131 ++++++++++++++++++
gcc/testsuite/gcc.dg/tree-ssa/pr126035-5.c | 51 +++++++
gcc/testsuite/gcc.dg/tree-ssa/pr126035-6.c | 55 ++++++++
gcc/testsuite/gcc.target/aarch64/pr126035-2.c | 65 +++++++++
gcc/testsuite/gcc.target/aarch64/pr126035-3.c | 69 +++++++++
gcc/tree-ssa-phiopt.cc | 5 +-
8 files changed, 451 insertions(+), 8 deletions(-)
create mode 100644 gcc/testsuite/gcc.dg/torture/pr126035-3.c
create mode 100644 gcc/testsuite/gcc.dg/tree-ssa/pr126035-5.c
create mode 100644 gcc/testsuite/gcc.dg/tree-ssa/pr126035-6.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/pr126035-2.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/pr126035-3.c
diff --git a/gcc/gimple-match-head.cc b/gcc/gimple-match-head.cc
index b2f26c71fc9..2423b977fb0 100644
--- a/gcc/gimple-match-head.cc
+++ b/gcc/gimple-match-head.cc
@@ -552,3 +552,34 @@ gimple_match_no_overflow_p (tree_code code, tree type,
tree op0, tree op1,
&& gimple_match_range_of_expr (vr1, op1, ctx)
&& range_op_handler (code).overflow_free_p (vr0, vr1));
}
+
+/* Return the value for a zero argument of COUNT, a count of trailing or
+ leading zeros that the ctz_nonzero or clz_nonzero predicate matched
+ with VALUEIZE, or NULL_TREE if it is undefined. */
+
+static inline tree
+gimple_count_value_at_zero (tree count, tree (*valueize)(tree))
+{
+ gimple *def = get_def (valueize, count);
+ tree zero;
+ if (gcall *call = dyn_cast <gcall *> (def))
+ {
+ if (gimple_call_num_args (call) == 1)
+ return NULL_TREE;
+ zero = do_valueize (valueize, gimple_call_arg (call, 1));
+ }
+ else if (gimple_assign_rhs_code (def) == COND_EXPR)
+ zero = do_valueize (valueize, gimple_assign_rhs2 (def));
+ else
+ {
+ gcc_checking_assert (gimple_assign_rhs_code (def) == BIT_AND_EXPR);
+ tree arg = do_valueize (valueize, gimple_assign_rhs1 (def));
+ tree mask = do_valueize (valueize, gimple_assign_rhs2 (def));
+ if (tree_swap_operands_p (arg, mask))
+ std::swap (arg, mask);
+ zero = int_const_binop (BIT_AND_EXPR,
+ gimple_count_value_at_zero (arg, valueize), mask);
+ }
+ gcc_checking_assert (TREE_CODE (zero) == INTEGER_CST);
+ return zero;
+}
diff --git a/gcc/match.pd b/gcc/match.pd
index d07c555baa6..6c04fbb1176 100644
--- a/gcc/match.pd
+++ b/gcc/match.pd
@@ -11876,16 +11876,48 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
&& TYPE_PRECISION (type) >= wi::to_widest (@1))))
/* Counts of the trailing and leading zeros of @0 that are exact for a
- nonzero @0. */
+ nonzero @0. forwprop emits table based counts as counts that are
+ defined at zero, possibly masked or selected for a zero @0. The select
+ can also test a right shift S if @0 is S converted and S has no more
+ significant bits than @0. */
(match (ctz_nonzero @0)
(CTZ @0))
+(match (ctz_nonzero @0)
+ (IFN_CTZ @0 INTEGER_CST))
+(match (ctz_nonzero @0)
+ (bit_and (IFN_CTZ @0 INTEGER_CST) INTEGER_CST@1)
+ (if (wi::ctz (~wi::to_wide (@1))
+ >= ceil_log2 (TYPE_PRECISION (TREE_TYPE (@0))))))
+(match (ctz_nonzero @0)
+ (cond (eq @0 integer_zerop) INTEGER_CST (ctz_nonzero @0)))
+(match (ctz_nonzero @0)
+ (cond (eq (rshift@1 @2 INTEGER_CST@3) integer_zerop) INTEGER_CST
+ (ctz_nonzero (convert@0 @1)))
+ (if (INTEGRAL_TYPE_P (TREE_TYPE (@2))
+ && (TYPE_PRECISION (TREE_TYPE (@2)) - wi::to_widest (@3)
+ <= TYPE_PRECISION (TREE_TYPE (@0))))))
(match (clz_nonzero @0)
(CLZ @0))
+(match (clz_nonzero @0)
+ (IFN_CLZ @0 INTEGER_CST))
+(match (clz_nonzero @0)
+ (bit_and (IFN_CLZ @0 INTEGER_CST) INTEGER_CST@1)
+ (if (wi::ctz (~wi::to_wide (@1))
+ >= ceil_log2 (TYPE_PRECISION (TREE_TYPE (@0))))))
+(match (clz_nonzero @0)
+ (cond (eq @0 integer_zerop) INTEGER_CST (clz_nonzero @0)))
+(match (clz_nonzero @0)
+ (cond (eq (rshift@1 @2 INTEGER_CST@3) integer_zerop) INTEGER_CST
+ (clz_nonzero (convert@0 @1)))
+ (if (INTEGRAL_TYPE_P (TREE_TYPE (@2))
+ && (TYPE_PRECISION (TREE_TYPE (@2)) - wi::to_widest (@3)
+ <= TYPE_PRECISION (TREE_TYPE (@0))))))
/* LO != 0 ? ctz (LO) : W + ctz (HI) is ctz (X) for a nonzero X, where LO
and HI are the low and high halves of the double-word integer X and W
is their precision. Likewise HI != 0 ? clz (HI) : W + clz (LO) is
- clz (X). */
+ clz (X). If the count in the second arm is defined at zero, so is the
+ result, and a select keeps its value. */
(for cnt (ctz_nonzero clz_nonzero)
tst (double_word_low_test double_word_high_test)
nzh (double_word_low double_word_high)
@@ -11894,14 +11926,22 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
(simplify
(cond (ne (tst @0) integer_zerop)
(convert1? (cnt (nzh @0)))
- (plus (convert? (cnt (zh@1 @0))) INTEGER_CST@2))
+ (plus (convert? (cnt@3 (zh@1 @0))) INTEGER_CST@2))
(with { int w = TYPE_PRECISION (TREE_TYPE (@1)); }
(if (wi::to_widest (@2) == w && 2 * w <= MAX_FIXED_MODE_SIZE)
- (with { tree utype = build_nonstandard_integer_type (2 * w, 1); }
+ (with { tree utype = build_nonstandard_integer_type (2 * w, 1);
+ /* The result for a zero X, if defined. */
+ tree zero = gimple_count_value_at_zero (@3, valueize);
+ widest_int zw = zero ? wi::to_widest (zero) + w : 0; }
(if (type_has_mode_precision_p (utype)
&& direct_internal_fn_supported_p (as_internal_fn (ifn), utype,
- OPTIMIZE_FOR_BOTH))
- (convert (ifn:integer_type_node (convert:utype @0)))))))))
+ OPTIMIZE_FOR_BOTH)
+ && wi::fits_to_tree_p (zw, integer_type_node))
+ (if (zero)
+ (convert (cond (ne (convert:utype @0) { build_zero_cst (utype); })
+ (ifn:integer_type_node (convert:utype @0))
+ { wide_int_to_tree (integer_type_node, zw); }))
+ (convert (ifn:integer_type_node (convert:utype @0))))))))))
#endif
/* Common POPCOUNT/PARITY simplifications. */
diff --git a/gcc/testsuite/gcc.dg/torture/pr126035-3.c
b/gcc/testsuite/gcc.dg/torture/pr126035-3.c
new file mode 100644
index 00000000000..4a17e67c312
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr126035-3.c
@@ -0,0 +1,131 @@
+/* PR tree-optimization/126035 */
+/* { dg-do run } */
+/* { dg-require-effective-target int32plus } */
+
+/* Check the combined counts of a double-word value that are defined at
+ zero, including the value at zero. */
+
+typedef __UINT64_TYPE__ U64;
+typedef __UINT32_TYPE__ U32;
+
+#define N __attribute__((noipa))
+
+static inline unsigned
+ctz32 (U32 val)
+{
+ static const U32 tab[32] = {
+ 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,
+ 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9 };
+ return tab[((U32) ((val & -val) * 0x077CB531U)) >> 27];
+}
+
+static inline unsigned
+clz32 (U32 val)
+{
+ static const U32 tab[32] = {
+ 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30,
+ 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
+ val |= val >> 1;
+ val |= val >> 2;
+ val |= val >> 4;
+ val |= val >> 8;
+ val |= val >> 16;
+ return 31 - tab[(val * 0x07C4ACDDU) >> 27];
+}
+
+/* ctz32 (0) is 0 and clz32 (0) is 31. */
+N unsigned
+ctz64 (U64 val)
+{
+ U32 msw = val >> 32, lsw = val;
+ return lsw == 0 ? 32 + ctz32 (msw) : ctz32 (lsw);
+}
+
+N unsigned
+clz64 (U64 val)
+{
+ U32 msw = val >> 32, lsw = val;
+ return msw == 0 ? 32 + clz32 (lsw) : clz32 (msw);
+}
+
+/* A named intermediate: with -g its debug bind is at the merge. */
+N unsigned
+ctz64_named (U64 val)
+{
+ U32 msw = val >> 32, lsw = val;
+ unsigned r;
+ if (lsw == 0)
+ r = 32 + ctz32 (msw);
+ else
+ r = ctz32 (lsw);
+ return r;
+}
+
+N unsigned
+ctzg64 (U64 val)
+{
+ U32 msw = val >> 32, lsw = val;
+ return lsw == 0 ? 32 + __builtin_ctzg (msw, 32) : __builtin_ctzg (lsw, 32);
+}
+
+N unsigned
+clzg64 (U64 val)
+{
+ U32 msw = val >> 32, lsw = val;
+ unsigned r = __builtin_clzg (msw, 32);
+ if (msw == 0)
+ r = 32 + __builtin_clzg (lsw, 32);
+ return r;
+}
+
+N unsigned
+mixed (U64 val)
+{
+ U32 msw = val >> 32, lsw = val;
+ return lsw ? __builtin_ctz (lsw) : 32 + __builtin_ctzg (msw, 32);
+}
+
+/* The value at zero does not fit a signed 7-bit field. */
+struct S { signed int c : 7; };
+
+N int
+narrow (U64 val)
+{
+ struct S s;
+ U32 msw = val >> 32, lsw = val;
+ if (lsw)
+ return __builtin_ctzg (lsw, 32);
+ s.c = 32 + __builtin_ctzg (msw, 32);
+ return s.c;
+}
+
+int
+main (void)
+{
+ if (ctz64 (0) != 32 || clz64 (0) != 63 || ctz64_named (0) != 32
+ || ctzg64 (0) != 64 || clzg64 (0) != 64 || mixed (0) != 64
+ || narrow (0) != -64)
+ __builtin_abort ();
+ U64 v = 0x9E3779B97F4A7C15ULL;
+ for (int i = 0; i < 100000; i++)
+ {
+ v = v * 6364136223846793005ULL + 1442695040888963407ULL;
+ U64 x = v;
+ switch (i & 7)
+ {
+ case 0: x = 1ULL << (v % 64); break;
+ case 1: x = (U32) v; break;
+ case 2: x = ((U64) (U32) v) << 32; break;
+ case 3: x = ~0ULL; break;
+ default: break;
+ }
+ if (x == 0)
+ x = 1;
+ unsigned t = __builtin_ctzll (x), l = __builtin_clzll (x);
+ if (ctz64 (x) != t || clz64 (x) != l || ctz64_named (x) != t
+ || ctzg64 (x) != t || clzg64 (x) != l || mixed (x) != t
+ || narrow (x) != (int) t)
+ __builtin_abort ();
+ }
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr126035-5.c
b/gcc/testsuite/gcc.dg/tree-ssa/pr126035-5.c
new file mode 100644
index 00000000000..d0bc5068fe4
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr126035-5.c
@@ -0,0 +1,51 @@
+/* PR tree-optimization/126035 */
+/* { dg-do compile { target aarch64*-*-* i?86-*-* x86_64-*-* } } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-options "-O2 -fdump-tree-optimized" } */
+
+/* The table based counts of the halves are recognized as counts that are
+ defined at zero. Their split double-word forms are still combined into
+ one count of the double-word value. */
+
+typedef __UINT64_TYPE__ U64;
+typedef __UINT32_TYPE__ U32;
+
+static inline unsigned
+ctz32 (U32 val)
+{
+ static const U32 tab[32] = {
+ 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,
+ 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9 };
+ return tab[((U32) ((val & -val) * 0x077CB531U)) >> 27];
+}
+
+static inline unsigned
+clz32 (U32 val)
+{
+ static const U32 tab[32] = {
+ 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30,
+ 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
+ val |= val >> 1;
+ val |= val >> 2;
+ val |= val >> 4;
+ val |= val >> 8;
+ val |= val >> 16;
+ return 31 - tab[(val * 0x07C4ACDDU) >> 27];
+}
+
+unsigned
+ctz64 (U64 val)
+{
+ U32 msw = val >> 32, lsw = val;
+ return lsw == 0 ? 32 + ctz32 (msw) : ctz32 (lsw);
+}
+
+unsigned
+clz64 (U64 val)
+{
+ U32 msw = val >> 32, lsw = val;
+ return msw == 0 ? 32 + clz32 (lsw) : clz32 (msw);
+}
+
+/* { dg-final { scan-tree-dump-times "\\.CTZ \\(" 1 "optimized" } } */
+/* { dg-final { scan-tree-dump-times "\\.CLZ \\(" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr126035-6.c
b/gcc/testsuite/gcc.dg/tree-ssa/pr126035-6.c
new file mode 100644
index 00000000000..73817bf0191
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr126035-6.c
@@ -0,0 +1,55 @@
+/* PR tree-optimization/126035 */
+/* { dg-do compile { target aarch64*-*-* i?86-*-* x86_64-*-* } } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-options "-O2 -fdump-tree-optimized" } */
+
+/* Early phiopt combines the table based split count in the out-of-line
+ body before inlining. After inlining, jump threading copies the test
+ of the result into the arms and late phiopt no longer sees the two
+ counts. */
+
+typedef __UINT64_TYPE__ U64;
+typedef __UINT32_TYPE__ U32;
+
+static inline unsigned
+ctz32 (U32 val)
+{
+ static const U32 tab[32] = {
+ 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,
+ 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9 };
+ return tab[((U32) ((val & -val) * 0x077CB531U)) >> 27];
+}
+
+static inline unsigned
+ctz64 (U64 val)
+{
+ U32 msw = val >> 32, lsw = val;
+ if (lsw == 0)
+ return 32 + ctz32 (msw);
+ return ctz32 (lsw);
+}
+
+static inline int
+ctz64_i (U64 val)
+{
+ U32 msw = val >> 32, lsw = val;
+ if (lsw == 0)
+ return 32 + (int) ctz32 (msw);
+ return ctz32 (lsw);
+}
+
+unsigned
+g1 (U64 d)
+{
+ unsigned l = ctz64 (d);
+ return l > 40 ? 0 : l;
+}
+
+int
+g2 (U64 d)
+{
+ int l = ctz64_i (d);
+ return l > 40 ? 0 : l;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.CTZ \\(" 2 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/pr126035-2.c
b/gcc/testsuite/gcc.target/aarch64/pr126035-2.c
new file mode 100644
index 00000000000..8dca163b7e6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/pr126035-2.c
@@ -0,0 +1,65 @@
+/* PR tree-optimization/126035 */
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#pragma GCC target "+nocssc"
+
+/* Split counts on top of table based counts. The double-word count is
+ defined at zero as 32 for ctz and 63 for clz, which needs a select. */
+
+typedef __UINT64_TYPE__ U64;
+typedef __UINT32_TYPE__ U32;
+
+static inline unsigned
+ctz32 (U32 val)
+{
+ static const U32 tab[32] = {
+ 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,
+ 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9 };
+ return tab[((U32) ((val & ((~val) + 1)) * 0x077CB531U)) >> 27];
+}
+
+static inline unsigned
+clz32 (U32 val)
+{
+ static const U32 tab[32] = {
+ 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30,
+ 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
+ val |= val >> 1;
+ val |= val >> 2;
+ val |= val >> 4;
+ val |= val >> 8;
+ val |= val >> 16;
+ return 31 - tab[(val * 0x07C4ACDDU) >> 27];
+}
+
+unsigned
+ctz64 (U64 val)
+{
+ U32 msw = val >> 32, lsw = val;
+ if (lsw == 0)
+ return 32 + ctz32 (msw);
+ return ctz32 (lsw);
+}
+
+unsigned
+clz64 (U64 val)
+{
+ U32 msw = val >> 32, lsw = val;
+ if (msw == 0)
+ return 32 + clz32 (lsw);
+ return clz32 (msw);
+}
+
+/* With a value of 64 at zero no select is needed. */
+unsigned
+ctzg64 (U64 val)
+{
+ U32 msw = val >> 32, lsw = val;
+ return lsw == 0 ? 32 + __builtin_ctzg (msw, 32) : __builtin_ctzg (lsw, 32);
+}
+
+/* { dg-final { scan-assembler-times {\trbit\tx[0-9]+, x[0-9]+} 2 } } */
+/* { dg-final { scan-assembler-times {\tclz\tx[0-9]+, x[0-9]+} 3 } } */
+/* { dg-final { scan-assembler-times {\tcsel\t} 2 } } */
+/* { dg-final { scan-assembler-not {\tldr} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/pr126035-3.c
b/gcc/testsuite/gcc.target/aarch64/pr126035-3.c
new file mode 100644
index 00000000000..733d4d43e4e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/pr126035-3.c
@@ -0,0 +1,69 @@
+/* PR tree-optimization/126035 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -fgimple -fdump-tree-optimized" } */
+
+/* FRE simplifies the select of the split counts. It looks through the
+ copy in f1 and matches the mask with the constant first in f2. */
+
+typedef __UINT64_TYPE__ U64;
+typedef __UINT32_TYPE__ U32;
+
+unsigned int __GIMPLE (ssa,startwith("fre1"))
+f1 (U64 x)
+{
+ U32 lo;
+ U32 hi;
+ U64 t;
+ int c1;
+ int c2;
+ int c3;
+ int c4;
+ int c5;
+ _Bool b;
+ int r2;
+ unsigned int r;
+
+ __BB(2):
+ lo_2 = (U32) x_1(D);
+ t_3 = x_1(D) >> 32;
+ hi_4 = (U32) t_3;
+ c1_5 = .CTZ (lo_2, 32);
+ c2_6 = .CTZ (hi_4, 32);
+ c3_7 = c2_6;
+ c4_8 = c3_7 & 31;
+ c5_9 = c4_8 + 32;
+ b_10 = lo_2 != 0u;
+ r2_11 = b_10 ? c1_5 : c5_9;
+ r_12 = (unsigned int) r2_11;
+ return r_12;
+}
+
+unsigned int __GIMPLE (ssa,startwith("fre1"))
+f2 (U64 x)
+{
+ U32 lo;
+ U32 hi;
+ U64 t;
+ int c1;
+ int c2;
+ int c4;
+ int c5;
+ _Bool b;
+ int r2;
+ unsigned int r;
+
+ __BB(2):
+ lo_2 = (U32) x_1(D);
+ t_3 = x_1(D) >> 32;
+ hi_4 = (U32) t_3;
+ c1_5 = .CTZ (lo_2, 32);
+ c2_6 = .CTZ (hi_4, 32);
+ c4_8 = 31 & c2_6;
+ c5_9 = c4_8 + 32;
+ b_10 = lo_2 != 0u;
+ r2_11 = b_10 ? c1_5 : c5_9;
+ r_12 = (unsigned int) r2_11;
+ return r_12;
+}
+
+/* { dg-final { scan-tree-dump-times {\.CTZ \(x_\d+\(D\)\)} 2 "optimized" } }
*/
diff --git a/gcc/tree-ssa-phiopt.cc b/gcc/tree-ssa-phiopt.cc
index 91f0d331b40..6d157850338 100644
--- a/gcc/tree-ssa-phiopt.cc
+++ b/gcc/tree-ssa-phiopt.cc
@@ -687,10 +687,11 @@ factor_out_conditional_operation (edge e0, edge e1,
basic_block merge,
static bool
phiopt_early_allow (gimple_seq &seq, gimple_match_op &op)
{
- /* Allow a count of leading or trailing zeros, possibly converted. */
+ /* Allow a count of leading or trailing zeros, possibly converted or
+ selected. */
if (op.code == CFN_CLZ || op.code == CFN_CTZ)
return true;
- if (op.code == NOP_EXPR)
+ if (op.code == SSA_NAME || op.code == NOP_EXPR || op.code == COND_EXPR)
for (gimple_stmt_iterator gsi = gsi_start (seq); !gsi_end_p (gsi);
gsi_next (&gsi))
if (gimple_call_internal_p (gsi_stmt (gsi), IFN_CLZ)
--
2.54.0 (Apple Git-157)
