BSR leaves its destination unmodified when the source is zero, so the
hardware treats the destination as an input. This creates a false
dependency on the previous value of the destination register, which
can delay BSR until that value is available. Clear the destination
before BSR via post-epilogue splitters when the destination is not
used by the source, controlled by the new
X86_TUNE_AVOID_FALSE_DEP_FOR_BSR tuning.
The tuning is enabled for processors where BSR is a single uop and the
dependency is longer than its throughput cost, and for generic.
Disable the new tuning in pr78103-*.c, which check the exact BSR
code sequence generated for generic tuning.
Assisted-by: Claude Sonnet 5.5
PR target/128662
gcc/ChangeLog:
* config/i386/i386.h (TARGET_AVOID_FALSE_DEP_FOR_BSR): New.
* config/i386/x86-tune.def (X86_TUNE_AVOID_FALSE_DEP_FOR_BSR):
New tuning.
* config/i386/i386.md (bsr_rex64): Convert to
define_insn_and_split. Split to *bsr_rex64_falsedep when
TARGET_AVOID_FALSE_DEP_FOR_BSR is set and the destination is not
mentioned in the source.
(bsr_rex64_1): Ditto. Split to *bsr_rex64_1_falsedep.
(bsr_rex64_1_zext): Ditto. Split to *bsr_rex64_1_zext_falsedep.
(bsr): Ditto. Split to *bsr_falsedep.
(bsr_1): Ditto. Split to *bsr_1_falsedep.
(bsr_zext_1): Ditto. Split to *bsr_zext_1_falsedep.
(*bsr_rex64_falsedep, *bsr_rex64_1_falsedep)
(*bsr_rex64_1_zext_falsedep, *bsr_falsedep, *bsr_1_falsedep)
(*bsr_zext_1_falsedep): New insns.
gcc/testsuite/ChangeLog:
* gcc.target/i386/pr128662.c: New test.
* gcc.target/i386/pr78103-1.c (dg-options): Add
-mtune-ctrl=^avoid_false_dep_for_bsr.
* gcc.target/i386/pr78103-2.c: Ditto.
* gcc.target/i386/pr78103-3.c: Ditto.
Bootstrapped and regression tested on x86_64-pc-linux-gnu {,-m32}.
Uros.
diff --git a/gcc/config/i386/i386.h b/gcc/config/i386/i386.h
index b28a3fc3b0e..31a36d2e274 100644
--- a/gcc/config/i386/i386.h
+++ b/gcc/config/i386/i386.h
@@ -489,7 +489,9 @@ extern unsigned char ix86_tune_features[X86_TUNE_LAST];
#define TARGET_SPLIT_MEM_OPND_FOR_FP_CONVERTS \
ix86_tune_features[X86_TUNE_SPLIT_MEM_OPND_FOR_FP_CONVERTS]
#define TARGET_ADJUST_UNROLL \
- ix86_tune_features[X86_TUNE_ADJUST_UNROLL]
+ ix86_tune_features[X86_TUNE_ADJUST_UNROLL]
+#define TARGET_AVOID_FALSE_DEP_FOR_BSR \
+ ix86_tune_features[X86_TUNE_AVOID_FALSE_DEP_FOR_BSR]
#define TARGET_AVOID_FALSE_DEP_FOR_BMI \
ix86_tune_features[X86_TUNE_AVOID_FALSE_DEP_FOR_BMI]
#define TARGET_AVOID_FALSE_DEP_FOR_TZCNT \
diff --git a/gcc/config/i386/i386.md b/gcc/config/i386/i386.md
index 2d2e620713f..23a9bc7fa59 100644
--- a/gcc/config/i386/i386.md
+++ b/gcc/config/i386/i386.md
@@ -22472,7 +22472,7 @@ (define_insn "*ctzsidi2_<s>ext_falsedep"
(set_attr "prefix_rep" "1")
(set_attr "mode" "SI")])
-(define_insn "bsr_rex64"
+(define_insn_and_split "bsr_rex64"
[(set (reg:CCZ FLAGS_REG)
(compare:CCZ (match_operand:DI 1 "nonimmediate_operand" "rm")
(const_int 0)))
@@ -22481,32 +22481,116 @@ (define_insn "bsr_rex64"
(clz:DI (match_dup 1))))]
"TARGET_64BIT"
"bsr{q}\t{%1, %0|%0, %1}"
+ "&& TARGET_AVOID_FALSE_DEP_FOR_BSR && epilogue_completed
+ && optimize_function_for_speed_p (cfun)
+ && !reg_mentioned_p (operands[0], operands[1])"
+ [(parallel
+ [(set (reg:CCZ FLAGS_REG)
+ (compare:CCZ (match_dup 1) (const_int 0)))
+ (set (match_dup 0)
+ (minus:DI (const_int 63)
+ (clz:DI (match_dup 1))))
+ (unspec [(match_dup 0)] UNSPEC_INSN_FALSE_DEP)])]
+ "ix86_expand_clear (operands[0]);"
+ [(set_attr "type" "alu1")
+ (set_attr "prefix_0f" "1")
+ (set_attr "znver1_decode" "vector")
+ (set_attr "c86_decode" "vector")
+ (set_attr "mode" "DI")])
+
+(define_insn "*bsr_rex64_falsedep"
+ [(set (reg:CCZ FLAGS_REG)
+ (compare:CCZ (match_operand:DI 1 "nonimmediate_operand" "rm")
+ (const_int 0)))
+ (set (match_operand:DI 0 "register_operand" "=r")
+ (minus:DI (const_int 63)
+ (clz:DI (match_dup 1))))
+ (unspec [(match_operand:DI 2 "register_operand" "0")]
+ UNSPEC_INSN_FALSE_DEP)]
+ "TARGET_64BIT"
+ "bsr{q}\t{%1, %0|%0, %1}"
+ [(set_attr "type" "alu1")
+ (set_attr "prefix_0f" "1")
+ (set_attr "znver1_decode" "vector")
+ (set_attr "c86_decode" "vector")
+ (set_attr "mode" "DI")])
+
+(define_insn_and_split "bsr_rex64_1"
+ [(set (match_operand:DI 0 "register_operand" "=r")
+ (minus:DI (const_int 63)
+ (clz:DI (match_operand:DI 1 "nonimmediate_operand" "rm"))))
+ (clobber (reg:CC FLAGS_REG))]
+ "!TARGET_LZCNT && TARGET_64BIT"
+ "bsr{q}\t{%1, %0|%0, %1}"
+ "&& TARGET_AVOID_FALSE_DEP_FOR_BSR && epilogue_completed
+ && optimize_function_for_speed_p (cfun)
+ && !reg_mentioned_p (operands[0], operands[1])"
+ [(parallel
+ [(set (match_dup 0)
+ (minus:DI (const_int 63)
+ (clz:DI (match_dup 1))))
+ (unspec [(match_dup 0)] UNSPEC_INSN_FALSE_DEP)
+ (clobber (reg:CC FLAGS_REG))])]
+ "ix86_expand_clear (operands[0]);"
[(set_attr "type" "alu1")
(set_attr "prefix_0f" "1")
(set_attr "znver1_decode" "vector")
(set_attr "c86_decode" "vector")
(set_attr "mode" "DI")])
-(define_insn "bsr_rex64_1"
+(define_insn "*bsr_rex64_1_falsedep"
[(set (match_operand:DI 0 "register_operand" "=r")
(minus:DI (const_int 63)
(clz:DI (match_operand:DI 1 "nonimmediate_operand" "rm"))))
+ (unspec [(match_operand:DI 2 "register_operand" "0")]
+ UNSPEC_INSN_FALSE_DEP)
+ (clobber (reg:CC FLAGS_REG))]
+ "!TARGET_LZCNT && TARGET_64BIT"
+ "bsr{q}\t{%1, %0|%0, %1}"
+ [(set_attr "type" "alu1")
+ (set_attr "prefix_0f" "1")
+ (set_attr "znver1_decode" "vector")
+ (set_attr "c86_decode" "vector")
+ (set_attr "mode" "DI")])
+
+(define_insn_and_split "bsr_rex64_1_zext"
+ [(set (match_operand:DI 0 "register_operand" "=r")
+ (zero_extend:DI
+ (minus:SI
+ (const_int 63)
+ (subreg:SI
+ (clz:DI (match_operand:DI 1 "nonimmediate_operand" "rm")) 0))))
(clobber (reg:CC FLAGS_REG))]
"!TARGET_LZCNT && TARGET_64BIT"
"bsr{q}\t{%1, %0|%0, %1}"
+ "&& TARGET_AVOID_FALSE_DEP_FOR_BSR && epilogue_completed
+ && optimize_function_for_speed_p (cfun)
+ && !reg_mentioned_p (operands[0], operands[1])"
+ [(parallel
+ [(set (match_dup 0)
+ (zero_extend:DI
+ (minus:SI
+ (const_int 63)
+ (subreg:SI
+ (clz:DI (match_dup 1)) 0))))
+ (unspec [(match_dup 0)] UNSPEC_INSN_FALSE_DEP)
+ (clobber (reg:CC FLAGS_REG))])]
+ "ix86_expand_clear (operands[0]);"
[(set_attr "type" "alu1")
(set_attr "prefix_0f" "1")
(set_attr "znver1_decode" "vector")
(set_attr "c86_decode" "vector")
(set_attr "mode" "DI")])
-(define_insn "bsr_rex64_1_zext"
+(define_insn "*bsr_rex64_1_zext_falsedep"
[(set (match_operand:DI 0 "register_operand" "=r")
(zero_extend:DI
- (minus:SI (const_int 63)
- (subreg:SI
- (clz:DI (match_operand:DI 1 "nonimmediate_operand" "rm"))
- 0))))
+ (minus:SI
+ (const_int 63)
+ (subreg:SI
+ (clz:DI (match_operand:DI 1 "nonimmediate_operand" "rm")) 0))))
+ (unspec [(match_operand:DI 2 "register_operand" "0")]
+ UNSPEC_INSN_FALSE_DEP)
(clobber (reg:CC FLAGS_REG))]
"!TARGET_LZCNT && TARGET_64BIT"
"bsr{q}\t{%1, %0|%0, %1}"
@@ -22516,7 +22600,7 @@ (define_insn "bsr_rex64_1_zext"
(set_attr "c86_decode" "vector")
(set_attr "mode" "DI")])
-(define_insn "bsr"
+(define_insn_and_split "bsr"
[(set (reg:CCZ FLAGS_REG)
(compare:CCZ (match_operand:SI 1 "nonimmediate_operand" "rm")
(const_int 0)))
@@ -22525,26 +22609,79 @@ (define_insn "bsr"
(clz:SI (match_dup 1))))]
""
"bsr{l}\t{%1, %0|%0, %1}"
+ "TARGET_AVOID_FALSE_DEP_FOR_BSR && epilogue_completed
+ && optimize_function_for_speed_p (cfun)
+ && !reg_mentioned_p (operands[0], operands[1])"
+ [(parallel
+ [(set (reg:CCZ FLAGS_REG)
+ (compare:CCZ (match_dup 1) (const_int 0)))
+ (set (match_dup 0)
+ (minus:SI (const_int 31)
+ (clz:SI (match_dup 1))))
+ (unspec [(match_dup 0)] UNSPEC_INSN_FALSE_DEP)])]
+ "ix86_expand_clear (operands[0]);"
+ [(set_attr "type" "alu1")
+ (set_attr "prefix_0f" "1")
+ (set_attr "znver1_decode" "vector")
+ (set_attr "c86_decode" "vector")
+ (set_attr "mode" "SI")])
+
+(define_insn "*bsr_falsedep"
+ [(set (reg:CCZ FLAGS_REG)
+ (compare:CCZ (match_operand:SI 1 "nonimmediate_operand" "rm")
+ (const_int 0)))
+ (set (match_operand:SI 0 "register_operand" "=r")
+ (minus:SI (const_int 31)
+ (clz:SI (match_dup 1))))
+ (unspec [(match_operand:SI 2 "register_operand" "0")]
+ UNSPEC_INSN_FALSE_DEP)]
+ ""
+ "bsr{l}\t{%1, %0|%0, %1}"
[(set_attr "type" "alu1")
(set_attr "prefix_0f" "1")
(set_attr "znver1_decode" "vector")
(set_attr "c86_decode" "vector")
(set_attr "mode" "SI")])
-(define_insn "bsr_1"
+(define_insn_and_split "bsr_1"
[(set (match_operand:SI 0 "register_operand" "=r")
(minus:SI (const_int 31)
(clz:SI (match_operand:SI 1 "nonimmediate_operand" "rm"))))
(clobber (reg:CC FLAGS_REG))]
"!TARGET_LZCNT"
"bsr{l}\t{%1, %0|%0, %1}"
+ "&& TARGET_AVOID_FALSE_DEP_FOR_BSR && epilogue_completed
+ && optimize_function_for_speed_p (cfun)
+ && !reg_mentioned_p (operands[0], operands[1])"
+ [(parallel
+ [(set (match_dup 0)
+ (minus:SI (const_int 31)
+ (clz:SI (match_dup 1))))
+ (unspec [(match_dup 0)] UNSPEC_INSN_FALSE_DEP)
+ (clobber (reg:CC FLAGS_REG))])]
+ "ix86_expand_clear (operands[0]);"
[(set_attr "type" "alu1")
(set_attr "prefix_0f" "1")
(set_attr "znver1_decode" "vector")
(set_attr "c86_decode" "vector")
(set_attr "mode" "SI")])
-(define_insn "bsr_zext_1"
+(define_insn "*bsr_1_falsedep"
+ [(set (match_operand:SI 0 "register_operand" "=r")
+ (minus:SI (const_int 31)
+ (clz:SI (match_operand:SI 1 "nonimmediate_operand" "rm"))))
+ (unspec [(match_operand:SI 2 "register_operand" "0")]
+ UNSPEC_INSN_FALSE_DEP)
+ (clobber (reg:CC FLAGS_REG))]
+ "!TARGET_LZCNT"
+ "bsr{l}\t{%1, %0|%0, %1}"
+ [(set_attr "type" "alu1")
+ (set_attr "prefix_0f" "1")
+ (set_attr "znver1_decode" "vector")
+ (set_attr "c86_decode" "vector")
+ (set_attr "mode" "SI")])
+
+(define_insn_and_split "bsr_zext_1"
[(set (match_operand:DI 0 "register_operand" "=r")
(zero_extend:DI
(minus:SI
@@ -22553,6 +22690,35 @@ (define_insn "bsr_zext_1"
(clobber (reg:CC FLAGS_REG))]
"!TARGET_LZCNT && TARGET_64BIT"
"bsr{l}\t{%1, %k0|%k0, %1}"
+ "&& TARGET_AVOID_FALSE_DEP_FOR_BSR && epilogue_completed
+ && optimize_function_for_speed_p (cfun)
+ && !reg_mentioned_p (operands[0], operands[1])"
+ [(parallel
+ [(set (match_dup 0)
+ (zero_extend:DI
+ (minus:SI
+ (const_int 31)
+ (clz:SI (match_dup 1)))))
+ (unspec [(match_dup 0)] UNSPEC_INSN_FALSE_DEP)
+ (clobber (reg:CC FLAGS_REG))])]
+ "ix86_expand_clear (operands[0]);"
+ [(set_attr "type" "alu1")
+ (set_attr "prefix_0f" "1")
+ (set_attr "znver1_decode" "vector")
+ (set_attr "c86_decode" "vector")
+ (set_attr "mode" "SI")])
+
+(define_insn "*bsr_zext_1_falsedep"
+ [(set (match_operand:DI 0 "register_operand" "=r")
+ (zero_extend:DI
+ (minus:SI
+ (const_int 31)
+ (clz:SI (match_operand:SI 1 "nonimmediate_operand" "rm")))))
+ (unspec [(match_operand:DI 2 "register_operand" "0")]
+ UNSPEC_INSN_FALSE_DEP)
+ (clobber (reg:CC FLAGS_REG))]
+ "!TARGET_LZCNT && TARGET_64BIT"
+ "bsr{l}\t{%1, %k0|%k0, %1}"
[(set_attr "type" "alu1")
(set_attr "prefix_0f" "1")
(set_attr "znver1_decode" "vector")
diff --git a/gcc/config/i386/x86-tune.def b/gcc/config/i386/x86-tune.def
index 11617220282..a219a688e7c 100644
--- a/gcc/config/i386/x86-tune.def
+++ b/gcc/config/i386/x86-tune.def
@@ -362,6 +362,12 @@ DEF_TUNE (X86_TUNE_USE_BT, "use_bt",
| m_AMD_MULTIPLE | m_ZHAOXIN | m_GOLDMONT | m_GOLDMONT_PLUS
| m_TREMONT | m_CORE_HYBRID | m_CORE_ATOM | m_C86_4G | m_GENERIC)
+/* X86_TUNE_AVOID_FALSE_DEP_FOR_BSR: BSR leaves its destination unmodified
+ for a zero source, so the hardware treats it as an input. */
+DEF_TUNE (X86_TUNE_AVOID_FALSE_DEP_FOR_BSR, "avoid_false_dep_for_bsr",
+ m_NEHALEM | m_SANDYBRIDGE | m_CORE_AVX2 | m_TREMONT | m_CORE_HYBRID
+ | m_CORE_ATOM | m_ZNVER4 | m_ZNVER5 | m_ZNVER6 | m_GENERIC)
+
/* X86_TUNE_AVOID_FALSE_DEP_FOR_BMI: Avoid false dependency
for bit-manipulation instructions. */
DEF_TUNE (X86_TUNE_AVOID_FALSE_DEP_FOR_BMI, "avoid_false_dep_for_bmi",
diff --git a/gcc/testsuite/gcc.target/i386/pr128662.c
b/gcc/testsuite/gcc.target/i386/pr128662.c
new file mode 100644
index 00000000000..fae9c0ae03a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr128662.c
@@ -0,0 +1,26 @@
+/* PR target/128662 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -mno-lzcnt -masm=att -mtune-ctrl=avoid_false_dep_for_bsr"
} */
+
+#define BITS (__SIZEOF_LONG__ * __CHAR_BIT__)
+
+unsigned long
+f1 (unsigned long x)
+{
+ return BITS - 1 - __builtin_clzl (x);
+}
+
+unsigned long
+f2 (unsigned long x)
+{
+ return __builtin_clzl (x) ^ (BITS - 1);
+}
+
+/* Zero-extending 32-bit variant (bsr_zext_1 on 64-bit targets). */
+unsigned long
+f3 (unsigned int x)
+{
+ return 31 - __builtin_clz (x);
+}
+
+/* { dg-final { scan-assembler-times {xorl\t(%\w+), \1\n\tbsr[lq]\t} 3 } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr78103-1.c
b/gcc/testsuite/gcc.target/i386/pr78103-1.c
index 95aea694c45..89f0282e971 100644
--- a/gcc/testsuite/gcc.target/i386/pr78103-1.c
+++ b/gcc/testsuite/gcc.target/i386/pr78103-1.c
@@ -1,6 +1,6 @@
/* PR target/78103 */
/* { dg-do compile { target { ! ia32 } } } */
-/* { dg-options "-O2 -mno-lzcnt" } */
+/* { dg-options "-O2 -mno-lzcnt -mtune-ctrl=^avoid_false_dep_for_bsr" } */
/* { dg-final { scan-assembler-not {\mcltq\M} } } */
long long
diff --git a/gcc/testsuite/gcc.target/i386/pr78103-2.c
b/gcc/testsuite/gcc.target/i386/pr78103-2.c
index 0183554389f..d5fb9303368 100644
--- a/gcc/testsuite/gcc.target/i386/pr78103-2.c
+++ b/gcc/testsuite/gcc.target/i386/pr78103-2.c
@@ -1,6 +1,6 @@
/* PR target/78103 */
/* { dg-do compile } */
-/* { dg-options "-O2 -mno-lzcnt" } */
+/* { dg-options "-O2 -mno-lzcnt -mtune-ctrl=^avoid_false_dep_for_bsr" } */
/* Cope with --enable-frame-pointer. */
/* { dg-additional-options "-fomit-frame-pointer" } */
/* { dg-final { scan-assembler-not {\mmovl\M} } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr78103-3.c
b/gcc/testsuite/gcc.target/i386/pr78103-3.c
index b8d82312a0e..69eb836d508 100644
--- a/gcc/testsuite/gcc.target/i386/pr78103-3.c
+++ b/gcc/testsuite/gcc.target/i386/pr78103-3.c
@@ -1,6 +1,6 @@
/* PR target/78103 */
/* { dg-do compile { target { ! ia32 } } } */
-/* { dg-options "-O2 -mno-lzcnt" } */
+/* { dg-options "-O2 -mno-lzcnt -mtune-ctrl=^avoid_false_dep_for_bsr" } */
/* { dg-final { scan-assembler-not {\mmovl\M} } } */
/* { dg-final { scan-assembler-not {\mmovslq\M} } } */
/* { dg-final { scan-assembler-not {\mxor[lq]\M} } } */
