https://gcc.gnu.org/g:9844856b32f318381ba0ded6da2ed2d548b778a7
commit r17-6230-g9844856b32f318381ba0ded6da2ed2d548b778a7 Author: Uros Bizjak <[email protected]> Date: Thu Nov 12 11:37:24 2026 +0100 i386: Avoid false dependency for BSR [PR128662] BSR leaves its destination unmodified when the source is zero, so the hardware treats the destination as an input. This creates a false dependency on the previous value of the destination register, which can delay BSR until that value is available. Clear the destination before BSR via post-epilogue splitters when the destination is not used by the source, controlled by the new X86_TUNE_AVOID_FALSE_DEP_FOR_BSR tuning. The tuning is enabled for processors where BSR is a single uop and the dependency is longer than its throughput cost, and for generic. Disable the new tuning in pr78103-*.c, which check the exact BSR code sequence generated for generic tuning. Assisted-by: Claude Sonnet 5.5 PR target/128662 gcc/ChangeLog: * config/i386/i386.h (TARGET_AVOID_FALSE_DEP_FOR_BSR): New. * config/i386/x86-tune.def (X86_TUNE_AVOID_FALSE_DEP_FOR_BSR): New tuning. * config/i386/i386.md (bsr_rex64): Convert to define_insn_and_split. Split to *bsr_rex64_falsedep when TARGET_AVOID_FALSE_DEP_FOR_BSR is set and the destination is not mentioned in the source. (bsr_rex64_1): Ditto. Split to *bsr_rex64_1_falsedep. (bsr_rex64_1_zext): Ditto. Split to *bsr_rex64_1_zext_falsedep. (bsr): Ditto. Split to *bsr_falsedep. (bsr_1): Ditto. Split to *bsr_1_falsedep. (bsr_zext_1): Ditto. Split to *bsr_zext_1_falsedep. (*bsr_rex64_falsedep, *bsr_rex64_1_falsedep) (*bsr_rex64_1_zext_falsedep, *bsr_falsedep, *bsr_1_falsedep) (*bsr_zext_1_falsedep): New insns. gcc/testsuite/ChangeLog: * gcc.target/i386/pr128662.c: New test. * gcc.target/i386/pr78103-1.c (dg-options): Add -mtune-ctrl=^avoid_false_dep_for_bsr. * gcc.target/i386/pr78103-2.c: Ditto. * gcc.target/i386/pr78103-3.c: Ditto. Diff: --- gcc/config/i386/i386.h | 4 +- gcc/config/i386/i386.md | 186 ++++++++++++++++++++++++++++-- gcc/config/i386/x86-tune.def | 6 + gcc/testsuite/gcc.target/i386/pr128662.c | 26 +++++ gcc/testsuite/gcc.target/i386/pr78103-1.c | 2 +- gcc/testsuite/gcc.target/i386/pr78103-2.c | 2 +- gcc/testsuite/gcc.target/i386/pr78103-3.c | 2 +- 7 files changed, 214 insertions(+), 14 deletions(-) diff --git a/gcc/config/i386/i386.h b/gcc/config/i386/i386.h index b28a3fc3b0e4..31a36d2e274f 100644 --- a/gcc/config/i386/i386.h +++ b/gcc/config/i386/i386.h @@ -489,7 +489,9 @@ extern unsigned char ix86_tune_features[X86_TUNE_LAST]; #define TARGET_SPLIT_MEM_OPND_FOR_FP_CONVERTS \ ix86_tune_features[X86_TUNE_SPLIT_MEM_OPND_FOR_FP_CONVERTS] #define TARGET_ADJUST_UNROLL \ - ix86_tune_features[X86_TUNE_ADJUST_UNROLL] + ix86_tune_features[X86_TUNE_ADJUST_UNROLL] +#define TARGET_AVOID_FALSE_DEP_FOR_BSR \ + ix86_tune_features[X86_TUNE_AVOID_FALSE_DEP_FOR_BSR] #define TARGET_AVOID_FALSE_DEP_FOR_BMI \ ix86_tune_features[X86_TUNE_AVOID_FALSE_DEP_FOR_BMI] #define TARGET_AVOID_FALSE_DEP_FOR_TZCNT \ diff --git a/gcc/config/i386/i386.md b/gcc/config/i386/i386.md index 2d2e620713f1..23a9bc7fa591 100644 --- a/gcc/config/i386/i386.md +++ b/gcc/config/i386/i386.md @@ -22472,7 +22472,7 @@ (set_attr "prefix_rep" "1") (set_attr "mode" "SI")]) -(define_insn "bsr_rex64" +(define_insn_and_split "bsr_rex64" [(set (reg:CCZ FLAGS_REG) (compare:CCZ (match_operand:DI 1 "nonimmediate_operand" "rm") (const_int 0))) @@ -22481,32 +22481,116 @@ (clz:DI (match_dup 1))))] "TARGET_64BIT" "bsr{q}\t{%1, %0|%0, %1}" + "&& TARGET_AVOID_FALSE_DEP_FOR_BSR && epilogue_completed + && optimize_function_for_speed_p (cfun) + && !reg_mentioned_p (operands[0], operands[1])" + [(parallel + [(set (reg:CCZ FLAGS_REG) + (compare:CCZ (match_dup 1) (const_int 0))) + (set (match_dup 0) + (minus:DI (const_int 63) + (clz:DI (match_dup 1)))) + (unspec [(match_dup 0)] UNSPEC_INSN_FALSE_DEP)])] + "ix86_expand_clear (operands[0]);" + [(set_attr "type" "alu1") + (set_attr "prefix_0f" "1") + (set_attr "znver1_decode" "vector") + (set_attr "c86_decode" "vector") + (set_attr "mode" "DI")]) + +(define_insn "*bsr_rex64_falsedep" + [(set (reg:CCZ FLAGS_REG) + (compare:CCZ (match_operand:DI 1 "nonimmediate_operand" "rm") + (const_int 0))) + (set (match_operand:DI 0 "register_operand" "=r") + (minus:DI (const_int 63) + (clz:DI (match_dup 1)))) + (unspec [(match_operand:DI 2 "register_operand" "0")] + UNSPEC_INSN_FALSE_DEP)] + "TARGET_64BIT" + "bsr{q}\t{%1, %0|%0, %1}" + [(set_attr "type" "alu1") + (set_attr "prefix_0f" "1") + (set_attr "znver1_decode" "vector") + (set_attr "c86_decode" "vector") + (set_attr "mode" "DI")]) + +(define_insn_and_split "bsr_rex64_1" + [(set (match_operand:DI 0 "register_operand" "=r") + (minus:DI (const_int 63) + (clz:DI (match_operand:DI 1 "nonimmediate_operand" "rm")))) + (clobber (reg:CC FLAGS_REG))] + "!TARGET_LZCNT && TARGET_64BIT" + "bsr{q}\t{%1, %0|%0, %1}" + "&& TARGET_AVOID_FALSE_DEP_FOR_BSR && epilogue_completed + && optimize_function_for_speed_p (cfun) + && !reg_mentioned_p (operands[0], operands[1])" + [(parallel + [(set (match_dup 0) + (minus:DI (const_int 63) + (clz:DI (match_dup 1)))) + (unspec [(match_dup 0)] UNSPEC_INSN_FALSE_DEP) + (clobber (reg:CC FLAGS_REG))])] + "ix86_expand_clear (operands[0]);" [(set_attr "type" "alu1") (set_attr "prefix_0f" "1") (set_attr "znver1_decode" "vector") (set_attr "c86_decode" "vector") (set_attr "mode" "DI")]) -(define_insn "bsr_rex64_1" +(define_insn "*bsr_rex64_1_falsedep" [(set (match_operand:DI 0 "register_operand" "=r") (minus:DI (const_int 63) (clz:DI (match_operand:DI 1 "nonimmediate_operand" "rm")))) + (unspec [(match_operand:DI 2 "register_operand" "0")] + UNSPEC_INSN_FALSE_DEP) + (clobber (reg:CC FLAGS_REG))] + "!TARGET_LZCNT && TARGET_64BIT" + "bsr{q}\t{%1, %0|%0, %1}" + [(set_attr "type" "alu1") + (set_attr "prefix_0f" "1") + (set_attr "znver1_decode" "vector") + (set_attr "c86_decode" "vector") + (set_attr "mode" "DI")]) + +(define_insn_and_split "bsr_rex64_1_zext" + [(set (match_operand:DI 0 "register_operand" "=r") + (zero_extend:DI + (minus:SI + (const_int 63) + (subreg:SI + (clz:DI (match_operand:DI 1 "nonimmediate_operand" "rm")) 0)))) (clobber (reg:CC FLAGS_REG))] "!TARGET_LZCNT && TARGET_64BIT" "bsr{q}\t{%1, %0|%0, %1}" + "&& TARGET_AVOID_FALSE_DEP_FOR_BSR && epilogue_completed + && optimize_function_for_speed_p (cfun) + && !reg_mentioned_p (operands[0], operands[1])" + [(parallel + [(set (match_dup 0) + (zero_extend:DI + (minus:SI + (const_int 63) + (subreg:SI + (clz:DI (match_dup 1)) 0)))) + (unspec [(match_dup 0)] UNSPEC_INSN_FALSE_DEP) + (clobber (reg:CC FLAGS_REG))])] + "ix86_expand_clear (operands[0]);" [(set_attr "type" "alu1") (set_attr "prefix_0f" "1") (set_attr "znver1_decode" "vector") (set_attr "c86_decode" "vector") (set_attr "mode" "DI")]) -(define_insn "bsr_rex64_1_zext" +(define_insn "*bsr_rex64_1_zext_falsedep" [(set (match_operand:DI 0 "register_operand" "=r") (zero_extend:DI - (minus:SI (const_int 63) - (subreg:SI - (clz:DI (match_operand:DI 1 "nonimmediate_operand" "rm")) - 0)))) + (minus:SI + (const_int 63) + (subreg:SI + (clz:DI (match_operand:DI 1 "nonimmediate_operand" "rm")) 0)))) + (unspec [(match_operand:DI 2 "register_operand" "0")] + UNSPEC_INSN_FALSE_DEP) (clobber (reg:CC FLAGS_REG))] "!TARGET_LZCNT && TARGET_64BIT" "bsr{q}\t{%1, %0|%0, %1}" @@ -22516,7 +22600,7 @@ (set_attr "c86_decode" "vector") (set_attr "mode" "DI")]) -(define_insn "bsr" +(define_insn_and_split "bsr" [(set (reg:CCZ FLAGS_REG) (compare:CCZ (match_operand:SI 1 "nonimmediate_operand" "rm") (const_int 0))) @@ -22525,26 +22609,79 @@ (clz:SI (match_dup 1))))] "" "bsr{l}\t{%1, %0|%0, %1}" + "TARGET_AVOID_FALSE_DEP_FOR_BSR && epilogue_completed + && optimize_function_for_speed_p (cfun) + && !reg_mentioned_p (operands[0], operands[1])" + [(parallel + [(set (reg:CCZ FLAGS_REG) + (compare:CCZ (match_dup 1) (const_int 0))) + (set (match_dup 0) + (minus:SI (const_int 31) + (clz:SI (match_dup 1)))) + (unspec [(match_dup 0)] UNSPEC_INSN_FALSE_DEP)])] + "ix86_expand_clear (operands[0]);" + [(set_attr "type" "alu1") + (set_attr "prefix_0f" "1") + (set_attr "znver1_decode" "vector") + (set_attr "c86_decode" "vector") + (set_attr "mode" "SI")]) + +(define_insn "*bsr_falsedep" + [(set (reg:CCZ FLAGS_REG) + (compare:CCZ (match_operand:SI 1 "nonimmediate_operand" "rm") + (const_int 0))) + (set (match_operand:SI 0 "register_operand" "=r") + (minus:SI (const_int 31) + (clz:SI (match_dup 1)))) + (unspec [(match_operand:SI 2 "register_operand" "0")] + UNSPEC_INSN_FALSE_DEP)] + "" + "bsr{l}\t{%1, %0|%0, %1}" [(set_attr "type" "alu1") (set_attr "prefix_0f" "1") (set_attr "znver1_decode" "vector") (set_attr "c86_decode" "vector") (set_attr "mode" "SI")]) -(define_insn "bsr_1" +(define_insn_and_split "bsr_1" [(set (match_operand:SI 0 "register_operand" "=r") (minus:SI (const_int 31) (clz:SI (match_operand:SI 1 "nonimmediate_operand" "rm")))) (clobber (reg:CC FLAGS_REG))] "!TARGET_LZCNT" "bsr{l}\t{%1, %0|%0, %1}" + "&& TARGET_AVOID_FALSE_DEP_FOR_BSR && epilogue_completed + && optimize_function_for_speed_p (cfun) + && !reg_mentioned_p (operands[0], operands[1])" + [(parallel + [(set (match_dup 0) + (minus:SI (const_int 31) + (clz:SI (match_dup 1)))) + (unspec [(match_dup 0)] UNSPEC_INSN_FALSE_DEP) + (clobber (reg:CC FLAGS_REG))])] + "ix86_expand_clear (operands[0]);" [(set_attr "type" "alu1") (set_attr "prefix_0f" "1") (set_attr "znver1_decode" "vector") (set_attr "c86_decode" "vector") (set_attr "mode" "SI")]) -(define_insn "bsr_zext_1" +(define_insn "*bsr_1_falsedep" + [(set (match_operand:SI 0 "register_operand" "=r") + (minus:SI (const_int 31) + (clz:SI (match_operand:SI 1 "nonimmediate_operand" "rm")))) + (unspec [(match_operand:SI 2 "register_operand" "0")] + UNSPEC_INSN_FALSE_DEP) + (clobber (reg:CC FLAGS_REG))] + "!TARGET_LZCNT" + "bsr{l}\t{%1, %0|%0, %1}" + [(set_attr "type" "alu1") + (set_attr "prefix_0f" "1") + (set_attr "znver1_decode" "vector") + (set_attr "c86_decode" "vector") + (set_attr "mode" "SI")]) + +(define_insn_and_split "bsr_zext_1" [(set (match_operand:DI 0 "register_operand" "=r") (zero_extend:DI (minus:SI @@ -22553,6 +22690,35 @@ (clobber (reg:CC FLAGS_REG))] "!TARGET_LZCNT && TARGET_64BIT" "bsr{l}\t{%1, %k0|%k0, %1}" + "&& TARGET_AVOID_FALSE_DEP_FOR_BSR && epilogue_completed + && optimize_function_for_speed_p (cfun) + && !reg_mentioned_p (operands[0], operands[1])" + [(parallel + [(set (match_dup 0) + (zero_extend:DI + (minus:SI + (const_int 31) + (clz:SI (match_dup 1))))) + (unspec [(match_dup 0)] UNSPEC_INSN_FALSE_DEP) + (clobber (reg:CC FLAGS_REG))])] + "ix86_expand_clear (operands[0]);" + [(set_attr "type" "alu1") + (set_attr "prefix_0f" "1") + (set_attr "znver1_decode" "vector") + (set_attr "c86_decode" "vector") + (set_attr "mode" "SI")]) + +(define_insn "*bsr_zext_1_falsedep" + [(set (match_operand:DI 0 "register_operand" "=r") + (zero_extend:DI + (minus:SI + (const_int 31) + (clz:SI (match_operand:SI 1 "nonimmediate_operand" "rm"))))) + (unspec [(match_operand:DI 2 "register_operand" "0")] + UNSPEC_INSN_FALSE_DEP) + (clobber (reg:CC FLAGS_REG))] + "!TARGET_LZCNT && TARGET_64BIT" + "bsr{l}\t{%1, %k0|%k0, %1}" [(set_attr "type" "alu1") (set_attr "prefix_0f" "1") (set_attr "znver1_decode" "vector") diff --git a/gcc/config/i386/x86-tune.def b/gcc/config/i386/x86-tune.def index 116172202821..a219a688e7c1 100644 --- a/gcc/config/i386/x86-tune.def +++ b/gcc/config/i386/x86-tune.def @@ -362,6 +362,12 @@ DEF_TUNE (X86_TUNE_USE_BT, "use_bt", | m_AMD_MULTIPLE | m_ZHAOXIN | m_GOLDMONT | m_GOLDMONT_PLUS | m_TREMONT | m_CORE_HYBRID | m_CORE_ATOM | m_C86_4G | m_GENERIC) +/* X86_TUNE_AVOID_FALSE_DEP_FOR_BSR: BSR leaves its destination unmodified + for a zero source, so the hardware treats it as an input. */ +DEF_TUNE (X86_TUNE_AVOID_FALSE_DEP_FOR_BSR, "avoid_false_dep_for_bsr", + m_NEHALEM | m_SANDYBRIDGE | m_CORE_AVX2 | m_TREMONT | m_CORE_HYBRID + | m_CORE_ATOM | m_ZNVER4 | m_ZNVER5 | m_ZNVER6 | m_GENERIC) + /* X86_TUNE_AVOID_FALSE_DEP_FOR_BMI: Avoid false dependency for bit-manipulation instructions. */ DEF_TUNE (X86_TUNE_AVOID_FALSE_DEP_FOR_BMI, "avoid_false_dep_for_bmi", diff --git a/gcc/testsuite/gcc.target/i386/pr128662.c b/gcc/testsuite/gcc.target/i386/pr128662.c new file mode 100644 index 000000000000..fae9c0ae03aa --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/pr128662.c @@ -0,0 +1,26 @@ +/* PR target/128662 */ +/* { dg-do compile } */ +/* { dg-options "-O2 -mno-lzcnt -masm=att -mtune-ctrl=avoid_false_dep_for_bsr" } */ + +#define BITS (__SIZEOF_LONG__ * __CHAR_BIT__) + +unsigned long +f1 (unsigned long x) +{ + return BITS - 1 - __builtin_clzl (x); +} + +unsigned long +f2 (unsigned long x) +{ + return __builtin_clzl (x) ^ (BITS - 1); +} + +/* Zero-extending 32-bit variant (bsr_zext_1 on 64-bit targets). */ +unsigned long +f3 (unsigned int x) +{ + return 31 - __builtin_clz (x); +} + +/* { dg-final { scan-assembler-times {xorl\t(%\w+), \1\n\tbsr[lq]\t} 3 } } */ diff --git a/gcc/testsuite/gcc.target/i386/pr78103-1.c b/gcc/testsuite/gcc.target/i386/pr78103-1.c index 95aea694c45a..89f0282e971c 100644 --- a/gcc/testsuite/gcc.target/i386/pr78103-1.c +++ b/gcc/testsuite/gcc.target/i386/pr78103-1.c @@ -1,6 +1,6 @@ /* PR target/78103 */ /* { dg-do compile { target { ! ia32 } } } */ -/* { dg-options "-O2 -mno-lzcnt" } */ +/* { dg-options "-O2 -mno-lzcnt -mtune-ctrl=^avoid_false_dep_for_bsr" } */ /* { dg-final { scan-assembler-not {\mcltq\M} } } */ long long diff --git a/gcc/testsuite/gcc.target/i386/pr78103-2.c b/gcc/testsuite/gcc.target/i386/pr78103-2.c index 0183554389f1..d5fb93033682 100644 --- a/gcc/testsuite/gcc.target/i386/pr78103-2.c +++ b/gcc/testsuite/gcc.target/i386/pr78103-2.c @@ -1,6 +1,6 @@ /* PR target/78103 */ /* { dg-do compile } */ -/* { dg-options "-O2 -mno-lzcnt" } */ +/* { dg-options "-O2 -mno-lzcnt -mtune-ctrl=^avoid_false_dep_for_bsr" } */ /* Cope with --enable-frame-pointer. */ /* { dg-additional-options "-fomit-frame-pointer" } */ /* { dg-final { scan-assembler-not {\mmovl\M} } } */ diff --git a/gcc/testsuite/gcc.target/i386/pr78103-3.c b/gcc/testsuite/gcc.target/i386/pr78103-3.c index b8d82312a0ed..69eb836d508f 100644 --- a/gcc/testsuite/gcc.target/i386/pr78103-3.c +++ b/gcc/testsuite/gcc.target/i386/pr78103-3.c @@ -1,6 +1,6 @@ /* PR target/78103 */ /* { dg-do compile { target { ! ia32 } } } */ -/* { dg-options "-O2 -mno-lzcnt" } */ +/* { dg-options "-O2 -mno-lzcnt -mtune-ctrl=^avoid_false_dep_for_bsr" } */ /* { dg-final { scan-assembler-not {\mmovl\M} } } */ /* { dg-final { scan-assembler-not {\mmovslq\M} } } */ /* { dg-final { scan-assembler-not {\mxor[lq]\M} } } */
