Since LCP stall peepholes are added after register allocation, each
peephole may use a different scratch register. For input:
extern void bar (void);
void
foo (short *dst)
{
dst[0] = 3;
asm volatile ("" : : : "memory");
dst[2] = 3;
bar ();
dst[1] = 3;
asm volatile ("" : : : "memory");
dst[4] = 3;
}
with LCP stall peepholes, GCC generates:
movl $3, %eax
pushq %rbx
movq %rdi, %rbx
movw %ax, (%rdi)
movl $3, %edx
movw %dx, 4(%rdi)
call bar
movl $3, %ecx
movw %cx, 2(%rbx)
movl $3, %esi
movw %si, 8(%rbx)
popq %rbx
using 4 different scratch registers vs without LCP stall peepholes:
pushq %rbx
movq %rdi, %rbx
movw $3, (%rdi)
movw $3, 4(%rdi)
call bar
movw $3, 2(%rbx)
movw $3, 8(%rbx)
popq %rbx
Add X86_CSE_LCP_STALL to x86_cse to run before the cprop_hardreg pass
if there are multiple LCP stall peepholes:
1. Collect all LCP stall peepholes along with their scratch register
definitions in the same basic block. Group peepholes together with the
same integer constant in their scratch registers if the first scratch
register definition doesn't become invalid beyond a barrier, which can
be a function call, a definition or a use, before other peepholes.
2. Track scratch register barriers after scratch register definitions.
3. Mark a scratch register definition redundant in the group if the first
scratch register doesn't become invalid beyond a barrier.
4. Replace the redundant scratch register in LCP stall peepholes in the
group with the first scratch register and mark the scratch register
definition redundant for deletion.
5. Delete redundant scratch register definitions at the end.
so that the same scratch register can be reused if possible:
movl $3, %eax
pushq %rbx
movq %rdi, %rbx
movw %ax, (%rdi)
movw %ax, 4(%rdi)
call bar
movl $3, %ecx
movw %cx, 2(%rbx)
movw %cx, 8(%rbx)
popq %rbx
gcc/
PR target/125893
* config/i386/i386-features.cc (x86_cse_kind): Add
X86_CSE_LCP_STALL.
(replace_vector_const): Skip if SRC == VECTOR_CONST.
(pass_x86_cse::gate): Also run if there are multiple LCP stall
peepholes and reload is completed.
(pass_x86_cse::clone): New.
(pass_x86_cse::candidate_lcp_stall_p): Likewise.
(pass_x86_cse::x86_cse): Replace the redundant scratch register
in LCP stall peepholes in the group with the first scratch
register and mark the scratch register definition redundant for
deletion. Delete redundant scratch register definitions at the
end.
* config/i386/i386-passes.def: Add pass_x86_cse before
pass_cprop_hardreg.
* config/i386/i386.h (machine_function): Add
lcp_stall_peephole_generated.
(ix86_lcp_stall_peephole_generated): New.
* config/i386/i386.md (TARGET_LCP_STALL peepholes): Update
ix86_lcp_stall_peephole_generated.
gcc/testsuite/
PR target/125893
* gcc.target/i386/pr125893-1.c: New test.
* gcc.target/i386/pr125893-2.c: Likewise.
* gcc.target/i386/pr125893-3.c: Likewise.
* gcc.target/i386/pr125893-4.c: Likewise.
* gcc.target/i386/pr125893-5.c: Likewise.
* gcc.target/i386/pr125893-6.c: Likewise.
--
H.J.
From 2f0b74f23a95b0f7323a0633d45736176fa19a1a Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <[email protected]>
Date: Fri, 19 Jun 2026 11:20:22 +0800
Subject: [PATCH] x86_cse: Add X86_CSE_LCP_STALL
Since LCP stall peepholes are added after register allocation, each
peephole may use a different scratch register. For input:
extern void bar (void);
void
foo (short *dst)
{
dst[0] = 3;
asm volatile ("" : : : "memory");
dst[2] = 3;
bar ();
dst[1] = 3;
asm volatile ("" : : : "memory");
dst[4] = 3;
}
with LCP stall peepholes, GCC generates:
movl $3, %eax
pushq %rbx
movq %rdi, %rbx
movw %ax, (%rdi)
movl $3, %edx
movw %dx, 4(%rdi)
call bar
movl $3, %ecx
movw %cx, 2(%rbx)
movl $3, %esi
movw %si, 8(%rbx)
popq %rbx
using 4 different scratch registers vs without LCP stall peepholes:
pushq %rbx
movq %rdi, %rbx
movw $3, (%rdi)
movw $3, 4(%rdi)
call bar
movw $3, 2(%rbx)
movw $3, 8(%rbx)
popq %rbx
Add X86_CSE_LCP_STALL to x86_cse to run before the cprop_hardreg pass
if there are multiple LCP stall peepholes:
1. Collect all LCP stall peepholes along with their scratch register
definitions in the same basic block. Group peepholes together with the
same integer constant in their scratch registers if the first scratch
register definition doesn't become invalid beyond a barrier, which can
be a function call, a definition or a use, before other peepholes.
2. Track scratch register barriers after scratch register definitions.
3. Mark a scratch register definition redundant in the group if the first
scratch register doesn't become invalid beyond a barrier.
4. Replace the redundant scratch register in LCP stall peepholes in the
group with the first scratch register and mark the scratch register
definition redundant for deletion.
5. Delete redundant scratch register definitions at the end.
so that the same scratch register can be reused if possible:
movl $3, %eax
pushq %rbx
movq %rdi, %rbx
movw %ax, (%rdi)
movw %ax, 4(%rdi)
call bar
movl $3, %ecx
movw %cx, 2(%rbx)
movw %cx, 8(%rbx)
popq %rbx
gcc/
PR target/125893
* config/i386/i386-features.cc (x86_cse_kind): Add
X86_CSE_LCP_STALL.
(replace_vector_const): Skip if SRC == VECTOR_CONST.
(pass_x86_cse::gate): Also run if there are multiple LCP stall
peepholes and reload is completed.
(pass_x86_cse::clone): New.
(pass_x86_cse::candidate_lcp_stall_p): Likewise.
(pass_x86_cse::x86_cse): Replace the redundant scratch register
in LCP stall peepholes in the group with the first scratch
register and mark the scratch register definition redundant for
deletion. Delete redundant scratch register definitions at the
end.
* config/i386/i386-passes.def: Add pass_x86_cse before
pass_cprop_hardreg.
* config/i386/i386.h (machine_function): Add
lcp_stall_peephole_generated.
(ix86_lcp_stall_peephole_generated): New.
* config/i386/i386.md (TARGET_LCP_STALL peepholes): Update
ix86_lcp_stall_peephole_generated.
gcc/testsuite/
PR target/125893
* gcc.target/i386/pr125893-1.c: New test.
* gcc.target/i386/pr125893-2.c: Likewise.
* gcc.target/i386/pr125893-3.c: Likewise.
* gcc.target/i386/pr125893-4.c: Likewise.
* gcc.target/i386/pr125893-5.c: Likewise.
* gcc.target/i386/pr125893-6.c: Likewise.
Signed-off-by: H.J. Lu <[email protected]>
---
gcc/config/i386/i386-features.cc | 378 +++++++++++++++++++--
gcc/config/i386/i386-passes.def | 1 +
gcc/config/i386/i386.h | 9 +
gcc/config/i386/i386.md | 12 +-
gcc/testsuite/gcc.target/i386/pr125893-1.c | 27 ++
gcc/testsuite/gcc.target/i386/pr125893-2.c | 33 ++
gcc/testsuite/gcc.target/i386/pr125893-3.c | 27 ++
gcc/testsuite/gcc.target/i386/pr125893-4.c | 33 ++
gcc/testsuite/gcc.target/i386/pr125893-5.c | 34 ++
gcc/testsuite/gcc.target/i386/pr125893-6.c | 34 ++
10 files changed, 551 insertions(+), 37 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/i386/pr125893-1.c
create mode 100644 gcc/testsuite/gcc.target/i386/pr125893-2.c
create mode 100644 gcc/testsuite/gcc.target/i386/pr125893-3.c
create mode 100644 gcc/testsuite/gcc.target/i386/pr125893-4.c
create mode 100644 gcc/testsuite/gcc.target/i386/pr125893-5.c
create mode 100644 gcc/testsuite/gcc.target/i386/pr125893-6.c
diff --git a/gcc/config/i386/i386-features.cc b/gcc/config/i386/i386-features.cc
index b0d6e066a43..810d0f6aab5 100644
--- a/gcc/config/i386/i386-features.cc
+++ b/gcc/config/i386/i386-features.cc
@@ -3319,7 +3319,8 @@ enum x86_cse_kind
X86_CSE_VEC_DUP,
X86_CSE_TLS_GD,
X86_CSE_TLS_LD_BASE,
- X86_CSE_TLSDESC
+ X86_CSE_TLSDESC,
+ X86_CSE_LCP_STALL
};
struct redundant_pattern
@@ -3757,6 +3758,11 @@ replace_vector_const (machine_mode vector_mode, rtx vector_const,
/* Get the single SET instruction. */
rtx set = single_set (insn);
rtx src = SET_SRC (set);
+
+ /* Skip if SRC == VECTOR_CONST. */
+ if (rtx_equal_p (src, vector_const))
+ continue;
+
rtx dest = SET_DEST (set);
machine_mode mode = GET_MODE (dest);
@@ -4620,7 +4626,12 @@ public:
/* opt_pass methods: */
bool gate (function *fun) final override
{
- return optimize && optimize_function_for_speed_p (fun);
+ /* Also run if there are multiple LCP stall peepholes and reload
+ is completed. */
+ return (optimize
+ && optimize_function_for_speed_p (fun)
+ && (!reload_completed
+ || ix86_lcp_stall_peephole_generated > 1));
}
unsigned int execute (function *) final override
@@ -4628,6 +4639,11 @@ public:
return x86_cse ();
}
+ opt_pass *clone () final override
+ {
+ return new pass_x86_cse (m_ctxt);
+ }
+
private:
/* The redundant source value. */
rtx val;
@@ -4646,6 +4662,7 @@ private:
bool candidate_gnu_tls_p (rtx_insn *, attr_tls64);
bool candidate_gnu2_tls_p (rtx, attr_tls64);
bool candidate_vector_p (rtx, rtx_insn *);
+ bool candidate_lcp_stall_p (rtx, rtx_insn *, auto_bitmap &);
rtx_insn *tls_set_insn_from_symbol (const_rtx, const_rtx);
}; // class pass_x86_cse
@@ -4846,6 +4863,127 @@ pass_x86_cse::candidate_vector_p (rtx set, rtx_insn *insn)
return val ? true : false;
}
+/* Return true, output DEF_INSN, VAL, MODE, SCALAR_MODE and KIND as
+ as well as populate LCP_DEF_BARRIERS if INSN is an instruction
+ generated by LCP stall peephole. */
+
+bool
+pass_x86_cse::candidate_lcp_stall_p (rtx set, rtx_insn *insn,
+ auto_bitmap &lcp_def_barriers)
+{
+ rtx src;
+ rtx_insn *lcp_def_insn;
+
+ /* Return true, output SRC and LCP_DEF_INSN as well as populate
+ LCP_DEF_BARRIERS if INSN is an instruction generated by LCP stall
+ peephole. If CHECK_ONLY is true, don't scan for scratch register
+ barriers. */
+ auto lcp_stall_p = [&src, &lcp_def_insn, &lcp_def_barriers]
+ (rtx set, rtx_insn *insn, bool check_only = false)
+ {
+ /* Skip non LCP stall peephole. */
+ rtx dest = SET_DEST (set);
+ if (!MEM_P (dest))
+ return false;
+
+ /* Get the possible scratch register. */
+ src = SET_SRC (set);
+ if (!REG_P (src))
+ return false;
+
+ /* LCP stall peephole must be in HImode. */
+ machine_mode mode = GET_MODE (src);
+ if (mode != HImode)
+ return false;
+
+ basic_block bb = BLOCK_FOR_INSN (insn);
+
+ bool lcp_def_p = false;
+
+ df_ref def;
+ unsigned int regno = REGNO (src);
+ /* Scan backward for the previous scratch register def in the same
+ basic block. */
+ for (; insn != BB_HEAD (bb); insn = PREV_INSN (insn))
+ if (NONDEBUG_INSN_P (insn))
+ FOR_EACH_INSN_DEF (def, insn)
+ if (HARD_REGISTER_P (DF_REF_REAL_REG (def))
+ && DF_REF_REGNO (def) == regno)
+ {
+ set = single_set (insn);
+ if (set)
+ {
+ /* Check for integer constant. */
+ if (!lcp_def_p)
+ {
+ /* Get constant in the scratch register. */
+ src = SET_SRC (set);
+ if (!CONST_INT_P (src))
+ return false;
+
+ if (check_only)
+ return true;
+
+ /* This is the scratch register definition. */
+ lcp_def_insn = insn;
+
+ /* Keep scan for scratch register barriers. */
+ lcp_def_p = true;
+ }
+ }
+ else if (!check_only)
+ {
+ /* Track INSN which is a barrier for scratch register
+ reuse so that the scratch register won't be reused
+ after the barrier. */
+ bitmap_set_bit (lcp_def_barriers, INSN_UID (insn));
+ }
+ }
+
+ return lcp_def_p;
+ };
+
+ if (!lcp_stall_p (set, insn))
+ return false;
+
+ /* Instruction to create the scratch register for peephole. */
+ def_insn = lcp_def_insn;
+ /* Constant value in the scratch register. */
+ val = src;
+
+ /* Since LCP stall peephole is generated after register allocation,
+ it is redundant only if the only uses of the LCP stall scratch
+ register are LCP stall peepholes in the same basic block. */
+ rtx reg = SET_SRC (set);
+ basic_block bb = BLOCK_FOR_INSN (insn);
+ for (df_ref ref = DF_REG_USE_CHAIN (REGNO (reg));
+ ref;
+ ref = DF_REF_NEXT_REG (ref))
+ if (!DF_REF_IS_ARTIFICIAL (ref))
+ {
+ insn = DF_REF_INSN (ref);
+
+ /* CALL has been tracked by lcp_def_barriers. Ignore uses in
+ other BBs. */
+ if (CALL_P (insn) && BLOCK_FOR_INSN (insn) != bb)
+ continue;
+
+ set = single_set (insn);
+ if (!set || !lcp_stall_p (set, insn, true))
+ {
+ /* Track other uses as barriers so that its value won't be
+ reused after them. */
+ bitmap_set_bit (lcp_def_barriers, INSN_UID (insn));
+ continue;
+ }
+ }
+
+ kind = X86_CSE_LCP_STALL;
+ scalar_mode = HImode;
+
+ return true;
+}
+
/* At entry of the nearest common dominator for basic blocks with
1. Vector CONST0_RTX patterns.
@@ -4858,6 +4996,10 @@ pass_x86_cse::candidate_vector_p (rtx set, rtx_insn *insn)
generate a single pattern whose destination is used to replace the
source in all identical patterns.
+ For LCP stall peepholes, replace the scratch register in LCP stall
+ peepholes in the same group with the first scratch register and
+ delete redundant scratch register definitions at the end.
+
NB: We want to generate a pattern, which is executed only once, to
cover the whole function. The LCM algorithm isn't appropriate here
since it may place a pattern inside the loop. */
@@ -4875,6 +5017,8 @@ pass_x86_cse::x86_cse (void)
auto_bitmap updated_gnu_tls_insns;
auto_bitmap updated_gnu2_tls_insns;
auto_bitmap call_bbs;
+ auto_bitmap lcp_def_barriers;
+ auto_bitmap lcp_def_redundant_insns;
df_set_flags (DF_DEFER_INSN_RESCAN);
@@ -4901,38 +5045,51 @@ pass_x86_cse::x86_cse (void)
attr_tls64 tls64 = get_attr_tls64 (insn);
- /* NB: TLS calls preserve all registers. */
- if (call_p && tls64 == TLS64_NONE)
- bitmap_set_bit (call_bbs, BLOCK_FOR_INSN (insn)->index);
-
- switch (tls64)
+ if (reload_completed)
{
- case TLS64_GD:
- case TLS64_LD_BASE:
- /* Verify UNSPEC_TLS_GD and UNSPEC_TLS_LD_BASE. */
- if (candidate_gnu_tls_p (insn, tls64))
- break;
- continue;
+ /* Check for LCP STALL peepholes. */
+ if (!set
+ || call_p
+ || tls64 != TLS64_NONE
+ || !candidate_lcp_stall_p (set, insn,
+ lcp_def_barriers))
+ continue;
+ }
+ else
+ {
+ /* NB: TLS calls preserve all registers. */
+ if (call_p && tls64 == TLS64_NONE)
+ bitmap_set_bit (call_bbs, BLOCK_FOR_INSN (insn)->index);
- case TLS64_CALL:
- case TLS64_COMBINE:
- /* Verify UNSPEC_TLSDESC. */
- if (candidate_gnu2_tls_p (set, tls64))
- break;
- continue;
+ switch (tls64)
+ {
+ case TLS64_GD:
+ case TLS64_LD_BASE:
+ /* Verify UNSPEC_TLS_GD and UNSPEC_TLS_LD_BASE. */
+ if (candidate_gnu_tls_p (insn, tls64))
+ break;
+ continue;
- case TLS64_LEA:
- /* Skip TLS64_LEA. */
- continue;
+ case TLS64_CALL:
+ case TLS64_COMBINE:
+ /* Verify UNSPEC_TLSDESC. */
+ if (candidate_gnu2_tls_p (set, tls64))
+ break;
+ continue;
- case TLS64_NONE:
- if (!set)
- continue;
+ case TLS64_LEA:
+ /* Skip TLS64_LEA. */
+ continue;
- /* Check for vector broadcast. */
- if (candidate_vector_p (set, insn))
- break;
- continue;
+ case TLS64_NONE:
+ if (!set)
+ continue;
+
+ /* Check for vector broadcast. */
+ if (candidate_vector_p (set, insn))
+ break;
+ continue;
+ }
}
/* Check if there is a matching redundant load. */
@@ -4941,13 +5098,123 @@ pass_x86_cse::x86_cse (void)
&& load->kind == kind
&& load->mode == scalar_mode
&& (load->bb == bb
- || (kind != X86_CSE_VEC_DUP
- && kind != X86_CSE_CONST_VECTOR)
- /* Non all 0s/1s vector load must be in the same
- basic block if it is in a recursive call. */
- || !recursive_call_p)
- && rtx_equal_p (load->val, val))
+ /* LCP stall peepholes must be in the same basic
+ block. */
+ || (kind != X86_CSE_LCP_STALL
+ && ((kind != X86_CSE_VEC_DUP
+ && kind != X86_CSE_CONST_VECTOR)
+ /* Non all 0s/1s vector load must be in the
+ same basic block if it is in a recursive
+ call. */
+ || !recursive_call_p)))
+ && rtx_equal_p (load->val, val))
{
+ if (kind == X86_CSE_LCP_STALL)
+ {
+ /* Scan forward between load->def_insn and def_insn
+ to check if the scratch register becomes invalid.
+ */
+ bool invalid = false;
+ bool redundant = false;
+ rtx_insn *lcp_insn;
+ rtx def_set = single_set (load->def_insn);
+ rtx def_dest = SET_DEST (def_set);
+ unsigned int def_regno = REGNO (def_dest);
+ for (lcp_insn = load->def_insn;
+ lcp_insn != NEXT_INSN (BB_END (bb));
+ lcp_insn = NEXT_INSN (lcp_insn))
+ {
+ if (!NONDEBUG_INSN_P (lcp_insn))
+ continue;
+
+ if (bitmap_bit_p (lcp_def_barriers,
+ INSN_UID (lcp_insn)))
+ {
+ /* The scratch register becomes invalid
+ beyond a scratch register barrier. */
+ invalid = true;
+ break;
+ }
+
+ /* A definition, which may set the scratch
+ register to a different value, makes the
+ scratch register invalid. */
+ if (lcp_insn != load->def_insn)
+ {
+ df_ref def;
+ FOR_EACH_INSN_DEF (def, lcp_insn)
+ if (HARD_REGISTER_P (DF_REF_REAL_REG (def))
+ && DF_REF_REGNO (def) == def_regno)
+ {
+ set = single_set (lcp_insn);
+ if (!set
+ || !rtx_equal_p (set, def_set))
+ {
+ /* SET may set the scratch register
+ to a different value. */
+ invalid = true;
+ break;
+ }
+ }
+
+ if (invalid)
+ break;
+ }
+
+ /* Stop when def_insn is reached. */
+ if (lcp_insn == def_insn)
+ {
+ redundant = true;
+ /* When the same scratch register is used
+ in different peepholes as in:
+
+ movl $3, %edx
+ movw %dx, (%eax)
+ movl $3, %ecx
+ movw %cx, 2(%eax)
+ movl $3, %edx
+ movw %dx, 4(%eax)
+ movl $3, %ecx
+ movw %cx, 6(%eax)
+
+ the rtl_dce pass won't remove the second
+ DX definition:
+
+ movl $3, %edx
+ movw %dx, (%eax)
+ movw %dx, 2(%eax)
+ movl $3, %edx
+ movw %dx, 4(%eax)
+ movw %dx, 6(%eax)
+
+ since technically DX isn't dead. Track
+ redundant scratch register definitions
+ and delete them at the end of the pass
+ to generate:
+
+ movl $3, %edx
+ movw %dx, (%eax)
+ movw %dx, 2(%eax)
+ movw %dx, 4(%eax)
+ movw %dx, 6(%eax)
+
+ */
+ bitmap_set_bit (lcp_def_redundant_insns,
+ INSN_UID (lcp_insn));
+ break;
+ }
+ }
+
+ /* If def_insn isn't redundant or def becomes invalid
+ in between, get a new entry. */
+ if (!redundant || invalid)
+ continue;
+
+ /* We must find the def_insn when scan forward
+ from load->def_insn. */
+ gcc_assert (insn != NEXT_INSN (BB_END (bb)));
+ }
+
/* Record instruction. */
bitmap_set_bit (load->insns, INSN_UID (insn));
@@ -5009,9 +5276,25 @@ pass_x86_cse::x86_cse (void)
machine_mode mode;
rtx reg, broadcast_reg;
rtx broadcast_source = nullptr;
+ rtx set;
replaced = true;
switch (load->kind)
{
+ default:
+ gcc_unreachable ();
+
+ case X86_CSE_LCP_STALL:
+ mode = load->mode;
+ set = single_set (load->def_insn);
+ broadcast_reg = SET_DEST (set);
+ if (GET_MODE (broadcast_reg) != mode)
+ broadcast_reg = gen_lowpart (mode, broadcast_reg);
+ replace_vector_const (mode, broadcast_reg,
+ load->insns, load->mode);
+ load->broadcast_source = broadcast_source;
+ load->broadcast_reg = broadcast_reg;
+ break;
+
case X86_CSE_TLS_GD:
case X86_CSE_TLS_LD_BASE:
case X86_CSE_TLSDESC:
@@ -5201,6 +5484,11 @@ pass_x86_cse::x86_cse (void)
if (load->def_insn)
switch (load->kind)
{
+ case X86_CSE_LCP_STALL:
+ /* Do nothing since redundant LCP stall defs will
+ be removed at the end. */
+ break;
+
case X86_CSE_TLSDESC:
ix86_place_single_tls_call (load->broadcast_reg,
load->tlsdesc_val,
@@ -5245,6 +5533,8 @@ pass_x86_cse::x86_cse (void)
else
switch (load->kind)
{
+ default:
+ gcc_unreachable ();
case X86_CSE_TLS_GD:
case X86_CSE_TLS_LD_BASE:
case X86_CSE_TLSDESC:
@@ -5291,6 +5581,24 @@ pass_x86_cse::x86_cse (void)
}
}
+ if (!bitmap_empty_p (lcp_def_redundant_insns))
+ {
+ /* Delete redundant LCP defs. */
+ bitmap_iterator bi;
+ unsigned int id;
+ EXECUTE_IF_SET_IN_BITMAP (lcp_def_redundant_insns, 0,
+ id, bi)
+ {
+ rtx_insn *insn = DF_INSN_UID_GET (id)->insn;
+ if (dump_file)
+ {
+ fprintf (dump_file, "\nDelete:\n\n");
+ print_rtl_single (dump_file, insn);
+ }
+ delete_insn (insn);
+ }
+ }
+
df_process_deferred_rescans ();
}
diff --git a/gcc/config/i386/i386-passes.def b/gcc/config/i386/i386-passes.def
index cd6ccb8d077..20120b2174e 100644
--- a/gcc/config/i386/i386-passes.def
+++ b/gcc/config/i386/i386-passes.def
@@ -39,3 +39,4 @@ along with GCC; see the file COPYING3. If not see
INSERT_PASS_AFTER (pass_late_combine, 1, pass_remove_partial_avx_dependency);
INSERT_PASS_AFTER (pass_rtl_ifcvt, 1, pass_apx_nf_convert);
INSERT_PASS_BEFORE (pass_convert_to_eh_region_ranges, 1, pass_fold_sibcall);
+ INSERT_PASS_BEFORE (pass_cprop_hardreg, 1, pass_x86_cse);
diff --git a/gcc/config/i386/i386.h b/gcc/config/i386/i386.h
index 5a2a2b9c51e..54a4f739d8f 100644
--- a/gcc/config/i386/i386.h
+++ b/gcc/config/i386/i386.h
@@ -2924,6 +2924,13 @@ struct GTY(()) machine_function {
/* True if TLS descriptor is called more than once. */
BOOL_BITFIELD tls_descriptor_call_multiple_p : 1;
+ /* TARGET_LCP_STALL peephole count:
+ 0: None.
+ 1: 1 peephole generated.
+ > 1: More than 1 peepholes generated.
+ */
+ unsigned int lcp_stall_peephole_generated : 2;
+
/* If true, the current function has a STATIC_CHAIN is placed on the
stack below the return address. */
BOOL_BITFIELD static_chain_on_stack : 1;
@@ -3025,6 +3032,8 @@ extern GTY(()) tree ms_va_list_type_node;
REG_SP is live. */
#define ix86_current_function_calls_tls_descriptor \
(ix86_tls_descriptor_calls_expanded_in_cfun && df_regs_ever_live_p (SP_REG))
+#define ix86_lcp_stall_peephole_generated \
+ (cfun->machine->lcp_stall_peephole_generated)
#define ix86_static_chain_on_stack (cfun->machine->static_chain_on_stack)
#define ix86_red_zone_used (cfun->machine->red_zone_used)
diff --git a/gcc/config/i386/i386.md b/gcc/config/i386/i386.md
index a837121c33f..de6c1153ac4 100644
--- a/gcc/config/i386/i386.md
+++ b/gcc/config/i386/i386.md
@@ -29032,7 +29032,11 @@ (define_peephole2
[(parallel [(set (match_dup 2) (const_int 0))
(clobber (reg:CC FLAGS_REG))])
(set (match_dup 0) (match_dup 1))]
- "operands[2] = gen_lowpart (SImode, operands[1]);")
+{
+ operands[2] = gen_lowpart (SImode, operands[1]);
+ ix86_lcp_stall_peephole_generated <<= 1;
+ ix86_lcp_stall_peephole_generated |= 1;
+})
(define_peephole2
[(match_scratch:SWI124 2 "<r>")
@@ -29044,7 +29048,11 @@ (define_peephole2
|| (TARGET_SPLIT_LONG_MOVES
&& get_attr_length (insn) >= ix86_cur_cost ()->large_insn))"
[(set (match_dup 2) (match_dup 1))
- (set (match_dup 0) (match_dup 2))])
+ (set (match_dup 0) (match_dup 2))]
+{
+ ix86_lcp_stall_peephole_generated <<= 1;
+ ix86_lcp_stall_peephole_generated |= 1;
+})
;; Don't compare memory with zero, load and use a test instead.
(define_peephole2
diff --git a/gcc/testsuite/gcc.target/i386/pr125893-1.c b/gcc/testsuite/gcc.target/i386/pr125893-1.c
new file mode 100644
index 00000000000..22749363c6b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr125893-1.c
@@ -0,0 +1,27 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mtune-ctrl=lcp_stall" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc'). */
+/* { dg-final { check-function-bodies "**" "" "" { target *-*-* } {^\t?\.} } } */
+
+/*
+**foo:
+**...
+** movl \$3, %[a-z0-9]+
+** movw %[a-z0-9]+, \(%[a-z0-9]+\)
+** movw %[a-z0-9]+, 2\(%[a-z0-9]+\)
+** movw %[a-z0-9]+, 4\(%[a-z0-9]+\)
+** movw %[a-z0-9]+, 6\(%[a-z0-9]+\)
+**...
+*/
+
+void
+foo (short *dst)
+{
+ dst[0] = 3;
+ asm volatile ("" : : : "memory");
+ dst[1] = 3;
+ asm volatile ("" : : : "memory");
+ dst[2] = 3;
+ asm volatile ("" : : : "memory");
+ dst[3] = 3;
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr125893-2.c b/gcc/testsuite/gcc.target/i386/pr125893-2.c
new file mode 100644
index 00000000000..e241633a3d4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr125893-2.c
@@ -0,0 +1,33 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mtune-ctrl=lcp_stall" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc'). */
+/* { dg-final { check-function-bodies "**" "" "" { target *-*-* } {^\t?\.} } } */
+
+/*
+**foo:
+**...
+** movl \$3, %[a-z0-9]+
+**...
+** movw %[a-z0-9]+, \(%[a-z0-9]+\)
+** movw %[a-z0-9]+, 4\(%[a-z0-9]+\)
+** call _?bar
+** movl \$3, %[a-z0-9]+
+** movw %[a-z0-9]+, 2\(%[a-z0-9]+\)
+** movw %[a-z0-9]+, 8\(%[a-z0-9]+\)
+**...
+*/
+
+
+extern void bar (void);
+
+void
+foo (short *dst)
+{
+ dst[0] = 3;
+ asm volatile ("" : : : "memory");
+ dst[2] = 3;
+ bar ();
+ dst[1] = 3;
+ asm volatile ("" : : : "memory");
+ dst[4] = 3;
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr125893-3.c b/gcc/testsuite/gcc.target/i386/pr125893-3.c
new file mode 100644
index 00000000000..7c01661e7d0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr125893-3.c
@@ -0,0 +1,27 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mtune-ctrl=lcp_stall" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc'). */
+/* { dg-final { check-function-bodies "**" "" "" { target *-*-* } {^\t?\.} } } */
+
+/*
+**foo:
+**...
+** xorl %[a-z0-9]+, %[a-z0-9]+
+** movw %[a-z0-9]+, \(%[a-z0-9]+\)
+** movw %[a-z0-9]+, 2\(%[a-z0-9]+\)
+** movw %[a-z0-9]+, 4\(%[a-z0-9]+\)
+** movw %[a-z0-9]+, 6\(%[a-z0-9]+\)
+**...
+*/
+
+void
+foo (short *dst)
+{
+ dst[0] = 0;
+ asm volatile ("" : : : "memory");
+ dst[1] = 0;
+ asm volatile ("" : : : "memory");
+ dst[2] = 0;
+ asm volatile ("" : : : "memory");
+ dst[3] = 0;
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr125893-4.c b/gcc/testsuite/gcc.target/i386/pr125893-4.c
new file mode 100644
index 00000000000..88488a5b6a5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr125893-4.c
@@ -0,0 +1,33 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mtune-ctrl=lcp_stall" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc'). */
+/* { dg-final { check-function-bodies "**" "" "" { target *-*-* } {^\t?\.} } } */
+
+/*
+**foo:
+**...
+** xorl %[a-z0-9]+, %[a-z0-9]+
+**...
+** movw %[a-z0-9]+, \(%[a-z0-9]+\)
+** movw %[a-z0-9]+, 4\(%[a-z0-9]+\)
+** call _?bar
+** xorl %[a-z0-9]+, %[a-z0-9]+
+** movw %[a-z0-9]+, 2\(%[a-z0-9]+\)
+** movw %[a-z0-9]+, 8\(%[a-z0-9]+\)
+**...
+*/
+
+
+extern void bar (void);
+
+void
+foo (short *dst)
+{
+ dst[0] = 0;
+ asm volatile ("" : : : "memory");
+ dst[2] = 0;
+ bar ();
+ dst[1] = 0;
+ asm volatile ("" : : : "memory");
+ dst[4] = 0;
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr125893-5.c b/gcc/testsuite/gcc.target/i386/pr125893-5.c
new file mode 100644
index 00000000000..fa8858533f4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr125893-5.c
@@ -0,0 +1,34 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mtune-ctrl=lcp_stall" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc'). */
+/* { dg-final { check-function-bodies "**" "" "" { target *-*-* } {^\t?\.} } } */
+
+/*
+**foo:
+**...
+** movl \$3, %[a-z0-9]+
+**...
+** movw %[a-z0-9]+, \(%[a-z0-9]+\)
+** movw %[a-z0-9]+, 4\(%[a-z0-9]+\)
+** call _?bar
+** movl \$3, %[a-z0-9]+
+** movw %[a-z0-9]+, 2\(%[a-z0-9]+\)
+** movw %[a-z0-9]+, 8\(%[a-z0-9]+\)
+**...
+*/
+
+
+extern int bar (void);
+
+int
+foo (short *dst)
+{
+ dst[0] = 3;
+ asm volatile ("" : : : "memory");
+ dst[2] = 3;
+ int ret = bar ();
+ dst[1] = 3;
+ asm volatile ("" : : : "memory");
+ dst[4] = 3;
+ return ret;
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr125893-6.c b/gcc/testsuite/gcc.target/i386/pr125893-6.c
new file mode 100644
index 00000000000..62584cf4d77
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr125893-6.c
@@ -0,0 +1,34 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mtune-ctrl=lcp_stall" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc'). */
+/* { dg-final { check-function-bodies "**" "" "" { target *-*-* } {^\t?\.} } } */
+
+/*
+**foo:
+**...
+** xorl %[a-z0-9]+, %[a-z0-9]+
+**...
+** movw %[a-z0-9]+, \(%[a-z0-9]+\)
+** movw %[a-z0-9]+, 4\(%[a-z0-9]+\)
+** call _?bar
+** xorl %[a-z0-9]+, %[a-z0-9]+
+** movw %[a-z0-9]+, 2\(%[a-z0-9]+\)
+** movw %[a-z0-9]+, 8\(%[a-z0-9]+\)
+**...
+*/
+
+
+extern int bar (void);
+
+int
+foo (short *dst)
+{
+ dst[0] = 0;
+ asm volatile ("" : : : "memory");
+ dst[2] = 0;
+ int ret = bar ();
+ dst[1] = 0;
+ asm volatile ("" : : : "memory");
+ dst[4] = 0;
+ return ret;
+}
--
2.54.0
