From: Kyrylo Tkachov <[email protected]>
Postreload GCSE searches single-predecessor chains and replaces loads by
copies on incoming edges. It misses a fully redundant load after a join,
or when an earlier use of its destination prevents an edge copy:
bb2: r0 = [sp + K]
... paths that preserve r0 and [sp + K] ...
bb5: use (r1)
r1 = [sp + K]
Replace the last load with r1 = r0 at its original position. Select the
nearest eligible dominating occurrence, then use a backward worklist to
check memory transparency and register preservation on every path. Reuse
the existing expression table, alias analysis and register-clobber checks.
Keep partial redundancy elimination as a fallback. In cold blocks,
allow full forwarding only when it does not increase the size cost.
This removes some vector spills and reloads from video filter code
compiled with SVE that motivates this series.
A reduced SVE test covers forwarding across joins. RTL tests cover a
join with an earlier destination use, a transparent loop, and register
alias, memory overwrite, address and loop clobbers.
Bootstrapped and tested on aarch64-none-linux-gnu.
Ok for trunk?
Thanks,
Kyrill
gcc/ChangeLog:
* postreload-gcse.cc (stats): Add loads_forwarded.
(available_from_dominator_p): New function.
(replace_fully_redundant_load): Likewise.
(eliminate_partially_redundant_load): Try full forwarding before
partial elimination. Keep the cold-block gate on partial elimination.
(eliminate_partially_redundant_loads): Scan cold blocks too.
(gcse_after_reload_main): Compute dominators when needed and record
forwarded loads.
gcc/testsuite/ChangeLog:
* gcc.dg/rtl/aarch64/postreload-gcse-forward.c: New test.
* gcc.target/aarch64/sve/spill-gcse.c: New test.
Signed-off-by: Kyrylo Tkachov <[email protected]>
---
gcc/postreload-gcse.cc | 119 +++++-
.../rtl/aarch64/postreload-gcse-forward.c | 342 ++++++++++++++++++
.../gcc.target/aarch64/sve/spill-gcse.c | 173 +++++++++
3 files changed, 629 insertions(+), 5 deletions(-)
create mode 100644 gcc/testsuite/gcc.dg/rtl/aarch64/postreload-gcse-forward.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/spill-gcse.c
diff --git a/gcc/postreload-gcse.cc b/gcc/postreload-gcse.cc
index 2203c60ae91..ba7adc1da9f 100644
--- a/gcc/postreload-gcse.cc
+++ b/gcc/postreload-gcse.cc
@@ -61,7 +61,8 @@ along with GCC; see the file COPYING3. If not see
change until the end of the basic block add this memory expression to
the hash table.
2. Perform Redundancy elimination:
- For each load instruction do the following:
+ For each load instruction, first try to reuse a register from a
+ dominating occurrence that remains available on all paths. Otherwise
perform partial redundancy elimination, check if it's worth adding
loads to make the load fully redundant. If so add loads and
register copies and delete the load.
@@ -81,6 +82,7 @@ static struct
int moves_inserted;
int copies_inserted;
int insns_deleted;
+ int loads_forwarded;
} stats;
/* We need to keep a hash table of expressions. The table entries are of
@@ -1002,6 +1004,101 @@ compute_expr_transp (expr **slot, FILE *dump_file
ATTRIBUTE_UNUSED)
return 1;
}
+/* Return true if REG still holds EXPR at the entry to BB after SOURCE.
+ SOURCE must dominate BB. Check every path backwards to SOURCE's block,
+ including queued edge instructions. */
+
+static bool
+available_from_dominator_p (basic_block bb, occr *source, expr *expr, rtx reg)
+{
+ basic_block source_bb = BLOCK_FOR_INSN (source->insn);
+ gcc_checking_assert (dominated_by_p (CDI_DOMINATORS, bb, source_bb));
+ auto_bitmap visited;
+ auto_vec<basic_block> worklist;
+ worklist.safe_push (bb);
+ while (!worklist.is_empty ())
+ {
+ basic_block next = worklist.pop ();
+ edge e;
+ edge_iterator ei;
+ FOR_EACH_EDGE (e, ei, next->preds)
+ {
+ if ((e->flags & EDGE_ABNORMAL)
+ || e->src == ENTRY_BLOCK_PTR_FOR_FN (cfun)
+ || reg_killed_on_edge (reg, e))
+ return false;
+ for (rtx_insn *i = e->insns.r; i; i = NEXT_INSN (i))
+ if (INSN_P (i) && modified_in_p (expr->expr, i))
+ return false;
+
+ basic_block pred = e->src;
+ if (pred == source_bb)
+ continue;
+ if (bitmap_set_bit (visited, pred->index))
+ {
+ if (!bitmap_bit_p (transp[pred->index], expr->bitmap_index)
+ || reg_set_between_p (reg, PREV_INSN (BB_HEAD (pred)),
+ NEXT_INSN (BB_END (pred))))
+ return false;
+ worklist.safe_push (pred);
+ }
+ }
+ }
+ return true;
+}
+
+/* Return true if INSN's load of EXPR in BB is replaced with a register
+ copy from a dominating occurrence. Keep the destination and instruction
+ in place so that its occurrence-table entry remains valid. */
+
+static bool
+replace_fully_redundant_load (basic_block bb, rtx_insn *insn, expr *expr)
+{
+ if (!transp)
+ return false;
+
+ occr *best = nullptr;
+ for (occr *o = expr->avail_occr; o; o = o->next)
+ {
+ basic_block obb = BLOCK_FOR_INSN (o->insn);
+ if (o->deleted_p || obb == bb
+ || !dominated_by_p (CDI_DOMINATORS, bb, obb))
+ continue;
+ rtx reg = get_avail_load_store_reg (o->insn);
+ if (reg_changed_after_insn_p (reg, 0)
+ || reg_set_between_p (reg, o->insn, NEXT_INSN (BB_END (obb))))
+ continue;
+ if (!best
+ || dominated_by_p (CDI_DOMINATORS, obb,
+ BLOCK_FOR_INSN (best->insn)))
+ best = o;
+ }
+ if (!best)
+ return false;
+
+ rtx reg = get_avail_load_store_reg (best->insn);
+ if (!available_from_dominator_p (bb, best, expr, reg))
+ return false;
+
+ rtx set = PATTERN (insn);
+ if (optimize_bb_for_size_p (bb)
+ && set_rtx_cost (gen_rtx_SET (SET_DEST (set), reg), false)
+ > set_rtx_cost (set, false))
+ return false;
+
+ if (!validate_change (insn, &SET_SRC (set), copy_rtx (reg), 0))
+ return false;
+
+ stats.loads_forwarded++;
+ if (dump_file)
+ {
+ fprintf (dump_file, "forwarding load from dominating insn %d: ",
+ INSN_UID (best->insn));
+ print_rtl_single (dump_file, insn);
+ }
+ return true;
+}
+
/* This handles the case where several stores feed a partially redundant
load. It checks if the redundancy elimination is possible and if it's
worth it.
@@ -1042,6 +1139,13 @@ eliminate_partially_redundant_load (basic_block bb,
rtx_insn *insn,
pat = PATTERN (insn);
dest = SET_DEST (pat);
+ if (replace_fully_redundant_load (bb, insn, expr))
+ return;
+
+ /* Avoid adding edge copies in cold blocks. */
+ if (optimize_bb_for_size_p (bb))
+ return;
+
/* Check that the loaded register is not used, set, or killed from the
beginning of the block. */
if (reg_changed_after_insn_p (dest, 0)
@@ -1257,10 +1361,6 @@ eliminate_partially_redundant_loads (void)
if (! bb_has_well_behaved_predecessors (bb))
continue;
- /* Do not try anything on cold basic blocks. */
- if (optimize_bb_for_size_p (bb))
- continue;
-
/* Reset the table of things changed since the start of the current
basic block. */
reset_opr_set_tables ();
@@ -1387,7 +1487,13 @@ gcse_after_reload_main (rtx f ATTRIBUTE_UNUSED)
}
else
transp = NULL;
+ bool own_dominators
+ = do_transp && !dom_info_available_p (CDI_DOMINATORS);
+ if (own_dominators)
+ calculate_dominance_info (CDI_DOMINATORS);
eliminate_partially_redundant_loads ();
+ if (own_dominators)
+ free_dominance_info (CDI_DOMINATORS);
delete_redundant_insns ();
if (do_transp)
sbitmap_vector_free (transp);
@@ -1395,12 +1501,15 @@ gcse_after_reload_main (rtx f ATTRIBUTE_UNUSED)
if (dump_file)
{
fprintf (dump_file, "GCSE AFTER RELOAD stats:\n");
+ fprintf (dump_file, "loads forwarded: %d\n", stats.loads_forwarded);
fprintf (dump_file, "copies inserted: %d\n", stats.copies_inserted);
fprintf (dump_file, "moves inserted: %d\n", stats.moves_inserted);
fprintf (dump_file, "insns deleted: %d\n", stats.insns_deleted);
fprintf (dump_file, "\n\n");
}
+ statistics_counter_event (cfun, "loads forwarded",
+ stats.loads_forwarded);
statistics_counter_event (cfun, "copies inserted",
stats.copies_inserted);
statistics_counter_event (cfun, "moves inserted",
diff --git a/gcc/testsuite/gcc.dg/rtl/aarch64/postreload-gcse-forward.c
b/gcc/testsuite/gcc.dg/rtl/aarch64/postreload-gcse-forward.c
new file mode 100644
index 00000000000..b911e7e0392
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/rtl/aarch64/postreload-gcse-forward.c
@@ -0,0 +1,342 @@
+/* { dg-do compile { target aarch64*-*-* } } */
+/* { dg-options "-O2 -fgcse-after-reload -mlittle-endian -fdump-rtl-gcse2" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+/* Q0 holds the stack value across a diamond. The join uses the old Q1
+ before reloading it, so a copy on the incoming edges is not valid. */
+
+/*
+** join_forward:
+** ...
+** str q0, \[x1\]
+** ...
+*/
+void __RTL (startwith ("ira"))
+join_forward (void)
+{
+(function "join_forward"
+ (insn-chain
+ (block 2
+ (edge-from entry (flags "FALLTHRU"))
+ (cnote 1 [bb 2] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 2 (set (reg:DI sp) (plus:DI (reg:DI sp) (const_int -32))))
+ (cinsn 3 (set (mem:V4SI (reg:DI sp) [0 S16 A128]) (reg:V4SI v0)))
+ (cinsn 4 (set (reg:CC cc) (compare:CC (reg:SI x0) (const_int 0))))
+ (cjump_insn 5 (set (pc) (if_then_else (eq (reg:CC cc) (const_int 0))
+ (label_ref 10) (pc))))
+ (edge-to 3 (flags "FALLTHRU"))
+ (edge-to 4)
+ )
+ (block 3
+ (edge-from 2 (flags "FALLTHRU"))
+ (cnote 6 [bb 3] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 7 (set (reg:DI x4) (plus:DI (reg:DI x5) (const_int 1))))
+ (cjump_insn 8 (set (pc) (label_ref 15)))
+ (edge-to 5)
+ )
+ (cbarrier 9)
+ (block 4
+ (edge-from 2)
+ (clabel 10 1000)
+ (cnote 11 [bb 4] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 13 (set (reg:DI x4) (plus:DI (reg:DI x5) (const_int 2))))
+ (edge-to 5 (flags "FALLTHRU"))
+ )
+ (block 5
+ (edge-from 3)
+ (edge-from 4 (flags "FALLTHRU"))
+ (clabel 15 1001)
+ (cnote 16 [bb 5] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 17 (set (reg:V4SI v2) (plus:V4SI (reg:V4SI v1) (reg:V4SI v3))))
+ (cinsn 18 (set (reg:V4SI v1) (mem:V4SI (reg:DI sp) [0 S16 A128])))
+ (cinsn 19 (set (mem:V4SI (reg:DI x1) [0 S16 A128]) (reg:V4SI v1)))
+ (cinsn 20 (set (mem:V4SI (reg:DI x2) [0 S16 A128]) (reg:V4SI v2)))
+ (cinsn 21 (set (mem:V4SI (reg:DI x3) [0 S16 A128]) (reg:V4SI v0)))
+ (cinsn 22 (set (mem:DI (reg:DI x7) [0 S8 A64]) (reg:DI x4)))
+ (cinsn 23 (set (reg:DI sp) (plus:DI (reg:DI sp) (const_int 32))))
+ (edge-to exit (flags "FALLTHRU"))
+ )
+ )
+)
+}
+
+/* A write to D0 also changes the value of Q0. */
+
+/*
+** source_clobber:
+** ...
+** ldr q[0-9]+, \[sp\]
+** ...
+*/
+void __RTL (startwith ("ira"))
+source_clobber (void)
+{
+(function "source_clobber"
+ (insn-chain
+ (block 2
+ (edge-from entry (flags "FALLTHRU"))
+ (cnote 1 [bb 2] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 2 (set (reg:DI sp) (plus:DI (reg:DI sp) (const_int -32))))
+ (cinsn 3 (set (mem:V4SI (reg:DI sp) [0 S16 A128]) (reg:V4SI v0)))
+ (cinsn 4 (set (reg:CC cc) (compare:CC (reg:SI x0) (const_int 0))))
+ (cjump_insn 5 (set (pc) (if_then_else (eq (reg:CC cc) (const_int 0))
+ (label_ref 10) (pc))))
+ (edge-to 3 (flags "FALLTHRU"))
+ (edge-to 4)
+ )
+ (block 3
+ (edge-from 2 (flags "FALLTHRU"))
+ (cnote 6 [bb 3] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 7 (set (reg:DI x4) (plus:DI (reg:DI x5) (const_int 1))))
+ (cjump_insn 8 (set (pc) (label_ref 15)))
+ (edge-to 5)
+ )
+ (cbarrier 9)
+ (block 4
+ (edge-from 2)
+ (clabel 10 2000)
+ (cnote 11 [bb 4] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 12 (set (reg:DI v0) (reg:DI v4)))
+ (cinsn 13 (set (reg:DI x4) (plus:DI (reg:DI x5) (const_int 2))))
+ (edge-to 5 (flags "FALLTHRU"))
+ )
+ (block 5
+ (edge-from 3)
+ (edge-from 4 (flags "FALLTHRU"))
+ (clabel 15 2001)
+ (cnote 16 [bb 5] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 17 (set (reg:V4SI v2) (plus:V4SI (reg:V4SI v1) (reg:V4SI v3))))
+ (cinsn 18 (set (reg:V4SI v1) (mem:V4SI (reg:DI sp) [0 S16 A128])))
+ (cinsn 19 (set (mem:V4SI (reg:DI x1) [0 S16 A128]) (reg:V4SI v1)))
+ (cinsn 20 (set (mem:V4SI (reg:DI x2) [0 S16 A128]) (reg:V4SI v2)))
+ (cinsn 21 (set (mem:V4SI (reg:DI x3) [0 S16 A128]) (reg:V4SI v0)))
+ (cinsn 22 (set (mem:DI (reg:DI x7) [0 S8 A64]) (reg:DI x4)))
+ (cinsn 23 (set (reg:DI sp) (plus:DI (reg:DI sp) (const_int 32))))
+ (edge-to exit (flags "FALLTHRU"))
+ )
+ )
+)
+}
+
+/* A partial store on one incoming path invalidates the stack value. */
+
+/*
+** alias_store:
+** ...
+** ldr q[0-9]+, \[sp\]
+** ...
+*/
+void __RTL (startwith ("ira"))
+alias_store (void)
+{
+(function "alias_store"
+ (insn-chain
+ (block 2
+ (edge-from entry (flags "FALLTHRU"))
+ (cnote 1 [bb 2] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 2 (set (reg:DI sp) (plus:DI (reg:DI sp) (const_int -32))))
+ (cinsn 3 (set (mem:V4SI (reg:DI sp) [0 S16 A128]) (reg:V4SI v0)))
+ (cinsn 4 (set (reg:CC cc) (compare:CC (reg:SI x0) (const_int 0))))
+ (cjump_insn 5 (set (pc) (if_then_else (eq (reg:CC cc) (const_int 0))
+ (label_ref 10) (pc))))
+ (edge-to 3 (flags "FALLTHRU"))
+ (edge-to 4)
+ )
+ (block 3
+ (edge-from 2 (flags "FALLTHRU"))
+ (cnote 6 [bb 3] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 7 (set (reg:DI x4) (plus:DI (reg:DI x5) (const_int 1))))
+ (cjump_insn 8 (set (pc) (label_ref 15)))
+ (edge-to 5)
+ )
+ (cbarrier 9)
+ (block 4
+ (edge-from 2)
+ (clabel 10 3000)
+ (cnote 11 [bb 4] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 12 (set (mem:SI (reg:DI sp) [0 S4 A32]) (reg:SI x6)))
+ (cinsn 13 (set (reg:DI x4) (plus:DI (reg:DI x5) (const_int 2))))
+ (edge-to 5 (flags "FALLTHRU"))
+ )
+ (block 5
+ (edge-from 3)
+ (edge-from 4 (flags "FALLTHRU"))
+ (clabel 15 3001)
+ (cnote 16 [bb 5] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 17 (set (reg:V4SI v2) (plus:V4SI (reg:V4SI v1) (reg:V4SI v3))))
+ (cinsn 18 (set (reg:V4SI v1) (mem:V4SI (reg:DI sp) [0 S16 A128])))
+ (cinsn 19 (set (mem:V4SI (reg:DI x1) [0 S16 A128]) (reg:V4SI v1)))
+ (cinsn 20 (set (mem:V4SI (reg:DI x2) [0 S16 A128]) (reg:V4SI v2)))
+ (cinsn 21 (set (mem:V4SI (reg:DI x3) [0 S16 A128]) (reg:V4SI v0)))
+ (cinsn 22 (set (mem:DI (reg:DI x7) [0 S8 A64]) (reg:DI x4)))
+ (cinsn 23 (set (reg:DI sp) (plus:DI (reg:DI sp) (const_int 32))))
+ (edge-to exit (flags "FALLTHRU"))
+ )
+ )
+)
+}
+
+/*
+** different_address:
+** ...
+** ldr q[0-9]+, \[sp, 16\]
+** ...
+*/
+void __RTL (startwith ("ira"))
+different_address (void)
+{
+(function "different_address"
+ (insn-chain
+ (block 2
+ (edge-from entry (flags "FALLTHRU"))
+ (cnote 1 [bb 2] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 2 (set (reg:DI sp) (plus:DI (reg:DI sp) (const_int -32))))
+ (cinsn 3 (set (mem:V4SI (reg:DI sp) [0 S16 A128]) (reg:V4SI v0)))
+ (cinsn 4 (set (reg:CC cc) (compare:CC (reg:SI x0) (const_int 0))))
+ (cjump_insn 5 (set (pc) (if_then_else (eq (reg:CC cc) (const_int 0))
+ (label_ref 10) (pc))))
+ (edge-to 3 (flags "FALLTHRU"))
+ (edge-to 4)
+ )
+ (block 3
+ (edge-from 2 (flags "FALLTHRU"))
+ (cnote 6 [bb 3] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 7 (set (reg:DI x4) (plus:DI (reg:DI x5) (const_int 1))))
+ (cjump_insn 8 (set (pc) (label_ref 15)))
+ (edge-to 5)
+ )
+ (cbarrier 9)
+ (block 4
+ (edge-from 2)
+ (clabel 10 4000)
+ (cnote 11 [bb 4] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 13 (set (reg:DI x4) (plus:DI (reg:DI x5) (const_int 2))))
+ (edge-to 5 (flags "FALLTHRU"))
+ )
+ (block 5
+ (edge-from 3)
+ (edge-from 4 (flags "FALLTHRU"))
+ (clabel 15 4001)
+ (cnote 16 [bb 5] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 17 (set (reg:V4SI v2) (plus:V4SI (reg:V4SI v1) (reg:V4SI v3))))
+ (cinsn 18 (set (reg:V4SI v1) (mem:V4SI (plus:DI (reg:DI sp) (const_int
16)) [0 S16 A128])))
+ (cinsn 19 (set (mem:V4SI (reg:DI x1) [0 S16 A128]) (reg:V4SI v1)))
+ (cinsn 20 (set (mem:V4SI (reg:DI x2) [0 S16 A128]) (reg:V4SI v2)))
+ (cinsn 21 (set (mem:V4SI (reg:DI x3) [0 S16 A128]) (reg:V4SI v0)))
+ (cinsn 22 (set (mem:DI (reg:DI x7) [0 S8 A64]) (reg:DI x4)))
+ (cinsn 23 (set (reg:DI sp) (plus:DI (reg:DI sp) (const_int 32))))
+ (edge-to exit (flags "FALLTHRU"))
+ )
+ )
+)
+}
+
+/* The loop can overwrite the stored value before the reload. */
+/*
+** loop_store:
+** ...
+** ldr q[0-9]+, \[sp\]
+** ...
+*/
+void __RTL (startwith ("ira"))
+loop_store (void)
+{
+(function "loop_store"
+ (insn-chain
+ (block 2
+ (edge-from entry (flags "FALLTHRU"))
+ (cnote 1 [bb 2] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 2 (set (reg:DI sp) (plus:DI (reg:DI sp) (const_int -32))))
+ (cinsn 3 (set (mem:V4SI (reg:DI sp) [0 S16 A128]) (reg:V4SI v0)))
+ (edge-to 3 (flags "FALLTHRU"))
+ )
+ (block 3
+ (edge-from 2 (flags "FALLTHRU"))
+ (edge-from 4)
+ (clabel 4 5000)
+ (cnote 5 [bb 3] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 6 (set (reg:CC cc) (compare:CC (reg:SI x0) (const_int 0))))
+ (cjump_insn 7 (set (pc) (if_then_else (eq (reg:CC cc) (const_int 0))
+ (label_ref 14) (pc))))
+ (edge-to 4 (flags "FALLTHRU"))
+ (edge-to 5)
+ )
+ (block 4
+ (edge-from 3 (flags "FALLTHRU"))
+ (cnote 8 [bb 4] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 9 (set (mem:SI (reg:DI sp) [0 S4 A32]) (reg:SI x6)))
+ (cinsn 10 (set (reg:SI x0) (plus:SI (reg:SI x0) (const_int -1))))
+ (cjump_insn 11 (set (pc) (label_ref 4)))
+ (edge-to 3)
+ )
+ (cbarrier 12)
+ (block 5
+ (edge-from 3)
+ (clabel 14 5001)
+ (cnote 15 [bb 5] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 16 (set (reg:V4SI v1) (mem:V4SI (reg:DI sp) [0 S16 A128])))
+ (cinsn 17 (set (mem:V4SI (reg:DI x1) [0 S16 A128]) (reg:V4SI v1)))
+ (cinsn 18 (set (reg:DI sp) (plus:DI (reg:DI sp) (const_int 32))))
+ (edge-to exit (flags "FALLTHRU"))
+ )
+ )
+)
+}
+
+/* The entry store anchors the value across a loop that preserves Q0 and
+ the stack slot. The exit uses Q1 before the reload. */
+
+/*
+** loop_forward:
+** ...
+** str q0, \[x1\]
+** ...
+*/
+void __RTL (startwith ("ira"))
+loop_forward (void)
+{
+(function "loop_forward"
+ (insn-chain
+ (block 2
+ (edge-from entry (flags "FALLTHRU"))
+ (cnote 1 [bb 2] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 2 (set (reg:DI sp) (plus:DI (reg:DI sp) (const_int -32))))
+ (cinsn 3 (set (mem:V4SI (reg:DI sp) [0 S16 A128]) (reg:V4SI v0)))
+ (edge-to 3 (flags "FALLTHRU"))
+ )
+ (block 3
+ (edge-from 2 (flags "FALLTHRU"))
+ (edge-from 4)
+ (clabel 4 6000)
+ (cnote 5 [bb 3] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 6 (set (reg:CC cc) (compare:CC (reg:SI x0) (const_int 0))))
+ (cjump_insn 7 (set (pc) (if_then_else (eq (reg:CC cc) (const_int 0))
+ (label_ref 14) (pc))))
+ (edge-to 4 (flags "FALLTHRU"))
+ (edge-to 5)
+ )
+ (block 4
+ (edge-from 3 (flags "FALLTHRU"))
+ (cnote 8 [bb 4] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 9 (set (reg:DI x4) (plus:DI (reg:DI x4) (const_int 1))))
+ (cinsn 10 (set (reg:SI x0) (plus:SI (reg:SI x0) (const_int -1))))
+ (cjump_insn 11 (set (pc) (label_ref 4)))
+ (edge-to 3)
+ )
+ (cbarrier 12)
+ (block 5
+ (edge-from 3)
+ (clabel 14 6001)
+ (cnote 15 [bb 5] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 19 (set (reg:V4SI v2) (plus:V4SI (reg:V4SI v1) (reg:V4SI v3))))
+ (cinsn 16 (set (reg:V4SI v1) (mem:V4SI (reg:DI sp) [0 S16 A128])))
+ (cinsn 17 (set (mem:V4SI (reg:DI x1) [0 S16 A128]) (reg:V4SI v1)))
+ (cinsn 20 (set (mem:V4SI (reg:DI x2) [0 S16 A128]) (reg:V4SI v2)))
+ (cinsn 21 (set (mem:V4SI (reg:DI x3) [0 S16 A128]) (reg:V4SI v0)))
+ (cinsn 22 (set (mem:DI (reg:DI x7) [0 S8 A64]) (reg:DI x4)))
+ (cinsn 18 (set (reg:DI sp) (plus:DI (reg:DI sp) (const_int 32))))
+ (edge-to exit (flags "FALLTHRU"))
+ )
+ )
+)
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/spill-gcse.c
b/gcc/testsuite/gcc.target/aarch64/sve/spill-gcse.c
new file mode 100644
index 00000000000..6a9223dd736
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/spill-gcse.c
@@ -0,0 +1,173 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -march=armv9-a+sve2 -mtune=generic-armv9-a
-msve-vector-bits=256 -mautovec-preference=sve-only -fno-ivopts
-fno-align-functions -fdump-rtl-gcse2-details" } */
+/* { dg-require-effective-target aarch64_little_endian } */
+
+/* Reuse an SVE register across a join after its value has been spilled. */
+
+#include <arm_sve.h>
+#include <stddef.h>
+typedef svuint8_t x102 __attribute__ ((arm_sve_vector_bits (256)));
+typedef svuint16_t x100;
+typedef svuint32_t x101;
+typedef svint16_t x99;
+typedef svbool_t x36;
+static inline void
+g (x102 x24[])
+{
+ for (int x58 = 0; x58 < 8; x58++)
+ {
+ x102 x27 = svtrn1_u8 (x24[2 * x58], x24[2]);
+ x24[2 * x58] = x27;
+ }
+ for (int x49 = 0; x49 < 16; x49 += 4)
+ {
+ x100 x28 = svtrn1_u16 (svreinterpret_u16 (x24[x49]),
+ svreinterpret_u16 (x24[x49 + 2]));
+ x24[x49] = svreinterpret_u8 (x28);
+ }
+ for (int x49 = 0; x49 < 16; x49 += 8)
+ for (int x57 = 0; x57 < 4; x57++)
+ {
+ x101 x34 = svtrn2_u32 (svreinterpret_u32 (x24[x49 + x57]),
+ svreinterpret_u32 (x24[x49 + 4 + x57]));
+ x24[x49 + 4 + x57] = svreinterpret_u8 (x34);
+ }
+ for (int x57 = 0; x57 < 8; x57++)
+ {
+ x102 x27 = svreinterpret_u8 (svtrn1_u64 (
+ svreinterpret_u64 (x24[x57]), svreinterpret_u64 (x24[x57 + 8])));
+ x24[x57] = x27;
+ }
+}
+void
+f (uint8_t *x39, ptrdiff_t x91, uint32_t *x98, uint64_t *x60)
+{
+ unsigned x97 = x98[1] | x98[2];
+ int x70 = 32;
+ x36 x83 = svptrue_pat_b8 (SV_VL32);
+ x36 x82 = svptrue_pat_b8 (SV_VL16);
+ x36 x79 = svptrue_pat_b32 (SV_VL8);
+ uint8_t x5[32] = { 7 };
+ x102 x42 = svld1_u8 (x83, x5);
+#define x6(x95) \
+ svtbl_u8 (svuzp1_u8 (svuzp1_u8 (svreinterpret_u8 (x95), svdup_u8 (0)), \
+ svdup_u8 (0)), \
+ x42)
+ uint32_t x88 = x60[1];
+ for (int x40 = 0; x40 < x70; x40 += 8)
+ {
+ x102 x59 = svreinterpret_u8 (svinsr_n_u64 (svdup_u64 (0), 0));
+ svuint32_t x10 = svunpklo_u32 (svunpklo_u16 (x59));
+ x101 x41 = svadd_n_u32_x (x79, svindex_u32 (0, 1), x40);
+ x101 x35 = (x41);
+ x36 x31 = svand_z (
+ x79, svcmpne_n_u32 (x79, svand_u32_x (x79, svdup_u32 (x97), x35), 0),
+ svcmpne_n_u32 (x79, x10, 0));
+ x101 x32 = svsel_u32 (x31, svdup_u32 (1), svdup_u32 (0));
+ x36 x77 = svcmpne_n_u8 (x83, x6 (x32), 0);
+ if (svptest_any (x83, x77))
+ continue;
+ uint8_t *x38 = x39;
+ x102 x3[16];
+ uint8_t *x73 = x38, *x80 = x38 - 7;
+ for (int x56 = 0; x56 < 16; x56++)
+ {
+ x3[x56]
+ = svsplice_u8 (x82, svld1_u8 (x82, x73), svld1_u8 (x82, x80));
+ x80 += x91;
+ }
+ g (x3);
+ x101 x8 = svmax_n_u32_x (x79, (svdup_u32 (x88)), 1);
+ x101 x4 = svadd_u32_x (
+ x79, svlsl_n_u32_x (x79, svadd_n_u32_x (x79, x10, 2), 1), x8);
+ x36 x55
+ = svcmpne_n_u32 (x79, svand_u32_x (x79, svdup_u32 (x98[2]), x35), 0);
+ x36 x54
+ = svcmpne_n_u32 (x79, svand_u32_x (x79, svdup_u32 (x98[1]), x35), 0);
+ x101 x103 = svsel_u32 (x55, svdup_u32 (16),
+ svsel_u32 (x54, svdup_u32 (8), svdup_u32 (4)));
+ x102 x7 = x6 (x4), x9 = x6 (x8), x26 = x6 (x103);
+#define x25(x71) x3[(x71) + 7]
+#define x0(x27, x34) svabd_u8_x (x83, (x27), (x34))
+#define x15(x27, x34) svmax_u8_x (x83, (x27), (x34))
+ x102 x75 = x25 (-2), x74 = x25 (-1), x84 = x25 (0), x85 = x25 (1),
+ x76 = x25 (-3), x86 = x25 (2);
+ x102 x30 = x0 (x75, x74), x29 = x0 (x85, x84);
+ x102 x65 = x15 (x30, x29);
+ x36 x48 = svcmple_u8 (x83, x65, x9);
+ x102 x43 = svqadd_u8 ((x84), x0 (x74, x84));
+ x36 x44 = svcmple_u8 (x83, x43, x7);
+ x36 x47 = svand_z (x83, svand_z (x83, x48, x44), x77);
+ if (svptest_any (x83, x47))
+ continue;
+ x36 x104 = svcmpge_n_u8 (x83, x26, 6);
+ x102 x67 = x15 (x15 (x0 (x76, x74), x0 (x75, x74)),
+ x15 (x0 (x85, x84), x0 (x86, x84)));
+ x36 x45 = svand_z (x83, x104, svcmple_n_u8 (x83, x67, 1));
+ x36 x46 = svpfalse_b ();
+ x36 x12 = (x45);
+ x36 x14 = (x46);
+ x36 x13 = (svand_z (x83, svcmpeq_n_u8 (x83, x26, 8), x45));
+ x36 x33 = (x13);
+ if (svptest_any (x83, x33))
+ {
+ x36 x78 = (x83);
+#define x11(x71) \
+ svreinterpret_s16 (svunpklo_u16 (x3[(x71) + 7]))
+#define x1(x27, x34) svadd_s16_x (x78, (x27), (x34))
+#define x23(x27, x34) svsub_s16_x (x78, (x27), (x34))
+ x36 x61 = (x12);
+ x36 x64 = (x14);
+ x36 x63 = svunpklo_b (x13);
+ x36 x62 = (x63);
+ int x50 = svptest_any (x78, x61), x52 = svptest_any (x78, x64);
+#define x18(x72, x66, x96) \
+ x102 x69 = svuzp1_u8 ( \
+ svreinterpret_u8 (svsel_s16 ((x66), (x96), x11 (x72))), svdup_u8 (0)); \
+ x3[(x72) + 7] \
+ = svsplice_u8 (x82, x69, svext_u8 (x3[(x72) + 7], x3[7], 16))
+
+#define x17 x11 (-2)
+#define x16 x11 (-1)
+#define x19 x11 (1)
+#define x20 x11 (3)
+#define x21 x11 (4)
+#define x22 x11 (6)
+ x99 x90 = x11 (7);
+ x36 x68 = x62;
+ if (x50 || x52)
+ {
+ x99 x93 = svdup_s16 (0), x94 = svdup_s16 (x60[0]);
+
+ x18 (-2, x68, (x94));
+ {
+ x18 (-1, x68, (x94));
+ }
+ {
+ x18 (0, x68, (x94));
+ }
+ {
+ if (x50)
+ x93 = (x90);
+ {
+ x99 x27 = x16;
+ x94 = (x27);
+ }
+ x18 (1, x68, svsel_s16 (x61, x93, (x94)));
+ }
+ }
+ x90 = x1 (x1 ((x19), x21), x22);
+ x90 = x1 (x23 (x23 (x90, x17), x20), (x22));
+ x18 (5, x61, (x90));
+ }
+ uint8_t *x87 = x38 + 7;
+ for (int x56 = 0; x56 < 16; x56++)
+ {
+ svst1_u8 (x82, x87, svext_u8 (x3[x56], x3[x56], 16));
+ x87 += x91;
+ }
+ svst1_u8 (x82, x87, x3[8]);
+ }
+}
+
+/* { dg-final { scan-rtl-dump {forwarding load from dominating insn
[^\n]*\(reg(/v)?:VNx16QI} "gcse2" } } */
--
2.50.1 (Apple Git-155)
