From: Kyrylo Tkachov <[email protected]>
A spilled vector can be stored in one mode and immediately reloaded in
another. CSELIB's memory lookup requires an exact mode match, so postreload
misses an available register value. For example:
mem:VNx4SI (addr) = v7:VNx4SI;
v23:VNx2DI = mem:VNx2DI (addr);
can use a register copy for the second set. The store must remain when
other paths still load its value.
Search the existing address-value list for an equal-sized vector value in
a register. Require the target's permission for a bit-preserving mode change
and a
valid subreg, then use the existing cost and instruction validation rules.
Retain volatile loads and honor float-store. Expose elt_list alongside
the other CSELIB list structures.
Bootstrapped and tested on aarch64-none-linux-gnu.
Ok for trunk?
Thanks,
Kyrill
gcc/ChangeLog:
* cselib.cc (elt_list): Move to cselib.h.
* cselib.h (elt_list): Move from cselib.cc.
* postreload.cc (reload_cse_simplify_mem): New.
(reload_cse_simplify_set): Try reload_cse_simplify_mem.
gcc/testsuite/ChangeLog:
* gcc.dg/rtl/aarch64/postreload-vector-forward.c: New test.
* gcc.dg/rtl/aarch64/postreload-vector-forward-be.c: Likewise.
* gcc.target/aarch64/sve/spill-mode-change.c: New test.
Signed-off-by: Kyrylo Tkachov <[email protected]>
---
gcc/cselib.cc | 7 --
gcc/cselib.h | 7 ++
gcc/postreload.cc | 75 ++++++++++++-
.../aarch64/postreload-vector-forward-be.c | 61 ++++++++++
.../rtl/aarch64/postreload-vector-forward.c | 105 ++++++++++++++++++
.../aarch64/sve/spill-mode-change.c | 94 ++++++++++++++++
6 files changed, 340 insertions(+), 9 deletions(-)
create mode 100644
gcc/testsuite/gcc.dg/rtl/aarch64/postreload-vector-forward-be.c
create mode 100644 gcc/testsuite/gcc.dg/rtl/aarch64/postreload-vector-forward.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/spill-mode-change.c
diff --git a/gcc/cselib.cc b/gcc/cselib.cc
index 3e48fe5dcb0..e86abf3c6dd 100644
--- a/gcc/cselib.cc
+++ b/gcc/cselib.cc
@@ -36,13 +36,6 @@ along with GCC; see the file COPYING3. If not see
#include "predict.h"
#include "rtl-iter.h"
-/* A list of cselib_val structures. */
-struct elt_list
-{
- struct elt_list *next;
- cselib_val *elt;
-};
-
static bool cselib_record_memory;
static bool cselib_preserve_constants;
static bool cselib_any_perm_equivs;
diff --git a/gcc/cselib.h b/gcc/cselib.h
index 6493303f7ff..29765d33f30 100644
--- a/gcc/cselib.h
+++ b/gcc/cselib.h
@@ -47,6 +47,13 @@ struct cselib_val
struct cselib_val *next_containing_mem;
};
+/* A list of cselib_val structures. */
+struct elt_list
+{
+ struct elt_list *next;
+ cselib_val *elt;
+};
+
/* A list of rtl expressions that hold the same value. */
struct elt_loc_list {
/* Next element in the list. */
diff --git a/gcc/postreload.cc b/gcc/postreload.cc
index 475b5889629..cb76f4851a5 100644
--- a/gcc/postreload.cc
+++ b/gcc/postreload.cc
@@ -355,6 +355,77 @@ reload_cse_regs_1 (void)
cleanup_cfg (0);
}
+/* Try to replace the memory source of SET in INSN with a register that
+ holds the same bytes in another mode. Return nonzero if a replacement
+ is added to the current change group. */
+
+static int
+reload_cse_simplify_mem (rtx set, rtx_insn *insn)
+{
+ rtx mem = SET_SRC (set);
+ if (!MEM_P (mem) || MEM_VOLATILE_P (mem))
+ return 0;
+
+ machine_mode mode = GET_MODE (mem);
+ if (!VECTOR_MODE_P (mode)
+ || (FLOAT_MODE_P (mode) && flag_float_store))
+ return 0;
+
+ rtx address = XEXP (mem, 0);
+ machine_mode address_mode = GET_MODE (address);
+ if (address_mode == VOIDmode)
+ address_mode = Pmode;
+ cselib_val *addr = cselib_lookup (address, address_mode, 0, mode);
+ if (!addr)
+ return 0;
+ addr = canonical_cselib_val (addr);
+
+ reg_class_t dclass = REGNO_REG_CLASS (true_regnum (SET_DEST (set)));
+ int best_cost = memory_move_cost (mode, dclass, true);
+ rtx best = NULL_RTX;
+ for (elt_list *l = addr->addr_list; l; l = l->next)
+ {
+ machine_mode from = GET_MODE (l->elt->val_rtx);
+ if (from == mode || !VECTOR_MODE_P (from)
+ || !known_eq (GET_MODE_SIZE (from), GET_MODE_SIZE (mode)))
+ continue;
+
+ bool same_address_space = false;
+ for (elt_loc_list *loc = l->elt->locs; loc; loc = loc->next)
+ if (MEM_P (loc->loc)
+ && MEM_ADDR_SPACE (loc->loc) == MEM_ADDR_SPACE (mem))
+ {
+ same_address_space = true;
+ break;
+ }
+ if (!same_address_space)
+ continue;
+
+ /* The target must permit a bitcast that preserves the common bits.
+ This excludes incompatible register layouts, such as big-endian
+ SVE modes with different element sizes. */
+ for (elt_loc_list *loc = l->elt->locs; loc; loc = loc->next)
+ if (REG_P (loc->loc)
+ && REG_CAN_CHANGE_MODE_P (REGNO (loc->loc), from, mode))
+ {
+ gcc_checking_assert (HARD_REGISTER_P (loc->loc));
+ rtx reg = simplify_subreg (mode, loc->loc, from, 0);
+ if (!reg || !REG_P (reg))
+ continue;
+ int cost = register_move_cost (mode,
+ REGNO_REG_CLASS (REGNO (reg)),
+ dclass);
+ if (cost <= best_cost)
+ best = reg, best_cost = cost;
+ }
+ }
+ if (!best)
+ return 0;
+
+ validate_unshare_change (insn, &SET_SRC (set), best, 1);
+ return 1;
+}
+
/* Try to simplify a single SET instruction. SET is the set pattern.
INSN is the instruction it came from.
This function only handles one case: if we set a register to a value
@@ -395,7 +466,7 @@ reload_cse_simplify_set (rtx set, rtx_insn *insn)
val = cselib_lookup (src, GET_MODE (SET_DEST (set)), 0, VOIDmode);
if (! val)
- return 0;
+ return reload_cse_simplify_mem (set, insn);
/* If memory loads are cheaper than register copies, don't change them. */
if (MEM_P (src))
@@ -476,7 +547,7 @@ reload_cse_simplify_set (rtx set, rtx_insn *insn)
}
}
- return did_change;
+ return did_change ? did_change : reload_cse_simplify_mem (set, insn);
}
/* Try to replace operands in INSN with equivalent values that are already
diff --git a/gcc/testsuite/gcc.dg/rtl/aarch64/postreload-vector-forward-be.c
b/gcc/testsuite/gcc.dg/rtl/aarch64/postreload-vector-forward-be.c
new file mode 100644
index 00000000000..bd7c1651b1e
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/rtl/aarch64/postreload-vector-forward-be.c
@@ -0,0 +1,61 @@
+/* { dg-do compile { target aarch64*-*-* } } */
+/* { dg-options "-O2 -mbig-endian -march=armv8.2-a+sve -msve-vector-bits=256"
} */
+
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+/* A full SVE vector and an Advanced SIMD pair have incompatible register
+ layouts. Advanced SIMD whole-register moves permit mode changes even
+ on big-endian targets. */
+
+#pragma GCC target "+sve"
+
+/*
+** forward_sve:
+** str z0, \[x0\]
+** ldp q1, q2, \[x0\]
+** stp q1, q2, \[x1\]
+** ret
+*/
+void __RTL (startwith ("ira"))
+forward_sve (void)
+{
+(function "forward_sve"
+ (insn-chain
+ (block 2
+ (edge-from entry (flags "FALLTHRU"))
+ (cnote 1 [bb 2] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 5 (set (reg:VNx16BI p0)
+ (const_vector:VNx16BI [(const_int 1) repeated x32])))
+ (cinsn 2 (set (mem:VNx16QI (reg:DI x0) [0 S32 A128])
+ (unspec:VNx16QI [(reg:VNx16BI p0) (reg:VNx16QI v0)]
+ UNSPEC_PRED_X)))
+ (cinsn 3 (set (reg:V2x16QI v1) (mem:V2x16QI (reg:DI x0) [0 S32 A128])))
+ (cinsn 4 (set (mem:V2x16QI (reg:DI x1) [0 S32 A128]) (reg:V2x16QI v1)))
+ (edge-to exit (flags "FALLTHRU"))
+ )
+ )
+)
+}
+
+/*
+** forward_advsimd:
+** str q0, \[x0\]
+** str q0, \[x1\]
+** ret
+*/
+void __RTL (startwith ("ira"))
+forward_advsimd (void)
+{
+(function "forward_advsimd"
+ (insn-chain
+ (block 2
+ (edge-from entry (flags "FALLTHRU"))
+ (cnote 1 [bb 2] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 2 (set (mem:V4SI (reg:DI x0) [0 S16 A128]) (reg:V4SI v0)))
+ (cinsn 3 (set (reg:V2DI v1) (mem:V2DI (reg:DI x0) [0 S16 A128])))
+ (cinsn 4 (set (mem:V2DI (reg:DI x1) [0 S16 A128]) (reg:V2DI v1)))
+ (edge-to exit (flags "FALLTHRU"))
+ )
+ )
+)
+}
diff --git a/gcc/testsuite/gcc.dg/rtl/aarch64/postreload-vector-forward.c
b/gcc/testsuite/gcc.dg/rtl/aarch64/postreload-vector-forward.c
new file mode 100644
index 00000000000..27b71889a23
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/rtl/aarch64/postreload-vector-forward.c
@@ -0,0 +1,105 @@
+/* { dg-do compile { target aarch64*-*-* } } */
+/* { dg-options "-O2 -mlittle-endian" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+/* Start before IRA so that reload initializes dataflow state. The hard
+ registers keep the store and reload modes unchanged until postreload. */
+
+/*
+** forward:
+** str q0, \[x0\]
+** str q0, \[x1\]
+** ret
+*/
+void __RTL (startwith ("ira"))
+forward (void)
+{
+(function "forward"
+ (insn-chain
+ (block 2
+ (edge-from entry (flags "FALLTHRU"))
+ (cnote 1 [bb 2] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 2 (set (mem:V4SI (reg:DI x0) [0 S16 A128]) (reg:V4SI v0)))
+ (cinsn 3 (set (reg:V2DI v1) (mem:V2DI (reg:DI x0) [0 S16 A128])))
+ (cinsn 4 (set (mem:V2DI (reg:DI x1) [0 S16 A128]) (reg:V2DI v1)))
+ (edge-to exit (flags "FALLTHRU"))
+ )
+ )
+)
+}
+
+/*
+** clobbered:
+** str q0, \[x0\]
+** ldr q1, \[x0\]
+** str q1, \[x1\]
+** ret
+*/
+void __RTL (startwith ("ira"))
+clobbered (void)
+{
+(function "clobbered"
+ (insn-chain
+ (block 2
+ (edge-from entry (flags "FALLTHRU"))
+ (cnote 1 [bb 2] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 2 (set (mem:V4SI (reg:DI x0) [0 S16 A128]) (reg:V4SI v0)))
+ (cinsn 7 (set (reg:V4SI v0) (reg:V4SI v2)))
+ (cinsn 3 (set (reg:V2DI v1) (mem:V2DI (reg:DI x0) [0 S16 A128])))
+ (cinsn 4 (set (mem:V2DI (reg:DI x1) [0 S16 A128]) (reg:V2DI v1)))
+ (edge-to exit (flags "FALLTHRU"))
+ )
+ )
+)
+}
+
+/*
+** alias_store:
+** str q0, \[x0\]
+** str w2, \[x0\]
+** ldr q1, \[x0\]
+** str q1, \[x1\]
+** ret
+*/
+void __RTL (startwith ("ira"))
+alias_store (void)
+{
+(function "alias_store"
+ (insn-chain
+ (block 2
+ (edge-from entry (flags "FALLTHRU"))
+ (cnote 1 [bb 2] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 2 (set (mem:V4SI (reg:DI x0) [0 S16 A128]) (reg:V4SI v0)))
+ (cinsn 7 (set (mem:SI (reg:DI x0) [0 S4 A32]) (reg:SI x2)))
+ (cinsn 3 (set (reg:V2DI v1) (mem:V2DI (reg:DI x0) [0 S16 A128])))
+ (cinsn 4 (set (mem:V2DI (reg:DI x1) [0 S16 A128]) (reg:V2DI v1)))
+ (edge-to exit (flags "FALLTHRU"))
+ )
+ )
+)
+}
+
+/*
+** volatile_load:
+** str q0, \[x0\]
+** ldr q1, \[x0\]
+** str q1, \[x1\]
+** ret
+*/
+void __RTL (startwith ("ira"))
+volatile_load (void)
+{
+(function "volatile_load"
+ (insn-chain
+ (block 2
+ (edge-from entry (flags "FALLTHRU"))
+ (cnote 1 [bb 2] NOTE_INSN_BASIC_BLOCK)
+ (cinsn 2 (set (mem:V4SI (reg:DI x0) [0 S16 A128]) (reg:V4SI v0)))
+ (cinsn 3 (set (reg:V2DI v1) (mem/v:V2DI (reg:DI x0) [0 S16 A128])))
+ (cinsn 4 (set (mem:V2DI (reg:DI x1) [0 S16 A128]) (reg:V2DI v1)))
+ (edge-to exit (flags "FALLTHRU"))
+ )
+ )
+)
+}
+
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/spill-mode-change.c
b/gcc/testsuite/gcc.target/aarch64/sve/spill-mode-change.c
new file mode 100644
index 00000000000..3230976af0c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/spill-mode-change.c
@@ -0,0 +1,94 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -march=armv9-a+sve2 -mtune=generic-armv9-a
-msve-vector-bits=256 -mautovec-preference=sve-only -fno-ivopts
-fno-align-functions" } */
+/* { dg-require-effective-target aarch64_little_endian } */
+
+/* Reuse a stored vector across a change in element size. */
+
+#include <arm_sve.h>
+
+typedef svuint8_t v8 __attribute__ ((arm_sve_vector_bits (256)));
+
+static inline void
+f (v8 *a)
+{
+ for (int i = 0; i < 16; i += 8)
+ for (int j = 0; j < 4; j++)
+ {
+ svuint32_t x = svtrn1_u32 (svreinterpret_u32 (a[i + j]),
+ svreinterpret_u32 (a[i + j + 4]));
+ svuint32_t y = svtrn2_u32 (svreinterpret_u32 (a[j]),
+ svreinterpret_u32 (a[i + j]));
+ a[i + j] = svreinterpret_u8 (x);
+ a[i + j + 4] = svreinterpret_u8 (y);
+ }
+ for (int i = 0; i < 8; i++)
+ {
+ v8 x = svreinterpret_u8 (svtrn1_u64 (svreinterpret_u64 (a[i]),
+ svreinterpret_u64 (a[i + 8])));
+ a[i + 8] = svreinterpret_u8 (svtrn2_u64 (svreinterpret_u64 (a[i]),
+ svreinterpret_u64 (a[i + 8])));
+ a[i] = x;
+ }
+}
+
+void
+g (uint8_t *a, long b, uint32_t *c, uint64_t *d, uint8_t *e)
+{
+ unsigned h = c[10];
+ svbool_t p = svptrue_pat_b8 (SV_VL32);
+ svbool_t q = svptrue_pat_b8 (SV_VL16);
+ svbool_t r = svptrue_pat_b32 (SV_VL8);
+ v8 t = svld1_u8 (p, e);
+#define T(x) svtbl_u8 (svuzp1_u8 (svuzp1_u8 (svreinterpret_u8 (x), \
+ svdup_u8 (0)), svdup_u8 (0)), t)
+ uint32_t k = d[1];
+ for (int i = 0; i < 32; i += 8)
+ {
+ v8 v0 = svreinterpret_u8 (svinsr_n_u64 (svdup_u64 (0), 0));
+ svuint32_t v1 = svunpklo_u32 (svunpklo_u16 (v0));
+ svuint32_t v2 = svadd_n_u32_x (r, svindex_u32 (0, 1), i);
+ svuint32_t v3 = svlsl_u32_x (r, svdup_u32 (1), v2);
+ svbool_t p0 = svand_z (
+ r, svcmpne_n_u32 (r, svand_u32_x (r, svdup_u32 (h), v3), 0),
+ svcmpne_n_u32 (r, v1, 0));
+ svuint32_t v4 = svsel_u32 (p0, svdup_u32 (1), svdup_u32 (0));
+ svbool_t p1 = svcmpne_n_u8 (p, T (v4), 0);
+ if (svptest_any (p, p1))
+ continue;
+ uint8_t *s = a + b, *u = s + 7;
+ v8 v[16];
+ for (int j = 0; j < 16; j++)
+ {
+ v[j] = svsplice_u8 (q, svld1_u8 (q, s), svld1_u8 (q, u));
+ u += b;
+ }
+ f (v);
+ svuint32_t v5 = svmax_n_u32_x (r, svdup_u32 (k), 1);
+ svuint32_t v6 = svlsr_n_u32_x (r, v1, 4);
+ svbool_t p2 = svcmpne_n_u32 (
+ r, svand_u32_x (r, svdup_u32 (c[1]), v3), 0);
+ svuint32_t v7 = svsel_u32 (p2, svdup_u32 (8), svdup_u32 (4));
+ v8 v15 = T (v5), v9 = T (v6), v10 = T (v7);
+ svbool_t p3 = svcmple_u8 (p, v[7], v15);
+ v8 v11 = svabd_u8_x (p, v[6], v[7]);
+ v11 = svqadd_u8 (v11, svlsr_n_u8_x (
+ p, svabd_u8_x (p, v[9], v[8]), 1));
+ svbool_t p4 = svcmple_u8 (p, v11, v15);
+ svbool_t p5 = svand_z (p, svand_z (p, p3, p4), p1);
+ if (svptest_any (p, p5))
+ continue;
+ svbool_t p6 = svcmpgt_u8 (p, v[7], v9);
+ svbool_t p7 = svcmpge_n_u8 (p, v10, 6);
+ v8 v12 = svdup_u8 (128);
+ svint8_t v13 = svqsub_s8_m (p6, svreinterpret_s8 (v12), svdup_s8 (0));
+ v8 v14 = sveor_u8_x (p, svreinterpret_u8 (v13), v12);
+ v[8] = svsel_u8 (p7, v14, v[8]);
+ for (int j = 0; j < 16; j++)
+ {
+ svst1_u8 (q, s, v[j]);
+ s += b;
+ }
+ }
+}
+
+/* { dg-final { scan-assembler-not {\tstr\t(z[0-9]+), (\[sp, #[0-9]+, mul
vl\])\n\tldr\t\1, \2} } } */
--
2.50.1 (Apple Git-155)
