From: e0001533 <[email protected]> On RV32 a 2*XLEN argument passed when only one register remains is split: the low word goes in the odd argument register (a7) and the high word on the incoming stack. When the middle-end later reconstructs the value as a single DImode, riscv_zilsd_valid_mem_p relies only on MEM_ALIGN, which is mis-marked A64 for the spilled slot, so a single misaligned zilsd "ld"/"sd" is emitted (e.g. ld s0,76(sp)) and faults under -mzilsd-strict-align.
Double-check the address: reject the zilsd access unless the offset is a multiple of the required alignment. For small frames the address is a constant offset from a frame-related pointer (sp/hfp/ap/fp, all 16-byte aligned). For larger frames address legitimization materializes the offset, either as two chained 12-bit adds (e.g. addi a4,sp,13; addi a5,a4,2047; ld a0,0(a5)) or via a LUI high part (e.g. addi a5,a4,12; add a5,sp,a5; ld a0,0(a5)). Follow the defining instructions of a materialized base register to recover the offset in either case, forcing riscv_split_64bit_move_p to use two 32-bit moves for the misaligned accesses. Signed-off-by: Ciyan Pan <[email protected]> gcc/ChangeLog: * config/riscv/riscv-protos.h (riscv_split_64bit_move_p) (riscv_zilsd_valid_mem_p): Add rtx_insn * parameter. * config/riscv/riscv.cc (riscv_zilsd_frame_ptr_p) (riscv_zilsd_find_def, riscv_zilsd_known_const) (riscv_zilsd_frame_offset): New helpers. (riscv_zilsd_valid_mem_p): Double-check address, including materialized frame offsets. (riscv_split_64bit_move_p, riscv_load_store_insns): Pass insn through to riscv_zilsd_valid_mem_p. * config/riscv/riscv.md (MOVE64 split): Pass insn to riscv_split_64bit_move_p. gcc/testsuite/ChangeLog: * gcc.target/riscv/zilsd-align-split-arg-1.c: New test. * gcc.target/riscv/zilsd-align-split-arg-2.c: New test. * gcc.target/riscv/zilsd-align-split-arg-3.c: New test. * gcc.target/riscv/zilsd-align-split-arg-4.c: New test. --- gcc/config/riscv/riscv-protos.h | 4 +- gcc/config/riscv/riscv.cc | 241 +++++++++++++++++- gcc/config/riscv/riscv.md | 2 +- .../riscv/zilsd-align-split-arg-1.c | 37 +++ .../riscv/zilsd-align-split-arg-2.c | 22 ++ .../riscv/zilsd-align-split-arg-3.c | 29 +++ .../riscv/zilsd-align-split-arg-4.c | 31 +++ 7 files changed, 355 insertions(+), 11 deletions(-) create mode 100644 gcc/testsuite/gcc.target/riscv/zilsd-align-split-arg-1.c create mode 100644 gcc/testsuite/gcc.target/riscv/zilsd-align-split-arg-2.c create mode 100644 gcc/testsuite/gcc.target/riscv/zilsd-align-split-arg-3.c create mode 100644 gcc/testsuite/gcc.target/riscv/zilsd-align-split-arg-4.c diff --git a/gcc/config/riscv/riscv-protos.h b/gcc/config/riscv/riscv-protos.h index b0c1d8d4d05c..930471ccd8d4 100644 --- a/gcc/config/riscv/riscv-protos.h +++ b/gcc/config/riscv/riscv-protos.h @@ -122,7 +122,7 @@ extern rtx riscv_unspec_address (rtx, enum riscv_symbol_type); extern void riscv_move_integer (rtx, rtx, HOST_WIDE_INT); extern bool riscv_legitimize_move (machine_mode, rtx, rtx); extern rtx riscv_subword (rtx, bool); -extern bool riscv_split_64bit_move_p (rtx, rtx); +extern bool riscv_split_64bit_move_p (rtx, rtx, rtx_insn * = nullptr); extern void riscv_split_doubleword_move (rtx, rtx); extern void riscv_split_quadword_move (rtx, rtx); extern const char *riscv_output_move (rtx, rtx); @@ -159,7 +159,7 @@ extern bool riscv_expand_conditional_move (rtx, rtx, rtx, rtx); extern rtx riscv_legitimize_call_address (rtx, bool); extern bool riscv_call_needs_lpad_p (rtx); extern bool riscv_expand_zilsd_misaligned_move (rtx, rtx); -extern bool riscv_zilsd_valid_mem_p (rtx, machine_mode); +extern bool riscv_zilsd_valid_mem_p (rtx, machine_mode, rtx_insn * = nullptr); extern void riscv_set_return_address (rtx, rtx); extern rtx riscv_return_addr (int, rtx); extern poly_int64 riscv_initial_elimination_offset (int, int); diff --git a/gcc/config/riscv/riscv.cc b/gcc/config/riscv/riscv.cc index f8b54fc4ae68..92cfdaab8e05 100644 --- a/gcc/config/riscv/riscv.cc +++ b/gcc/config/riscv/riscv.cc @@ -1265,15 +1265,239 @@ riscv_zilsd_required_align (void) return riscv_zilsd_required_align (riscv_zilsd_align, TARGET_STRICT_ALIGN); } +/* Return true if REGNO is a frame-related pointer that is known to be + 16-byte aligned on RISC-V. */ + +static bool +riscv_zilsd_frame_ptr_p (unsigned int regno) +{ + return (regno == STACK_POINTER_REGNUM + || regno == HARD_FRAME_POINTER_REGNUM + || regno == ARG_POINTER_REGNUM + || regno == FRAME_POINTER_REGNUM); +} + +/* Find the most recent instruction before INSN that sets register REG and + return it, or NULL if none is found. The search is limited to the + current basic block, which is where reload materializes frame offsets. */ + +static rtx_insn * +riscv_zilsd_find_def (rtx reg, rtx_insn *insn) +{ + basic_block bb = BLOCK_FOR_INSN (insn); + + for (rtx_insn *it = prev_real_insn (insn); it; it = prev_real_insn (it)) + { + if (bb && BLOCK_FOR_INSN (it) != bb) + break; + rtx set = single_set (it); + if (set && REG_P (SET_DEST (set)) + && REGNO (SET_DEST (set)) == REGNO (reg)) + return it; + } + + return NULL; +} + +/* Return true if register REG holds a known constant at the point of INSN, + following materialization of large constants which can involve multiple + instructions plus REG_EQUAL notes. */ + +static bool +riscv_zilsd_known_const (rtx reg, rtx_insn *insn, HOST_WIDE_INT *val, + int depth) +{ + if (depth > 4) + return false; + + rtx_insn *def = riscv_zilsd_find_def (reg, insn); + if (!def) + return false; + + rtx set = single_set (def); + if (!set) + return false; + + rtx note = find_reg_equal_equiv_note (def); + if (note && CONST_INT_P (XEXP (note, 0))) + { + *val = INTVAL (XEXP (note, 0)); + return true; + } + + rtx src = SET_SRC (set); + if (CONST_INT_P (src)) + { + *val = INTVAL (src); + return true; + } + + if (GET_CODE (src) == PLUS + && CONST_INT_P (XEXP (src, 1)) + && REG_P (XEXP (src, 0))) + { + HOST_WIDE_INT sub; + if (riscv_zilsd_known_const (XEXP (src, 0), def, &sub, depth + 1)) + { + *val = sub + INTVAL (XEXP (src, 1)); + return true; + } + } + + return false; +} + +/* Return true if ADDR, at instruction INSN, is provably a stack access of + the form (frame_pointer + OFFSET), storing the result in OFFSET. Handle + both the direct form and the forms produced by address legitimization + when the offset does not fit in a single I-type immediate, e.g.: + + addi a5, sp, 13 # a5 = sp + 13 (two s12 adds) + addi a5, a5, 2047 # a5 = sp + 2060 + ld a0, 0(a5) + + or, for offsets that also need a high part: + + lui a4, 1 # a4 = 4096 + addi a5, a4, 12 # a5 = 4108 + add a5, sp, a5 # a5 = sp + 4108 + ld a0, 0(a5) + + DEPTH bounds the recursion. Return false if OFFSET cannot be proven. */ + +static bool +riscv_zilsd_frame_offset (rtx addr, rtx_insn *insn, HOST_WIDE_INT *offset, + int depth) +{ + if (depth > 8) + return false; + + /* Direct frame pointer plus constant offset. */ + if (GET_CODE (addr) == PLUS + && REG_P (XEXP (addr, 0)) + && CONST_INT_P (XEXP (addr, 1)) + && riscv_zilsd_frame_ptr_p (REGNO (XEXP (addr, 0)))) + { + *offset = INTVAL (XEXP (addr, 1)); + return true; + } + + if (REG_P (addr) && riscv_zilsd_frame_ptr_p (REGNO (addr))) + { + *offset = 0; + return true; + } + + if (!insn) + return false; + + /* Split ADDR into (base + small) or a bare register. */ + rtx base = addr; + HOST_WIDE_INT small = 0; + if (GET_CODE (addr) == PLUS + && REG_P (XEXP (addr, 0)) + && CONST_INT_P (XEXP (addr, 1))) + { + base = XEXP (addr, 0); + small = INTVAL (XEXP (addr, 1)); + } + else if (!REG_P (addr)) + return false; + + rtx_insn *def = riscv_zilsd_find_def (base, insn); + if (!def) + return false; + + rtx set = single_set (def); + if (!set) + return false; + + rtx src = SET_SRC (set); + if (GET_CODE (src) == PLUS) + { + rtx op0 = XEXP (src, 0); + rtx op1 = XEXP (src, 1); + + /* (frame_pointer + (const | reg)): the base is found, the other + operand is either an immediate or a register holding a known + constant. */ + if (REG_P (op0) && riscv_zilsd_frame_ptr_p (REGNO (op0))) + { + HOST_WIDE_INT mat; + if (CONST_INT_P (op1)) + mat = INTVAL (op1); + else if (REG_P (op1) + && riscv_zilsd_known_const (op1, def, &mat, 0)) + ; + else + return false; + *offset = mat + small; + return true; + } + + /* (reg + const): recurse on the register, accumulating the + immediate. */ + if (REG_P (op0) && CONST_INT_P (op1)) + { + HOST_WIDE_INT sub; + if (riscv_zilsd_frame_offset (op0, def, &sub, depth + 1)) + { + *offset = sub + INTVAL (op1) + small; + return true; + } + return false; + } + + /* (reg + reg): recurse on the register, adding the constant held by + the second operand. */ + if (REG_P (op0) && REG_P (op1)) + { + HOST_WIDE_INT mat, sub; + if (riscv_zilsd_known_const (op1, def, &mat, 0) + && riscv_zilsd_frame_offset (op0, def, &sub, depth + 1)) + { + *offset = sub + mat + small; + return true; + } + } + } + + /* Bare register source: recurse. */ + if (REG_P (src)) + { + HOST_WIDE_INT sub; + if (riscv_zilsd_frame_offset (src, def, &sub, depth + 1)) + { + *offset = sub + small; + return true; + } + } + + return false; +} + /* Return true if MEM can be accessed by a Zilsd 2 * XLEN load/store. */ bool -riscv_zilsd_valid_mem_p (rtx mem, machine_mode mode) +riscv_zilsd_valid_mem_p (rtx mem, machine_mode mode, rtx_insn *insn) { - return (TARGET_ZILSD - && riscv_2x_xlen_mode_p (mode) - && MEM_P (mem) - && MEM_ALIGN (mem) >= riscv_zilsd_required_align () * BITS_PER_UNIT); + if (!TARGET_ZILSD || !riscv_2x_xlen_mode_p (mode) || !MEM_P (mem)) + return false; + + if (MEM_ALIGN (mem) < riscv_zilsd_required_align () * BITS_PER_UNIT) + return false; + + /* MEM_ALIGN can be inaccurate for reconstructed split arguments where the + low word is held in an odd register and the high word lives on the + incoming stack. Such accesses materialize at a 4-byte-aligned stack + offset, so double-check the address when it is a constant offset from a + frame-related pointer (all 16-byte aligned on RISC-V). Large frame + offsets may be materialized into a register before the access. */ + HOST_WIDE_INT offset; + if (riscv_zilsd_frame_offset (XEXP (mem, 0), insn, &offset, 0)) + return (offset % riscv_zilsd_required_align ()) == 0; + + return true; } /* Return the effective Zilsd memory access alignment policy. */ @@ -3091,7 +3315,8 @@ riscv_load_store_insns (rtx mem, rtx_insn *insn) else if (GET_MODE_BITSIZE (mode).to_constant () == 64) { set = single_set (insn); - if (set && !riscv_split_64bit_move_p (SET_DEST (set), SET_SRC (set))) + if (set && !riscv_split_64bit_move_p (SET_DEST (set), SET_SRC (set), + insn)) might_split_p = false; } @@ -5396,7 +5621,7 @@ spill_subreg_move (rtx dest, rtx src) /* Return true if a 64-bit move from SRC to DEST should be split into two. */ bool -riscv_split_64bit_move_p (rtx dest, rtx src) +riscv_split_64bit_move_p (rtx dest, rtx src, rtx_insn *insn) { if (TARGET_64BIT) return false; @@ -5413,7 +5638,7 @@ riscv_split_64bit_move_p (rtx dest, rtx src) if (GP_REG_P (regno)) { rtx mem = MEM_P (dest) ? dest : src; - if (!riscv_zilsd_valid_mem_p (mem, GET_MODE (mem))) + if (!riscv_zilsd_valid_mem_p (mem, GET_MODE (mem), insn)) return true; return (regno < FIRST_PSEUDO_REGISTER) && ((regno % 2) != 0); diff --git a/gcc/config/riscv/riscv.md b/gcc/config/riscv/riscv.md index bd308e3d056d..88bc55d0e4ad 100644 --- a/gcc/config/riscv/riscv.md +++ b/gcc/config/riscv/riscv.md @@ -2870,7 +2870,7 @@ [(set (match_operand:MOVE64 0 "nonimmediate_operand") (match_operand:MOVE64 1 "move_operand"))] "reload_completed - && riscv_split_64bit_move_p (operands[0], operands[1])" + && riscv_split_64bit_move_p (operands[0], operands[1], insn)" [(const_int 0)] { riscv_split_doubleword_move (operands[0], operands[1]); diff --git a/gcc/testsuite/gcc.target/riscv/zilsd-align-split-arg-1.c b/gcc/testsuite/gcc.target/riscv/zilsd-align-split-arg-1.c new file mode 100644 index 000000000000..b3a5785862b3 --- /dev/null +++ b/gcc/testsuite/gcc.target/riscv/zilsd-align-split-arg-1.c @@ -0,0 +1,37 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=rv32gcv_zilsd -mabi=ilp32d -mrvv-vector-bits=scalable -fno-vect-cost-model" } */ +/* { dg-additional-options "-mzilsd-strict-align" } */ + +#include <stdint-gcc.h> + +/* On RV32 an int64_t argument passed after seven pointer arguments is + split: the low word lands in the last available argument register (a7) + and the high word on the incoming stack. When the vectoriser later + materialises n as a single DImode value, the low word is spilled to a + 4-byte-aligned stack slot. With -mzilsd-strict-align the backend must + not reload that value with a single misaligned zilsd "ld"; it must split + the reload into two 32-bit loads instead, otherwise the access faults on + targets that trap on misaligned 64-bit loads. */ + +void __attribute__ ((noinline, noclone)) +f6 (uint64_t *__restrict a, uint64_t *__restrict b, uint64_t *__restrict c, + uint64_t *__restrict d, uint64_t *__restrict e, uint64_t *__restrict f, + uint64_t *__restrict g, int64_t n) +{ + for (int64_t i = 0; i < n; ++i) + { + a[i] = g[i * 6]; + b[i] = g[i * 6 + 1]; + c[i] = g[i * 6 + 2]; + d[i] = g[i * 6 + 3]; + e[i] = g[i * 6 + 4]; + f[i] = g[i * 6 + 5]; + } +} + +/* The spilled low word of n sits at a 4-byte-aligned stack offset (76); + it must be reloaded with "lw", never with a single misaligned zilsd + "ld". The vector loop counter, in contrast, lives at an 8-byte-aligned + offset (8) and may legitimately use ld/sd. */ +/* { dg-final { scan-assembler-not {ld\t[a-z0-9]+,76\(sp\)} } } */ +/* { dg-final { scan-assembler-not {sd\t[a-z0-9]+,76\(sp\)} } } */ diff --git a/gcc/testsuite/gcc.target/riscv/zilsd-align-split-arg-2.c b/gcc/testsuite/gcc.target/riscv/zilsd-align-split-arg-2.c new file mode 100644 index 000000000000..248b16b0686e --- /dev/null +++ b/gcc/testsuite/gcc.target/riscv/zilsd-align-split-arg-2.c @@ -0,0 +1,22 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=rv32i_zilsd -mabi=ilp32" } */ +/* { dg-additional-options "-mzilsd-strict-align" } */ + +/* On RV32 an int64_t argument passed after seven pointer arguments is + split between registers and the stack: the low word lands in the last + available argument register (a7) and the high word on the incoming + stack. Reloading n as a single 64-bit value therefore materialises a + zilsd "ld" from a 4-byte-aligned stack slot (offset 12). Under + -mzilsd-strict-align the backend must not emit such a misaligned + doubleword load; it must split the reload into two 32-bit loads. */ + +long long +f7 (long long *a, long long *b, long long *c, long long *d, + long long *e, long long *f, long long *g, long long n) +{ + return n; +} + +/* The misaligned reload must use two 32-bit "lw", never a single zilsd + "ld" from the 4-byte-aligned stack offset 12. */ +/* { dg-final { scan-assembler-not {ld\t[a-z0-9]+,12\(sp\)} } } */ diff --git a/gcc/testsuite/gcc.target/riscv/zilsd-align-split-arg-3.c b/gcc/testsuite/gcc.target/riscv/zilsd-align-split-arg-3.c new file mode 100644 index 000000000000..467839a5dd6b --- /dev/null +++ b/gcc/testsuite/gcc.target/riscv/zilsd-align-split-arg-3.c @@ -0,0 +1,29 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=rv32i_zilsd -mabi=ilp32" } */ +/* { dg-additional-options "-mzilsd-strict-align" } */ + +/* Same split-argument reconstruction as zilsd-align-split-arg-2.c, but with + a large stack frame. The stack offset of the spilled high word of n is + then too large for a single I-type immediate, so address legitimization + materializes the offset into a register before the access: + + addi a5, a4, 12 # a5 = 4108 + add a5, sp, a5 # a5 = sp + 4108 + ld a0, 0(a5) # misaligned zilsd load (would fault) + + Under -mzilsd-strict-align the backend must recognise this materialized + form and split the reload into two 32-bit loads. */ + +long long +f7 (long long *a, long long *b, long long *c, long long *d, + long long *e, long long *f, long long *g, long long n) +{ + volatile char buf[4096]; + buf[0] = 1; + return n; +} + +/* The only doubleword access in this function is the misaligned reload of + n, so no zilsd ld/sd may appear at all; the reload must use two lw. */ +/* { dg-final { scan-assembler-not {ld\t} } } */ +/* { dg-final { scan-assembler-not {sd\t} } } */ diff --git a/gcc/testsuite/gcc.target/riscv/zilsd-align-split-arg-4.c b/gcc/testsuite/gcc.target/riscv/zilsd-align-split-arg-4.c new file mode 100644 index 000000000000..1a57881e3b62 --- /dev/null +++ b/gcc/testsuite/gcc.target/riscv/zilsd-align-split-arg-4.c @@ -0,0 +1,31 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=rv32i_zilsd -mabi=ilp32" } */ +/* { dg-additional-options "-mzilsd-strict-align" } */ + +/* Same split-argument reconstruction as zilsd-align-split-arg-2.c, but with + a frame whose stack offset of the spilled high word of n no longer fits in + a single I-type immediate, yet is still small enough to be formed from two + 12-bit adds (unlike zilsd-align-split-arg-3.c, which needs a LUI high + part). Address legitimization then materializes the offset with a chained + addi: + + addi a4, sp, 13 # a4 = sp + 13 + addi t1, a4, 2047 # t1 = sp + 2060 + ld a0, 0(t1) # misaligned zilsd load (would fault) + + Under -mzilsd-strict-align the backend must recognise this two-step form + and split the reload into two 32-bit loads. */ + +long long +f7 (long long *a, long long *b, long long *c, long long *d, + long long *e, long long *f, long long *g, long long n) +{ + volatile char buf[2040]; + buf[0] = 1; + return n; +} + +/* The only doubleword access in this function is the misaligned reload of + n, so no zilsd ld/sd may appear at all; the reload must use two lw. */ +/* { dg-final { scan-assembler-not {ld\t} } } */ +/* { dg-final { scan-assembler-not {sd\t} } } */ -- 2.43.0
