Group scalar load/store-pair candidates before sched2 and order them by
offset, preferring decreasing offsets when both INC and DEC are eligible.
Share eligibility, direction and alignment constraints through a rule
table used by the priority hook and the final pair matcher. Require
offsets aligned to the access size for integer and floating-point pairs.
ALIGNED_STD requires the lower offset aligned to twice the access size.
Enable scheduling fusion by default for supported tunes, including after
restoring per-function optimization options, while preserving explicit
-f[no-]schedule-fusion options. Keep positive tests as XFAIL until the
tuning patch enables the pairs.
Reference static sched2 macro-fusion counts for SPEC CPU 2017 Integer
built with -O3 -flto, measured independently of ready-list reordering
against a baseline with xt-c9501fdvt fusion pairs enabled. These
measurements have not been repeated for this patch:
Benchmark Baseline With patch Change
500.perlbench_r 15302 17077 +11.6%
502.gcc_r 72806 78135 +7.3%
505.mcf_r 63 85 +34.9%
520.omnetpp_r 20479 22084 +7.8%
523.xalancbmk_r 42442 44651 +5.2%
525.x264_r 6232 7033 +12.9%
531.deepsjeng_r 299 328 +9.7%
541.leela_r 794 851 +7.2%
548.exchange2_r 353 426 +20.7%
557.xz_r 1217 1321 +8.5%
SUM 159987 171991 +7.5%
gcc/ChangeLog:
* config/riscv/riscv-fusion.cc (sched_fusion_type): Use bitmasks.
(riscv_fusion_mem_info): Add reg.
(riscv_fuse_mem_p): Record the register operand.
(riscv_fusion_mem_rule): New struct.
(riscv_fusion_mem_rules): New table.
(riscv_fuse_mem_candidate_p): New function.
(riscv_fuse_mem_direction): Likewise.
(riscv_fuse_mem_pair_p): Check complete pairs using an operation
parameter and the shared rule table and eligibility check.
(riscv_fuse_ldst_pair_p): Remove.
(riscv_sched_fusion_priority): New function.
(riscv_fuse_aligned_std): Use riscv_fuse_mem_pair_p.
(riscv_fuse_ldst_pair_inc): Update call.
(riscv_fuse_ldst_pair_dec): Likewise.
(riscv_fuse_fldfst_pair_inc): Likewise.
(riscv_fuse_fldfst_pair_dec): Likewise.
* config/riscv/riscv-protos.h (riscv_fusion_direction): New enum.
(riscv_sched_fusion_priority): Declare.
(riscv_fuse_mem_direction): Likewise.
* config/riscv/riscv.cc (riscv_override_options_internal): Enable
scheduling fusion by default using riscv_fuse_mem_direction.
(riscv_override_options_after_change): Refresh the scheduling-fusion
default.
(TARGET_SCHED_FUSION_PRIORITY): Define.
gcc/testsuite/ChangeLog:
* gcc.target/riscv/fusion-memory-rtl-shapes-ldst-pair.c: Check
access-size alignment.
* gcc.target/riscv/sched-fusion-priority.c: New test.
Signed-off-by: Jin Ma <[email protected]>
---
gcc/config/riscv/riscv-fusion.cc | 301 +++++++++++++-----
gcc/config/riscv/riscv-protos.h | 11 +
gcc/config/riscv/riscv.cc | 13 +
.../fusion-memory-rtl-shapes-ldst-pair.c | 70 +++-
.../gcc.target/riscv/sched-fusion-priority.c | 92 ++++++
5 files changed, 402 insertions(+), 85 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/riscv/sched-fusion-priority.c
diff --git a/gcc/config/riscv/riscv-fusion.cc b/gcc/config/riscv/riscv-fusion.cc
index 3e1e466dd8e..608cd2e7a38 100644
--- a/gcc/config/riscv/riscv-fusion.cc
+++ b/gcc/config/riscv/riscv-fusion.cc
@@ -633,13 +633,13 @@ riscv_fuse_shift_pair_p (rtx_insn *prev, rtx_insn *curr,
return true;
}
-/* Load/store classes used by fusion checks. */
+/* Load/store classes used by fusion checks and rule masks. */
enum sched_fusion_type
{
- SCHED_FUSION_LD_SIGN_EXTEND = 0,
- SCHED_FUSION_LD_ZERO_EXTEND,
- SCHED_FUSION_LD,
- SCHED_FUSION_ST
+ SCHED_FUSION_LD_SIGN_EXTEND = 1 << 0,
+ SCHED_FUSION_LD_ZERO_EXTEND = 1 << 1,
+ SCHED_FUSION_LD = 1 << 2,
+ SCHED_FUSION_ST = 1 << 3
};
/* Fusion-relevant information about a scalar load or store. */
@@ -648,6 +648,8 @@ struct riscv_fusion_mem_info
{
enum sched_fusion_type type;
struct riscv_address_info addr;
+ /* The register operand, or const0_rtx for a zero store. */
+ rtx reg;
machine_mode mode;
bool fp_p;
};
@@ -768,9 +770,111 @@ riscv_fuse_mem_p (rtx_insn *insn, struct
riscv_fusion_mem_info *info)
else
return false;
+ /* Retain the register operand for pair dependency checks. */
+ info->reg = isload_p ? dest : src;
return true;
}
+/* Constraints shared by memory-pair classification and matching. */
+
+struct riscv_fusion_mem_rule
+{
+ enum riscv_fusion_pairs op;
+ /* Bit N permits an N-byte access. */
+ unsigned HOST_WIDE_INT access_sizes;
+ /* Mask of sched_fusion_type values. */
+ unsigned int types;
+ /* Whether the access uses the floating-point register file. */
+ bool fp_p;
+ /* Whether the memory mode must be a scalar integer mode. */
+ bool scalar_int_p;
+ /* Supported address order, independent of scheduling preferences. */
+ enum riscv_fusion_direction direction;
+ /* Positive multiple of the access size used to align the lower offset. */
+ unsigned int alignment_factor;
+};
+
+static const struct riscv_fusion_mem_rule riscv_fusion_mem_rules[] =
+{
+ { RISCV_FUSE_LDST_PAIR_INC, (1U << 4) | (1U << 8),
+ SCHED_FUSION_LD | SCHED_FUSION_LD_SIGN_EXTEND | SCHED_FUSION_ST,
+ false, false, RISCV_FUSION_INC, 1 },
+ { RISCV_FUSE_LDST_PAIR_DEC, (1U << 4) | (1U << 8),
+ SCHED_FUSION_LD | SCHED_FUSION_LD_SIGN_EXTEND | SCHED_FUSION_ST,
+ false, false, RISCV_FUSION_DEC, 1 },
+ { RISCV_FUSE_FLDFST_PAIR_INC, (1U << 4) | (1U << 8),
+ SCHED_FUSION_LD | SCHED_FUSION_LD_SIGN_EXTEND
+ | SCHED_FUSION_LD_ZERO_EXTEND | SCHED_FUSION_ST,
+ true, false, RISCV_FUSION_INC, 1 },
+ { RISCV_FUSE_FLDFST_PAIR_DEC, (1U << 4) | (1U << 8),
+ SCHED_FUSION_LD | SCHED_FUSION_LD_SIGN_EXTEND
+ | SCHED_FUSION_LD_ZERO_EXTEND | SCHED_FUSION_ST,
+ true, false, RISCV_FUSION_DEC, 1 },
+ /* Allow 1-, 2-, 4- and 8-byte stores. */
+ { RISCV_FUSE_ALIGNED_STD, (1U << 1) | (1U << 2) | (1U << 4) | (1U << 8),
+ SCHED_FUSION_ST, false, true, RISCV_FUSION_ANY, 2 }
+};
+
+/* Return true if MEM satisfies RULE's single-instruction constraints.
+ Leave scheduling phases and pair constraints to the operation checker. */
+
+static bool
+riscv_fuse_mem_candidate_p (const struct riscv_fusion_mem_info *mem,
+ const struct riscv_fusion_mem_rule &rule)
+{
+ /* Check the memory mode independently of the register file. */
+ if (mem->addr.type != ADDRESS_REG
+ || !CONST_INT_P (mem->addr.offset)
+ || mem->fp_p != rule.fp_p
+ || !(rule.types & mem->type)
+ || (rule.scalar_int_p && !SCALAR_INT_MODE_P (mem->mode)))
+ return false;
+
+ /* Bound the shift used to test the access-size mask. */
+ unsigned int access_size = GET_MODE_SIZE (mem->mode).to_constant ();
+ if (access_size >= HOST_BITS_PER_WIDE_INT
+ || !(rule.access_sizes & (HOST_WIDE_INT_1U << access_size)))
+ return false;
+
+ /* Both members of an aligned pair have access-size-aligned offsets. */
+ return INTVAL (mem->addr.offset) % access_size == 0;
+}
+
+/* Return a memory-pair scheduling direction for INSN under the current tune.
+ A null INSN queries support without checking a specific instruction. */
+
+enum riscv_fusion_direction
+riscv_fuse_mem_direction (rtx_insn *insn)
+{
+ struct riscv_fusion_mem_info mem;
+ if (insn && !riscv_fuse_mem_p (insn, &mem))
+ return RISCV_FUSION_NONE;
+
+ unsigned HOST_WIDE_INT fusible_ops = riscv_get_fusible_ops ();
+ bool inc_p = false, dec_p = false, any_p = false;
+ /* Collect directions from all enabled rules that accept this access. */
+ for (const riscv_fusion_mem_rule &rule : riscv_fusion_mem_rules)
+ {
+ if (!(fusible_ops & rule.op)
+ || (insn && !riscv_fuse_mem_candidate_p (&mem, rule)))
+ continue;
+
+ inc_p |= rule.direction == RISCV_FUSION_INC;
+ dec_p |= rule.direction == RISCV_FUSION_DEC;
+ any_p |= rule.direction == RISCV_FUSION_ANY;
+ }
+
+ /* Prefer directional rules over bidirectional rules to preserve their
+ scheduling order when both are eligible. */
+ if (inc_p && dec_p)
+ return RISCV_FUSION_ANY;
+ if (inc_p)
+ return RISCV_FUSION_INC;
+ if (dec_p)
+ return RISCV_FUSION_DEC;
+ return any_p ? RISCV_FUSION_ANY : RISCV_FUSION_NONE;
+}
+
/* Extract an add-type instruction followed by an integer load or store that
uses the add result as an undisplaced address. */
@@ -839,72 +943,120 @@ riscv_fuse_indexed_mem_p (rtx_insn *prev, rtx_insn *curr,
return !riscv_fuse_same_reg_p (SET_SRC (mem_set), update_dest);
}
-/* Extract a pair of scalar loads or stores with the same mode and base
- register. Store whether they are loads in *LOAD_P. */
+/* Check scalar load/store pair operation OP, sharing candidate eligibility
+ with the priority hook. Leave scheduling phase checks to the caller. */
static bool
riscv_fuse_mem_pair_p (rtx_insn *prev, rtx_insn *curr,
- struct riscv_fusion_mem_info *prev_mem,
- struct riscv_fusion_mem_info *curr_mem,
- bool *load_p)
-{
- if (!riscv_fuse_sets_p (prev, curr)
- || !riscv_fuse_mem_p (prev, prev_mem)
- || !riscv_fuse_mem_p (curr, curr_mem))
+ enum riscv_fusion_pairs op)
+{
+ /* Match the caller's operation, regardless of other enabled rules. */
+ const riscv_fusion_mem_rule *rule = nullptr;
+ for (const riscv_fusion_mem_rule &candidate : riscv_fusion_mem_rules)
+ if (candidate.op == op)
+ {
+ rule = &candidate;
+ break;
+ }
+ gcc_assert (rule);
+
+ struct riscv_fusion_mem_info prev_mem, curr_mem;
+ if (!riscv_fuse_mem_p (prev, &prev_mem)
+ || !riscv_fuse_mem_candidate_p (&prev_mem, *rule)
+ || !riscv_fuse_mem_p (curr, &curr_mem)
+ || !riscv_fuse_mem_candidate_p (&curr_mem, *rule))
return false;
- bool prev_load_p = prev_mem->type != SCHED_FUSION_ST;
- bool curr_load_p = curr_mem->type != SCHED_FUSION_ST;
+ /* Check the relationship between the two eligible accesses. */
+ bool prev_load_p = prev_mem.type != SCHED_FUSION_ST;
+ bool curr_load_p = curr_mem.type != SCHED_FUSION_ST;
if (prev_load_p != curr_load_p
- || prev_mem->fp_p != curr_mem->fp_p
- || prev_mem->mode != curr_mem->mode
- || prev_mem->addr.type != ADDRESS_REG
- || curr_mem->addr.type != ADDRESS_REG
- || !CONST_INT_P (prev_mem->addr.offset)
- || !CONST_INT_P (curr_mem->addr.offset)
- || !riscv_fuse_same_reg_p (prev_mem->addr.reg,
- curr_mem->addr.reg))
+ || prev_mem.fp_p != curr_mem.fp_p
+ || prev_mem.mode != curr_mem.mode
+ || !riscv_fuse_same_reg_p (prev_mem.addr.reg, curr_mem.addr.reg))
return false;
- *load_p = prev_load_p;
- return true;
-}
-
-/* Check common adjacent load/store-pair constraints. INC_P selects ascending
- offsets and FP_P selects floating-point rather than integer accesses. */
+ /* Loads need distinct results, and the first must preserve the base. */
+ if (prev_load_p
+ && (riscv_fuse_same_reg_p (prev_mem.reg, curr_mem.reg)
+ || riscv_fuse_same_reg_p (prev_mem.addr.reg, prev_mem.reg)))
+ return false;
-static bool
-riscv_fuse_ldst_pair_p (rtx_insn *prev, rtx_insn *curr,
- bool inc_p, bool fp_p)
-{
- struct riscv_fusion_mem_info prev_mem, curr_mem;
- bool load_p;
- if (!riscv_fuse_mem_pair_p (prev, curr, &prev_mem, &curr_mem,
- &load_p)
- || prev_mem.fp_p != fp_p
- || prev_mem.type == SCHED_FUSION_LD_ZERO_EXTEND
- || curr_mem.type == SCHED_FUSION_LD_ZERO_EXTEND)
+ HOST_WIDE_INT prev_offset = INTVAL (prev_mem.addr.offset);
+ HOST_WIDE_INT curr_offset = INTVAL (curr_mem.addr.offset);
+ bool inc_p = prev_offset < curr_offset;
+ if (rule->direction != RISCV_FUSION_ANY
+ && inc_p != (rule->direction == RISCV_FUSION_INC))
return false;
+ /* Use the lower offset for the pair alignment check. */
+ if (!inc_p)
+ std::swap (prev_offset, curr_offset);
+
+ /* Use unsigned subtraction to avoid overflow for opposite-signed offsets.
*/
+ unsigned HOST_WIDE_INT diff = ((unsigned HOST_WIDE_INT) curr_offset
+ - (unsigned HOST_WIDE_INT) prev_offset);
HOST_WIDE_INT access_size = GET_MODE_SIZE (prev_mem.mode).to_constant ();
- if (access_size != 4 && access_size != 8)
- return false;
+ HOST_WIDE_INT alignment = rule->alignment_factor * access_size;
+ return (diff == (unsigned HOST_WIDE_INT) access_size
+ && prev_offset % alignment == 0);
+}
- if (load_p)
- {
- rtx prev_dest = SET_DEST (single_set (prev));
- rtx curr_dest = SET_DEST (single_set (curr));
- if (riscv_fuse_same_reg_p (prev_dest, curr_dest)
- || riscv_fuse_same_reg_p (prev_mem.addr.reg, prev_dest))
- return false;
- }
+/* Implement TARGET_SCHED_FUSION_PRIORITY. Group load/store pair candidates
+ by register file, access kind, mode and base register, then by offset. */
- HOST_WIDE_INT diff = inc_p
- ? INTVAL (curr_mem.addr.offset)
- - INTVAL (prev_mem.addr.offset)
- : INTVAL (prev_mem.addr.offset)
- - INTVAL (curr_mem.addr.offset);
- return diff == access_size;
+void
+riscv_sched_fusion_priority (rtx_insn *insn, int max_pri,
+ int *fusion_pri, int *pri)
+{
+ struct riscv_fusion_mem_info mem;
+ unsigned int base_regno;
+ bool isload_p, fp_p, inc_p;
+ int fusion_type, tmp;
+
+ gcc_assert (INSN_P (insn));
+
+ /* Keep default priorities unless this instruction is a pair candidate. */
+ tmp = max_pri - 1;
+ *fusion_pri = tmp;
+ *pri = tmp;
+
+ enum riscv_fusion_direction direction
+ = riscv_fuse_mem_direction (insn);
+ if (direction == RISCV_FUSION_NONE || !riscv_fuse_mem_p (insn, &mem))
+ return;
+
+ base_regno = riscv_regno (mem.addr.reg);
+ if (base_regno >= FIRST_PSEUDO_REGISTER)
+ return;
+
+ isload_p = mem.type != SCHED_FUSION_ST;
+ fp_p = mem.fp_p;
+ /* Prefer decreasing offsets by default when either direction is possible,
+ matching frame save/restore order. */
+ inc_p = direction == RISCV_FUSION_INC;
+
+ /* Give each load/store class and base register a distinct priority below
+ that of unrelated instructions. */
+ fusion_type = (fp_p ? 2 : 0) + (isload_p ? 0 : 1);
+ fusion_type *= NUM_MACHINE_MODES;
+ fusion_type += (int) mem.mode + 1;
+ *fusion_pri -= (fusion_type * FIRST_PSEUDO_REGISTER
+ + (int) base_regno);
+
+ tmp /= 2;
+ HOST_WIDE_INT off_val = INTVAL (mem.addr.offset);
+ /* Use unsigned arithmetic to handle the most negative offset. */
+ unsigned HOST_WIDE_INT magnitude = off_val < 0
+ ? -(unsigned HOST_WIDE_INT) off_val
+ : off_val;
+ int offset_pri = magnitude & 0xfffff;
+
+ /* Order offsets in the preferred pair direction. */
+ if (inc_p == (off_val >= 0))
+ *pri = tmp - offset_pri;
+ else
+ *pri = tmp + offset_pri;
}
/* Check the common RTL for ZEXTW, ZEXTWS and ZEXTH fusion. */
@@ -1488,6 +1640,7 @@ riscv_fuse_auipc_ld (rtx_insn *prev, rtx_insn *curr)
curr (store) == (set (mem (rs1, offset2)) (reg rs3))
Constraints:
+ access size is 1, 2, 4 or 8 bytes
both stores use the same scalar integer mode
min (offset1, offset2) is aligned to twice the access size
abs (offset1 - offset2) equals the access size. */
@@ -1495,27 +1648,7 @@ riscv_fuse_auipc_ld (rtx_insn *prev, rtx_insn *curr)
static bool
riscv_fuse_aligned_std (rtx_insn *prev, rtx_insn *curr)
{
- struct riscv_fusion_mem_info prev_mem, curr_mem;
- bool load_p;
- if (!riscv_fuse_mem_pair_p (prev, curr, &prev_mem, &curr_mem,
- &load_p)
- || load_p
- || prev_mem.fp_p
- || !SCALAR_INT_MODE_P (prev_mem.mode))
- return false;
-
- unsigned int mode_size
- = estimated_poly_value (GET_MODE_SIZE (curr_mem.mode));
- HOST_WIDE_INT prev_offset = INTVAL (prev_mem.addr.offset);
- HOST_WIDE_INT curr_offset = INTVAL (curr_mem.addr.offset);
- if (prev_offset > curr_offset)
- std::swap (prev_offset, curr_offset);
-
- if (prev_offset % (2 * mode_size) == 0
- && prev_offset + mode_size == curr_offset)
- return true;
-
- return false;
+ return riscv_fuse_mem_pair_p (prev, curr, RISCV_FUSE_ALIGNED_STD);
}
/* Check for RISCV_FUSE_LDST_PAIR_INC fusion.
@@ -1529,13 +1662,14 @@ riscv_fuse_aligned_std (rtx_insn *prev, rtx_insn *curr)
Constraints:
access size is 4 or 8 bytes
offset2 - offset1 equals the access size
+ offsets are aligned to the access size
loads are not zero-extending
for loads, rd1 != rd2 and rd1 != rs1. */
static bool
riscv_fuse_ldst_pair_inc (rtx_insn *prev, rtx_insn *curr)
{
- return riscv_fuse_ldst_pair_p (prev, curr, true, false);
+ return riscv_fuse_mem_pair_p (prev, curr, RISCV_FUSE_LDST_PAIR_INC);
}
/* Check for RISCV_FUSE_LDST_PAIR_DEC fusion.
@@ -1549,13 +1683,14 @@ riscv_fuse_ldst_pair_inc (rtx_insn *prev, rtx_insn
*curr)
Constraints:
access size is 4 or 8 bytes
offset1 - offset2 equals the access size
+ offsets are aligned to the access size
loads are not zero-extending
for loads, rd1 != rd2 and rd1 != rs1. */
static bool
riscv_fuse_ldst_pair_dec (rtx_insn *prev, rtx_insn *curr)
{
- return riscv_fuse_ldst_pair_p (prev, curr, false, false);
+ return riscv_fuse_mem_pair_p (prev, curr, RISCV_FUSE_LDST_PAIR_DEC);
}
/* Check for RISCV_FUSE_FLDFST_PAIR_INC fusion.
@@ -1569,12 +1704,13 @@ riscv_fuse_ldst_pair_dec (rtx_insn *prev, rtx_insn
*curr)
Constraints:
access size is 4 or 8 bytes
offset2 - offset1 equals the access size
+ offsets are aligned to the access size
for loads, frd1 != frd2. */
static bool
riscv_fuse_fldfst_pair_inc (rtx_insn *prev, rtx_insn *curr)
{
- return riscv_fuse_ldst_pair_p (prev, curr, true, true);
+ return riscv_fuse_mem_pair_p (prev, curr, RISCV_FUSE_FLDFST_PAIR_INC);
}
/* Check for RISCV_FUSE_FLDFST_PAIR_DEC fusion.
@@ -1588,12 +1724,13 @@ riscv_fuse_fldfst_pair_inc (rtx_insn *prev, rtx_insn
*curr)
Constraints:
access size is 4 or 8 bytes
offset1 - offset2 equals the access size
+ offsets are aligned to the access size
for loads, frd1 != frd2. */
static bool
riscv_fuse_fldfst_pair_dec (rtx_insn *prev, rtx_insn *curr)
{
- return riscv_fuse_ldst_pair_p (prev, curr, false, true);
+ return riscv_fuse_mem_pair_p (prev, curr, RISCV_FUSE_FLDFST_PAIR_DEC);
}
/* Check for RISCV_FUSE_BFEXT fusion.
diff --git a/gcc/config/riscv/riscv-protos.h b/gcc/config/riscv/riscv-protos.h
index f9e9ad51ab5..0a0ba6a0442 100644
--- a/gcc/config/riscv/riscv-protos.h
+++ b/gcc/config/riscv/riscv-protos.h
@@ -883,8 +883,19 @@ enum riscv_fusion_pairs
RISCV_FUSE_FLDFST_PAIR_DEC = HOST_WIDE_INT_1U << 26,
};
+/* Possible directions for a memory-pair candidate. */
+enum riscv_fusion_direction
+{
+ RISCV_FUSION_NONE,
+ RISCV_FUSION_INC,
+ RISCV_FUSION_DEC,
+ RISCV_FUSION_ANY
+};
+
extern bool riscv_macro_fusion_p (void);
extern bool riscv_macro_fusion_pair_p (rtx_insn *, rtx_insn *);
+extern void riscv_sched_fusion_priority (rtx_insn *, int, int *, int *);
+extern enum riscv_fusion_direction riscv_fuse_mem_direction (rtx_insn *);
extern unsigned HOST_WIDE_INT riscv_get_fusible_ops (void);
/* Routines implemented in thead.cc. */
diff --git a/gcc/config/riscv/riscv.cc b/gcc/config/riscv/riscv.cc
index f8b54fc4ae6..281080f282c 100644
--- a/gcc/config/riscv/riscv.cc
+++ b/gcc/config/riscv/riscv.cc
@@ -12319,6 +12319,11 @@ riscv_override_options_internal (struct gcc_options
*opts)
? &optimize_size_tune_info
: cpu->tune_param;
+ /* Enable scheduling fusion for load/store-pair tunes. */
+ if (!OPTION_SET_P (flag_schedule_fusion))
+ opts->x_flag_schedule_fusion
+ = riscv_fuse_mem_direction (nullptr) != RISCV_FUSION_NONE;
+
/* If not optimizing for size, set the default
alignment to what the target wants. */
if (!opts->x_optimize_size)
@@ -12427,6 +12432,12 @@ riscv_override_options_after_change (void)
}
else if (!OPTION_SET_P (flag_cunroll_grow_size))
flag_cunroll_grow_size = flag_peel_loops || optimize >= 3;
+
+ /* Restore the tune's fusion default after optimization option changes.
+ The initial option override sets tune_param later. */
+ if (tune_param && !OPTION_SET_P (flag_schedule_fusion))
+ flag_schedule_fusion
+ = riscv_fuse_mem_direction (nullptr) != RISCV_FUSION_NONE;
}
/* Implement TARGET_OPTION_OVERRIDE. */
@@ -16835,6 +16846,8 @@ riscv_memtag_tag_bitsize ()
#define TARGET_SCHED_MACRO_FUSION_P riscv_macro_fusion_p
#undef TARGET_SCHED_MACRO_FUSION_PAIR_P
#define TARGET_SCHED_MACRO_FUSION_PAIR_P riscv_macro_fusion_pair_p
+#undef TARGET_SCHED_FUSION_PRIORITY
+#define TARGET_SCHED_FUSION_PRIORITY riscv_sched_fusion_priority
#undef TARGET_SCHED_INIT
#define TARGET_SCHED_INIT riscv_sched_init
diff --git
a/gcc/testsuite/gcc.target/riscv/fusion-memory-rtl-shapes-ldst-pair.c
b/gcc/testsuite/gcc.target/riscv/fusion-memory-rtl-shapes-ldst-pair.c
index 2a19c6f3d97..ff59079fcc9 100644
--- a/gcc/testsuite/gcc.target/riscv/fusion-memory-rtl-shapes-ldst-pair.c
+++ b/gcc/testsuite/gcc.target/riscv/fusion-memory-rtl-shapes-ldst-pair.c
@@ -3,7 +3,8 @@
/* { dg-options "-march=rv64gc -mabi=lp64d -O2 -mtune=xt-c9501fdvt
-fdump-rtl-sched2-details" } */
/* { dg-final { scan-rtl-dump-times "RISCV_FUSE_LDST_PAIR_INC" 1 "sched2" {
xfail *-*-* } } } */
-/* The second load destination may be the shared base register. */
+/* The second load destination may be the shared base register. Offsets
+ aligned to the access size need not be aligned to twice that size. */
long __RTL (startwith ("sched2"))
test_load_pair_inc_second_destination_is_base (void)
{
@@ -14,11 +15,13 @@ test_load_pair_inc_second_destination_is_base (void)
(cnote 1 [bb 2] NOTE_INSN_BASIC_BLOCK)
(cnote 2 NOTE_INSN_FUNCTION_BEG)
(cinsn 3 (set (reg:DI a1)
- (mem:DI (reg:DI a0) [0 S8 A64])))
- (cinsn 4 (set (reg:DI a0)
(mem:DI
(plus:DI (reg:DI a0)
(const_int 8)) [0 S8 A64])))
+ (cinsn 4 (set (reg:DI a0)
+ (mem:DI
+ (plus:DI (reg:DI a0)
+ (const_int 16)) [0 S8 A64])))
(cinsn 5 (use (reg:DI a1)))
(cinsn 6 (use (reg:DI a0)))
(cjump_insn 7 (simple_return))
@@ -31,3 +34,64 @@ test_load_pair_inc_second_destination_is_base (void)
) ;; crtl
) ;; function "test_load_pair_inc_second_destination_is_base"
}
+
+/* Reject unaligned loads and stores, covering both widths and directions. */
+long __RTL (startwith ("sched2"))
+test_unaligned_load_store_pairs (void)
+{
+(function "test_unaligned_load_store_pairs"
+ (insn-chain
+ (block 2
+ (edge-from entry (flags "FALLTHRU"))
+ (cnote 1 [bb 2] NOTE_INSN_BASIC_BLOCK)
+ (cnote 2 NOTE_INSN_FUNCTION_BEG)
+ (cinsn 3 (set (reg:DI a1)
+ (sign_extend:DI
+ (mem:SI
+ (plus:DI (reg:DI a0)
+ (const_int 2)) [0 S4 A16]))))
+ (cinsn 4 (set (reg:DI a2)
+ (sign_extend:DI
+ (mem:SI
+ (plus:DI (reg:DI a0)
+ (const_int 6)) [0 S4 A16]))))
+ (cinsn 5 (set (reg:DI a3)
+ (mem:DI
+ (plus:DI (reg:DI a0)
+ (const_int 12)) [0 S8 A32])))
+ (cinsn 6 (set (reg:DI a4)
+ (mem:DI
+ (plus:DI (reg:DI a0)
+ (const_int 4)) [0 S8 A32])))
+ (cinsn 7 (set (mem:SI
+ (plus:DI (reg:DI a0)
+ (const_int 18)) [0 S4 A16])
+ (reg:SI a5)))
+ (cinsn 8 (set (mem:SI
+ (plus:DI (reg:DI a0)
+ (const_int 22)) [0 S4 A16])
+ (reg:SI a6)))
+ (cinsn 9 (set (mem:DI
+ (plus:DI (reg:DI a0)
+ (const_int 44)) [0 S8 A32])
+ (reg:DI a5)))
+ (cinsn 10 (set (mem:DI
+ (plus:DI (reg:DI a0)
+ (const_int 36)) [0 S8 A32])
+ (reg:DI a6)))
+ (cinsn 11 (use (reg:DI a1)))
+ (cinsn 12 (use (reg:DI a2)))
+ (cinsn 13 (use (reg:DI a3)))
+ (cinsn 14 (use (reg:DI a4)))
+ (cjump_insn 15 (simple_return))
+ (edge-to exit)
+ ) ;; block 2
+ (cbarrier 16)
+ ) ;; insn-chain
+ (crtl
+ (return_rtx (reg/i:DI a0))
+ ) ;; crtl
+) ;; function "test_unaligned_load_store_pairs"
+}
+
+/* { dg-final { scan-rtl-dump-not "RISCV_FUSE_LDST_PAIR_DEC" "sched2" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sched-fusion-priority.c
b/gcc/testsuite/gcc.target/riscv/sched-fusion-priority.c
new file mode 100644
index 00000000000..f153e45e9aa
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sched-fusion-priority.c
@@ -0,0 +1,92 @@
+/* { dg-do compile { target { rv64 } } } */
+/* { dg-skip-if "" { *-*-* } { "-O0" "-O1" "-O3" "-O[sgz]" "-flto" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -mtune=xt-c9501fdvt -O2" } */
+/* { dg-additional-options "-fno-schedule-insns2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+/* Start before sched_fusion and disable sched2 to isolate the priority
+ hook. */
+
+/*
+**sched_fusion_load_pair: { xfail *-*-* }
+** ...
+** ld [a-z][0-9]+,8\(a0\)
+** ld [a-z][0-9]+,0\(a0\)
+** ...
+** ret
+*/
+long __RTL (startwith ("compgotos"))
+sched_fusion_load_pair (void)
+{
+(function "sched_fusion_load_pair"
+ (insn-chain
+ (block 2
+ (edge-from entry (flags "FALLTHRU"))
+ (cnote 1 [bb 2] NOTE_INSN_BASIC_BLOCK)
+ (cnote 2 NOTE_INSN_FUNCTION_BEG)
+ (cinsn 3 (set (reg:DI a1)
+ (mem:DI (reg:DI a0) [0 S8 A64])))
+ (cinsn 4 (set (reg:DI a3)
+ (plus:DI (reg:DI a4) (reg:DI a5))))
+ (cinsn 5 (set (reg:DI a2)
+ (mem:DI
+ (plus:DI (reg:DI a0)
+ (const_int 8)) [0 S8 A64])))
+ (cinsn 6 (use (reg:DI a1)))
+ (cinsn 7 (use (reg:DI a2)))
+ (cinsn 8 (use (reg:DI a3)))
+ (cjump_insn 9 (simple_return))
+ (edge-to exit)
+ ) ;; block 2
+ (cbarrier 10)
+ ) ;; insn-chain
+ (crtl
+ (return_rtx
+ (reg/i:DI a0)
+ ) ;; return_rtx
+ ) ;; crtl
+) ;; function "sched_fusion_load_pair"
+}
+
+/* Unaligned offsets must not receive memory-pair priorities. */
+/*
+**sched_fusion_unaligned_loads:
+** ...
+** ld [a-z][0-9]+,4\(a0\)
+** add [a-z][0-9]+,[a-z][0-9]+,[a-z][0-9]+
+** ld [a-z][0-9]+,12\(a0\)
+** ...
+** ret
+*/
+long __RTL (startwith ("compgotos"))
+sched_fusion_unaligned_loads (void)
+{
+(function "sched_fusion_unaligned_loads"
+ (insn-chain
+ (block 2
+ (edge-from entry (flags "FALLTHRU"))
+ (cnote 1 [bb 2] NOTE_INSN_BASIC_BLOCK)
+ (cnote 2 NOTE_INSN_FUNCTION_BEG)
+ (cinsn 3 (set (reg:DI a1)
+ (mem:DI
+ (plus:DI (reg:DI a0)
+ (const_int 4)) [0 S8 A32])))
+ (cinsn 4 (set (reg:DI a3)
+ (plus:DI (reg:DI a4) (reg:DI a5))))
+ (cinsn 5 (set (reg:DI a2)
+ (mem:DI
+ (plus:DI (reg:DI a0)
+ (const_int 12)) [0 S8 A32])))
+ (cinsn 6 (use (reg:DI a1)))
+ (cinsn 7 (use (reg:DI a2)))
+ (cinsn 8 (use (reg:DI a3)))
+ (cjump_insn 9 (simple_return))
+ (edge-to exit)
+ ) ;; block 2
+ (cbarrier 10)
+ ) ;; insn-chain
+ (crtl
+ (return_rtx (reg/i:DI a0))
+ ) ;; crtl
+) ;; function "sched_fusion_unaligned_loads"
+}
--
2.52.0
