https://gcc.gnu.org/g:0e2e9e48ca5683e175ab842cc5e75fed6d69dcd2
commit r17-4172-g0e2e9e48ca5683e175ab842cc5e75fed6d69dcd2 Author: Robin Dapp <[email protected]> Date: Fri Sep 4 17:24:09 2026 +0200 RISC-V: Add scalar memset. Currently, if we compile with vector support and we cannot straight-line expand a memset using vectors, the expander fails and we don't fall back to the by-pieces infrastructure but to a libcall instead. That's because the guard is more permissive than the actual expander. But even if they agree, memset of a nonzero value expands to a libcall because by-pieces refuses a 5-insn synthesized splat constant. This patch adds two things: - A testing_p param for memset expansion that asks the expanders to not emit any code but just report if they could expand. This is used in riscv_use_by_pieces_infrastructure_p to decide whether or not to fall back to by-pieces. - A scalar memset expander for nonzero values and known lengths. The implementation is intentionally kept simple for now and just stores splatted Pmode values when unaligned accesses are fast. gcc/ChangeLog: * config/riscv/riscv-protos.h (riscv_expand_block_clear): Remove. (expand_vec_setmem): Add testing param. (riscv_expand_setmem): Likewise. * config/riscv/riscv-string.cc (riscv_expand_block_clear_zicboz_zic64b): Likewise. (riscv_expand_block_clear): Make static. (expand_scalar_setmem): New function. (riscv_expand_setmem): New function. (expand_vec_setmem): Add testing param. * config/riscv/riscv.cc (riscv_use_by_pieces_infrastructure_p): Query riscv_expand_setmem for support. * config/riscv/riscv.md: Use riscv_expand_setmem. gcc/testsuite/ChangeLog: * gcc.target/riscv/setmem-nolibcall.c: New test. * gcc.target/riscv/setmem-scalar-32.c: New test. * gcc.target/riscv/setmem-scalar-64.c: New test. * gcc.target/riscv/setmem-scalar-run.c: New test. Diff: --- gcc/config/riscv/riscv-protos.h | 4 +- gcc/config/riscv/riscv-string.cc | 155 ++++++++++++++++++++- gcc/config/riscv/riscv.cc | 21 ++- gcc/config/riscv/riscv.md | 11 +- gcc/testsuite/gcc.target/riscv/setmem-nolibcall.c | 33 +++++ gcc/testsuite/gcc.target/riscv/setmem-scalar-32.c | 29 ++++ gcc/testsuite/gcc.target/riscv/setmem-scalar-64.c | 31 +++++ gcc/testsuite/gcc.target/riscv/setmem-scalar-run.c | 58 ++++++++ 8 files changed, 317 insertions(+), 25 deletions(-) diff --git a/gcc/config/riscv/riscv-protos.h b/gcc/config/riscv/riscv-protos.h index 121714014c6e..72b7b4823399 100644 --- a/gcc/config/riscv/riscv-protos.h +++ b/gcc/config/riscv/riscv-protos.h @@ -225,7 +225,6 @@ extern bool vsetvl_insn_p (rtx_insn *); /* Routines implemented in riscv-string.c. */ extern bool riscv_expand_block_compare (rtx, rtx, rtx, rtx); extern bool riscv_expand_block_move (rtx, rtx, rtx, bool); -extern bool riscv_expand_block_clear (rtx, rtx); /* Information about one CPU we know about. */ struct riscv_cpu_info { @@ -768,7 +767,7 @@ void expand_popcount (rtx *); void expand_rawmemchr (machine_mode, rtx, rtx, rtx, bool = false); bool expand_strcmp (rtx, rtx, rtx, rtx, unsigned HOST_WIDE_INT, bool); void emit_vec_extract (rtx, rtx, rtx); -bool expand_vec_setmem (rtx, rtx, rtx); +bool expand_vec_setmem (rtx, rtx, rtx, bool); bool expand_vec_cmpmem (rtx, rtx, rtx, rtx); void expand_strided_load (machine_mode, rtx *); void expand_strided_store (machine_mode, rtx *); @@ -849,6 +848,7 @@ const unsigned int RISCV_BUILTIN_CLASS = (1 << RISCV_BUILTIN_SHIFT) - 1; /* Routines implemented in riscv-string.cc. */ extern bool riscv_expand_strcmp (rtx, rtx, rtx, rtx, rtx); extern bool riscv_expand_strlen (rtx, rtx, rtx, rtx); +extern bool riscv_expand_setmem (rtx, rtx, rtx, bool); /* Routines implemented in riscv-fusion.cc. */ enum riscv_fusion_pairs diff --git a/gcc/config/riscv/riscv-string.cc b/gcc/config/riscv/riscv-string.cc index b434609a781d..5ca9f4e1f829 100644 --- a/gcc/config/riscv/riscv-string.cc +++ b/gcc/config/riscv/riscv-string.cc @@ -1043,10 +1043,12 @@ riscv_expand_block_move (rtx dest, rtx src, rtx length, bool overlap_p) return false; } -/* Expand a block-clear instruction via cbo.zero instructions. */ +/* Expand a block-clear instruction via cbo.zero instructions. + If TESTING_P is true, don't actually emit instructions, but only + return whether we can. */ static bool -riscv_expand_block_clear_zicboz_zic64b (rtx dest, rtx length) +riscv_expand_block_clear_zicboz_zic64b (rtx dest, rtx length, bool testing_p) { unsigned HOST_WIDE_INT hwi_length; unsigned HOST_WIDE_INT align; @@ -1071,6 +1073,9 @@ riscv_expand_block_clear_zicboz_zic64b (rtx dest, rtx length) if (hwi_length > max_bytes) return false; + if (testing_p) + return true; + unsigned HOST_WIDE_INT offset = 0; while (offset + cbo_bytes <= hwi_length) { @@ -1092,8 +1097,8 @@ riscv_expand_block_clear_zicboz_zic64b (rtx dest, rtx length) return true; } -bool -riscv_expand_block_clear (rtx dest, rtx length) +static bool +riscv_expand_block_clear (rtx dest, rtx length, bool testing_p) { /* Only use setmem-zero expansion for Zicboz + Zic64b. */ if (!TARGET_ZICBOZ || !TARGET_ZIC64B) @@ -1102,9 +1107,138 @@ riscv_expand_block_clear (rtx dest, rtx length) if (optimize_function_for_size_p (cfun)) return false; - return riscv_expand_block_clear_zicboz_zic64b (dest, length); + return riscv_expand_block_clear_zicboz_zic64b (dest, length, testing_p); } +/* Expand a suitable memset to straightline Xmode moves. + Only expand if unaligned access is fast and the length does not + exceed the memset inline threshold. + Return true on success and false otherwise. If TESTING_P is true, + don't actually emit anything, but just return whether we could. */ + +static bool +expand_scalar_setmem (rtx dst, rtx len, rtx val, bool testing_p) +{ + if (!CONST_INT_P (len)) + return false; + + /* By-pieces can handle this better. */ + if (val == const0_rtx) + return false; + + unsigned HOST_WIDE_INT hwilen = UINTVAL (len); + + if (riscv_memset_size_threshold >= 0 + && hwilen > (unsigned HOST_WIDE_INT) riscv_memset_size_threshold) + return false; + + /* Unlike memcpy, don't emit loops here for now, irrespective of the + user-provided threshold. */ + if (hwilen > RISCV_MAX_MOVE_BYTES_STRAIGHT) + return false; + + /* Don't expand if misaligned stores are slow. */ + if (riscv_slow_unaligned_access_p) + return false; + + /* A libcall is smaller. */ + if (optimize_function_for_size_p (cfun)) + return false; + + /* ??? Needed for the splat but we could still emulate it. */ + if (!CONST_INT_P (val) && !TARGET_MUL) + return false; + + if (testing_p) + return true; + + rtx splat; + if (!CONST_INT_P (val)) + { + rtx val2 = gen_reg_rtx (Xmode); + do_zero_extendqi2 (val2, force_reg (QImode, val)); + + /* Splat the value to an Xmode register. */ + splat = gen_reg_rtx (Xmode); + if (TARGET_64BIT) + { + rtx valmul = force_reg (Xmode, GEN_INT + (HOST_WIDE_INT_C (0x0101010101010101))); + emit_insn (gen_muldi3 (splat, val2, valmul)); + } + else + { + rtx valmul = force_reg (Xmode, GEN_INT + (HOST_WIDE_INT_C (0x01010101))); + emit_insn (gen_mulsi3 (splat, val2, valmul)); + } + } + else + splat = force_reg (Xmode, gen_int_mode + ((INTVAL (val) & 0xff) + * HOST_WIDE_INT_UC (0x0101010101010101), Xmode)); + + unsigned HOST_WIDE_INT i = 0; + for (; i + UNITS_PER_WORD <= hwilen; i += UNITS_PER_WORD) + { + rtx mem = adjust_address (dst, Xmode, i); + riscv_emit_move (mem, splat); + } + + /* Use an overlapping store for the rest. */ + if (i < hwilen) + { + if (exact_log2 (hwilen - i) != -1) + { + scalar_int_mode elmode + = int_mode_for_size ((hwilen - i) * BITS_PER_UNIT, 0).require (); + rtx mem = adjust_address (dst, elmode, i); + riscv_emit_move (mem, gen_lowpart (elmode, splat)); + } + else + { + /* Get the next-smaller power of 2 of length. */ + HOST_WIDE_INT elsz = HOST_WIDE_INT_1U << (ceil_log2 (hwilen - i) - 1); + scalar_int_mode elmode + = int_mode_for_size (elsz * BITS_PER_UNIT, 0).require (); + rtx mem = adjust_address (dst, elmode, i); + riscv_emit_move (mem, gen_lowpart (elmode, splat)); + mem = adjust_address (dst, elmode, hwilen - elsz); + riscv_emit_move (mem, gen_lowpart (elmode, splat)); + } + } + + return true; +} + +/* Expand a memset operation. Return true on success and false + otherwise. TESTING_P is true if we're only querying whether + we could expand but not actually perform the expansion. */ + +bool +riscv_expand_setmem (rtx dst, rtx len, rtx val, bool testing_p) +{ + if (TARGET_VECTOR && stringop_strategy & STRATEGY_VECTOR) + { + bool ok = riscv_vector::expand_vec_setmem (dst, len, val, testing_p); + if (ok) + return true; + } + + if (val == const0_rtx && riscv_expand_block_clear (dst, len, testing_p)) + return true; + + if (stringop_strategy & STRATEGY_SCALAR) + { + bool ok = expand_scalar_setmem (dst, len, val, testing_p); + if (ok) + return true; + } + + return false; +} + + /* --- Vector expanders --- */ namespace riscv_vector { @@ -1686,9 +1820,13 @@ check_vectorise_memory_operation (rtx length_in, HOST_WIDE_INT &lmul_out) return true; } -/* Used by setmemdi in riscv.md. */ +/* Expand a vector memset with the given parameters. If TESTING_P is + true, return true if we could expand but don't actually perform + the expansion. */ + bool -expand_vec_setmem (rtx dst_in, rtx length_in, rtx fill_value_in) +expand_vec_setmem (rtx dst_in, rtx length_in, rtx fill_value_in, + bool testing_p) { stringop_info info; @@ -1707,6 +1845,9 @@ expand_vec_setmem (rtx dst_in, rtx length_in, rtx fill_value_in) else if (riscv_memset_size_threshold != -1) return false; + if (testing_p) + return true; + rtx dst_addr = copy_addr_to_reg (XEXP (dst_in, 0)); rtx dst = change_address (dst_in, info.vmode, dst_addr); diff --git a/gcc/config/riscv/riscv.cc b/gcc/config/riscv/riscv.cc index 126370755565..11d3c5a32dcd 100644 --- a/gcc/config/riscv/riscv.cc +++ b/gcc/config/riscv/riscv.cc @@ -15736,13 +15736,22 @@ riscv_use_by_pieces_infrastructure_p (unsigned HOST_WIDE_INT size, unsigned alignment, enum by_pieces_operation op, bool speed_p) { - /* For set/clear with size > UNITS_PER_WORD, by pieces uses vector broadcasts - with UNITS_PER_WORD size pieces. Use setmem<mode> instead which can use - bigger chunks. */ - if (TARGET_VECTOR && stringop_strategy & STRATEGY_VECTOR - && (op == CLEAR_BY_PIECES || op == SET_BY_PIECES) + /* Query the expanders whether they can handle the given operation. */ + if ((op == CLEAR_BY_PIECES || op == SET_BY_PIECES) && speed_p && size > UNITS_PER_WORD) - return false; + { + /* Use dummy RTX as the checks need reasonable values. */ + rtx val; + if (op == CLEAR_BY_PIECES) + val = const0_rtx; + else + val = gen_rtx_REG (QImode, GP_REG_FIRST + 1); + rtx dst = gen_rtx_MEM (BLKmode, stack_pointer_rtx); + set_mem_align (dst, alignment); + if (riscv_expand_setmem (dst, gen_int_mode (size, Xmode), val, + /* TESTING_P */ true)) + return false; + } return default_use_by_pieces_infrastructure_p (size, alignment, op, speed_p); } diff --git a/gcc/config/riscv/riscv.md b/gcc/config/riscv/riscv.md index 45d79c914286..8420b3d29e7b 100644 --- a/gcc/config/riscv/riscv.md +++ b/gcc/config/riscv/riscv.md @@ -2960,16 +2960,7 @@ (use (match_operand:SI 3 "const_int_operand"))])] "" { - /* If TARGET_VECTOR is false, this routine will return false and we will - try scalar expansion. */ - if (riscv_vector::expand_vec_setmem (operands[0], operands[1], operands[2])) - DONE; - - /* If value to set is not zero, use the library routine. */ - if (operands[2] != const0_rtx) - FAIL; - - if (riscv_expand_block_clear (operands[0], operands[1])) + if (riscv_expand_setmem (operands[0], operands[1], operands[2], false)) DONE; else FAIL; diff --git a/gcc/testsuite/gcc.target/riscv/setmem-nolibcall.c b/gcc/testsuite/gcc.target/riscv/setmem-nolibcall.c new file mode 100644 index 000000000000..a475cff287fd --- /dev/null +++ b/gcc/testsuite/gcc.target/riscv/setmem-nolibcall.c @@ -0,0 +1,33 @@ +/* { dg-do compile } */ +/* { dg-options "-march=rv64gcv -mabi=lp64d -mtune=generic-ooo" } */ +/* { dg-skip-if "" { *-*-* } { "-O0" "-Og" "-Os" "-Oz" } } */ + +#define SET_VAR(N) \ +void set_var_##N (void *dst, int c) \ +{ \ + __builtin_memset (dst, c, N); \ +} + +#define SET_CST(N) \ +void set_cst_##N (void *dst) \ +{ \ + __builtin_memset (dst, 3, N); \ +} + +#define SET_ZERO(N) \ +void set_zero_##N (void *dst) \ +{ \ + __builtin_memset (dst, 0, N); \ +} + +#define FOR_EACH_LEN(M) \ + M(17) M(18) M(20) M(24) M(25) M(31) M(32) M(33) \ + M(40) M(63) M(64) M(65) M(96) + +FOR_EACH_LEN (SET_VAR) +FOR_EACH_LEN (SET_CST) +FOR_EACH_LEN (SET_ZERO) + +/* { dg-final { scan-assembler-not "memset" } } */ +/* { dg-final { scan-assembler-not "\tcall\t" } } */ +/* { dg-final { scan-assembler-not "\ttail\t" } } */ diff --git a/gcc/testsuite/gcc.target/riscv/setmem-scalar-32.c b/gcc/testsuite/gcc.target/riscv/setmem-scalar-32.c new file mode 100644 index 000000000000..3f174c1ffbf1 --- /dev/null +++ b/gcc/testsuite/gcc.target/riscv/setmem-scalar-32.c @@ -0,0 +1,29 @@ +/* { dg-do compile } */ +/* { dg-options "-march=rv32gc -mabi=ilp32d -mtune=generic-ooo -mstringop-strategy=scalar" } */ +/* { dg-skip-if "" { *-*-* } { "-O0" "-Os" "-Oz" "-Og" "-flto" } } */ + +#define SET_VAR(N) \ +void set_var_##N (void *dst, int c) \ +{ \ + dst = __builtin_assume_aligned (dst, 4); \ + __builtin_memset (dst, c, N); \ +} + +#define SET_CST(N) \ +void set_cst_##N (void *dst) \ +{ \ + dst = __builtin_assume_aligned (dst, 4); \ + __builtin_memset (dst, 3, N); \ +} + +SET_VAR(16) +SET_VAR(32) +SET_VAR(48) + +SET_CST(16) +SET_CST(32) +SET_CST(48) + +/* { dg-final { scan-assembler-times "sw\t" 48 } } */ +/* { dg-final { scan-assembler-not "sb\t" } } */ +/* { dg-final { scan-assembler-not "memset" } } */ diff --git a/gcc/testsuite/gcc.target/riscv/setmem-scalar-64.c b/gcc/testsuite/gcc.target/riscv/setmem-scalar-64.c new file mode 100644 index 000000000000..948f59a3962a --- /dev/null +++ b/gcc/testsuite/gcc.target/riscv/setmem-scalar-64.c @@ -0,0 +1,31 @@ +/* { dg-do compile } */ +/* { dg-options "-march=rv64gc -mabi=lp64d -mtune=generic-ooo -mstringop-strategy=scalar" } */ +/* { dg-skip-if "" { *-*-* } { "-O0" "-Os" "-Oz" "-Og" "-flto" } } */ + +#define SET_VAR(N) \ +void set_var_##N (void *dst, int c) \ +{ \ + dst = __builtin_assume_aligned (dst, 8); \ + __builtin_memset (dst, c, N); \ +} + +#define SET_CST(N) \ +void set_cst_##N (void *dst) \ +{ \ + dst = __builtin_assume_aligned (dst, 8); \ + __builtin_memset (dst, 3, N); \ +} + +SET_VAR(8) +SET_VAR(16) +SET_VAR(24) +SET_VAR(32) + +SET_CST(8) +SET_CST(16) +SET_CST(24) +SET_CST(32) + +/* { dg-final { scan-assembler-times "sd\t" 20 } } */ +/* { dg-final { scan-assembler-not "sb\t" } } */ +/* { dg-final { scan-assembler-not "memset" } } */ diff --git a/gcc/testsuite/gcc.target/riscv/setmem-scalar-run.c b/gcc/testsuite/gcc.target/riscv/setmem-scalar-run.c new file mode 100644 index 000000000000..766a091a3930 --- /dev/null +++ b/gcc/testsuite/gcc.target/riscv/setmem-scalar-run.c @@ -0,0 +1,58 @@ +/* { dg-do run } */ +/* { dg-additional-options "-O2 -mtune=generic-ooo -mstringop-strategy=scalar" } */ + +#define EMPTY 16 +#define SZ 128 +#define FILL 0xa5 + +#define FOR_EACH_LEN(M) \ + M(1) M(2) M(3) M(4) M(5) M(7) M(8) M(9) M(15) M(16) \ + M(17) M(23) M(24) M(25) M(31) M(32) M(33) M(40) M(63) \ + M(64) M(65) M(71) M(96) M(127) M(128) + +#define DEF(N) \ + __attribute__((noinline)) static void \ + setmem_##N (void *dst, int c) \ + { \ + __builtin_memset (dst, c, N); \ + } + +FOR_EACH_LEN (DEF) + +static unsigned char buf[EMPTY + SZ + EMPTY]; + +static void +test (unsigned off, unsigned len, int c) +{ + unsigned i; + + for (i = 0; i < sizeof (buf); i++) + { + unsigned char want = (i >= off && i < off + len) + ? (unsigned char) c : FILL; + if (buf[i] != want) + __builtin_abort (); + } +} + +#define RUN(N) \ + __builtin_memset (buf, FILL, sizeof (buf)); \ + setmem_##N (buf + EMPTY + 1, c); \ + test (EMPTY + 1, N, c); + +static void +run (int c) +{ + FOR_EACH_LEN (RUN) +} + +int +main (void) +{ + run (0); + run (1); + run (3); + run (0x5a); + run (0xff); + return 0; +}
