Hi,
This patch adds a scalar memmove and tests for it. So far we only had a
vector implementation but most of the plumbing was already in place.
In order to wire up the scalar implementation, the patch adds a new
dispatcher riscv_expand_block_move and an overlap parameter to the
riscv_block_move routines. It adds overlap handling to
riscv_block_move_straight but not to riscv_block_move_loop. Thus,
expansion is limited to known lengths up to 96 bytes.
Regtested on rv64gcv_zvl512b.
Regards
Robin
gcc/ChangeLog:
* config/riscv/riscv-protos.h (riscv_expand_block_move):
Declare.
* config/riscv/riscv-string.cc (riscv_block_move_straight):
Handle overlap.
(riscv_adjust_block_mem): Add overlap param.
(riscv_block_move_loop): Likewise.
(riscv_expand_block_move_scalar): Add overlap param.
(riscv_expand_block_move): Likewise.
(expand_block_move): Likewise.
* config/riscv/riscv.md: Use riscv_expand_block_move.
gcc/testsuite/ChangeLog:
* gcc.target/riscv/movmem-scalar-32.c: New test.
* gcc.target/riscv/movmem-scalar-64.c: New test.
* gcc.target/riscv/movmem-scalar-run.c: New test.
---
gcc/config/riscv/riscv-protos.h | 2 +-
gcc/config/riscv/riscv-string.cc | 134 ++++++++++++------
gcc/config/riscv/riscv.md | 7 +-
.../gcc.target/riscv/movmem-scalar-32.c | 19 +++
.../gcc.target/riscv/movmem-scalar-64.c | 20 +++
.../gcc.target/riscv/movmem-scalar-run.c | 100 +++++++++++++
6 files changed, 234 insertions(+), 48 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/riscv/movmem-scalar-32.c
create mode 100644 gcc/testsuite/gcc.target/riscv/movmem-scalar-64.c
create mode 100644 gcc/testsuite/gcc.target/riscv/movmem-scalar-run.c
diff --git a/gcc/config/riscv/riscv-protos.h b/gcc/config/riscv/riscv-protos.h
index b0c1d8d4d05..ca04b8f6c09 100644
--- a/gcc/config/riscv/riscv-protos.h
+++ b/gcc/config/riscv/riscv-protos.h
@@ -224,7 +224,7 @@ extern bool vsetvl_insn_p (rtx_insn *);
/* Routines implemented in riscv-string.c. */
extern bool riscv_expand_block_compare (rtx, rtx, rtx, rtx);
-extern bool riscv_expand_block_move (rtx, rtx, rtx);
+extern bool riscv_expand_block_move (rtx, rtx, rtx, bool);
extern bool riscv_expand_block_clear (rtx, rtx);
/* Information about one CPU we know about. */
diff --git a/gcc/config/riscv/riscv-string.cc b/gcc/config/riscv/riscv-string.cc
index fcafae61f5a..7e3c0c01ae0 100644
--- a/gcc/config/riscv/riscv-string.cc
+++ b/gcc/config/riscv/riscv-string.cc
@@ -798,15 +798,14 @@ riscv_expand_block_compare (rtx result, rtx src1, rtx
src2, rtx nbytes)
}
/* Emit straight-line code to move LENGTH bytes from SRC to DEST
with accesses that are ALIGN bytes aligned.
- Assume that the areas do not overlap. */
+ Only assume overlap may happen if OVERLAP_P is true. */
static void
riscv_block_move_straight (rtx dest, rtx src, unsigned HOST_WIDE_INT length,
- unsigned HOST_WIDE_INT align)
+ unsigned HOST_WIDE_INT align, bool overlap_p)
{
unsigned HOST_WIDE_INT offset = 0, delta;
unsigned HOST_WIDE_INT bits;
- int i;
enum machine_mode mode;
rtx *regs;
@@ -815,32 +814,71 @@ riscv_block_move_straight (rtx dest, rtx src, unsigned
HOST_WIDE_INT length,
mode = mode_for_size (bits, MODE_INT, 0).require ();
delta = bits / BITS_PER_UNIT;
- if (2 * delta <= length)
+ if (overlap_p)
{
- /* Allocate a buffer for the temporary registers. */
- regs = XALLOCAVEC (rtx, length / delta - 1);
+ /* For overlap, read everything including the remainder into
+ registers first, then store. The last store can overlap the
+ one before it. */
- /* Load as many BITS-sized chunks as possible. Use a normal load if
- the source has enough alignment, otherwise use left/right pairs. */
- for (offset = 0, i = 0; offset + 2 * delta <= length;
- offset += delta, i++)
+ /* The element size is the next power of two smaller than length
+ for small lengths or DELTA. */
+ unsigned HOST_WIDE_INT elsz = HOST_WIDE_INT_1U << floor_log2 (length);
+ elsz = std::min (elsz, delta);
+
+ HOST_WIDE_INT nregs = CEIL (length, elsz);
+ regs = XALLOCAVEC (rtx, nregs);
+
+ scalar_int_mode elmode
+ = int_mode_for_size (elsz * BITS_PER_UNIT, 0).require ();
+
+ for (int i = 0; i < nregs; i++)
{
- regs[i] = gen_reg_rtx (mode);
- riscv_emit_move (regs[i], adjust_address (src, mode, offset));
+ regs[i] = gen_reg_rtx (elmode);
+ HOST_WIDE_INT off = i * elsz;
+ if (i == nregs - 1)
+ off = length - elsz;
+ riscv_emit_move (regs[i], adjust_address (src, elmode, off));
+ }
+ for (int i = 0; i < nregs; i++)
+ {
+ HOST_WIDE_INT off = i * elsz;
+ if (i == nregs - 1)
+ off = length - elsz;
+ riscv_emit_move (adjust_address (dest, elmode, off), regs[i]);
}
-
- /* Copy the chunks to the destination. */
- for (offset = 0, i = 0; offset + 2 * delta <= length;
- offset += delta, i++)
- riscv_emit_move (adjust_address (dest, mode, offset), regs[i]);
}
-
- /* Mop up any left-over bytes. */
- if (offset < length)
+ else
{
- src = adjust_address (src, BLKmode, offset);
- dest = adjust_address (dest, BLKmode, offset);
- move_by_pieces (dest, src, length - offset, align, RETURN_BEGIN);
+ unsigned HOST_WIDE_INT step = 2 * delta;
+ if (step <= length)
+ {
+ int i;
+ /* Allocate a buffer for the temporary registers. */
+ regs = XALLOCAVEC (rtx, length / delta - 1);
+
+ /* Load as many BITS-sized chunks as possible. Use a normal load
+ if the source has enough alignment, otherwise use left/right
+ pairs. */
+ for (offset = 0, i = 0; offset + step <= length;
+ offset += delta, i++)
+ {
+ regs[i] = gen_reg_rtx (mode);
+ riscv_emit_move (regs[i], adjust_address (src, mode, offset));
+ }
+
+ /* Copy the chunks to the destination. */
+ for (offset = 0, i = 0; offset + step <= length;
+ offset += delta, i++)
+ riscv_emit_move (adjust_address (dest, mode, offset), regs[i]);
+ }
+
+ /* Mop up any left-over bytes. */
+ if (offset < length)
+ {
+ src = adjust_address (src, BLKmode, offset);
+ dest = adjust_address (dest, BLKmode, offset);
+ move_by_pieces (dest, src, length - offset, align, RETURN_BEGIN);
+ }
}
}
@@ -866,8 +904,7 @@ riscv_adjust_block_mem (rtx mem, unsigned HOST_WIDE_INT
align,
/* Move LENGTH bytes from SRC to DEST using a loop that moves BYTES_PER_ITER
bytes at a time. LENGTH must be at least BYTES_PER_ITER. The alignment
- of the access can be set by ALIGN. Assume that the memory regions do not
- overlap. */
+ of the access can be set by ALIGN. */
static void
riscv_block_move_loop (rtx dest, rtx src, unsigned HOST_WIDE_INT length,
@@ -894,7 +931,7 @@ riscv_block_move_loop (rtx dest, rtx src, unsigned
HOST_WIDE_INT length,
emit_label (label);
/* Emit the loop body. */
- riscv_block_move_straight (dest, src, bytes_per_iter, align);
+ riscv_block_move_straight (dest, src, bytes_per_iter, align, false);
/* Move on to the next block. */
riscv_emit_move (src_reg, plus_constant (Pmode, src_reg, bytes_per_iter));
@@ -906,16 +943,17 @@ riscv_block_move_loop (rtx dest, rtx src, unsigned
HOST_WIDE_INT length,
/* Mop up any left-over bytes. */
if (leftover)
- riscv_block_move_straight (dest, src, leftover, align);
+ riscv_block_move_straight (dest, src, leftover, align, false);
else
emit_insn (gen_nop ());
}
-/* Expand a cpymemsi instruction, which copies LENGTH bytes from
- memory reference SRC to memory reference DEST. */
+/* Expand a cpymemsi/movmemsi instruction, which copies or moves LENGTH bytes
+ from memory reference SRC to memory reference DEST. Handle overlap
+ for short sequences that won't loop if OVERLAP_P is true. */
static bool
-riscv_expand_block_move_scalar (rtx dest, rtx src, rtx length)
+riscv_expand_block_move_scalar (rtx dest, rtx src, rtx length, bool overlap_p)
{
if (!CONST_INT_P (length))
return false;
@@ -923,10 +961,19 @@ riscv_expand_block_move_scalar (rtx dest, rtx src, rtx
length)
unsigned HOST_WIDE_INT hwi_length = UINTVAL (length);
unsigned HOST_WIDE_INT factor, align;
- if (riscv_memcpy_size_threshold >= 0
+ if (overlap_p
+ && riscv_memmove_size_threshold >= 0
+ && hwi_length > (unsigned HOST_WIDE_INT) riscv_memmove_size_threshold)
+ return false;
+ if (!overlap_p
+ && riscv_memcpy_size_threshold >= 0
&& hwi_length > (unsigned HOST_WIDE_INT) riscv_memcpy_size_threshold)
return false;
+ /* Don't bother with overlap for slow misaligned access. */
+ if (overlap_p && riscv_slow_unaligned_access_p)
+ return false;
+
if (riscv_slow_unaligned_access_p)
{
align = MIN (MIN (MEM_ALIGN (src), MEM_ALIGN (dest)), BITS_PER_WORD);
@@ -945,10 +992,10 @@ riscv_expand_block_move_scalar (rtx dest, rtx src, rtx
length)
if (hwi_length <= (RISCV_MAX_MOVE_BYTES_STRAIGHT / factor))
{
- riscv_block_move_straight (dest, src, hwi_length, align);
+ riscv_block_move_straight (dest, src, hwi_length, align, overlap_p);
return true;
}
- else if (optimize && align >= BITS_PER_WORD)
+ else if (optimize && align >= BITS_PER_WORD && !overlap_p)
{
unsigned min_iter_words
= RISCV_MAX_MOVE_BYTES_PER_LOOP_ITER / UNITS_PER_WORD;
@@ -975,21 +1022,22 @@ riscv_expand_block_move_scalar (rtx dest, rtx src, rtx
length)
/* This function delegates block-move expansion to either the vector
implementation or the scalar one. Return TRUE if successful or FALSE
- otherwise. Assume that the memory regions do not overlap. */
+ otherwise. Assume that the memory regions may overlap if OVERLAP_P
+ is true. */
bool
-riscv_expand_block_move (rtx dest, rtx src, rtx length)
+riscv_expand_block_move (rtx dest, rtx src, rtx length, bool overlap_p)
{
if (TARGET_VECTOR
&& stringop_strategy & STRATEGY_VECTOR)
{
- bool ok = riscv_vector::expand_block_move (dest, src, length, false);
+ bool ok = riscv_vector::expand_block_move (dest, src, length, overlap_p);
if (ok)
return true;
}
if (stringop_strategy & STRATEGY_SCALAR)
- return riscv_expand_block_move_scalar (dest, src, length);
+ return riscv_expand_block_move_scalar (dest, src, length, overlap_p);
return false;
}
@@ -1223,7 +1271,7 @@ use_vector_stringop_p (struct stringop_info &info,
HOST_WIDE_INT max_ew,
/* Used by cpymemsi in riscv.md . */
bool
-expand_block_move (rtx dst_in, rtx src_in, rtx length_in, bool movmem_p)
+expand_block_move (rtx dst_in, rtx src_in, rtx length_in, bool overlap_p)
{
/*
memcpy:
@@ -1250,13 +1298,13 @@ expand_block_move (rtx dst_in, rtx src_in, rtx
length_in, bool movmem_p)
if (CONST_INT_P (length_in))
{
HOST_WIDE_INT length = INTVAL (length_in);
- if (movmem_p
+ if (overlap_p
&& riscv_memmove_size_threshold >= 0
&& length > riscv_memmove_size_threshold)
return false;
- else if (!movmem_p
- && riscv_memcpy_size_threshold >= 0
- && length > riscv_memcpy_size_threshold)
+ if (!overlap_p
+ && riscv_memcpy_size_threshold >= 0
+ && length > riscv_memcpy_size_threshold)
return false;
}
else if (riscv_memmove_size_threshold != -1)
@@ -1267,7 +1315,7 @@ expand_block_move (rtx dst_in, rtx src_in, rtx length_in,
bool movmem_p)
count however for situations where the entire move fits in one vector
operation we can do all reads before doing any writes so we don't have to
worry so generate the inline vector code in such situations. */
- if (info.need_loop && movmem_p)
+ if (info.need_loop && overlap_p)
return false;
rtx src, dst;
diff --git a/gcc/config/riscv/riscv.md b/gcc/config/riscv/riscv.md
index bd308e3d056..45d79c91428 100644
--- a/gcc/config/riscv/riscv.md
+++ b/gcc/config/riscv/riscv.md
@@ -2941,7 +2941,7 @@ (define_expand "cpymem<mode>"
(use (match_operand:SI 3 "const_int_operand"))])]
""
{
- if (riscv_expand_block_move (operands[0], operands[1], operands[2]))
+ if (riscv_expand_block_move (operands[0], operands[1], operands[2], false))
DONE;
else
FAIL;
@@ -2980,10 +2980,9 @@ (define_expand "movmem<mode>"
(match_operand:BLK 1 "general_operand"))
(use (match_operand:P 2 "const_int_operand"))
(use (match_operand:SI 3 "const_int_operand"))])]
- "TARGET_VECTOR"
+ ""
{
- if (riscv_vector::expand_block_move (operands[0], operands[1], operands[2],
- true))
+ if (riscv_expand_block_move (operands[0], operands[1], operands[2], true))
DONE;
else
FAIL;
diff --git a/gcc/testsuite/gcc.target/riscv/movmem-scalar-32.c
b/gcc/testsuite/gcc.target/riscv/movmem-scalar-32.c
new file mode 100644
index 00000000000..323beb2ccb4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/movmem-scalar-32.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv32gc -mabi=ilp32d -mtune=generic-ooo
-mstringop-strategy=scalar" } */
+/* { dg-skip-if "" { *-*-* } { "-O0" "-Os" "-Oz" "-Og" "-flto" } } */
+
+#define MEMMOVE(N) \
+void movmem_##N (void *dst, void *src) \
+{ \
+ __builtin_memmove (dst, src, N); \
+}
+
+MEMMOVE(16)
+MEMMOVE(32)
+MEMMOVE(48)
+
+/* { dg-final { scan-assembler-times "lw\t" 24 } } */
+/* { dg-final { scan-assembler-times "sw\t" 24 } } */
+/* { dg-final { scan-assembler-not "sb\t" } } */
+/* { dg-final { scan-assembler-not "lbu\t" } } */
+/* { dg-final { scan-assembler-not "memmove" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/movmem-scalar-64.c
b/gcc/testsuite/gcc.target/riscv/movmem-scalar-64.c
new file mode 100644
index 00000000000..9558c9eb14e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/movmem-scalar-64.c
@@ -0,0 +1,20 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -mtune=generic-ooo
-mstringop-strategy=scalar" } */
+/* { dg-skip-if "" { *-*-* } { "-O0" "-Os" "-Oz" "-Og" "-flto" } } */
+
+#define MEMMOVE(N) \
+void movmem_##N (void *dst, void *src) \
+{ \
+ __builtin_memmove (dst, src, N); \
+}
+
+MEMMOVE(8)
+MEMMOVE(16)
+MEMMOVE(24)
+MEMMOVE(32)
+
+/* { dg-final { scan-assembler-times "ld\t" 10 } } */
+/* { dg-final { scan-assembler-times "sd\t" 10 } } */
+/* { dg-final { scan-assembler-not "sb\t" } } */
+/* { dg-final { scan-assembler-not "lbu\t" } } */
+/* { dg-final { scan-assembler-not "memmove" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/movmem-scalar-run.c
b/gcc/testsuite/gcc.target/riscv/movmem-scalar-run.c
new file mode 100644
index 00000000000..64a35fb89dd
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/movmem-scalar-run.c
@@ -0,0 +1,100 @@
+/* { dg-do run } */
+/* { dg-additional-options "-O2 -mtune=generic-ooo -mstringop-strategy=scalar"
} */
+
+#define ALIGN 8
+#define SZ 160
+#define EMPTY 32
+
+#define ARR (2 * SZ + 2 * ALIGN)
+#define BUFSZ (EMPTY + ARR + EMPTY)
+
+#define FOR_EACH_LEN(M) \
+ M(1) M(2) M(3) M(4) M(5) M(6) M(7) M(8) M(9) M(15) M(16) \
+ M(17) M(23) M(24) M(25) M(31) M(32) M(33) M(40) M(47) \
+ M(48) M(49) M(63) M(64) M(65) M(95) M(96) M(97) M(128) M(160)
+
+#define DEF(N) \
+ __attribute__((noinline)) static void \
+ move_##N (void *dst, void *src) \
+ { \
+ __builtin_memmove (dst, src, N); \
+ }
+
+FOR_EACH_LEN (DEF)
+
+static unsigned char arr[BUFSZ];
+static unsigned char ref[BUFSZ];
+static unsigned char tmp[SZ];
+
+static void
+init (void)
+{
+ unsigned i;
+
+ for (i = 0; i < BUFSZ; i++)
+ arr[i] = ref[i] = (unsigned char) (i * 7 + 1);
+}
+
+static void
+ref_move (unsigned off1, unsigned off2, unsigned len)
+{
+ unsigned i;
+
+ for (i = 0; i < len; i++)
+ tmp[i] = ref[off2 + i];
+ for (i = 0; i < len; i++)
+ ref[off1 + i] = tmp[i];
+}
+
+typedef void (*fun) (void *, void *);
+
+static void
+test (fun f, unsigned len, unsigned off1, unsigned off2)
+{
+ unsigned i;
+
+ init ();
+ ref_move (off1, off2, len);
+ f (arr + off1, arr + off2);
+
+ for (i = 0; i < BUFSZ; i++)
+ if (arr[i] != ref[i])
+ __builtin_abort ();
+}
+
+static void
+test_len (fun f, unsigned len)
+{
+ unsigned deltas[6];
+ unsigned delta, di, si, k;
+
+ for (delta = 0; delta <= len + ALIGN; delta++)
+ {
+ test (f, len, EMPTY, EMPTY + delta);
+ test (f, len, EMPTY + delta, EMPTY);
+ }
+
+ deltas[0] = 0;
+ deltas[1] = 1;
+ deltas[2] = len / 2;
+ deltas[3] = len > 0 ? len - 1 : 0;
+ deltas[4] = len;
+ deltas[5] = len + 1;
+
+ for (di = 0; di < ALIGN; di++)
+ for (si = 0; si < ALIGN; si++)
+ for (k = 0; k < 6; k++)
+ {
+ test (f, len, EMPTY + di, EMPTY + si + deltas[k]);
+ test (f, len, EMPTY + di + deltas[k], EMPTY + si);
+ }
+}
+
+#define RUN(N) test_len (move_##N, N);
+
+int
+main (void)
+{
+ FOR_EACH_LEN (RUN)
+ return 0;
+}
--
2.55.0
