LD1 and ST1 of one vector have a register post-index form; LDR and STR do
not, so a row loop at a runtime stride pays an ADD per row. Define
HAVE_POST_MODIFY_REG so that auto_inc_dec tries the form, and match the
MEM in full, since aarch64_classify_address still rejects a register
POST_MODIFY. The base is "+r": the insn writes it. aarch64_address_cost
charges the form post_modify, not the jump-table default. TARGET_SIMD
keeps it out of streaming mode; big-endian is excluded because LD1/ST1
use a different lane order from a big-endian vector move.
The rowsum16 loop in ld1-post-reg-1.c goes from 7 insns to 6 at -O2.
Over 71 loop kernels from x264, x265 and FFmpeg (-O2 and -O3,
-mcpu=cortex-a76) the form appears 23 times; 4 kernels shrink by 18
insns in total and none grows.
Bootstrapped and regression tested on x86_64-pc-linux-gnu; regression
tested on aarch64-linux-gnu (cross, under qemu).
Assisted-by: Claude Opus 5 (Anthropic)
gcc/ChangeLog:
* config/aarch64/aarch64.h (HAVE_POST_MODIFY_REG): Define.
* config/aarch64/aarch64.cc (aarch64_address_cost): Cost a register
POST_MODIFY as post_modify.
* config/aarch64/aarch64-simd.md (*aarch64_ld1_post_reg<mode>): New
pattern.
(*aarch64_st1_post_reg<mode>): Likewise.
gcc/testsuite/ChangeLog:
* gcc.dg/torture/post-modify-reg-1.c: New test.
* gcc.target/aarch64/ld1-post-reg-1.c: New test.
* gcc.target/aarch64/ld1-post-reg-2.c: New test.
* gcc.target/aarch64/ld1-post-reg-3.c: New test.
Signed-off-by: Dominic P <[email protected]>
---
OK for trunk?
gcc/config/aarch64/aarch64-simd.md | 24 +++++++
gcc/config/aarch64/aarch64.cc | 2 +
gcc/config/aarch64/aarch64.h | 1 +
.../gcc.dg/torture/post-modify-reg-1.c | 69 +++++++++++++++++++
.../gcc.target/aarch64/ld1-post-reg-1.c | 45 ++++++++++++
.../gcc.target/aarch64/ld1-post-reg-2.c | 11 +++
.../gcc.target/aarch64/ld1-post-reg-3.c | 26 +++++++
7 files changed, 178 insertions(+)
create mode 100644 gcc/testsuite/gcc.dg/torture/post-modify-reg-1.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/ld1-post-reg-1.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/ld1-post-reg-2.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/ld1-post-reg-3.c
diff --git a/gcc/config/aarch64/aarch64-simd.md
b/gcc/config/aarch64/aarch64-simd.md
index 98c2da034c5..58e431960c1 100644
--- a/gcc/config/aarch64/aarch64-simd.md
+++ b/gcc/config/aarch64/aarch64-simd.md
@@ -8985,6 +8985,30 @@
[(set_attr "type" "neon_store1_1reg<q>")]
)
+(define_insn "*aarch64_ld1_post_reg<mode>"
+ [(set (match_operand:VALL_F16 0 "register_operand" "=w")
+ (mem:VALL_F16
+ (post_modify:DI
+ (match_operand:DI 1 "register_operand" "+r")
+ (plus:DI (match_dup 1)
+ (match_operand:DI 2 "register_operand" "r")))))]
+ "TARGET_SIMD && !BYTES_BIG_ENDIAN"
+ "ld1\t{%0.<Vtype>}, [%1], %2"
+ [(set_attr "type" "neon_load1_1reg<q>")]
+)
+
+(define_insn "*aarch64_st1_post_reg<mode>"
+ [(set (mem:VALL_F16
+ (post_modify:DI
+ (match_operand:DI 0 "register_operand" "+r")
+ (plus:DI (match_dup 0)
+ (match_operand:DI 1 "register_operand" "r"))))
+ (match_operand:VALL_F16 2 "register_operand" "w"))]
+ "TARGET_SIMD && !BYTES_BIG_ENDIAN"
+ "st1\t{%2.<Vtype>}, [%0], %1"
+ [(set_attr "type" "neon_store1_1reg<q>")]
+)
+
(define_insn "*aarch64_mov<mode>"
[(set (match_operand:VSTRUCT_2D 0 "nonimmediate_operand")
(match_operand:VSTRUCT_2D 1 "general_operand"))]
diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index 107271b0404..711b0836e3d 100644
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -14933,6 +14933,8 @@ aarch64_address_cost (rtx x,
followed by an immediate (possibly 0) offset. */
return cost_symbol_ref + addr_cost->imm_offset;
}
+ else if (c == POST_MODIFY)
+ return addr_cost->post_modify;
else
{
/* This is most likely a jump table from a case
diff --git a/gcc/config/aarch64/aarch64.h b/gcc/config/aarch64/aarch64.h
index 39f51d4e92d..3c3503969c7 100644
--- a/gcc/config/aarch64/aarch64.h
+++ b/gcc/config/aarch64/aarch64.h
@@ -1349,6 +1349,7 @@ typedef struct
#define HAVE_PRE_DECREMENT 1
#define HAVE_POST_MODIFY_DISP 1
#define HAVE_PRE_MODIFY_DISP 1
+#define HAVE_POST_MODIFY_REG 1
#define MAX_REGS_PER_ADDRESS 2
diff --git a/gcc/testsuite/gcc.dg/torture/post-modify-reg-1.c
b/gcc/testsuite/gcc.dg/torture/post-modify-reg-1.c
new file mode 100644
index 00000000000..8c0f01c4d42
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/post-modify-reg-1.c
@@ -0,0 +1,69 @@
+/* { dg-do run } */
+/* { dg-additional-options "-Wno-psabi" } */
+
+/* The pointers advanced by a register post-increment stay live across a loop
+ with enough other values to spill them. */
+
+typedef unsigned char v16 __attribute__ ((vector_size (16)));
+
+#define N 40
+#define LS 48
+#define SS 80
+
+unsigned char src[N * LS] __attribute__ ((aligned (16)));
+unsigned char dst[N * SS] __attribute__ ((aligned (16)));
+unsigned long k[24];
+
+__attribute__ ((noipa)) v16
+f (const v16 *p, long ls, v16 *q, long ss, int n)
+{
+ unsigned long a0 = k[0], a1 = k[1], a2 = k[2], a3 = k[3], a4 = k[4],
+ a5 = k[5], a6 = k[6], a7 = k[7], a8 = k[8], a9 = k[9], a10 = k[10],
+ a11 = k[11], a12 = k[12], a13 = k[13], a14 = k[14], a15 = k[15],
+ a16 = k[16], a17 = k[17], a18 = k[18], a19 = k[19], a20 = k[20],
+ a21 = k[21], a22 = k[22], a23 = k[23];
+ v16 s = { 0 };
+ for (int i = 0; i < n; i++)
+ {
+ v16 x = *p;
+ s += x;
+ *q = x;
+ p = (const v16 *) ((const char *) p + ls);
+ q = (v16 *) ((char *) q + ss);
+ a0 = a0 * 3 + x[0]; a1 = a1 * 5 + a0; a2 = a2 * 7 + a1;
+ a3 = a3 * 9 + a2; a4 = a4 * 11 + a3; a5 = a5 * 13 + a4;
+ a6 = a6 * 15 + a5; a7 = a7 * 17 + a6; a8 = a8 * 19 + a7;
+ a9 = a9 * 21 + a8; a10 = a10 * 23 + a9; a11 = a11 * 25 + a10;
+ a12 = a12 * 27 + a11; a13 = a13 * 29 + a12; a14 = a14 * 31 + a13;
+ a15 = a15 * 33 + a14; a16 = a16 * 35 + a15; a17 = a17 * 37 + a16;
+ a18 = a18 * 39 + a17; a19 = a19 * 41 + a18; a20 = a20 * 43 + a19;
+ a21 = a21 * 45 + a20; a22 = a22 * 47 + a21; a23 = a23 * 49 + a22;
+ }
+ k[0] = a0; k[1] = a1; k[2] = a2; k[3] = a3; k[4] = a4; k[5] = a5;
+ k[6] = a6; k[7] = a7; k[8] = a8; k[9] = a9; k[10] = a10; k[11] = a11;
+ k[12] = a12; k[13] = a13; k[14] = a14; k[15] = a15; k[16] = a16;
+ k[17] = a17; k[18] = a18; k[19] = a19; k[20] = a20; k[21] = a21;
+ k[22] = a22; k[23] = a23;
+ return s;
+}
+
+int
+main (void)
+{
+ for (int i = 0; i < N * LS; i++)
+ src[i] = i * 7 + (i >> 5);
+ v16 s = f ((const v16 *) src, LS, (v16 *) dst, SS, N);
+ for (int j = 0; j < 16; j++)
+ {
+ unsigned char t = 0;
+ for (int i = 0; i < N; i++)
+ {
+ t += src[i * LS + j];
+ if (dst[i * SS + j] != src[i * LS + j])
+ __builtin_abort ();
+ }
+ if (s[j] != t)
+ __builtin_abort ();
+ }
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/ld1-post-reg-1.c
b/gcc/testsuite/gcc.target/aarch64/ld1-post-reg-1.c
new file mode 100644
index 00000000000..d041ce3b125
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/ld1-post-reg-1.c
@@ -0,0 +1,45 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" "" { target { le } } } } */
+
+typedef unsigned char pixel;
+
+/*
+** rowsum16:
+** ...
+** ld1 {v[0-9]+\.16b}, \[x[0-9]+\], x1
+** ...
+** bne \.L[0-9]+
+** ret
+*/
+int
+rowsum16 (const pixel *p, long stride)
+{
+ int s = 0;
+ for (int y = 0; y < 16; y++)
+ {
+ for (int x = 0; x < 16; x++)
+ s += p[x];
+ p += stride;
+ }
+ return s;
+}
+
+/*
+** rowstore16:
+** ...
+** st1 {v[0-9]+\.16b}, \[x[0-9]+\], x1
+** subs w[0-9]+, w[0-9]+, #1
+** bne \.L[0-9]+
+** ret
+*/
+void
+rowstore16 (pixel *p, long stride, pixel v)
+{
+ for (int y = 0; y < 16; y++)
+ {
+ for (int x = 0; x < 16; x++)
+ p[x] = v;
+ p += stride;
+ }
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/ld1-post-reg-2.c
b/gcc/testsuite/gcc.target/aarch64/ld1-post-reg-2.c
new file mode 100644
index 00000000000..a1e5dd7ddd5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/ld1-post-reg-2.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target aarch64_mbig_endian } */
+/* { dg-options "-O2 -mbig-endian" } */
+
+/* LD1 and ST1 transfer in element order, so they cannot replace the LDR and
+ STR of a big-endian vector move. */
+
+#include "ld1-post-reg-1.c"
+
+/* { dg-final { scan-assembler-not {\tld1\t} } } */
+/* { dg-final { scan-assembler-not {\tst1\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/ld1-post-reg-3.c
b/gcc/testsuite/gcc.target/aarch64/ld1-post-reg-3.c
new file mode 100644
index 00000000000..8fba24d8126
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/ld1-post-reg-3.c
@@ -0,0 +1,26 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+/* Advanced SIMD LD1 and ST1 are not available in streaming mode. */
+
+#pragma GCC target "+sme"
+
+typedef unsigned char v16 __attribute__ ((vector_size (16)));
+
+v16
+f (const unsigned char *p, unsigned char *q, long st, long st2, int n)
+ __arm_streaming_compatible
+{
+ v16 s = { 0 };
+ for (int y = 0; y < n; y++)
+ {
+ s += *(const v16 *) p;
+ *(v16 *) q = s;
+ p += st;
+ q += st2;
+ }
+ return s;
+}
+
+/* { dg-final { scan-assembler-not {\tld1\t} } } */
+/* { dg-final { scan-assembler-not {\tst1\t} } } */
--
2.55.0
