Advanced SIMD tuple types such as uint64x2x2_t have a vector tuple mode
(V2x2DImode) and are passed whole to the ld<N>/st<N> lane builtins.
When such a tuple is carried around a loop, ESRA scalarizes it into its
member vectors, and the tuple is then rebuilt before and split after
every builtin call:
va01.val[0] = va01$val$0_219;
va01.val[1] = va01$val$1_218;
D.19979 = __builtin_aarch64_ld2_lanev2di_usus (a0_176, va01, 0);
va01$val$0_9 = D.19979.val[0];
va01$val$1_8 = D.19979.val[1];
The ld2 lane instructions need the tuple in consecutive registers, so
RTL ends up with vector moves around every lane load and, under register
pressure, stack spills inside the loop.
This patch disqualifies such a tuple (a record whose only field is an
array of vectors and whose TYPE_MODE is a vector mode) from SRA when it
is passed whole as a call argument inside a loop. Tuples that are only
accessed element-wise in the loop are still scalarized; that matters for
gcc.target/aarch64/pr109078.c, where the accumulator tuple is updated
per element and only stored whole after the loop, and which regresses
if all vector tuples are excluded from SRA.
Results with -O3 -march=armv8.2-a+i8mm on the XNNPACK 4x16c8 I8MM GEMM
micro-kernel (qs8-qc8w-gemm-4x16c8-minmax-fp32-neoni8mm.c), which loads
activations with vld2q_lane_u64 into two loop-carried uint64x2x2_t:
inner-loop insns [sp] ld/st vector movs
GCC 11 70 0 18
GCC 12 67 3 5
GCC 14 62 3 2
GCC 16.2 64 2 11
trunk 63 2 10
trunk + patch 51 0 0
clang 14 48 0 0
For the PR89967 testcase at -O2 the vector moves go from 15 to 3; the
3 left are the zero initialisation outside the loop, and the loop is
8 back-to-back ld2 with no copies.
On Graviton4 (Neoverse-V2) that XNNPACK kernel built with GCC 15.2 is
about 2x slower than with clang 18/21/23, and mobilenet_v2's qs8
convolutions are about 1.4x slower in total. The patched compiler has
not been benchmarked on I8MM hardware yet.
What remains is the RA side of PR89967: for a minimal loop-carried tuple
(v = vld2q_lane (a, v, 0); v = vld2q_lane (b, v, 1); then uses of
v.val[]) the expanded RTL is clean but IRA/LRA do not tie the V2x2DI
pseudos, so a pair of moves remains around each ld2. -mearly-ra=all
does not change that.
PR tree-optimization/89967
gcc/ChangeLog:
* tree-sra.cc: Include cfgloop.h.
(vector_tuple_type_p): New function.
(scan_function): Disqualify vector tuples that are passed whole
to a call inside a loop.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/pr89967.c: New test.
Assisted-by: Claude Code (claude-opus-5-5)
Signed-off-by: Rajeev Gadgil <[email protected]>
---
This is an RFC because the patch was written with AI assistance (see the
Assisted-by: tag), and the GCC AI policy declines legally significant
LLM-generated contributions. If it cannot be accepted as is, please treat
it as a proof of concept for PR89967: the root-cause analysis and the
numbers below are meant to make it straightforward to reimplement, and the
new testcase is offered under the policy's test-case exception. I have
reviewed the change and can answer questions about it.
Testing: bootstrapped and regtested on aarch64-linux-gnu (Ampere Altra,
Neoverse-N1; default languages c, c++, fortran, lto, objc) on top of
trunk 453e99a9d (2026-10-24), with no new failures. The new
gcc.target/aarch64/pr89967.c fails without the patch and passes with it.
Context: we hit this in XNNPACK. Its 4x16c8 I8MM GEMM micro-kernels load
two rows of activations with vld2q_lane_u64 into a loop-carried
uint64x2x2_t. On AWS Graviton4 (Neoverse-V2), single thread, master
XNNPACK vs a variant that uses vld1q + vzip1q/vzip2q_u64 instead
(speedup of the zip variant):
GCC 15.2 clang 18.1 clang 21.1 clang 23.1
qs8_qc8w 196x1024x1024 2.08x 1.07x 1.07x 1.13x
qs8_qc8w 3136x64x576 2.09x 1.06x 1.07x 1.06x
qd8_f32 3136x64x576 2.18x 1.11x 1.12x 1.12x
mobilenet_v2 qs8 convs (total) 1.35x 1.04x 1.04x 1.05x
Most of the GCC gap is this PR: master built with GCC 15.2 spills the
tuple to the stack in the inner loop, and is ~1.4x slower than master
built with clang on mobilenet_v2's qs8 convolutions. XNNPACK has since
taken the zip variant:
https://github.com/google/XNNPACK/pull/11779
https://github.com/google/XNNPACK/pull/11794
A self-contained reproducer of that kernel's loop (repro-lane.c, 16 SMMLA
accumulators) is attached; repro-zip.c is the vld1q+zip control, which
GCC already compiles optimally (52 insns, no copies).
gcc/testsuite/gcc.target/aarch64/pr89967.c | 49 ++++++++++++++++++++++
gcc/tree-sra.cc | 30 +++++++++++++
2 files changed, 79 insertions(+)
create mode 100644 gcc/testsuite/gcc.target/aarch64/pr89967.c
diff --git a/gcc/testsuite/gcc.target/aarch64/pr89967.c
b/gcc/testsuite/gcc.target/aarch64/pr89967.c
new file mode 100644
index 000000000..0195ccdf1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/pr89967.c
@@ -0,0 +1,49 @@
+/* PR tree-optimization/89967 */
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+/* The uint8x16x2_t tuples are passed whole to the ld2 lane intrinsics, so
+ SRA should leave them in their register pairs instead of rebuilding them
+ around every call. */
+
+#include <arm_neon.h>
+
+void func(unsigned char *in, int *out, int n, int32x4_t o1v, int32x4_t o2v,
+ int32x4_t s1v, int32x4_t s2v, int32x4_t m1v, int32x4_t m2v,
+ int32x4_t m3v, int32x4_t m4v)
+{
+ int i;
+ int32x4_t l1v, l2v, g1v, g2v, g3v, g4v, gv;
+ uint8x16x2_t g1b, g2b;
+
+ l1v = o1v;
+ l2v = o2v;
+ g1b.val[0] = vdupq_n_u8(0);
+ g1b.val[1] = vdupq_n_u8(0);
+ g2b.val[0] = vdupq_n_u8(0);
+ g2b.val[1] = vdupq_n_u8(0);
+ for (i=0; i<n; i+=4)
+ {
+ g1b = vld2q_lane_u8(&in[vgetq_lane_s32(l1v, 0)], g1b, 0);
+ g1b = vld2q_lane_u8(&in[vgetq_lane_s32(l1v, 1)], g1b, 4);
+ g1b = vld2q_lane_u8(&in[vgetq_lane_s32(l1v, 2)], g1b, 8);
+ g1b = vld2q_lane_u8(&in[vgetq_lane_s32(l1v, 3)], g1b, 12);
+ g2b = vld2q_lane_u8(&in[vgetq_lane_s32(l2v, 0)], g2b, 0);
+ g2b = vld2q_lane_u8(&in[vgetq_lane_s32(l2v, 1)], g2b, 4);
+ g2b = vld2q_lane_u8(&in[vgetq_lane_s32(l2v, 2)], g2b, 8);
+ g2b = vld2q_lane_u8(&in[vgetq_lane_s32(l2v, 3)], g2b, 12);
+ g1v = vreinterpretq_s32_u8(g1b.val[0]);
+ g2v = vreinterpretq_s32_u8(g1b.val[1]);
+ g3v = vreinterpretq_s32_u8(g2b.val[0]);
+ g4v = vreinterpretq_s32_u8(g2b.val[1]);
+ gv = vmlaq_s32(vmlaq_s32(vmlaq_s32(vmulq_s32(m4v, g4v), m3v, g3v),
+ m2v, g2v), m1v, g1v);
+ vst1q_s32(&out[i], gv);
+ vaddq_s32(l1v, s1v);
+ vaddq_s32(l2v, s2v);
+ }
+}
+
+/* { dg-final { scan-assembler-times {\tld2\t} 8 } } */
+/* { dg-final { scan-assembler-not {\tld2\t[^\n]*\n\tmov\tv} } } */
+/* { dg-final { scan-assembler-not {\tmov\tv[^\n]*\n\tld2\t} } } */
diff --git a/gcc/tree-sra.cc b/gcc/tree-sra.cc
index f951a3844..b213265a0 100644
--- a/gcc/tree-sra.cc
+++ b/gcc/tree-sra.cc
@@ -99,6 +99,7 @@ along with GCC; see the file COPYING3. If not see
#include "dbgcnt.h"
#include "builtins.h"
#include "tree-sra.h"
+#include "cfgloop.h"
#include "opts.h"
#include "tree-ssa-alias-compare.h"
@@ -1609,6 +1610,23 @@ scan_visit_addr (gimple *, tree op, tree, void *)
return false;
}
+/* Return true if TYPE is a structure whose only field is an array of vectors
+ and which the target gives a vector mode, such as the AArch64 Advanced SIMD
+ uint64x2x2_t family. Such tuples live in consecutive vector registers. */
+
+static bool
+vector_tuple_type_p (tree type)
+{
+ if (TREE_CODE (type) != RECORD_TYPE || !VECTOR_MODE_P (TYPE_MODE (type)))
+ return false;
+ tree fld = first_field (type);
+ if (!fld || DECL_CHAIN (fld))
+ return false;
+ tree ftype = TREE_TYPE (fld);
+ return (TREE_CODE (ftype) == ARRAY_TYPE
+ && VECTOR_TYPE_P (TREE_TYPE (ftype)));
+}
+
/* Scan function and look for interesting expressions and create access
structures for them. Return true iff any access is created. */
@@ -1653,6 +1671,18 @@ scan_function (void)
gcall *call = as_a <gcall *> (stmt);
for (i = 0; i < gimple_call_num_args (call); i++)
{
+ /* A vector tuple that is passed whole to a call inside a
+ loop, such as to the ld<N> lane builtins, would have to
+ be rebuilt from its scalar replacements on every
+ iteration. */
+ tree arg = gimple_call_arg (call, i);
+ if (DECL_P (arg)
+ && current_loops
+ && bb_loop_depth (bb) > 0
+ && vector_tuple_type_p (TREE_TYPE (arg))
+ && bitmap_bit_p (candidate_bitmap, DECL_UID (arg)))
+ disqualify_candidate (arg, "vector tuple passed whole to "
+ "a call in a loop");
bool can_be_returned;
if (gimple_call_lhs (call))
{
--
2.34.1
