On Thu, Nov 12, 2026 at 1:21 PM liuhongt <[email protected]> wrote:
>
> Unnamed 256-bit and 512-bit vector arguments must be passed on the
> stack, since the register save area set up by a variadic callee only
> holds 16 bytes per SSE register. function_arg_advance_64,
> function_arg_64 and ix86_gimplify_va_arg implement this rule with an
> explicit check for the 256-bit and 512-bit vector modes.
>
> The explicit mode check is not enough. Whether an argument goes in a
> single YMM/ZMM register is decided by classify_argument (SSE followed
> by all SSEUP), and that classification also applies to BLKmode
> aggregates without a vector mode, e.g. a union whose widest member is
> a 256-bit or 512-bit vector. Such unnamed arguments slip past the
> mode check: the caller passes them in %ymm/%zmm, and va_arg reads them
> from consecutive 16-byte slots of the register save area, which yields
> wrong upper lanes, or segfaults on an aligned 32/64-byte load from a
> 16-byte aligned slot.
>
> Replace the explicit mode check with classify_argument: it only returns
> more than 2 classes for SSE followed by all SSEUP, i.e. exactly when
> the argument would be passed in a single 256-bit or 512-bit vector
> register. Assert that invariant in the classify_argument wrapper with
> checking enabled.
>
> Bootstrapped and regtested on x86_64-pc-linux-gnu{-m32,}.
> Ok for trunk?
>
> gcc/ChangeLog:
>
> PR target/128640
> * config/i386/i386.cc (classify_argument): In the wrapper,
> assert that more than 2 classes means SSE followed by all SSEUP.
> (function_arg_advance_64): Use classify_argument instead of an
> explicit vector mode check to pass unnamed 256-bit and 512-bit
> vector arguments on stack.
> (function_arg_64): Likewise.
> (ix86_gimplify_va_arg): Likewise.
>
> gcc/testsuite/ChangeLog:
>
> PR target/128640
> * gcc.target/i386/pr128640-1.c: New test.
> * gcc.target/i386/pr128640-2.c: New test.
> * gcc.target/i386/pr128640-3.c: New test.
OK for mainline and backports.
No -Wpsabi note is needed here: the old behaviour was already
inconsistent. The caller passed the union in %ymm/%zmm while va_arg
read the register save area, so the value was never recoverable, and a
compliant callee (e.g. clang’s) already read the overflow area. Only
unnamed arguments change; named parameters still go in %ymm/%zmm.
Thanks,
Uros.
> ---
> gcc/config/i386/i386.cc | 90 ++++++++--------------
> gcc/testsuite/gcc.target/i386/pr128640-1.c | 54 +++++++++++++
> gcc/testsuite/gcc.target/i386/pr128640-2.c | 52 +++++++++++++
> gcc/testsuite/gcc.target/i386/pr128640-3.c | 27 +++++++
> 4 files changed, 166 insertions(+), 57 deletions(-)
> create mode 100644 gcc/testsuite/gcc.target/i386/pr128640-1.c
> create mode 100644 gcc/testsuite/gcc.target/i386/pr128640-2.c
> create mode 100644 gcc/testsuite/gcc.target/i386/pr128640-3.c
>
> diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc
> index 8943da8b068..283602d2bdc 100644
> --- a/gcc/config/i386/i386.cc
> +++ b/gcc/config/i386/i386.cc
> @@ -2747,6 +2747,18 @@ classify_argument (machine_mode mode, const_tree type,
> static bool warned = false;
> int n = classify_argument (mode, type, classes, bit_offset,
> zero_width_bitfields);
> + /* More than 2 classes is only possible for a single 256-bit or
> + 512-bit vector register: SSE followed by all SSEUP. This is
> + relied upon by function_arg_advance_64, function_arg_64 and
> + ix86_gimplify_va_arg, which pass unnamed 256-bit and 512-bit
> + vector parameters on stack when classify_argument returns more
> + than 2 classes. */
> + if (flag_checking && n > 2)
> + {
> + gcc_assert ((n == 4 || n == 8) && classes[0] == X86_64_SSE_CLASS);
> + for (int i = 1; i < n; i++)
> + gcc_assert (classes[i] == X86_64_SSEUP_CLASS);
> + }
> if (!zero_width_bitfields || warned || !warn_psabi)
> return n;
> enum x86_64_reg_class alt_classes[MAX_CLASSES];
> @@ -3239,10 +3251,11 @@ function_arg_advance_64 (CUMULATIVE_ARGS *cum,
> machine_mode mode,
> const_tree type, HOST_WIDE_INT words, bool named)
> {
> int int_nregs, sse_nregs;
> + enum x86_64_reg_class regclass[MAX_CLASSES];
>
> - /* Unnamed 512 and 256bit vector mode parameters are passed on stack. */
> - if (!named && (VALID_AVX512F_REG_MODE (mode)
> - || VALID_AVX256_REG_MODE (mode)))
> + /* Unnamed 512 and 256bit vector parameters are passed on stack. See
> + function_arg_64. */
> + if (!named && classify_argument (mode, type, regclass, 0) > 2)
> return 0;
>
> if (!examine_argument (mode, type, false, &int_nregs, &sse_nregs)
> @@ -3492,32 +3505,14 @@ function_arg_64 (const CUMULATIVE_ARGS *cum,
> machine_mode mode,
> : cum->sse_regno)
> : -1);
>
> - switch (mode)
> - {
> - default:
> - break;
> -
> - case E_V16HFmode:
> - case E_V16BFmode:
> - case E_V8SFmode:
> - case E_V8SImode:
> - case E_V32QImode:
> - case E_V16HImode:
> - case E_V4DFmode:
> - case E_V4DImode:
> - case E_V32HFmode:
> - case E_V32BFmode:
> - case E_V16SFmode:
> - case E_V16SImode:
> - case E_V64QImode:
> - case E_V32HImode:
> - case E_V8DFmode:
> - case E_V8DImode:
> - /* Unnamed 256 and 512bit vector mode parameters are passed on stack.
> */
> - if (!named)
> - return NULL;
> - break;
> - }
> + /* Unnamed 256 and 512bit vector parameters are passed on stack.
> + classify_argument returns more than 2 classes only for SSE followed
> + by all SSEUP, i.e. a single 256-bit or 512-bit vector register.
> + Besides the vector modes, this covers BLKmode aggregates such as a
> + union with such a vector member. */
> + enum x86_64_reg_class regclass[MAX_CLASSES];
> + if (!named && classify_argument (mode, type, regclass, 0) > 2)
> + return NULL;
>
> const int *parm_regs;
> if (cum->preserve_none_abi)
> @@ -5035,34 +5030,15 @@ ix86_gimplify_va_arg (tree valist, tree type,
> gimple_seq *pre_p,
> rsize = CEIL (size, UNITS_PER_WORD);
>
> nat_mode = type_natural_mode (type, NULL, false);
> - switch (nat_mode)
> - {
> - case E_V16HFmode:
> - case E_V16BFmode:
> - case E_V8SFmode:
> - case E_V8SImode:
> - case E_V32QImode:
> - case E_V16HImode:
> - case E_V4DFmode:
> - case E_V4DImode:
> - case E_V32HFmode:
> - case E_V32BFmode:
> - case E_V16SFmode:
> - case E_V16SImode:
> - case E_V64QImode:
> - case E_V32HImode:
> - case E_V8DFmode:
> - case E_V8DImode:
> - /* Unnamed 256 and 512bit vector mode parameters are passed on stack.
> */
> - container = NULL;
> - break;
> -
> - default:
> - container = construct_container (nat_mode, TYPE_MODE (type),
> - type, false, X86_64_REGPARM_MAX,
> - X86_64_SSE_REGPARM_MAX, intreg, 0);
> - break;
> - }
> + /* Unnamed 256 and 512bit vector parameters are passed on stack. See
> + function_arg_64. */
> + enum x86_64_reg_class regclass[MAX_CLASSES];
> + if (classify_argument (nat_mode, type, regclass, 0) > 2)
> + container = NULL;
> + else
> + container = construct_container (nat_mode, TYPE_MODE (type),
> + type, false, X86_64_REGPARM_MAX,
> + X86_64_SSE_REGPARM_MAX, intreg, 0);
>
> /* Pull the value out of the saved registers. */
>
> diff --git a/gcc/testsuite/gcc.target/i386/pr128640-1.c
> b/gcc/testsuite/gcc.target/i386/pr128640-1.c
> new file mode 100644
> index 00000000000..50019ca2d9b
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/pr128640-1.c
> @@ -0,0 +1,54 @@
> +/* PR target/128640 */
> +/* { dg-do run { target { ! ia32 } } } */
> +/* { dg-require-effective-target avx } */
> +/* { dg-options "-O2 -mavx" } */
> +
> +#include <stdarg.h>
> +#include "avx-check.h"
> +
> +typedef float v8sf __attribute__((vector_size (32)));
> +typedef long long v4di __attribute__((vector_size (32)));
> +
> +union u1 { v8sf f; };
> +union u2 { v8sf f; float x; };
> +struct s3 { union u1 u; };
> +union u4 { v4di f; };
> +
> +#ifndef CHECK
> +#define CHECK(X, N) \
> + for (int i = 0; i < N; i++) \
> + if ((X)[i] != i + 1) \
> + abort ()
> +#endif
> +
> +__attribute__((noipa)) void
> +foo (int n, ...)
> +{
> + va_list ap;
> + va_start (ap, n);
> + if (va_arg (ap, double) != 0.0)
> + abort ();
> + union u1 a = va_arg (ap, union u1);
> + CHECK (a.f, 8);
> + if (va_arg (ap, double) != 5.0)
> + abort ();
> + union u2 b = va_arg (ap, union u2);
> + CHECK (b.f, 8);
> + struct s3 c = va_arg (ap, struct s3);
> + CHECK (c.u.f, 8);
> + union u4 d = va_arg (ap, union u4);
> + CHECK (d.f, 4);
> + if (va_arg (ap, double) != 6.0)
> + abort ();
> + va_end (ap);
> +}
> +
> +static void
> +avx_test (void)
> +{
> + union u1 a = { { 1, 2, 3, 4, 5, 6, 7, 8 } };
> + union u2 b = { { 1, 2, 3, 4, 5, 6, 7, 8 } };
> + struct s3 c = { { { 1, 2, 3, 4, 5, 6, 7, 8 } } };
> + union u4 d = { { 1, 2, 3, 4 } };
> + foo (1, 0.0, a, 5.0, b, c, d, 6.0);
> +}
> diff --git a/gcc/testsuite/gcc.target/i386/pr128640-2.c
> b/gcc/testsuite/gcc.target/i386/pr128640-2.c
> new file mode 100644
> index 00000000000..99480fbab47
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/pr128640-2.c
> @@ -0,0 +1,52 @@
> +/* PR target/128640 */
> +/* { dg-do run { target { ! ia32 } } } */
> +/* { dg-require-effective-target avx512f } */
> +/* { dg-options "-O2 -mavx512f" } */
> +
> +#include <stdarg.h>
> +#include "avx512f-check.h"
> +
> +typedef float v16sf __attribute__((vector_size (64)));
> +typedef long long v8di __attribute__((vector_size (64)));
> +
> +union u1 { v16sf f; };
> +union u2 { v16sf f; float x; };
> +struct s3 { union u1 u; };
> +union u4 { v8di f; };
> +
> +#define CHECK(X, N) \
> + for (int i = 0; i < N; i++) \
> + if ((X)[i] != i + 1) \
> + abort ()
> +
> +__attribute__((noipa)) void
> +foo (int n, ...)
> +{
> + va_list ap;
> + va_start (ap, n);
> + if (va_arg (ap, double) != 0.0)
> + abort ();
> + union u1 a = va_arg (ap, union u1);
> + CHECK (a.f, 16);
> + if (va_arg (ap, double) != 5.0)
> + abort ();
> + union u2 b = va_arg (ap, union u2);
> + CHECK (b.f, 16);
> + struct s3 c = va_arg (ap, struct s3);
> + CHECK (c.u.f, 16);
> + union u4 d = va_arg (ap, union u4);
> + CHECK (d.f, 8);
> + if (va_arg (ap, double) != 6.0)
> + abort ();
> + va_end (ap);
> +}
> +
> +static void
> +avx512f_test (void)
> +{
> + union u1 a = { { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 } };
> + union u2 b = { { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 } };
> + struct s3 c = { { { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16
> } } };
> + union u4 d = { { 1, 2, 3, 4, 5, 6, 7, 8 } };
> + foo (1, 0.0, a, 5.0, b, c, d, 6.0);
> +}
> diff --git a/gcc/testsuite/gcc.target/i386/pr128640-3.c
> b/gcc/testsuite/gcc.target/i386/pr128640-3.c
> new file mode 100644
> index 00000000000..390e0d5b83a
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/pr128640-3.c
> @@ -0,0 +1,27 @@
> +/* PR target/128640 */
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O2 -mavx512f" } */
> +
> +typedef float v8sf __attribute__((vector_size (32)));
> +typedef float v16sf __attribute__((vector_size (64)));
> +union u1 { v8sf f; };
> +union u2 { v16sf f; };
> +
> +void bar (int, ...);
> +
> +/* Unnamed 256-bit and 512-bit unions are passed on the stack, so no
> + SSE register is used for the variadic arguments. */
> +void
> +foo1 (union u1 *p)
> +{
> + bar (1, *p);
> +}
> +
> +void
> +foo2 (union u2 *p)
> +{
> + bar (1, *p);
> +}
> +
> +/* { dg-final { scan-assembler-times "xorl\[\\t \]+%eax, %eax" 2 } } */
> +/* { dg-final { scan-assembler-not "movl\[\\t \]+\\\$1, %eax" } } */
> --
> 2.34.1
>
