https://gcc.gnu.org/g:b1ed5e88f55539db555b990a7a8e0f2ba00588e1
commit r17-4914-gb1ed5e88f55539db555b990a7a8e0f2ba00588e1 Author: Dhruv Chawla <[email protected]> Date: Tue Aug 25 06:16:24 2026 +0000 aarch64: Port NEON halving-add intrinsics to pragma-based framework Port the following intrinsics to the pragma-based framework: * vhadd * vrhadd The arm_neon_{1,2,3}.c test cases now expect the ACLE streaming-mode diagnostic instead of an inlining failure. The inlining_10.c and inlining_11.c test cases are updated to use the vqtbl intrinsic instead of vhadd as the latter is no longer implemented as an always_inline function in arm_neon.h. Bootstrapped and regtested on aarch64-linux-gnu. Signed-off-by: Dhruv Chawla <[email protected]> gcc/ChangeLog: * config/aarch64/aarch64-neon-builtins-base.cc (vhadd, vhaddq, vrhadd, vrhaddq): New function bases. * config/aarch64/aarch64-neon-builtins-base.def (vhadd, vhaddq, vrhadd, vrhaddq): New function groups. * config/aarch64/arm_neon.h (vhadd_s8, vhadd_s16, vhadd_s32, vhadd_u8, vhadd_u16, vhadd_u32, vhaddq_s8, vhaddq_s16, vhaddq_s32, vhaddq_u8, vhaddq_u16, vhaddq_u32, vrhadd_s8, vrhadd_s16, vrhadd_s32, vrhadd_u8, vrhadd_u16, vrhadd_u32, vrhaddq_s8, vrhaddq_s16, vrhaddq_s32, vrhaddq_u8, vrhaddq_u16, vrhaddq_u32): Delete functions. gcc/testsuite/ChangeLog: * gcc.target/aarch64/sme/arm_neon_1.c: Update the expected dg-error message. * gcc.target/aarch64/sme/arm_neon_2.c: Likewise. * gcc.target/aarch64/sme/arm_neon_3.c: Likewise. * gcc.target/aarch64/sme/inlining_10.c: Replace vhadd with vqtbl as vhadd is no longer always_inline. * gcc.target/aarch64/sme/inlining_11.c: Likewise. * gcc.target/aarch64/neon/vhadd.c: New test. * gcc.target/aarch64/neon/vrhadd.c: Likewise. Diff: --- gcc/config/aarch64/aarch64-neon-builtins-base.cc | 4 + gcc/config/aarch64/aarch64-neon-builtins-base.def | 8 + gcc/config/aarch64/arm_neon.h | 168 --------------------- gcc/testsuite/gcc.target/aarch64/neon/vhadd.c | 88 +++++++++++ gcc/testsuite/gcc.target/aarch64/neon/vrhadd.c | 88 +++++++++++ gcc/testsuite/gcc.target/aarch64/sme/arm_neon_1.c | 4 +- gcc/testsuite/gcc.target/aarch64/sme/arm_neon_2.c | 4 +- gcc/testsuite/gcc.target/aarch64/sme/arm_neon_3.c | 4 +- gcc/testsuite/gcc.target/aarch64/sme/inlining_10.c | 6 +- gcc/testsuite/gcc.target/aarch64/sme/inlining_11.c | 6 +- 10 files changed, 197 insertions(+), 183 deletions(-) diff --git a/gcc/config/aarch64/aarch64-neon-builtins-base.cc b/gcc/config/aarch64/aarch64-neon-builtins-base.cc index ff53bd837489..3a5ba21eb61d 100644 --- a/gcc/config/aarch64/aarch64-neon-builtins-base.cc +++ b/gcc/config/aarch64/aarch64-neon-builtins-base.cc @@ -793,6 +793,10 @@ NEON_FUNCTION (vminv, vminvq, gimple_ifn, (IFN_REDUC_MIN)) NEON_FUNCTION (vmaxnmv, vmaxnmvq, gimple_ifn, (IFN_REDUC_FMAX)) NEON_FUNCTION (vminnmv, vminnmvq, gimple_ifn, (IFN_REDUC_FMIN)) +// Halving add +NEON_FUNCTION (vhadd, vhaddq, gimple_ifn, (IFN_AVG_FLOOR)) +NEON_FUNCTION (vrhadd, vrhaddq, gimple_ifn, (IFN_AVG_CEIL)) + // Bitwise operations NEON_FUNCTION (vand, vandq, gimple_expr, (BIT_AND_EXPR)) NEON_FUNCTION (vbic, vbicq, gimple_not_rhs, (BIT_AND_EXPR)) diff --git a/gcc/config/aarch64/aarch64-neon-builtins-base.def b/gcc/config/aarch64/aarch64-neon-builtins-base.def index 759944b307d0..095932c85343 100644 --- a/gcc/config/aarch64/aarch64-neon-builtins-base.def +++ b/gcc/config/aarch64/aarch64-neon-builtins-base.def @@ -114,6 +114,14 @@ DEF_NEON_FUNCTION (vminnmv, h_float, ("s0,D0")) DEF_NEON_FUNCTION (vminnmvq, h_float, ("s0,Q0")) #undef REQUIRED_EXTENSIONS +// Halving add +#define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_SIMD) +DEF_NEON_FUNCTION (vhadd, bhs_integer, ("D0,D0,D0")) +DEF_NEON_FUNCTION (vhaddq, bhs_integer, ("Q0,Q0,Q0")) +DEF_NEON_FUNCTION (vrhadd, bhs_integer, ("D0,D0,D0")) +DEF_NEON_FUNCTION (vrhaddq, bhs_integer, ("Q0,Q0,Q0")) +#undef REQUIRED_EXTENSIONS + // Bitwise operations #define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_SIMD) DEF_NEON_FUNCTION (vand, all_integer, ("D0,D0,D0")) diff --git a/gcc/config/aarch64/arm_neon.h b/gcc/config/aarch64/arm_neon.h index c32d5e2c0594..0edccdec49b2 100644 --- a/gcc/config/aarch64/arm_neon.h +++ b/gcc/config/aarch64/arm_neon.h @@ -273,174 +273,6 @@ vaddw_high_u32 (uint64x2_t __a, uint32x4_t __b) return __builtin_aarch64_uaddw2v4si_uuu (__a, __b); } -__extension__ extern __inline int8x8_t -__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) -vhadd_s8 (int8x8_t __a, int8x8_t __b) -{ - return __builtin_aarch64_shaddv8qi (__a, __b); -} - -__extension__ extern __inline int16x4_t -__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) -vhadd_s16 (int16x4_t __a, int16x4_t __b) -{ - return __builtin_aarch64_shaddv4hi (__a, __b); -} - -__extension__ extern __inline int32x2_t -__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) -vhadd_s32 (int32x2_t __a, int32x2_t __b) -{ - return __builtin_aarch64_shaddv2si (__a, __b); -} - -__extension__ extern __inline uint8x8_t -__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) -vhadd_u8 (uint8x8_t __a, uint8x8_t __b) -{ - return __builtin_aarch64_uhaddv8qi_uuu (__a, __b); -} - -__extension__ extern __inline uint16x4_t -__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) -vhadd_u16 (uint16x4_t __a, uint16x4_t __b) -{ - return __builtin_aarch64_uhaddv4hi_uuu (__a, __b); -} - -__extension__ extern __inline uint32x2_t -__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) -vhadd_u32 (uint32x2_t __a, uint32x2_t __b) -{ - return __builtin_aarch64_uhaddv2si_uuu (__a, __b); -} - -__extension__ extern __inline int8x16_t -__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) -vhaddq_s8 (int8x16_t __a, int8x16_t __b) -{ - return __builtin_aarch64_shaddv16qi (__a, __b); -} - -__extension__ extern __inline int16x8_t -__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) -vhaddq_s16 (int16x8_t __a, int16x8_t __b) -{ - return __builtin_aarch64_shaddv8hi (__a, __b); -} - -__extension__ extern __inline int32x4_t -__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) -vhaddq_s32 (int32x4_t __a, int32x4_t __b) -{ - return __builtin_aarch64_shaddv4si (__a, __b); -} - -__extension__ extern __inline uint8x16_t -__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) -vhaddq_u8 (uint8x16_t __a, uint8x16_t __b) -{ - return __builtin_aarch64_uhaddv16qi_uuu (__a, __b); -} - -__extension__ extern __inline uint16x8_t -__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) -vhaddq_u16 (uint16x8_t __a, uint16x8_t __b) -{ - return __builtin_aarch64_uhaddv8hi_uuu (__a, __b); -} - -__extension__ extern __inline uint32x4_t -__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) -vhaddq_u32 (uint32x4_t __a, uint32x4_t __b) -{ - return __builtin_aarch64_uhaddv4si_uuu (__a, __b); -} - -__extension__ extern __inline int8x8_t -__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) -vrhadd_s8 (int8x8_t __a, int8x8_t __b) -{ - return __builtin_aarch64_srhaddv8qi (__a, __b); -} - -__extension__ extern __inline int16x4_t -__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) -vrhadd_s16 (int16x4_t __a, int16x4_t __b) -{ - return __builtin_aarch64_srhaddv4hi (__a, __b); -} - -__extension__ extern __inline int32x2_t -__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) -vrhadd_s32 (int32x2_t __a, int32x2_t __b) -{ - return __builtin_aarch64_srhaddv2si (__a, __b); -} - -__extension__ extern __inline uint8x8_t -__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) -vrhadd_u8 (uint8x8_t __a, uint8x8_t __b) -{ - return __builtin_aarch64_urhaddv8qi_uuu (__a, __b); -} - -__extension__ extern __inline uint16x4_t -__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) -vrhadd_u16 (uint16x4_t __a, uint16x4_t __b) -{ - return __builtin_aarch64_urhaddv4hi_uuu (__a, __b); -} - -__extension__ extern __inline uint32x2_t -__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) -vrhadd_u32 (uint32x2_t __a, uint32x2_t __b) -{ - return __builtin_aarch64_urhaddv2si_uuu (__a, __b); -} - -__extension__ extern __inline int8x16_t -__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) -vrhaddq_s8 (int8x16_t __a, int8x16_t __b) -{ - return __builtin_aarch64_srhaddv16qi (__a, __b); -} - -__extension__ extern __inline int16x8_t -__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) -vrhaddq_s16 (int16x8_t __a, int16x8_t __b) -{ - return __builtin_aarch64_srhaddv8hi (__a, __b); -} - -__extension__ extern __inline int32x4_t -__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) -vrhaddq_s32 (int32x4_t __a, int32x4_t __b) -{ - return __builtin_aarch64_srhaddv4si (__a, __b); -} - -__extension__ extern __inline uint8x16_t -__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) -vrhaddq_u8 (uint8x16_t __a, uint8x16_t __b) -{ - return __builtin_aarch64_urhaddv16qi_uuu (__a, __b); -} - -__extension__ extern __inline uint16x8_t -__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) -vrhaddq_u16 (uint16x8_t __a, uint16x8_t __b) -{ - return __builtin_aarch64_urhaddv8hi_uuu (__a, __b); -} - -__extension__ extern __inline uint32x4_t -__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) -vrhaddq_u32 (uint32x4_t __a, uint32x4_t __b) -{ - return __builtin_aarch64_urhaddv4si_uuu (__a, __b); -} - __extension__ extern __inline int8x8_t __attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) vaddhn_s16 (int16x8_t __a, int16x8_t __b) diff --git a/gcc/testsuite/gcc.target/aarch64/neon/vhadd.c b/gcc/testsuite/gcc.target/aarch64/neon/vhadd.c new file mode 100644 index 000000000000..8411f6a4e30d --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/neon/vhadd.c @@ -0,0 +1,88 @@ +/* { dg-do compile } */ +/* { dg-final { check-function-bodies "**" "" } } */ + +#include "arm_neon_test.h" + +/* +** test_vhadd_u8: +** uhadd v0\.8b, (v0\.8b, v1\.8b|v1\.8b, v0\.8b) +** ret +*/ +TEST_UNIFORM_BINARY (vhadd_u8, uint8x8_t) + +/* +** test_vhadd_s8: +** shadd v0\.8b, (v0\.8b, v1\.8b|v1\.8b, v0\.8b) +** ret +*/ +TEST_UNIFORM_BINARY (vhadd_s8, int8x8_t) + +/* +** test_vhadd_u16: +** uhadd v0\.4h, (v0\.4h, v1\.4h|v1\.4h, v0\.4h) +** ret +*/ +TEST_UNIFORM_BINARY (vhadd_u16, uint16x4_t) + +/* +** test_vhadd_s16: +** shadd v0\.4h, (v0\.4h, v1\.4h|v1\.4h, v0\.4h) +** ret +*/ +TEST_UNIFORM_BINARY (vhadd_s16, int16x4_t) + +/* +** test_vhadd_u32: +** uhadd v0\.2s, (v0\.2s, v1\.2s|v1\.2s, v0\.2s) +** ret +*/ +TEST_UNIFORM_BINARY (vhadd_u32, uint32x2_t) + +/* +** test_vhadd_s32: +** shadd v0\.2s, (v0\.2s, v1\.2s|v1\.2s, v0\.2s) +** ret +*/ +TEST_UNIFORM_BINARY (vhadd_s32, int32x2_t) + +/* +** test_vhaddq_u8: +** uhadd v0\.16b, (v0\.16b, v1\.16b|v1\.16b, v0\.16b) +** ret +*/ +TEST_UNIFORM_BINARY (vhaddq_u8, uint8x16_t) + +/* +** test_vhaddq_s8: +** shadd v0\.16b, (v0\.16b, v1\.16b|v1\.16b, v0\.16b) +** ret +*/ +TEST_UNIFORM_BINARY (vhaddq_s8, int8x16_t) + +/* +** test_vhaddq_u16: +** uhadd v0\.8h, (v0\.8h, v1\.8h|v1\.8h, v0\.8h) +** ret +*/ +TEST_UNIFORM_BINARY (vhaddq_u16, uint16x8_t) + +/* +** test_vhaddq_s16: +** shadd v0\.8h, (v0\.8h, v1\.8h|v1\.8h, v0\.8h) +** ret +*/ +TEST_UNIFORM_BINARY (vhaddq_s16, int16x8_t) + +/* +** test_vhaddq_u32: +** uhadd v0\.4s, (v0\.4s, v1\.4s|v1\.4s, v0\.4s) +** ret +*/ +TEST_UNIFORM_BINARY (vhaddq_u32, uint32x4_t) + +/* +** test_vhaddq_s32: +** shadd v0\.4s, (v0\.4s, v1\.4s|v1\.4s, v0\.4s) +** ret +*/ +TEST_UNIFORM_BINARY (vhaddq_s32, int32x4_t) diff --git a/gcc/testsuite/gcc.target/aarch64/neon/vrhadd.c b/gcc/testsuite/gcc.target/aarch64/neon/vrhadd.c new file mode 100644 index 000000000000..46e3cb864c8d --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/neon/vrhadd.c @@ -0,0 +1,88 @@ +/* { dg-do compile } */ +/* { dg-final { check-function-bodies "**" "" } } */ + +#include "arm_neon_test.h" + +/* +** test_vrhadd_u8: +** urhadd v0\.8b, (v0\.8b, v1\.8b|v1\.8b, v0\.8b) +** ret +*/ +TEST_UNIFORM_BINARY (vrhadd_u8, uint8x8_t) + +/* +** test_vrhadd_s8: +** srhadd v0\.8b, (v0\.8b, v1\.8b|v1\.8b, v0\.8b) +** ret +*/ +TEST_UNIFORM_BINARY (vrhadd_s8, int8x8_t) + +/* +** test_vrhadd_u16: +** urhadd v0\.4h, (v0\.4h, v1\.4h|v1\.4h, v0\.4h) +** ret +*/ +TEST_UNIFORM_BINARY (vrhadd_u16, uint16x4_t) + +/* +** test_vrhadd_s16: +** srhadd v0\.4h, (v0\.4h, v1\.4h|v1\.4h, v0\.4h) +** ret +*/ +TEST_UNIFORM_BINARY (vrhadd_s16, int16x4_t) + +/* +** test_vrhadd_u32: +** urhadd v0\.2s, (v0\.2s, v1\.2s|v1\.2s, v0\.2s) +** ret +*/ +TEST_UNIFORM_BINARY (vrhadd_u32, uint32x2_t) + +/* +** test_vrhadd_s32: +** srhadd v0\.2s, (v0\.2s, v1\.2s|v1\.2s, v0\.2s) +** ret +*/ +TEST_UNIFORM_BINARY (vrhadd_s32, int32x2_t) + +/* +** test_vrhaddq_u8: +** urhadd v0\.16b, (v0\.16b, v1\.16b|v1\.16b, v0\.16b) +** ret +*/ +TEST_UNIFORM_BINARY (vrhaddq_u8, uint8x16_t) + +/* +** test_vrhaddq_s8: +** srhadd v0\.16b, (v0\.16b, v1\.16b|v1\.16b, v0\.16b) +** ret +*/ +TEST_UNIFORM_BINARY (vrhaddq_s8, int8x16_t) + +/* +** test_vrhaddq_u16: +** urhadd v0\.8h, (v0\.8h, v1\.8h|v1\.8h, v0\.8h) +** ret +*/ +TEST_UNIFORM_BINARY (vrhaddq_u16, uint16x8_t) + +/* +** test_vrhaddq_s16: +** srhadd v0\.8h, (v0\.8h, v1\.8h|v1\.8h, v0\.8h) +** ret +*/ +TEST_UNIFORM_BINARY (vrhaddq_s16, int16x8_t) + +/* +** test_vrhaddq_u32: +** urhadd v0\.4s, (v0\.4s, v1\.4s|v1\.4s, v0\.4s) +** ret +*/ +TEST_UNIFORM_BINARY (vrhaddq_u32, uint32x4_t) + +/* +** test_vrhaddq_s32: +** srhadd v0\.4s, (v0\.4s, v1\.4s|v1\.4s, v0\.4s) +** ret +*/ +TEST_UNIFORM_BINARY (vrhaddq_s32, int32x4_t) diff --git a/gcc/testsuite/gcc.target/aarch64/sme/arm_neon_1.c b/gcc/testsuite/gcc.target/aarch64/sme/arm_neon_1.c index 5b5346cf4350..9f50bb927227 100644 --- a/gcc/testsuite/gcc.target/aarch64/sme/arm_neon_1.c +++ b/gcc/testsuite/gcc.target/aarch64/sme/arm_neon_1.c @@ -4,10 +4,8 @@ #pragma GCC target "+nosme" -// { dg-error {inlining failed.*'vhaddq_s32'} "" { target *-*-* } 0 } - int32x4_t foo (int32x4_t x, int32x4_t y) [[arm::streaming_compatible]] { - return vhaddq_s32 (x, y); + return vhaddq_s32 (x, y); // { dg-error {ACLE function 'vhaddq_s32' cannot be called when SME streaming mode is enabled} } } diff --git a/gcc/testsuite/gcc.target/aarch64/sme/arm_neon_2.c b/gcc/testsuite/gcc.target/aarch64/sme/arm_neon_2.c index 2092c4471f06..9a4fff5e98c8 100644 --- a/gcc/testsuite/gcc.target/aarch64/sme/arm_neon_2.c +++ b/gcc/testsuite/gcc.target/aarch64/sme/arm_neon_2.c @@ -2,10 +2,8 @@ #include <arm_neon.h> -// { dg-error {inlining failed.*'vhaddq_s32'} "" { target *-*-* } 0 } - int32x4_t foo (int32x4_t x, int32x4_t y) [[arm::streaming_compatible]] { - return vhaddq_s32 (x, y); + return vhaddq_s32 (x, y); // { dg-error {ACLE function 'vhaddq_s32' cannot be called when SME streaming mode is enabled} } } diff --git a/gcc/testsuite/gcc.target/aarch64/sme/arm_neon_3.c b/gcc/testsuite/gcc.target/aarch64/sme/arm_neon_3.c index 36794e5b0dfe..19f633e18492 100644 --- a/gcc/testsuite/gcc.target/aarch64/sme/arm_neon_3.c +++ b/gcc/testsuite/gcc.target/aarch64/sme/arm_neon_3.c @@ -2,10 +2,8 @@ #include <arm_neon.h> -// { dg-error {inlining failed.*'vhaddq_s32'} "" { target *-*-* } 0 } - int32x4_t foo (int32x4_t x, int32x4_t y) [[arm::streaming]] { - return vhaddq_s32 (x, y); + return vhaddq_s32 (x, y); // { dg-error {ACLE function 'vhaddq_s32' cannot be called when SME streaming mode is enabled} } } diff --git a/gcc/testsuite/gcc.target/aarch64/sme/inlining_10.c b/gcc/testsuite/gcc.target/aarch64/sme/inlining_10.c index 131fb7a66379..3e9af18265fc 100644 --- a/gcc/testsuite/gcc.target/aarch64/sme/inlining_10.c +++ b/gcc/testsuite/gcc.target/aarch64/sme/inlining_10.c @@ -18,9 +18,9 @@ call_vadd () } inline void __attribute__((always_inline)) -call_vhadd () // { dg-error "inlining failed" } +call_vqtbl () // { dg-error "inlining failed" } { - neon[0] = vhaddq_u8 (neon[1], neon[2]); + neon[0] = vqtbl1q_u8 (neon[1], neon[2]); } inline void __attribute__((always_inline)) @@ -51,7 +51,7 @@ void sc_caller () [[arm::inout("za"), arm::streaming_compatible]] { call_vadd (); - call_vhadd (); + call_vqtbl (); call_svadd (); call_svld1_gather (); call_svzero (); diff --git a/gcc/testsuite/gcc.target/aarch64/sme/inlining_11.c b/gcc/testsuite/gcc.target/aarch64/sme/inlining_11.c index d500a62743d0..e60b357dfa24 100644 --- a/gcc/testsuite/gcc.target/aarch64/sme/inlining_11.c +++ b/gcc/testsuite/gcc.target/aarch64/sme/inlining_11.c @@ -18,9 +18,9 @@ call_vadd () } inline void __attribute__((always_inline)) -call_vhadd () // { dg-error "inlining failed" } +call_vqtbl () // { dg-error "inlining failed" } { - neon[0] = vhaddq_u8 (neon[1], neon[2]); + neon[0] = vqtbl1q_u8 (neon[1], neon[2]); } inline void __attribute__((always_inline)) @@ -51,7 +51,7 @@ void sc_caller () [[arm::inout("za"), arm::streaming]] { call_vadd (); - call_vhadd (); + call_vqtbl (); call_svadd (); call_svld1_gather (); call_svzero ();
