https://gcc.gnu.org/g:9cf7c3266406e909891d2a40aeb468a091ccdaf2
commit r17-3034-g9cf7c3266406e909891d2a40aeb468a091ccdaf2 Author: Kyrylo Tkachov <[email protected]> Date: Thu Aug 6 07:13:11 2026 +0000 vect: Recognise high-part multiply when the wide type has no vector form vect_recog_mulhs_pattern turns (a * b) >> N into IFN_MULH and casts the result back to the type of the shift. It records a vector type for that cast in *type_out and gives up when there is none. For a 64-bit high-part multiply the shift result is 128 bits wide, and no target has a vector of 128-bit integers, so on AArch64 the pattern never fires for scalable SVE and the loop stays scalar. The cast itself is still wanted, so that the pattern statement has the same type as the statement it replaces, and the over-widening machinery removes it later anyway. A vector type for it is not needed though, so leave *type_out null in that case rather than giving up. for (int i = 0; i < n; i++) d[i] = (uint64_t) (((unsigned __int128) a[i] * b[i]) >> 64); on -march=armv8.2-a+sve2 now gives: ld1d z31.d, p7/z, [x1, x4, lsl 3] ld1d z30.d, p7/z, [x2, x4, lsl 3] umulh z30.d, z30.d, z31.d st1d z30.d, p7, [x0, x4, lsl 3] rather than: ldr x5, [x1, x4] ldr x6, [x2, x4] umulh x5, x5, x6 str x5, [x0, x4] The new vect_mulh_di effective target lists the targets I could find with the relevant optab at DImode, but I have only tested AArch64 myself. Bootstrapped and regression-tested on aarch64-unknown-linux-gnu. gcc/ChangeLog: * tree-vect-patterns.cc (vect_recog_mulhs_pattern): Do not give up when LHS_TYPE has no vector type. gcc/testsuite/ChangeLog: * lib/target-supports.exp (check_effective_target_vect_mulh_di): New procedure. * gcc.dg/vect/vect-mulh-1.c: New test. * gcc.dg/vect/vect-mulh-2.c: New test. * gcc.target/aarch64/sve/mul_highpart_scalable_1.c: New test. Signed-off-by: Kyrylo Tkachov <[email protected]> Diff: --- gcc/testsuite/gcc.dg/vect/vect-mulh-1.c | 52 ++++++++++++++++++++++ gcc/testsuite/gcc.dg/vect/vect-mulh-2.c | 9 ++++ .../aarch64/sve/mul_highpart_scalable_1.c | 36 +++++++++++++++ gcc/testsuite/lib/target-supports.exp | 14 ++++++ gcc/tree-vect-patterns.cc | 8 ++-- 5 files changed, 115 insertions(+), 4 deletions(-) diff --git a/gcc/testsuite/gcc.dg/vect/vect-mulh-1.c b/gcc/testsuite/gcc.dg/vect/vect-mulh-1.c new file mode 100644 index 000000000000..06330cc08901 --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/vect-mulh-1.c @@ -0,0 +1,52 @@ +/* { dg-require-effective-target vect_int } */ +/* { dg-require-effective-target int128 } */ + +#include "tree-vect.h" + +#ifndef SIGNEDNESS +#define SIGNEDNESS signed +#endif + +void __attribute__ ((noipa)) +f (SIGNEDNESS long long *restrict a, SIGNEDNESS long long *restrict b, + SIGNEDNESS long long *restrict c, __INTPTR_TYPE__ n) +{ + for (__INTPTR_TYPE__ i = 0; i < n; ++i) + a[i] = ((SIGNEDNESS __int128) b[i] * c[i]) >> 64; +} + +#define N 50 +#define BASE1 0x1234567890abcdefULL +#define BASE2 0x0fedcba098765432ULL +#define CONST1 0x0123456789abcdefULL +#define CONST2 0x0f0e0d0c0b0a0908ULL + +int +main (void) +{ + check_vect (); + + SIGNEDNESS long long a[N], b[N], c[N]; + /* Compute the inputs with wrapping unsigned arithmetic so that they cover + the whole 64-bit range without overflowing a signed type. */ + for (int i = 0; i < N; ++i) + { + b[i] = (SIGNEDNESS long long) (BASE1 + (unsigned long long) i * CONST1); + c[i] = (SIGNEDNESS long long) (BASE2 + (unsigned long long) i * CONST2); + asm volatile ("" ::: "memory"); + } + b[0] = 0; + c[0] = -1; + b[1] = -1; + c[1] = -1; + f (a, b, c, N); +#pragma GCC novector + for (int i = 0; i < N; ++i) + if (a[i] != (SIGNEDNESS long long) (((SIGNEDNESS __int128) b[i] * c[i]) + >> 64)) + __builtin_abort (); + return 0; +} + +/* { dg-final { scan-tree-dump {\.MULH} "vect" { target vect_mulh_di } } } */ +/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" { target vect_mulh_di } } } */ diff --git a/gcc/testsuite/gcc.dg/vect/vect-mulh-2.c b/gcc/testsuite/gcc.dg/vect/vect-mulh-2.c new file mode 100644 index 000000000000..0249caea3ded --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/vect-mulh-2.c @@ -0,0 +1,9 @@ +/* { dg-require-effective-target vect_int } */ +/* { dg-require-effective-target int128 } */ + +#define SIGNEDNESS unsigned + +#include "vect-mulh-1.c" + +/* { dg-final { scan-tree-dump {\.MULH} "vect" { target vect_mulh_di } } } */ +/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" { target vect_mulh_di } } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_scalable_1.c b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_scalable_1.c new file mode 100644 index 000000000000..2d19cde259ef --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_scalable_1.c @@ -0,0 +1,36 @@ +/* A 64-bit high-part multiply has a 128-bit product type, and there is no + scalable vector of 128-bit integers. Check that the vectoriser recognises + the high-part multiply anyway. */ +/* { dg-do compile } */ +/* { dg-options "-O2 -ftree-vectorize -march=armv8.2-a+sve2 -mautovec-preference=sve-only -msve-vector-bits=scalable" } */ +/* { dg-final { check-function-bodies "**" "" } } */ + +#include <stdint.h> + +/* +** mulh_s64: +** ... +** smulh z[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d +** ... +*/ +void __attribute__ ((noipa)) +mulh_s64 (int64_t *restrict dst, int64_t *restrict a, int64_t *restrict b, + int count) +{ + for (int i = 0; i < count; ++i) + dst[i] = (int64_t) (((__int128) a[i] * b[i]) >> 64); +} + +/* +** mulh_u64: +** ... +** umulh z[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d +** ... +*/ +void __attribute__ ((noipa)) +mulh_u64 (uint64_t *restrict dst, uint64_t *restrict a, uint64_t *restrict b, + int count) +{ + for (int i = 0; i < count; ++i) + dst[i] = (uint64_t) (((unsigned __int128) a[i] * b[i]) >> 64); +} diff --git a/gcc/testsuite/lib/target-supports.exp b/gcc/testsuite/lib/target-supports.exp index e57e98b76e89..99f134e9b691 100644 --- a/gcc/testsuite/lib/target-supports.exp +++ b/gcc/testsuite/lib/target-supports.exp @@ -9302,6 +9302,20 @@ proc check_effective_target_vect_mulhrs_hi {} { && [check_effective_target_aarch64_sve2] }] } +# Return 1 if the target plus current options supports both signed +# and unsigned high-part multiplication on vectors of 8-byte integers. + +proc check_effective_target_vect_mulh_di {} { + return [expr { ([istarget aarch64*-*-*] + && [check_effective_target_aarch64_sve]) + || ([istarget riscv*-*-*] + && [check_effective_target_riscv_v]) + || ([istarget powerpc*-*-*] + && [check_effective_target_has_arch_pwr10]) + || ([istarget loongarch*-*-*] + && [check_effective_target_loongarch_sx]) }] +} + # Return 1 if the target plus current options supports signed division # by power-of-2 operations on vectors of 4-byte integers. diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc index c57e215be577..b921ae94848e 100644 --- a/gcc/tree-vect-patterns.cc +++ b/gcc/tree-vect-patterns.cc @@ -3412,11 +3412,11 @@ vect_recog_mulhs_pattern (vec_info *vinfo, (ifn, new_vectype, OPTIMIZE_FOR_SPEED)) return NULL; - /* The IR requires a valid vector type for the cast result, even though - it's likely to be discarded. */ + /* The result is cast back to LHS_TYPE, a cast that the over-widening + machinery then removes. LHS_TYPE need not have a vector type, as for the + 128-bit product of a 64-bit high-part multiply, so leave *TYPE_OUT null + rather than giving up. */ *type_out = get_vectype_for_scalar_type (vinfo, lhs_type); - if (!*type_out) - return NULL; /* Generate the IFN_MULHRS call. */ tree new_var = vect_recog_temp_ssa_var (new_type, NULL);
