From: Kyrylo Tkachov <[email protected]>
vect_recog_mulhs_pattern turns (a * b) >> N into IFN_MULH and casts the
result back to the type of the shift, which the over-widening machinery
then drops. Building that cast needs a vector type for the wide type, and
the pattern gives up when there is none.
For a 64-bit high-part multiply the wide type is 128 bits, and no target
has a vector of 128-bit integers, so on AArch64 the pattern never fires
for scalable SVE and the loop stays scalar. It does fire under
-msve-vector-bits=128, where the prevailing vector size is 128 bits and a
one-element TImode vector exists, which is what makes the failure specific
to variable-length vectors.
The pattern has already established that the users of the result need at
most min_output_precision bits and that NEW_TYPE holds at least that many,
so when the wide vector type is unavailable we can simply leave the result
at NEW_TYPE precision and skip the cast.
for (int i = 0; i < n; i++)
d[i] = (uint64_t) (((unsigned __int128) a[i] * b[i]) >> 64);
on -march=armv8.2-a+sve2 -msve-vector-bits=scalable now gives:
ld1d z31.d, p7/z, [x1, x4, lsl 3]
ld1d z30.d, p7/z, [x2, x4, lsl 3]
umulh z30.d, z30.d, z31.d
st1d z30.d, p7, [x0, x4, lsl 3]
rather than:
ldr x5, [x1, x4]
ldr x6, [x2, x4]
umulh x5, x5, x6
str x5, [x0, x4]
Added a gcc.dg/vect/ test. I've added a new vect_mulh_di effective
target and added what targets I could find that support the relevant
optab at DImode but I've only tested aarch64 myself.
Bootstrapped and regression-tested on aarch64-unknown-linux-gnu.
Ok for trunk?
Thanks,
Kyrill
gcc/ChangeLog:
* tree-vect-patterns.cc (vect_recog_mulhs_pattern): Keep the
pattern result at NEW_TYPE precision when LHS_TYPE has no vector
type.
gcc/testsuite/ChangeLog:
* lib/target-supports.exp (check_effective_target_vect_mulh_di):
New procedure.
* gcc.dg/vect/vect-mulh-1.c: New test.
* gcc.dg/vect/vect-mulh-2.c: New test.
* gcc.target/aarch64/sve/mul_highpart_scalable_1.c: New test.
Signed-off-by: Kyrylo Tkachov <[email protected]>
---
gcc/testsuite/gcc.dg/vect/vect-mulh-1.c | 52 +++++++++++++++++++
gcc/testsuite/gcc.dg/vect/vect-mulh-2.c | 9 ++++
.../aarch64/sve/mul_highpart_scalable_1.c | 36 +++++++++++++
gcc/testsuite/lib/target-supports.exp | 14 +++++
gcc/tree-vect-patterns.cc | 15 ++++--
5 files changed, 122 insertions(+), 4 deletions(-)
create mode 100644 gcc/testsuite/gcc.dg/vect/vect-mulh-1.c
create mode 100644 gcc/testsuite/gcc.dg/vect/vect-mulh-2.c
create mode 100644
gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_scalable_1.c
diff --git a/gcc/testsuite/gcc.dg/vect/vect-mulh-1.c
b/gcc/testsuite/gcc.dg/vect/vect-mulh-1.c
new file mode 100644
index 00000000000..06330cc0890
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-mulh-1.c
@@ -0,0 +1,52 @@
+/* { dg-require-effective-target vect_int } */
+/* { dg-require-effective-target int128 } */
+
+#include "tree-vect.h"
+
+#ifndef SIGNEDNESS
+#define SIGNEDNESS signed
+#endif
+
+void __attribute__ ((noipa))
+f (SIGNEDNESS long long *restrict a, SIGNEDNESS long long *restrict b,
+ SIGNEDNESS long long *restrict c, __INTPTR_TYPE__ n)
+{
+ for (__INTPTR_TYPE__ i = 0; i < n; ++i)
+ a[i] = ((SIGNEDNESS __int128) b[i] * c[i]) >> 64;
+}
+
+#define N 50
+#define BASE1 0x1234567890abcdefULL
+#define BASE2 0x0fedcba098765432ULL
+#define CONST1 0x0123456789abcdefULL
+#define CONST2 0x0f0e0d0c0b0a0908ULL
+
+int
+main (void)
+{
+ check_vect ();
+
+ SIGNEDNESS long long a[N], b[N], c[N];
+ /* Compute the inputs with wrapping unsigned arithmetic so that they cover
+ the whole 64-bit range without overflowing a signed type. */
+ for (int i = 0; i < N; ++i)
+ {
+ b[i] = (SIGNEDNESS long long) (BASE1 + (unsigned long long) i * CONST1);
+ c[i] = (SIGNEDNESS long long) (BASE2 + (unsigned long long) i * CONST2);
+ asm volatile ("" ::: "memory");
+ }
+ b[0] = 0;
+ c[0] = -1;
+ b[1] = -1;
+ c[1] = -1;
+ f (a, b, c, N);
+#pragma GCC novector
+ for (int i = 0; i < N; ++i)
+ if (a[i] != (SIGNEDNESS long long) (((SIGNEDNESS __int128) b[i] * c[i])
+ >> 64))
+ __builtin_abort ();
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump {\.MULH} "vect" { target vect_mulh_di } } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" { target
vect_mulh_di } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-mulh-2.c
b/gcc/testsuite/gcc.dg/vect/vect-mulh-2.c
new file mode 100644
index 00000000000..0249caea3de
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-mulh-2.c
@@ -0,0 +1,9 @@
+/* { dg-require-effective-target vect_int } */
+/* { dg-require-effective-target int128 } */
+
+#define SIGNEDNESS unsigned
+
+#include "vect-mulh-1.c"
+
+/* { dg-final { scan-tree-dump {\.MULH} "vect" { target vect_mulh_di } } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" { target
vect_mulh_di } } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_scalable_1.c
b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_scalable_1.c
new file mode 100644
index 00000000000..2d19cde259e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_scalable_1.c
@@ -0,0 +1,36 @@
+/* A 64-bit high-part multiply has a 128-bit product type, and there is no
+ scalable vector of 128-bit integers. Check that the vectoriser recognises
+ the high-part multiply anyway. */
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8.2-a+sve2
-mautovec-preference=sve-only -msve-vector-bits=scalable" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <stdint.h>
+
+/*
+** mulh_s64:
+** ...
+** smulh z[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d
+** ...
+*/
+void __attribute__ ((noipa))
+mulh_s64 (int64_t *restrict dst, int64_t *restrict a, int64_t *restrict b,
+ int count)
+{
+ for (int i = 0; i < count; ++i)
+ dst[i] = (int64_t) (((__int128) a[i] * b[i]) >> 64);
+}
+
+/*
+** mulh_u64:
+** ...
+** umulh z[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d
+** ...
+*/
+void __attribute__ ((noipa))
+mulh_u64 (uint64_t *restrict dst, uint64_t *restrict a, uint64_t *restrict b,
+ int count)
+{
+ for (int i = 0; i < count; ++i)
+ dst[i] = (uint64_t) (((unsigned __int128) a[i] * b[i]) >> 64);
+}
diff --git a/gcc/testsuite/lib/target-supports.exp
b/gcc/testsuite/lib/target-supports.exp
index 42cbb5ce3df..89784ca68cd 100644
--- a/gcc/testsuite/lib/target-supports.exp
+++ b/gcc/testsuite/lib/target-supports.exp
@@ -9302,6 +9302,20 @@ proc check_effective_target_vect_mulhrs_hi {} {
&& [check_effective_target_aarch64_sve2] }]
}
+# Return 1 if the target plus current options supports both signed
+# and unsigned high-part multiplication on vectors of 8-byte integers.
+
+proc check_effective_target_vect_mulh_di {} {
+ return [expr { ([istarget aarch64*-*-*]
+ && [check_effective_target_aarch64_sve])
+ || ([istarget riscv*-*-*]
+ && [check_effective_target_riscv_v])
+ || ([istarget powerpc*-*-*]
+ && [check_effective_target_has_arch_pwr10])
+ || ([istarget loongarch*-*-*]
+ && [check_effective_target_loongarch_sx]) }]
+}
+
# Return 1 if the target plus current options supports signed division
# by power-of-2 operations on vectors of 4-byte integers.
diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc
index c57e215be57..185b28302d4 100644
--- a/gcc/tree-vect-patterns.cc
+++ b/gcc/tree-vect-patterns.cc
@@ -3412,11 +3412,18 @@ vect_recog_mulhs_pattern (vec_info *vinfo,
(ifn, new_vectype, OPTIMIZE_FOR_SPEED))
return NULL;
- /* The IR requires a valid vector type for the cast result, even though
- it's likely to be discarded. */
+ /* *TYPE_OUT is the vector type of the trailing cast back to LHS_TYPE, which
+ the over-widening machinery removes anyway. Drop the cast when LHS_TYPE
+ has no vector type, as for the 128-bit product of a 64-bit high-part
+ multiply. The uses need at most TARGET_PRECISION bits, which NEW_TYPE
+ has. */
+ tree out_type = lhs_type;
*type_out = get_vectype_for_scalar_type (vinfo, lhs_type);
if (!*type_out)
- return NULL;
+ {
+ out_type = new_type;
+ *type_out = new_vectype;
+ }
/* Generate the IFN_MULHRS call. */
tree new_var = vect_recog_temp_ssa_var (new_type, NULL);
@@ -3432,7 +3439,7 @@ vect_recog_mulhs_pattern (vec_info *vinfo,
dump_printf_loc (MSG_NOTE, vect_location,
"created pattern stmt: %G", (gimple *) mulhrs_stmt);
- return vect_convert_output (vinfo, last_stmt_info, lhs_type,
+ return vect_convert_output (vinfo, last_stmt_info, out_type,
mulhrs_stmt, new_vectype);
}
--
2.50.1 (Apple Git-155)
