https://gcc.gnu.org/g:8835caacb1b5d929b9df3ec873570b73e41a6847
commit r17-5837-g8835caacb1b5d929b9df3ec873570b73e41a6847 Author: liuhongt <[email protected]> Date: Thu Oct 29 19:44:35 2026 -0700 match.pd: Fold masked vector multiply to widening even/odd multiply [PR124271] (x & lowhalf_mask) * (y & lowhalf_mask) on an integer vector only multiplies the zero-extended low halves of each element. That is VEC_WIDEN_MULT_EVEN_EXPR of x and y viewed as vectors of half-width unsigned elements (VEC_WIDEN_MULT_ODD_EXPR on big-endian, where the low halves are the odd lanes). On x86 this replaces the V*DImode multiply emulation (3 pmuludq plus shifts and adds) with a single pmuludq, and replaces vpmullq when AVX512DQ is available. gcc/ChangeLog: PR target/124271 * match.pd ((x & lowhalf_mask) * (y & lowhalf_mask)): New simplification to VEC_WIDEN_MULT_{EVEN,ODD}_EXPR. gcc/testsuite/ChangeLog: PR target/124271 * gcc.dg/torture/pr124271.c: New test. * gcc.target/i386/pr124271-1.c: New test. * gcc.target/i386/pr124271-2.c: New test. Diff: --- gcc/match.pd | 39 ++++++++++++++++++ gcc/testsuite/gcc.dg/torture/pr124271.c | 63 ++++++++++++++++++++++++++++++ gcc/testsuite/gcc.target/i386/pr124271-1.c | 32 +++++++++++++++ gcc/testsuite/gcc.target/i386/pr124271-2.c | 40 +++++++++++++++++++ 4 files changed, 174 insertions(+) diff --git a/gcc/match.pd b/gcc/match.pd index d1eeb283bcbd..f25952c6fd88 100644 --- a/gcc/match.pd +++ b/gcc/match.pd @@ -474,6 +474,45 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT) { zeros; }) { ones; } { zeros; }))))))))) +#if GIMPLE +/* (x & lowhalf_mask) * (y & lowhalf_mask) + -> VEC_WIDEN_MULT_EVEN (VIEW_CONVERT (x), VIEW_CONVERT (y)), or + VEC_WIDEN_MULT_ODD on big-endian targets where the low halves are the + odd lanes. */ +(simplify + (mult (bit_and @0 VECTOR_CST@2) + (bit_and @1 @2)) + (if (uniform_vector_p (@2) + /* Single-element results are excluded since targets disagree + on the mode of a vector(1) widening multiply result (V2DI -> TI + on s390 but V2DI -> V1TI on rs6000). */ + && known_gt (TYPE_VECTOR_SUBPARTS (type), 1U)) + (with + { + auto elem = wi::to_wide (uniform_vector_p (@2)); + unsigned int outer_prec = TYPE_PRECISION (TREE_TYPE (type)); + unsigned int inner_prec = outer_prec / 2; + /* The half-width lanes are zero-extended, so the multiply is + unsigned even for a signed TYPE. */ + tree inner_scalar = build_nonstandard_integer_type (inner_prec, 1); + tree inner_type + = build_vector_type (inner_scalar, TYPE_VECTOR_SUBPARTS (type) * 2); + optab op = (BYTES_BIG_ENDIAN + ? vec_widen_umult_odd_optab : vec_widen_umult_even_optab); + } + (if (elem == wi::mask (inner_prec, false, outer_prec) + /* Without a mode of exactly INNER_PREC bits (e.g. for QImode + elements) the half-width vector would be wider than TYPE. */ + && type_has_mode_precision_p (inner_scalar) + && GET_MODE_CLASS (TYPE_MODE (inner_type)) == MODE_VECTOR_INT + && optab_handler (op, TYPE_MODE (inner_type)) != CODE_FOR_nothing) + (if (BYTES_BIG_ENDIAN) + (vec_widen_mult_odd (view_convert:inner_type @0) + (view_convert:inner_type @1)) + (vec_widen_mult_even (view_convert:inner_type @0) + (view_convert:inner_type @1))))))) +#endif + (for cmp (gt ge lt le) outp (convert convert negate negate) outn (negate negate convert convert) diff --git a/gcc/testsuite/gcc.dg/torture/pr124271.c b/gcc/testsuite/gcc.dg/torture/pr124271.c new file mode 100644 index 000000000000..911a50f71edc --- /dev/null +++ b/gcc/testsuite/gcc.dg/torture/pr124271.c @@ -0,0 +1,63 @@ +/* PR target/124271 */ +/* { dg-do run } */ + +typedef unsigned long long v2du __attribute__ ((vector_size (16))); +typedef unsigned long long v4du __attribute__ ((vector_size (32))); +typedef long long v2di __attribute__ ((vector_size (16))); +typedef unsigned int v4su __attribute__ ((vector_size (16))); +typedef unsigned short v8hu __attribute__ ((vector_size (16))); +typedef unsigned char v8qu __attribute__ ((vector_size (8))); + +#define TEST(NAME, VT, ET, UT, M) \ + __attribute__ ((noipa)) void \ + NAME (VT *r, VT *a, VT *b) \ + { \ + *r = (*a & (ET) M) * (*b & (ET) M); \ + } \ + \ + void \ + check_##NAME (void) \ + { \ + VT a, b, r; \ + for (unsigned i = 0; i < sizeof (VT) / sizeof (ET); i++) \ + { \ + a[i] = (ET) (0x9e3779b97f4a7c15ULL * (i + 1)); \ + b[i] = (ET) (0xc2b2ae3d27d4eb4fULL * (i + 3)); \ + /* Keep the signed product from overflowing. */ \ + if ((ET) -1 < 0) \ + { \ + a[i] &= (ET) ((UT) M >> 1) | ~(ET) M; \ + b[i] &= (ET) ((UT) M >> 1) | ~(ET) M; \ + } \ + } \ + NAME (&r, &a, &b); \ + for (unsigned i = 0; i < sizeof (VT) / sizeof (ET); i++) \ + if ((UT) r[i] != (UT) ((UT) (a[i] & (ET) M) \ + * (UT) (b[i] & (ET) M))) \ + __builtin_abort (); \ + } + +TEST (f1, v2du, unsigned long long, unsigned long long, 0xffffffffULL) +TEST (f2, v4du, unsigned long long, unsigned long long, 0xffffffffULL) +TEST (f3, v2di, long long, unsigned long long, 0xffffffffULL) +TEST (f4, v4su, unsigned int, unsigned int, 0xffffU) +TEST (f5, v8hu, unsigned short, unsigned short, 0xffU) +/* Not a low-half mask. */ +TEST (f6, v2du, unsigned long long, unsigned long long, 0x1ffffffffULL) +TEST (f7, v2du, unsigned long long, unsigned long long, 0x7fffffffULL) +/* No integer mode for the half-width element. */ +TEST (f8, v8qu, unsigned char, unsigned char, 0xfU) + +int +main () +{ + check_f1 (); + check_f2 (); + check_f3 (); + check_f4 (); + check_f5 (); + check_f6 (); + check_f7 (); + check_f8 (); + return 0; +} diff --git a/gcc/testsuite/gcc.target/i386/pr124271-1.c b/gcc/testsuite/gcc.target/i386/pr124271-1.c new file mode 100644 index 000000000000..00fcb5619895 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/pr124271-1.c @@ -0,0 +1,32 @@ +/* PR target/124271 */ +/* { dg-do compile } */ +/* { dg-options "-O3 -mavx2 -mno-avx512f -fdump-tree-optimized" } */ + +typedef __UINT64_TYPE__ uint64_t; +typedef __UINT32_TYPE__ uint32_t; + +static inline uint64_t +mul32 (uint64_t a, uint64_t b) +{ + return (uint64_t) (uint32_t) a * (uint64_t) (uint32_t) b; +} + +void +foo (uint64_t *__restrict a, const uint64_t *__restrict b) +{ + for (int i = 0; i < 1024; i++) + a[i] = mul32 (a[i], b[i]); +} + +void +bar (long long *__restrict a, const long long *__restrict b) +{ + for (int i = 0; i < 1024; i++) + a[i] = (a[i] & 0xffffffff) * (b[i] & 0xffffffff); +} + +/* { dg-final { scan-tree-dump-times "WIDEN_MULT_EVEN_EXPR" 2 "optimized" } } */ +/* { dg-final { scan-assembler-times "vpmuludq\[ \\t\]+\[^\n\]*%ymm" 2 } } */ +/* { dg-final { scan-assembler-not "vpsrlq" } } */ +/* { dg-final { scan-assembler-not "vpsllq" } } */ +/* { dg-final { scan-assembler-not "vpand" } } */ diff --git a/gcc/testsuite/gcc.target/i386/pr124271-2.c b/gcc/testsuite/gcc.target/i386/pr124271-2.c new file mode 100644 index 000000000000..fd9ed7ac6992 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/pr124271-2.c @@ -0,0 +1,40 @@ +/* PR target/124271 */ +/* { dg-do compile } */ +/* { dg-options "-O2 -mavx512f -mno-avx512dq" } */ + +typedef unsigned long long v2du __attribute__ ((vector_size (16))); +typedef unsigned long long v4du __attribute__ ((vector_size (32))); +typedef unsigned long long v8du __attribute__ ((vector_size (64))); +typedef long long v2di __attribute__ ((vector_size (16))); + +#define M 0xffffffffULL + +v2du +f1 (v2du x, v2du y) +{ + return (x & M) * (y & M); +} + +v4du +f2 (v4du x, v4du y) +{ + return (x & M) * (y & M); +} + +v8du +f3 (v8du x, v8du y) +{ + return (x & M) * (y & M); +} + +v2di +f4 (v2di x, v2di y) +{ + return (x & (long long) M) * (y & (long long) M); +} + +/* { dg-final { scan-assembler-times "vpmuludq\[ \\t\]+\[^\n\]*%xmm" 2 } } */ +/* { dg-final { scan-assembler-times "vpmuludq\[ \\t\]+\[^\n\]*%ymm" 1 } } */ +/* { dg-final { scan-assembler-times "vpmuludq\[ \\t\]+\[^\n\]*%zmm" 1 } } */ +/* { dg-final { scan-assembler-not "vpsrlq" } } */ +/* { dg-final { scan-assembler-not "vpsllq" } } */
