https://gcc.gnu.org/g:8835caacb1b5d929b9df3ec873570b73e41a6847

commit r17-5837-g8835caacb1b5d929b9df3ec873570b73e41a6847
Author: liuhongt <[email protected]>
Date:   Thu Oct 29 19:44:35 2026 -0700

    match.pd: Fold masked vector multiply to widening even/odd multiply 
[PR124271]
    
    (x & lowhalf_mask) * (y & lowhalf_mask) on an integer vector only
    multiplies the zero-extended low halves of each element.  That is
    VEC_WIDEN_MULT_EVEN_EXPR of x and y viewed as vectors of half-width
    unsigned elements (VEC_WIDEN_MULT_ODD_EXPR on big-endian, where the low
    halves are the odd lanes).
    
    On x86 this replaces the V*DImode multiply emulation (3 pmuludq plus
    shifts and adds) with a single pmuludq, and replaces vpmullq when
    AVX512DQ is available.
    
    gcc/ChangeLog:
    
            PR target/124271
            * match.pd ((x & lowhalf_mask) * (y & lowhalf_mask)): New
            simplification to VEC_WIDEN_MULT_{EVEN,ODD}_EXPR.
    
    gcc/testsuite/ChangeLog:
    
            PR target/124271
            * gcc.dg/torture/pr124271.c: New test.
            * gcc.target/i386/pr124271-1.c: New test.
            * gcc.target/i386/pr124271-2.c: New test.

Diff:
---
 gcc/match.pd                               | 39 ++++++++++++++++++
 gcc/testsuite/gcc.dg/torture/pr124271.c    | 63 ++++++++++++++++++++++++++++++
 gcc/testsuite/gcc.target/i386/pr124271-1.c | 32 +++++++++++++++
 gcc/testsuite/gcc.target/i386/pr124271-2.c | 40 +++++++++++++++++++
 4 files changed, 174 insertions(+)

diff --git a/gcc/match.pd b/gcc/match.pd
index d1eeb283bcbd..f25952c6fd88 100644
--- a/gcc/match.pd
+++ b/gcc/match.pd
@@ -474,6 +474,45 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
                                     { zeros; })
                           { ones; } { zeros; })))))))))
 
+#if GIMPLE
+/* (x & lowhalf_mask) * (y & lowhalf_mask)
+   -> VEC_WIDEN_MULT_EVEN (VIEW_CONVERT (x), VIEW_CONVERT (y)), or
+   VEC_WIDEN_MULT_ODD on big-endian targets where the low halves are the
+   odd lanes.  */
+(simplify
+ (mult (bit_and @0 VECTOR_CST@2)
+       (bit_and @1 @2))
+ (if (uniform_vector_p (@2)
+      /* Single-element results are excluded since targets disagree
+        on the mode of a vector(1) widening multiply result (V2DI -> TI
+        on s390 but V2DI -> V1TI on rs6000).  */
+      && known_gt (TYPE_VECTOR_SUBPARTS (type), 1U))
+  (with
+   {
+     auto elem = wi::to_wide (uniform_vector_p (@2));
+     unsigned int outer_prec = TYPE_PRECISION (TREE_TYPE (type));
+     unsigned int inner_prec = outer_prec / 2;
+     /* The half-width lanes are zero-extended, so the multiply is
+       unsigned even for a signed TYPE.  */
+     tree inner_scalar = build_nonstandard_integer_type (inner_prec, 1);
+     tree inner_type
+       = build_vector_type (inner_scalar, TYPE_VECTOR_SUBPARTS (type) * 2);
+     optab op = (BYTES_BIG_ENDIAN
+                ? vec_widen_umult_odd_optab : vec_widen_umult_even_optab);
+   }
+   (if (elem == wi::mask (inner_prec, false, outer_prec)
+       /* Without a mode of exactly INNER_PREC bits (e.g. for QImode
+          elements) the half-width vector would be wider than TYPE.  */
+       && type_has_mode_precision_p (inner_scalar)
+       && GET_MODE_CLASS (TYPE_MODE (inner_type)) == MODE_VECTOR_INT
+       && optab_handler (op, TYPE_MODE (inner_type)) != CODE_FOR_nothing)
+    (if (BYTES_BIG_ENDIAN)
+     (vec_widen_mult_odd (view_convert:inner_type @0)
+                        (view_convert:inner_type @1))
+     (vec_widen_mult_even (view_convert:inner_type @0)
+                         (view_convert:inner_type @1)))))))
+#endif
+
 (for cmp (gt ge lt le)
      outp (convert convert negate negate)
      outn (negate negate convert convert)
diff --git a/gcc/testsuite/gcc.dg/torture/pr124271.c 
b/gcc/testsuite/gcc.dg/torture/pr124271.c
new file mode 100644
index 000000000000..911a50f71edc
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr124271.c
@@ -0,0 +1,63 @@
+/* PR target/124271 */
+/* { dg-do run } */
+
+typedef unsigned long long v2du __attribute__ ((vector_size (16)));
+typedef unsigned long long v4du __attribute__ ((vector_size (32)));
+typedef long long v2di __attribute__ ((vector_size (16)));
+typedef unsigned int v4su __attribute__ ((vector_size (16)));
+typedef unsigned short v8hu __attribute__ ((vector_size (16)));
+typedef unsigned char v8qu __attribute__ ((vector_size (8)));
+
+#define TEST(NAME, VT, ET, UT, M)                              \
+  __attribute__ ((noipa)) void                                 \
+  NAME (VT *r, VT *a, VT *b)                                   \
+  {                                                            \
+    *r = (*a & (ET) M) * (*b & (ET) M);                                \
+  }                                                            \
+                                                               \
+  void                                                         \
+  check_##NAME (void)                                          \
+  {                                                            \
+    VT a, b, r;                                                        \
+    for (unsigned i = 0; i < sizeof (VT) / sizeof (ET); i++)   \
+      {                                                                \
+       a[i] = (ET) (0x9e3779b97f4a7c15ULL * (i + 1));          \
+       b[i] = (ET) (0xc2b2ae3d27d4eb4fULL * (i + 3));          \
+       /* Keep the signed product from overflowing.  */        \
+       if ((ET) -1 < 0)                                        \
+         {                                                     \
+           a[i] &= (ET) ((UT) M >> 1) | ~(ET) M;               \
+           b[i] &= (ET) ((UT) M >> 1) | ~(ET) M;               \
+         }                                                     \
+      }                                                                \
+    NAME (&r, &a, &b);                                         \
+    for (unsigned i = 0; i < sizeof (VT) / sizeof (ET); i++)   \
+      if ((UT) r[i] != (UT) ((UT) (a[i] & (ET) M)              \
+                            * (UT) (b[i] & (ET) M)))           \
+       __builtin_abort ();                                     \
+  }
+
+TEST (f1, v2du, unsigned long long, unsigned long long, 0xffffffffULL)
+TEST (f2, v4du, unsigned long long, unsigned long long, 0xffffffffULL)
+TEST (f3, v2di, long long, unsigned long long, 0xffffffffULL)
+TEST (f4, v4su, unsigned int, unsigned int, 0xffffU)
+TEST (f5, v8hu, unsigned short, unsigned short, 0xffU)
+/* Not a low-half mask.  */
+TEST (f6, v2du, unsigned long long, unsigned long long, 0x1ffffffffULL)
+TEST (f7, v2du, unsigned long long, unsigned long long, 0x7fffffffULL)
+/* No integer mode for the half-width element.  */
+TEST (f8, v8qu, unsigned char, unsigned char, 0xfU)
+
+int
+main ()
+{
+  check_f1 ();
+  check_f2 ();
+  check_f3 ();
+  check_f4 ();
+  check_f5 ();
+  check_f6 ();
+  check_f7 ();
+  check_f8 ();
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr124271-1.c 
b/gcc/testsuite/gcc.target/i386/pr124271-1.c
new file mode 100644
index 000000000000..00fcb5619895
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr124271-1.c
@@ -0,0 +1,32 @@
+/* PR target/124271 */
+/* { dg-do compile } */
+/* { dg-options "-O3 -mavx2 -mno-avx512f -fdump-tree-optimized" } */
+
+typedef __UINT64_TYPE__ uint64_t;
+typedef __UINT32_TYPE__ uint32_t;
+
+static inline uint64_t
+mul32 (uint64_t a, uint64_t b)
+{
+  return (uint64_t) (uint32_t) a * (uint64_t) (uint32_t) b;
+}
+
+void
+foo (uint64_t *__restrict a, const uint64_t *__restrict b)
+{
+  for (int i = 0; i < 1024; i++)
+    a[i] = mul32 (a[i], b[i]);
+}
+
+void
+bar (long long *__restrict a, const long long *__restrict b)
+{
+  for (int i = 0; i < 1024; i++)
+    a[i] = (a[i] & 0xffffffff) * (b[i] & 0xffffffff);
+}
+
+/* { dg-final { scan-tree-dump-times "WIDEN_MULT_EVEN_EXPR" 2 "optimized" } } 
*/
+/* { dg-final { scan-assembler-times "vpmuludq\[ \\t\]+\[^\n\]*%ymm" 2 } } */
+/* { dg-final { scan-assembler-not "vpsrlq" } } */
+/* { dg-final { scan-assembler-not "vpsllq" } } */
+/* { dg-final { scan-assembler-not "vpand" } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr124271-2.c 
b/gcc/testsuite/gcc.target/i386/pr124271-2.c
new file mode 100644
index 000000000000..fd9ed7ac6992
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr124271-2.c
@@ -0,0 +1,40 @@
+/* PR target/124271 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -mavx512f -mno-avx512dq" } */
+
+typedef unsigned long long v2du __attribute__ ((vector_size (16)));
+typedef unsigned long long v4du __attribute__ ((vector_size (32)));
+typedef unsigned long long v8du __attribute__ ((vector_size (64)));
+typedef long long v2di __attribute__ ((vector_size (16)));
+
+#define M 0xffffffffULL
+
+v2du
+f1 (v2du x, v2du y)
+{
+  return (x & M) * (y & M);
+}
+
+v4du
+f2 (v4du x, v4du y)
+{
+  return (x & M) * (y & M);
+}
+
+v8du
+f3 (v8du x, v8du y)
+{
+  return (x & M) * (y & M);
+}
+
+v2di
+f4 (v2di x, v2di y)
+{
+  return (x & (long long) M) * (y & (long long) M);
+}
+
+/* { dg-final { scan-assembler-times "vpmuludq\[ \\t\]+\[^\n\]*%xmm" 2 } } */
+/* { dg-final { scan-assembler-times "vpmuludq\[ \\t\]+\[^\n\]*%ymm" 1 } } */
+/* { dg-final { scan-assembler-times "vpmuludq\[ \\t\]+\[^\n\]*%zmm" 1 } } */
+/* { dg-final { scan-assembler-not "vpsrlq" } } */
+/* { dg-final { scan-assembler-not "vpsllq" } } */

Reply via email to