https://gcc.gnu.org/g:f2aa060578cd67b733558e0bef1bdc2158372475
commit r17-5393-gf2aa060578cd67b733558e0bef1bdc2158372475 Author: liuhongt <[email protected]> Date: Mon Oct 19 19:21:17 2026 -0700 i386: Use cost->mulss and ix86_vec_cost for VnDI umul_highpart cost Commit 82787e703f97 ("[x86][v2] umul<mode>3_highpart for VnDI") regressed 750.sealcrypto_r on Clearwater Forest, the hot NTT loop is now vectorized with V4DI which is slower than the scalar code. The cost of the V4DI .MULH emulation is too low: 1) The 4 pmuludq are costed with ix86_multiplication_cost (ix86_cost, SImode), which is the scalar imul cost. Use mulss instead, like the other VnDI multiply emulation. 2) The multiply part doesn't go through ix86_vec_cost, so X86_TUNE_AVX256_SPLIT_REGS is not honored. vect-umulv4di_high-2.c is reduced from 750.sealcrypto_r, with -march=clearwaterforest it's no longer vectorized, with generic tuning it's still vectorized. The simple loop in vect-umulv4di_high.c is no longer vectorized with generic tuning, add -mtune=znver5 to it. The cost for znver5 is not impacted since its scalar integer multiply cost is the same as mulss. gcc/ChangeLog: * config/i386/i386.cc (ix86_vector_costs::add_stmt_cost): Use mulss for the widening multiplies of VnDI MULT_HIGHPART_EXPR and CFN_MULH, and apply ix86_vec_cost to the whole cost. gcc/testsuite/ChangeLog: * gcc.target/i386/vect-umulv4di_high.c: Add -mtune=znver5. * gcc.target/i386/vect-umulv4di_high-2.c: New test. Diff: --- gcc/config/i386/i386.cc | 16 +++---- .../gcc.target/i386/vect-umulv4di_high-2.c | 49 ++++++++++++++++++++++ gcc/testsuite/gcc.target/i386/vect-umulv4di_high.c | 2 +- 3 files changed, 58 insertions(+), 9 deletions(-) diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc index eda00c137b40..d2e80a4401b7 100644 --- a/gcc/config/i386/i386.cc +++ b/gcc/config/i386/i386.cc @@ -26752,10 +26752,10 @@ ix86_vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind, if (kind == vector_stmt && GET_MODE_INNER (mode) == DImode) { /* ix86_expand_umulvndi_highpart, 4 SImode -> DImode - widening multiplies. */ - stmt_cost = ix86_multiplication_cost (ix86_cost, SImode) * 4; - /* 5 shifts by 32bit, 1 and, 4 adds. */ - stmt_cost += ix86_vec_cost (mode, ix86_cost->sse_op) * 10; + widening multiplies plus 5 shifts by 32bit, 1 and, + 4 adds. */ + stmt_cost = ix86_vec_cost (mode, ix86_cost->mulss * 4 + + ix86_cost->sse_op * 10); } else /* For MULT_HIGHPART_EXPR, x86 only supports pmulhw, @@ -27021,10 +27021,10 @@ ix86_vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind, if (kind == vector_stmt && GET_MODE_INNER (mode) == DImode) { /* ix86_expand_umulvndi_highpart, 4 SImode -> DImode - widening multiplies. */ - stmt_cost = ix86_multiplication_cost (ix86_cost, SImode) * 4; - /* 5 shifts by 32bit, 1 and, 4 adds. */ - stmt_cost += ix86_vec_cost (mode, ix86_cost->sse_op) * 10; + widening multiplies plus 5 shifts by 32bit, 1 and, + 4 adds. */ + stmt_cost = ix86_vec_cost (mode, ix86_cost->mulss * 4 + + ix86_cost->sse_op * 10); } else stmt_cost = ix86_multiplication_cost (ix86_cost, mode); diff --git a/gcc/testsuite/gcc.target/i386/vect-umulv4di_high-2.c b/gcc/testsuite/gcc.target/i386/vect-umulv4di_high-2.c new file mode 100644 index 000000000000..46df4d7242f1 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/vect-umulv4di_high-2.c @@ -0,0 +1,49 @@ +/* { dg-do compile { target { ! ia32 } } } */ +/* { dg-options "-O3 -march=clearwaterforest" } */ + +#include <stdint.h> + +/* 64x64 -> high 64 open-coded with 32-bit halves, as in SEAL's + multiply_uint64_hw64_generic. */ +static inline uint64_t +mulhi64 (uint64_t a, uint64_t b) +{ + uint64_t a_lo = a & 0xffffffff, b_lo = b & 0xffffffff; + a >>= 32; + b >>= 32; + uint64_t mid1 = a * b_lo; + uint64_t mid = mid1 + b * a_lo; + uint64_t left = a * b + ((uint64_t) (mid < mid1) << 32); + uint64_t right = a_lo * b_lo; + uint64_t tmp = (right >> 32) + (mid & 0xffffffff); + return left + (mid >> 32) + (tmp >> 32); +} + +struct mod_operand { uint64_t operand, quotient; }; +struct arith { uint64_t modulus, two_times_modulus; }; + +/* NTT butterfly with Barrett multiply_uint_mod_lazy, unrolled by 4, + reduced from 750.sealcrypto_r. */ +void +ntt_butterfly (const struct arith *a, uint64_t *x, uint64_t *y, + struct mod_operand r, unsigned long gap) +{ + for (unsigned long j = 0; j < gap; j += 4) + { +#define BFLY \ + { \ + uint64_t u = *x >= a->two_times_modulus \ + ? *x - a->two_times_modulus : *x; \ + uint64_t v = r.operand * *y \ + - mulhi64 (*y, r.quotient) * a->modulus; \ + *x++ = u + v; \ + *y++ = u + a->two_times_modulus - v; \ + } + BFLY BFLY BFLY BFLY + } +} + +/* With 256-bit ops split into two 128-bit uops, vectorizing the + 64bit unsigned highpart multiply with 4 vpmuludq plus the emulated + 64bit multiplies is slower than the scalar mulx/imul sequence. */ +/* { dg-final { scan-assembler-times "pmuludq" 0 } } */ diff --git a/gcc/testsuite/gcc.target/i386/vect-umulv4di_high.c b/gcc/testsuite/gcc.target/i386/vect-umulv4di_high.c index ef966b4ec07a..686f54b8288a 100644 --- a/gcc/testsuite/gcc.target/i386/vect-umulv4di_high.c +++ b/gcc/testsuite/gcc.target/i386/vect-umulv4di_high.c @@ -1,5 +1,5 @@ /* { dg-do compile { target { ! ia32 } } } */ -/* { dg-options "-O2 -mavx2 -mprefer-vector-width=256" } */ +/* { dg-options "-O2 -mavx2 -mprefer-vector-width=256 -mtune=znver5" } */ #include <stdint.h>
