https://gcc.gnu.org/g:18d2e513be7832fb08df59a5a1ee8dd6dadd2bd2
commit r17-5871-g18d2e513be7832fb08df59a5a1ee8dd6dadd2bd2 Author: Haochen Jiang <[email protected]> Date: Thu Oct 29 15:15:32 2026 +0800 i386: Emit mask fma for conditional fma [PR128221] Previosly on trunk, we will emit fma + masked mov for conditional fma, it will get the correct result, but those masked values causing exception will be calculated under this scenario. Those thrown out excpetions are not expected. Use mask fma instead of fma to do that. After using mask fma, we lose some ablitiy to merge, do it directly in the new ix86_expand_cond_fma for some scenarios to avoid extra moves. gcc/ChangeLog: PR target/128221 * config/i386/i386-expand.cc (ix86_expand_cond_fma): New function to expand conditional fma. * config/i386/i386-protos.h (ix86_expand_cond_fma): Ditto. * config/i386/sse.md (cond_fma<mode>): Use new ix86_expand_cond_fma. (cond_fms<mode>): Ditto. (cond_fnma<mode>): Ditto. (cond_fnms<mode>): Ditto. gcc/testsuite/ChangeLog: PR target/128221 * gcc.target/i386/pr128221-1a.c: New test. * gcc.target/i386/pr128221-1b.c: Ditto. * gcc.target/i386/pr128221-2.c: Ditto. Diff: --- gcc/config/i386/i386-expand.cc | 37 +++++++++++++++++++ gcc/config/i386/i386-protos.h | 1 + gcc/config/i386/sse.md | 40 +++------------------ gcc/testsuite/gcc.target/i386/pr128221-1a.c | 26 ++++++++++++++ gcc/testsuite/gcc.target/i386/pr128221-1b.c | 39 ++++++++++++++++++++ gcc/testsuite/gcc.target/i386/pr128221-2.c | 55 +++++++++++++++++++++++++++++ 6 files changed, 162 insertions(+), 36 deletions(-) diff --git a/gcc/config/i386/i386-expand.cc b/gcc/config/i386/i386-expand.cc index a9423388cf85..12ccc9a03dff 100644 --- a/gcc/config/i386/i386-expand.cc +++ b/gcc/config/i386/i386-expand.cc @@ -4767,6 +4767,43 @@ ix86_expand_fp_movcc (rtx operands[]) return true; } +/* Expand a conditional fma: operands[0] = operands[1] + ? fma (operands[2], operands[3], operands[4]) : operands[5]. */ + +void +ix86_expand_cond_fma (rtx operands[], bool neg_mult, bool neg_add) +{ + machine_mode mode = GET_MODE (operands[0]); + rtx mask = operands[1]; + rtx op1 = operands[2], op2 = operands[3], op3 = operands[4]; + rtx op4 = operands[5]; + rtx tmp, fma; + + /* The multiplication is commutative, so an else value equal to the + second operand for mult can be merged just like first operand + after swapping. */ + if (rtx_equal_p (op4, op2)) + std::swap (op1, op2); + + fma = gen_rtx_FMA (mode, neg_mult ? gen_rtx_NEG (mode, op1) : op1, + op2, neg_add ? gen_rtx_NEG (mode, op3) : op3); + + /* Do not emit an extra move. */ + if (rtx_equal_p (op4, op1) + || rtx_equal_p (op4, op3) + || op4 == CONST0_RTX (mode)) + { + emit_insn (gen_rtx_SET (operands[0], + gen_rtx_VEC_MERGE (mode, fma, op4, mask))); + return; + } + + tmp = gen_reg_rtx (mode); + emit_insn (gen_rtx_SET (tmp, gen_rtx_VEC_MERGE (mode, fma, + CONST0_RTX (mode), mask))); + emit_move_insn (operands[0], gen_rtx_VEC_MERGE (mode, tmp, op4, mask)); +} + /* Helper for ix86_cmp_code_to_pcmp_immediate for int modes. */ static int diff --git a/gcc/config/i386/i386-protos.h b/gcc/config/i386/i386-protos.h index 949f6296462b..788ad443b44b 100644 --- a/gcc/config/i386/i386-protos.h +++ b/gcc/config/i386/i386-protos.h @@ -158,6 +158,7 @@ extern void ix86_expand_branch (enum rtx_code, rtx, rtx, rtx); extern void ix86_expand_setcc (rtx, enum rtx_code, rtx, rtx); extern bool ix86_expand_int_movcc (rtx[]); extern bool ix86_expand_fp_movcc (rtx[]); +extern void ix86_expand_cond_fma (rtx[], bool, bool); extern bool ix86_expand_fp_vcond (rtx[]); extern bool ix86_expand_int_vcond (rtx[]); extern void ix86_expand_vec_perm (rtx[]); diff --git a/gcc/config/i386/sse.md b/gcc/config/i386/sse.md index 8b45f6dbbcdc..ea710dddc6be 100644 --- a/gcc/config/i386/sse.md +++ b/gcc/config/i386/sse.md @@ -6415,15 +6415,7 @@ (match_operand:<avx512fmaskmode> 1 "register_operand")))] "TARGET_AVX512F" { - rtx tmp = gen_reg_rtx (<MODE>mode); - emit_insn (gen_fma<mode>4 (tmp, - operands[2], - operands[3], - operands[4])); - emit_move_insn (operands[0], gen_rtx_VEC_MERGE (<MODE>mode, - tmp, - operands[5], - operands[1])); + ix86_expand_cond_fma (operands, false, false); DONE; }) @@ -6520,15 +6512,7 @@ (match_operand:<avx512fmaskmode> 1 "register_operand")))] "TARGET_AVX512F" { - rtx tmp = gen_reg_rtx (<MODE>mode); - emit_insn (gen_fms<mode>4 (tmp, - operands[2], - operands[3], - operands[4])); - emit_move_insn (operands[0], gen_rtx_VEC_MERGE (<MODE>mode, - tmp, - operands[5], - operands[1])); + ix86_expand_cond_fma (operands, false, true); DONE; }) @@ -6627,15 +6611,7 @@ (match_operand:<avx512fmaskmode> 1 "register_operand")))] "TARGET_AVX512F" { - rtx tmp = gen_reg_rtx (<MODE>mode); - emit_insn (gen_fnma<mode>4 (tmp, - operands[2], - operands[3], - operands[4])); - emit_move_insn (operands[0], gen_rtx_VEC_MERGE (<MODE>mode, - tmp, - operands[5], - operands[1])); + ix86_expand_cond_fma (operands, true, false); DONE; }) @@ -6737,15 +6713,7 @@ (match_operand:<avx512fmaskmode> 1 "register_operand")))] "TARGET_AVX512F" { - rtx tmp = gen_reg_rtx (<MODE>mode); - emit_insn (gen_fnms<mode>4 (tmp, - operands[2], - operands[3], - operands[4])); - emit_move_insn (operands[0], gen_rtx_VEC_MERGE (<MODE>mode, - tmp, - operands[5], - operands[1])); + ix86_expand_cond_fma (operands, true, true); DONE; }) diff --git a/gcc/testsuite/gcc.target/i386/pr128221-1a.c b/gcc/testsuite/gcc.target/i386/pr128221-1a.c new file mode 100644 index 000000000000..1cdc64d7ee76 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/pr128221-1a.c @@ -0,0 +1,26 @@ +/* { dg-do compile } */ +/* { dg-options "-O3 -mavx512f -mprefer-vector-width=512" } */ + +#define N 1024 + +double a[N], b[N], c[N]; +int m[N]; + +#define DEF_COND_FMA(NAME, EXPR) \ + void \ + NAME (double w, double e) \ + { \ + for (int i = 0; i < N; i++) \ + a[i] = m[i] ? (EXPR) : e; \ + } + +DEF_COND_FMA (fma_test, a[i] + b[i] * w) +DEF_COND_FMA (fms_test, b[i] * w - c[i]) +DEF_COND_FMA (fnma_test, c[i] - b[i] * w) +DEF_COND_FMA (fnms_test, -b[i] * w - c[i]) + +/* { dg-final { scan-assembler "vfmadd\[0-9\]+pd\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" } } */ +/* { dg-final { scan-assembler "vfmsub\[0-9\]+pd\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" } } */ +/* { dg-final { scan-assembler "vfnmadd\[0-9\]+pd\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" } } */ +/* { dg-final { scan-assembler "vfnmsub\[0-9\]+pd\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" } } */ +/* { dg-final { scan-assembler-not "vfn?m(?:add|sub)\[0-9\]+pd\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+(?:\n|\[ \\t\]+#)" } } */ diff --git a/gcc/testsuite/gcc.target/i386/pr128221-1b.c b/gcc/testsuite/gcc.target/i386/pr128221-1b.c new file mode 100644 index 000000000000..e876b78d8e1f --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/pr128221-1b.c @@ -0,0 +1,39 @@ +/* PR target/128221 */ +/* { dg-do compile } */ +/* { dg-options "-O3 -mavx512f -mprefer-vector-width=512" } */ + +#define N 1024 + +double a[N], b[N], c[N], d[N]; +int m[N]; + +#define DEF_COND_FMA(NAME, EXPR, ELSE) \ + void \ + NAME (void) \ + { \ + for (int i = 0; i < N; i++) \ + a[i] = m[i] ? (EXPR) : (ELSE); \ + } + +#define DEF_COND_FMA_ALL(NAME, EXPR) \ + DEF_COND_FMA (NAME##_z, EXPR, 0.0) \ + DEF_COND_FMA (NAME##_mul1, EXPR, b[i]) \ + DEF_COND_FMA (NAME##_mul2, EXPR, c[i]) \ + DEF_COND_FMA (NAME##_add, EXPR, d[i]) + +DEF_COND_FMA_ALL (fma, b[i] * c[i] + d[i]) +DEF_COND_FMA_ALL (fms, b[i] * c[i] - d[i]) +DEF_COND_FMA_ALL (fnma, d[i] - b[i] * c[i]) +DEF_COND_FMA_ALL (fnms, -b[i] * c[i] - d[i]) + +/* { dg-final { scan-assembler "vfmadd\[0-9\]+pd\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" } } */ +/* { dg-final { scan-assembler "vfmadd\[0-9\]+pd\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" } } */ +/* { dg-final { scan-assembler "vfmsub\[0-9\]+pd\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" } } */ +/* { dg-final { scan-assembler "vfmsub\[0-9\]+pd\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" } } */ +/* { dg-final { scan-assembler "vfnmadd\[0-9\]+pd\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" } } */ +/* { dg-final { scan-assembler "vfnmadd\[0-9\]+pd\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" } } */ +/* { dg-final { scan-assembler "vfnmsub\[0-9\]+pd\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" } } */ +/* { dg-final { scan-assembler "vfnmsub\[0-9\]+pd\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" } } */ +/* { dg-final { scan-assembler-not "vfn?m(?:add|sub)\[0-9\]+pd\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+(?:\n|\[ \\t\]+#)" } } */ +/* { dg-final { scan-assembler-not "vblendm" } } */ +/* { dg-final { scan-assembler-not "vmov\[a-z0-9\]+\[ \\t\]+%zmm\[0-9\]+, %zmm\[0-9\]+\{%k" } } */ diff --git a/gcc/testsuite/gcc.target/i386/pr128221-2.c b/gcc/testsuite/gcc.target/i386/pr128221-2.c new file mode 100644 index 000000000000..5fbc9410a0d6 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/pr128221-2.c @@ -0,0 +1,55 @@ +/* { dg-do run } */ +/* { dg-options "-O3 -mavx512f -mprefer-vector-width=512" } */ +/* { dg-require-effective-target avx512f } */ +/* { dg-require-effective-target fenv_exceptions } */ + +#include <fenv.h> +#include "avx512f-check.h" +#include "pr128221-1.c" + +#define W 1800.0 +#define E -1e34 + +static void +check (void (*fn) (double, double), int op) +{ + for (int i = 0; i < N; i++) + { + m[i] = i & 1; + /* Overflows only in the lanes where m[i] is zero. */ + b[i] = m[i] ? 1.0 : 1e307; + a[i] = 2.0; + c[i] = 3.0; + } + + feclearexcept (FE_ALL_EXCEPT); + fn (W, E); + if (fetestexcept (FE_OVERFLOW)) + abort (); + + for (int i = 0; i < N; i++) + { + double r; + if (!m[i]) + r = E; + else if (op == 0) + r = 2.0 + W; + else if (op == 1) + r = W - 3.0; + else if (op == 2) + r = 3.0 - W; + else + r = -W - 3.0; + if (a[i] != r) + abort (); + } +} + +static void +avx512f_test (void) +{ + check (fma_test, 0); + check (fms_test, 1); + check (fnma_test, 2); + check (fnms_test, 3); +}
