https://gcc.gnu.org/g:2b5fbca63825ef96d71c4bca44c6b0811c668f89
commit 2b5fbca63825ef96d71c4bca44c6b0811c668f89 Author: Michael Meissner <[email protected]> Date: Fri Sep 18 16:13:51 2026 -0400 Rework optimizing bfloat16 a + b + c 2026-09-18 Michael Meissner <[email protected]> gcc/ * config/rs6000/float16.cc (bfloat16_operation_as_v4sf_inner): Split inner part from bfloat16_operation_as_v4sf to do the operation in V4SFmode. Add support for chained binary operations. (bfloat16_operation_as_v4sf): Call bfloat16_operation_as_v4sf_inner and then convert the result to SFmode or BFmode. * config/rs6000/float16.md (bfloat16 peephole2): Delete peephole2 that tried to allow chained operations. * config/rs6000/predicates.md (bfloat16_v4sf_operand): Allow chained operations of plus, subtract, and multiply. (bfloat16_bf_operand): Allow SFmode CONST_DOUBLE. gcc/testsuite/ * gcc.target/powerpc/bf16-3.c: New test. * gcc.target/powerpc/bf16-4.c: Likewise. Diff: --- gcc/config/rs6000/float16.cc | 82 +++++++++++++++++++++++++------ gcc/config/rs6000/float16.md | 79 ----------------------------- gcc/config/rs6000/predicates.md | 22 +++++++-- gcc/testsuite/gcc.target/powerpc/bf16-3.c | 32 ++++++++++++ gcc/testsuite/gcc.target/powerpc/bf16-4.c | 31 ++++++++++++ 5 files changed, 150 insertions(+), 96 deletions(-) diff --git a/gcc/config/rs6000/float16.cc b/gcc/config/rs6000/float16.cc index 2c7b6278a16a..e5206b910694 100644 --- a/gcc/config/rs6000/float16.cc +++ b/gcc/config/rs6000/float16.cc @@ -187,7 +187,6 @@ fp16_vectorization (enum rtx_code icode, /* Expand a bfloat16 scalar floating point operation: ICODE: Operation to perform. - RESULT: Result of the operation. OP1: Input operand1. OP2: Input operand2. OP3: Input operand3 or NULL_RTX. @@ -198,15 +197,17 @@ fp16_vectorization (enum rtx_code icode, back again takes quite a bit of time. GCC will only generate the native operation if -Ofast is used. The float16.md code that calls this function adds various combine operations to do the operation in V4SFmode instead of - SFmode. */ - -void -bfloat16_operation_as_v4sf (enum rtx_code icode, - rtx result, - rtx op1, - rtx op2, - rtx op3, - enum fp16_operation subtype) + SFmode. + + Inner part that does the operation in V4SFmode, and returns the pseudo + with the results. */ + +static rtx +bfloat16_operation_as_v4sf_inner (enum rtx_code icode, + rtx op1, + rtx op2, + rtx op3, + enum fp16_operation subtype) { gcc_assert (can_create_pseudo_p ()); @@ -237,13 +238,31 @@ bfloat16_operation_as_v4sf (enum rtx_code icode, for (size_t i = 0; i < n_opts; i++) { rtx op = ops_orig[i]; - rtx tmp = ops_v4sf[i] = gen_reg_rtx (V4SFmode); gcc_assert (op != NULL_RTX); - /* Remove truncation/extend added. */ - if (GET_CODE (op) == FLOAT_EXTEND || GET_CODE (op) == FLOAT_TRUNCATE) - op = XEXP (op, 0); + switch (GET_CODE (op)) + { + /* Handle chained operations. */ + case PLUS: + case MINUS: + case MULT: + ops_v4sf[i] = bfloat16_operation_as_v4sf_inner (GET_CODE (op), + XEXP (op, 0), + XEXP (op, 1), + NULL_RTX, + FP16_BINARY); + continue; + + /* Remove truncation/extend added. */ + case FLOAT_EXTEND: + case FLOAT_TRUNCATE: + op = XEXP (op, 0); + break; + + default: + break; + } /* Convert operands to V4SFmode format. We use SPLAT for registers to get the value into the upper 32-bits. We can use XXSPLTW to splat @@ -254,6 +273,7 @@ bfloat16_operation_as_v4sf (enum rtx_code icode, XXSPLTIW or XXSPLTIB to load the constant the other bits are duplicated. */ + rtx tmp = ops_v4sf[i] = gen_reg_rtx (V4SFmode); if (op == CONST0_RTX (SFmode) || op == CONST0_RTX (BFmode)) emit_move_insn (tmp, CONST0_RTX (V4SFmode)); @@ -320,6 +340,40 @@ bfloat16_operation_as_v4sf (enum rtx_code icode, gcc_unreachable (); } + return result_v4sf; +} + +/* Expand a bfloat16 scalar floating point operation: + + ICODE: Operation to perform. + RESULT: Result of the operation. + OP1: Input operand1. + OP2: Input operand2. + OP3: Input operand3 or NULL_RTX. + SUBTYPE: Describe the operation. + + The operation is done as a V4SFmode vector operation. This is because + converting BFmode from a scalar BFmode to SFmode to do the operation and + back again takes quite a bit of time. GCC will only generate the native + operation if -Ofast is used. The float16.md code that calls this function + adds various combine operations to do the operation in V4SFmode instead of + SFmode. + + External part that calls bfloat16_operation_as_v4sf_inner to do the + calculation in V4SFmode, and then converts the result to SFmode or + BFmode. */ + +void +bfloat16_operation_as_v4sf (enum rtx_code icode, + rtx result, + rtx op1, + rtx op2, + rtx op3, + enum fp16_operation subtype) +{ + rtx result_v4sf + = bfloat16_operation_as_v4sf_inner (icode, op1, op2, op3, subtype); + /* Convert V4SF result back to scalar mode. */ if (GET_MODE (result) == BFmode) emit_insn (gen_xvcvspbf16_bf (result, result_v4sf)); diff --git a/gcc/config/rs6000/float16.md b/gcc/config/rs6000/float16.md index 9c88bb10d331..abe88c4ab632 100644 --- a/gcc/config/rs6000/float16.md +++ b/gcc/config/rs6000/float16.md @@ -1450,82 +1450,3 @@ operands[3], FP16_NFMS); DONE; }) - -;; Eliminate converts from SF to DF and back to SF done within V4SF -;; that shows up in optimizing BFmode to use V4SFmode. -;; -;; For example: -;; __bf16 a, b, c, d; -;; // ... -;; a = b + c + d; -;; -;; would generate: -;; (set (reg:SF temp1) -;; (plus:SF (float_extend:SF (reg:BF b)) -;; (float_extend:SF (reg:BF c)))) -;; -;; (set (reg:SF temp2) -;; (plus:SF (reg:SF temp1) -;; (float_extend:SF (reg:BF d)))) -;; -;; (set (reg:BF a) -;; (float_truncte:BF -;; (reg:SF temp2))) -;; -;; After split1 this would become: -;; (set (reg:V4SF temp3) -;; (unspec:V4SF [(reg:BF b)] UNSPEC_XXSPLTW_FP16)) -;; -;; (set (reg:V4SF temp4) -;; (unspec:V4SF [(reg:V4SF temp3)] UNSPEC_XXSPLTW_FP16)) -;; -;; (set (reg:V4SF temp5) -;; (unspec:V4SF [(reg:BF c)] UNSPEC_XXSPLTW_FP16)) -;; -;; (set (reg:V4SF temp6) -;; (unspec:V4SF [(reg:V4SF temp5)] UNSPEC_XXSPLTW_FP16)) -;; -;; (set (reg:V4SF temp7) -;; (unspec:V4SF [(reg:BF d)] UNSPEC_XXSPLTW_FP16)) -;; -;; (set (reg:V4SF temp8) -;; (unspec:V4SF [(reg:V4SF temp7)] UNSPEC_XXSPLTW_FP16)) -;; -;; (set (reg:V4SF temp9) -;; (plus:V4SF (reg:V4SF temp4) -;; (reg:V4SF temp6))) -;; -;; (set (reg:SF temp10) -;; (unspec:SF [(reg:V4SF temp9)] UNSPEC_VSX_CVSPDP)) -;; -;; (set (reg:V4SF temp11) -;; (unspec:V4SF [(reg:SF temp10)] UNSPEC_VSX_CVDPSPN)) -;; -;; (set (reg:V4SF temp12) -;; (unspec:V4SF [(reg:V4SF temp1) -;; (const_int 0)] UNSPEC_VSX_XXSPLTW)) -;; -;; (set (reg:V4SF temp13) -;; (plus:V4SF (reg:V4SF temp8) -;; (reg:V4SF temp12))) -;; -;; (set (reg:BF a) -;; (unspec:BF [(reg:V4SF temp13)] UNSPEC_XVCVSPBF16_BF)) - -(define_peephole2 - [(set (match_operand:SF 0 "vsx_register_operand") - (unspec:SF [(match_operand:V4SF 1 "vsx_register_operand")] - UNSPEC_VSX_CVSPDP)) - (set (match_operand:V4SF 2 "vsx_register_operand") - (unspec:V4SF [(match_dup 0)] - UNSPEC_VSX_CVDPSPN)) - (set (match_operand:V4SF 3 "vsx_register_operand") - (unspec:V4SF [(match_dup 2) - (const_int 0)] UNSPEC_VSX_XXSPLTW))] - "((REGNO (operands[0]) == REGNO (operands[2])) - || peep2_reg_dead_p (2, operands[0])) - && (REGNO (operands[2]) == REGNO (operands[3]) - || peep2_reg_dead_p (3, operands[2]))" - [(set (match_dup 3) (match_dup 1))] -{ -}) diff --git a/gcc/config/rs6000/predicates.md b/gcc/config/rs6000/predicates.md index f9c6fdff0f57..221fd9171629 100644 --- a/gcc/config/rs6000/predicates.md +++ b/gcc/config/rs6000/predicates.md @@ -2318,7 +2318,7 @@ ;; Match bfloat16/float operands that can be optimized to do the ;; operation in V4SFmode. (define_predicate "bfloat16_v4sf_operand" - (match_code "reg,subreg,const_double,float_extend,float_truncate") + (match_code "reg,subreg,const_double,float_extend,float_truncate,plus,minus,mult") { if (mode != BFmode && mode != SFmode) return false; @@ -2329,7 +2329,8 @@ if (CONST_DOUBLE_P (op)) return true; - if (GET_CODE (op) == FLOAT_EXTEND) + enum rtx_code icode = GET_CODE (op); + if (icode == FLOAT_EXTEND) { rtx op_arg = XEXP (op, 0); return (mode == SFmode @@ -2337,7 +2338,7 @@ && (REG_P (op_arg) || SUBREG_P (op_arg))); } - if (GET_CODE (op) == FLOAT_TRUNCATE) + if (icode == FLOAT_TRUNCATE) { rtx op_arg = XEXP (op, 0); return (mode == BFmode @@ -2345,6 +2346,18 @@ && (REG_P (op_arg) || SUBREG_P (op_arg))); } + /* Allow for chained operations. */ + if (icode == PLUS || icode == MINUS || icode == MULT) + { + rtx op0 = XEXP (op, 0); + rtx op1 = XEXP (op, 1); + + return (GET_MODE (op0) == mode + && GET_MODE (op1) == mode + && bfloat16_v4sf_operand (op0, mode) + && bfloat16_v4sf_operand (op1, mode)); + } + return false; }) @@ -2365,5 +2378,8 @@ && GET_MODE (XEXP (op, 0)) == BFmode) return true; + if (CONST_DOUBLE_P (op)) + return true; + return false; }) diff --git a/gcc/testsuite/gcc.target/powerpc/bf16-3.c b/gcc/testsuite/gcc.target/powerpc/bf16-3.c new file mode 100644 index 000000000000..3b70cb2cba78 --- /dev/null +++ b/gcc/testsuite/gcc.target/powerpc/bf16-3.c @@ -0,0 +1,32 @@ +/* { dg-require-effective-target power10_ok } */ +/* { dg-require-effective-target ppc_bf16_runtime } */ +/* { dg-options "-mdejagnu-cpu=power10 -O2 -mfloat16" } */ + +/* Check that we can do multiple __bf16 adds all in V4SFmode. */ + +/* Expected code: + + lxsihzx 0,0,4 + lxsihzx 11,0,5 + lxsihzx 12,0,6 + xxspltw 0,0,1 + xxspltw 11,11,1 + xxspltw 12,12,1 + xvcvbf16spn 0,0 + xvcvbf16spn 11,11 + xvcvbf16spn 12,12 + xvaddsp 0,0,11 + xvaddsp 0,0,12 + xvcvspbf16 0,0 + stxsihx 0,0,3 */ + +void +add3 (__bf16 *p, __bf16 *q, __bf16 *r, __bf16 *s) +{ + *p = *q + *r + *s; +} + +/* { dg-final { scan-assembler-times {\mxxspltw\M|\mvspltw\M|\mmtvsrws\M} 3 } } */ +/* { dg-final { scan-assembler-times {\mxvcvbf16spn\M} 3 } } */ +/* { dg-final { scan-assembler-times {\mxvaddsp\M} 2 } } */ +/* { dg-final { scan-assembler-times {\mxvcvspbf16\M} 1 } } */ diff --git a/gcc/testsuite/gcc.target/powerpc/bf16-4.c b/gcc/testsuite/gcc.target/powerpc/bf16-4.c new file mode 100644 index 000000000000..760e4751a75f --- /dev/null +++ b/gcc/testsuite/gcc.target/powerpc/bf16-4.c @@ -0,0 +1,31 @@ +/* { dg-require-effective-target power10_ok } */ +/* { dg-require-effective-target ppc_bf16_runtime } */ +/* { dg-options "-mdejagnu-cpu=power10 -O2 -mfloat16" } */ + +/* Check that we can do multiple __bf16 adds all in V4SFmode. */ + +/* Expected code: + + lxsihzx 0,0,4 + lxsihzx 12,0,5 + xxspltiw 11,1065353216 + xxspltw 0,0,1 + xxspltw 12,12,1 + xvcvbf16spn 0,0 + xvcvbf16spn 12,12 + xvaddsp 0,0,12 + xvaddsp 0,0,11 + xvcvspbf16 0,0 + stxsihx 0,0,3 */ + +void +add3 (__bf16 *p, __bf16 *q, __bf16 *r) +{ + *p = *q + *r + (__bf16)1.0; +} + +/* { dg-final { scan-assembler-times {\mxxspltw\M|\mvspltw\M|\mmtvsrws\M} 2 } } */ +/* { dg-final { scan-assembler-times {\mxvcvbf16spn\M} 2 } } */ +/* { dg-final { scan-assembler-times {\mxvaddsp\M} 2 } } */ +/* { dg-final { scan-assembler-times {\mxxspltiw\M} 1 } } */ +/* { dg-final { scan-assembler-times {\mxvcvspbf16\M} 1 } } */
