https://gcc.gnu.org/g:909dd97b0970d162fafee78b1ad2db8122ede32a
commit 909dd97b0970d162fafee78b1ad2db8122ede32a Author: Michael Meissner <[email protected]> Date: Thu Oct 22 01:25:24 2026 -0400 Optimize __bf16 operations. Optimize __bf16 binary and fma operations. Unlike _Float16 where we have instructions to convert between HFmode and SFmode as scalar values, with BFmode, we only have vector conversions. Thus to do: __bf16 a, b, c; a = b + c; the GCC compiler generates the following code: lhz 9,0(4) // load __bf16 value b lhz 2,0(5) // load __bf16 value c sldi 9,9,48 // prepare to call xscvspdpn sldi 2,2,48 // prepare to call xscvspdpn mtvsrd 12,9 // move to VSR register mtvsrd 11,2 // move to VSR register xscvspdpn 0,12 // convert to SF/DFmode scalar xscvspdpn 12,11 // convert to SF/DFmode scalar fadds 0,0,12 // SFmode add xscvdpsp 0,0 // convert to V4SF format xvcvspbf16 0,0 // convert to V8BF format stxsihx 0,0,3 // store b+c Using the following combiner patterns that are defined in this patch, the code generated would be: lxsihzx 12,4,2 // load __bf16 value b lxsihzx 0,5,2 // load __bf16 value c xxspltw 12,12,1 // shift b into bits 16..31 xxspltw 0,0,1 // shift c into bits 16..31 xvcvbf16spn 12,12 // vector convert b into V4SFmode xvcvbf16spn 0,0 // vector convert c into V4SFmode xvaddsp 0,0,12 // vector b+c in V4SFmode xvcvspbf16 0,0 // convert b+c into BFmode memory format stxsihx 0,3,2 // store b+c We cannot just define insns like 'addbf3' to keep the operation as BFmode because GCC will not generate these patterns unless the user uses -Ofast. Without -Ofast, it will always convert BFmode into SFmode. I have measured that these changes can improve __bf16 code by 20 to 45 times. 2026-10-22 Michael Meissner <[email protected]> * config/rs6000/float16.cc (toplevel): Include emit-rtl.h. (struct bfloat16_to_v4sf_stats): New structure. (bfloat16_to_v4sf_inner): New function to optimize BFmode to use V4SFmode instructions. (bfloat16_to_v4sf_p): Likewise. (bfloat16_to_v4sf_split_inner): Likewise. (bfloat16_to_v4sf_split): Likewise. * config/rs6000/float16.md (xvcvbf16spn_bf): New insns to optimize BFmode to use V4SFmode instructions. (bfloat16_as_v4sf_1): Likewise. (bfloat16_as_v4sf_2): Likewise. (bfloat16_as_v4sf_3): Likewise. * config/rs6000/predicates.md (bfloat16_to_v4sf_operand): New predicate. * config/rs6000/rs6000-protos.h (bfloat16_operation_as_v4sf): New declaration. (bfloat16_to_v4sf_p): Likewise. (bfloat16_to_v4sf_split): Likewise. * config/rs6000/rs6000.opt (-mfloat16-bf-v4sf): New debug switch. (-mfloat16-debug1): Likewise. (-mfloat16-debug2): Likewise. Diff: --- gcc/config/rs6000/float16.cc | 472 ++++++++++++++++++++++++++++++++++++++ gcc/config/rs6000/float16.md | 92 ++++++++ gcc/config/rs6000/predicates.md | 15 ++ gcc/config/rs6000/rs6000-protos.h | 4 + gcc/config/rs6000/rs6000.opt | 12 + 5 files changed, 595 insertions(+) diff --git a/gcc/config/rs6000/float16.cc b/gcc/config/rs6000/float16.cc index 5274a0df962f..035f96719fe2 100644 --- a/gcc/config/rs6000/float16.cc +++ b/gcc/config/rs6000/float16.cc @@ -37,6 +37,7 @@ #include "insn-attr.h" #include "flags.h" #include "attribs.h" +#include "emit-rtl.h" #include "explow.h" #include "expr.h" #include "common/common-target.h" @@ -183,3 +184,474 @@ fp16_vectorization (enum rtx_code icode, return; } + +/* Decide whether an expression can optimize BFmode to use V4SFmode. + + On Power10, we do not have scalar conversion functions between BFmode and + SFmode/DFmode. + + To convert from BFmode to the DFmode scalar format we have to do a shift + left by 48 bits to convert the value into the SFmode format in the top 32 + bits of a VSX register, and then do a XSCVSPDPN instruction to convert it to + the DFmode used for scalar instructions. + + To convert from DFmode to BFmode, we have to convert the DFmode value into + the SFmode memory format via the XSCVDPSP instruction, and then use the + vector XVCVSPBF16 instruction to convert the V4SFmode vector to a V8BFmode + vector and use the appropriate element. + + This code determines whether a given RTX can be converted to use the V4SF + vector instructions. + + We allow multiple operations (a + b + c) to be folded together to try to do + as much possible in the vector unit without having to convert back to scalar + mode. + + We also fold in loads from memory to simplify the conversion. + + The expression that is combined is split before register allocation. + + For example: + + __bf16 *p, *q, *r; + *p = *q + *r; + + the old vs. new code would be: + + old code new code + ======== ======== + + lhz 9,0(4) lxsihzx 0,0,4 + lhz 2,0(5) lxsihzx 12,0,5 + sldi 9,9,48 xxspltw 0,0,1 + sldi 2,2,48 xxspltw 12,12,1 + mtvsrd 12,9 xvcvbf16spn 0,0 + mtvsrd 11,2 xvcvbf16spn 12,12 + xscvspdpn 0,12 xvaddsp 0,0,12 + xscvspdpn 12,11 xvcvspbf16 0,0 + fadds 0,0,12 stxsihx 0,0,3 + xscvdpsp 0,0 + xvcvspbf16 0,0 + stxsihx 0,0,3 + + We can combine more operations to do as much of the calculation using + V4SFmode arithmetic, for example: + + __bf16 *p, *q, *r, *s; + *p = *q + *r + *s; + + The old vs. new code would be: + + old code new code + ======== ======== + + lhz 10,0(5) lxsihzx 0,0,5 + lhz 9,0(6) lxsihzx 11,0,4 + lhz 2,0(4) lxsihzx 12,0,6 + sldi 10,10,48 xxspltw 0,0,1 + sldi 9,9,48 xxspltw 11,11,1 + sldi 2,2,48 xxspltw 12,12,1 + mtvsrd 12,10 xvcvbf16spn 0,0 + mtvsrd 10,2 xvcvbf16spn 11,11 + xscvspdpn 0,12 xvcvbf16spn 12,12 + mtvsrd 12,9 xvaddsp 0,0,11 + xscvspdpn 11,12 xvaddsp 0,0,12 + xscvspdpn 12,10 xvcvspbf 16 0,0 + fadds 0,0,11 stxsihx 0,0,3 + fadds 0,0,12 + xscvdpsp 0,0 + xvcvspbf16 0,0 + stxsihx 0,0,3 + + For non-vectorized power10 code, I measured that using the V4SFmode vector + instructions were 20-45 times faster than using the scalar conversions and + scalar SFmode instructions. The HFmode code which has the appropriate + scalar conversions operates at roughly the same speed as the BFmode code + using the V4SFmode vector instructions. */ + +/* Stats for whole expression. */ + +struct bfloat16_to_v4sf_stats +{ + unsigned num_bf_regs; /* # BF registers used. */ + unsigned num_bf_loads; /* # BF values loaded from memory. */ + unsigned num_sf_regs; /* # SF registers used. */ + unsigned num_const; /* # constants used. */ + unsigned num_binary; /* # binary operations done. */ + unsigned num_fma; /* # FMA (fused multiply-adds) done. */ + unsigned num_unary; /* # unary ops not folded into FMA/ABS. */ + unsigned num_neg_folded; /* # negates folded into a FMA/ABS insn. */ + unsigned num_extend; /* # FLOAT_EXTENDs. */ +}; + +/* Recursively evaluate whether a RTX expression can be optimized to use + V4SFmode operations. Return true if the expression might be able to be + optimized, and update the stats. */ + +static bool +bfloat16_to_v4sf_inner (rtx op, struct bfloat16_to_v4sf_stats *stats) +{ + machine_mode mode = GET_MODE (op); + rtx arg; + rtx_code code; + + if (mode != BFmode && mode != SFmode) + return false; + + switch (GET_CODE (op)) + { + default: + break; + + // Update register counts. + case REG: + case SUBREG: + if (mode == BFmode) + stats->num_bf_regs++; + else + stats->num_sf_regs++; + + return true; + + // Update memory counts. + case MEM: + if (mode != BFmode) + return false; + + stats->num_bf_loads++; + return true; + + // Update constants + case CONST_DOUBLE: + stats->num_const++; + return true; + + // Process absolute value + case ABS: + stats->num_unary++; + return bfloat16_to_v4sf_inner (XEXP (op, 0), stats); + + // Process negation, deal with negate being folded into FMA or ABS. + case NEG: + arg = XEXP (op, 0); + code = GET_CODE (arg); + + if (code == FMA || code == ABS) + stats->num_neg_folded++; + else + stats->num_unary++; + + return bfloat16_to_v4sf_inner (arg, stats); + + // Process binary operations + case PLUS: + case MINUS: + case MULT: + case SMIN: + case SMAX: + stats->num_binary++; + return (bfloat16_to_v4sf_inner (XEXP (op, 0), stats) + && bfloat16_to_v4sf_inner (XEXP (op, 1), stats)); + + // Process fused multiply-adds + case FMA: + stats->num_fma++; + return (bfloat16_to_v4sf_inner (XEXP (op, 0), stats) + && bfloat16_to_v4sf_inner (XEXP (op, 1), stats) + && bfloat16_to_v4sf_inner (XEXP (op, 2), stats)); + + // Process extend BF -> SF + case FLOAT_EXTEND: + stats->num_extend++; + arg = XEXP (op, 0); + if (mode != SFmode || GET_MODE (arg) != BFmode) + return false; + + return bfloat16_to_v4sf_inner (arg, stats); + } + + return false; +} + +bool +bfloat16_to_v4sf_p (rtx op, machine_mode mode) +{ + static struct bfloat16_to_v4sf_stats zero_bfloat16_to_v4sf_stats; + + if (mode != BFmode && mode != SFmode) + return false; + + rtx orig_op = op; + struct bfloat16_to_v4sf_stats stats = zero_bfloat16_to_v4sf_stats; + + // Handle top level FLOAT_TRUNCATE + if (GET_CODE (op) == FLOAT_TRUNCATE + && mode == BFmode + && GET_MODE (XEXP (op, 0)) == SFmode) + op = XEXP (op, 0); + + bool ret = bfloat16_to_v4sf_inner (op, &stats); + + // Make sure we had enough BFmode operations to do the optimization + if (ret) + { + unsigned num_bf = stats.num_bf_regs + stats.num_bf_loads; + + // If we had no BFmode values, don't do the optimization. + if (!num_bf) + ret = false; + + // If we had no binary or fma opertors, don't do the optimization. + else if (!stats.num_binary && !stats.num_fma) + ret = false; + + // If the expression has much more SFmode elements than BFmode, prefer + // converting BFmode to SFmode instead of using V4SFmode. + else if (stats.num_sf_regs > (num_bf + 2)) + ret = false; + } + + // Debugging. + if (TARGET_FLOAT16_DEBUG1) + { + if (ret) + { + fprintf (stderr, "\n========== bfloat16_to_v4sf_p, return = true"); + + if (stats.num_bf_regs) + fprintf (stderr, ", # BF regs = %u", stats.num_bf_regs); + + if (stats.num_bf_loads) + fprintf (stderr, ", # BF values loaded = %u", stats.num_bf_loads); + + if (stats.num_sf_regs) + fprintf (stderr, ", # SF regs = %u", stats.num_sf_regs); + + if (stats.num_const) + fprintf (stderr, ", # constants = %u", stats.num_const); + + if (stats.num_extend) + fprintf (stderr, ", # extends = %u", stats.num_extend); + + if (stats.num_unary) + fprintf (stderr, ", # unary = %u", stats.num_unary); + + if (stats.num_binary) + fprintf (stderr, ", # binary = %u", stats.num_binary); + + if (stats.num_fma) + fprintf (stderr, ", # fma = %u", stats.num_fma); + + if (stats.num_neg_folded) + fprintf (stderr, ", # neg fold = %u", stats.num_neg_folded); + + fprintf (stderr, "\n"); + } + else + fprintf (stderr, "\n========== bfloat16_to_v4sf_p, return = false\n"); + + debug_rtx (orig_op); + } + + return ret; +} + + +/* Expand a bfloat16 scalar floating point operation. + + The operation is done as a V4SFmode vector operation. This is because + converting BFmode from a scalar BFmode to SFmode to do the operation and + back again takes quite a bit of time. GCC will only generate the native + operation if -Ofast is used. The float16.md code that calls this function + adds various combine operations to do the operation in V4SFmode instead of + SFmode. + + Inner part that does the operation in V4SFmode, and returns the pseudo + with the results. */ + +static rtx +bfloat16_to_v4sf_split_inner (rtx op) +{ + gcc_assert (can_create_pseudo_p ()); + + rtx result_v4sf = gen_reg_rtx (V4SFmode); + enum rtx_code icode = GET_CODE (op); + size_t n_opts = GET_RTX_LENGTH (icode); + rtx ops_orig[3]; + rtx ops_v4sf[3]; + + switch (n_opts) + { + case 1: + ops_orig[0] = XEXP (op, 0); + ops_orig[1] = NULL_RTX; + ops_orig[2] = NULL_RTX; + break; + + case 2: + ops_orig[0] = XEXP (op, 0); + ops_orig[1] = XEXP (op, 1); + ops_orig[2] = NULL_RTX; + break; + + case 3: + ops_orig[0] = XEXP (op, 0); + ops_orig[1] = XEXP (op, 1); + ops_orig[2] = XEXP (op, 2); + break; + + default: + gcc_unreachable (); + } + + for (size_t i = 0; i < n_opts; i++) + { + rtx op = ops_orig[i]; + + gcc_assert (op != NULL_RTX); + + switch (GET_CODE (op)) + { + /* Handle chained operations. */ + case NEG: + case ABS: + case PLUS: + case MINUS: + case MULT: + case SMIN: + case SMAX: + case FMA: + ops_v4sf[i] = bfloat16_to_v4sf_split_inner (op); + continue; + + /* Remove truncation/extend added. */ + case FLOAT_EXTEND: + case FLOAT_TRUNCATE: + op = XEXP (op, 0); + break; + + default: + break; + } + + /* Convert operands to V4SFmode format. We use SPLAT for registers to + get the value into the upper 32-bits. We can use XXSPLTW to splat + words instead of VSPLTIH since the XVCVBF16SPN instruction ignores the + odd half-words, and XXSPLTW can operate on all VSX registers instead + of just the Altivec registers. Using SPLAT instead of a shift also + insure that other bits are not a signalling NaN. If we are using + XXSPLTIW or XXSPLTIB to load the constant the other bits are + duplicated. */ + + rtx tmp = ops_v4sf[i] = gen_reg_rtx (V4SFmode); + if (op == CONST0_RTX (SFmode) || op == CONST0_RTX (BFmode)) + emit_move_insn (tmp, CONST0_RTX (V4SFmode)); + + else if (GET_MODE (op) == BFmode) + { + emit_insn (gen_xxspltw_bf (tmp, force_reg (BFmode, op))); + emit_insn (gen_xvcvbf16spn_bf (tmp, tmp)); + } + + else if (GET_MODE (op) == SFmode) + { + if (GET_CODE (op) == CONST_DOUBLE) + { + rtvec v = gen_rtvec (4, op, op, op, op); + emit_move_insn (tmp, gen_rtx_CONST_VECTOR (V4SFmode, v)); + } + + else + emit_insn (gen_vsx_splat_v4sf (tmp, + force_reg (SFmode, op))); + } + + else + gcc_unreachable (); + } + + /* Do the operation in V4SFmode. */ + switch (n_opts) + { + case 1: + emit_insn (gen_rtx_SET (result_v4sf, + gen_rtx_fmt_e (icode, V4SFmode, + ops_v4sf[0]))); + + break; + + case 2: + emit_insn (gen_rtx_SET (result_v4sf, + gen_rtx_fmt_ee (icode, V4SFmode, + ops_v4sf[0], + ops_v4sf[1]))); + break; + + case 3: + emit_insn (gen_rtx_SET (result_v4sf, + gen_rtx_fmt_eee (icode, V4SFmode, + ops_v4sf[0], + ops_v4sf[1], + ops_v4sf[2]))); + break; + + default: + fatal_insn_not_found (op); + } + + return result_v4sf; +} + +/* Split the isnsns tht optimize BFmode to use V4SFmode. */ + +void +bfloat16_to_v4sf_split (rtx dest, rtx src) +{ + rtx orig_src = src; + enum rtx_code code = GET_CODE (src); + rtx dest_v4sf; + + if (TARGET_FLOAT16_DEBUG2) + { + fprintf (stderr, "\n========== bfloat16_to_v4sf_split:\n"); + debug_rtx (gen_rtx_SET (dest, src)); + } + + if (code == FLOAT_EXTEND || code == FLOAT_TRUNCATE) + { + src = XEXP (src, 0); + code = GET_CODE (src); + } + + switch (code) + { + case ABS: + case NEG: + case PLUS: + case MINUS: + case MULT: + case SMIN: + case SMAX: + case FMA: + dest_v4sf = bfloat16_to_v4sf_split_inner (src); + break; + + default: + fatal_insn_not_found (gen_rtx_SET (dest, orig_src)); + } + + /* Convert V4SF result back to scalar mode. */ + if (GET_MODE (dest) == BFmode) + emit_insn (gen_xvcvspbf16_bf (dest, dest_v4sf)); + + else if (GET_MODE (dest) == SFmode) + { + rtx element = GEN_INT (WORDS_BIG_ENDIAN ? 2 : 3); + emit_insn (gen_vsx_extract_v4sf (dest, dest_v4sf, element)); + } + + else + gcc_unreachable (); + + return; +} diff --git a/gcc/config/rs6000/float16.md b/gcc/config/rs6000/float16.md index 7180747cbae6..1bbad9729c5e 100644 --- a/gcc/config/rs6000/float16.md +++ b/gcc/config/rs6000/float16.md @@ -1234,3 +1234,95 @@ "TARGET_BFLOAT16_HW" "xvcvbf16spn %x0,%x1" [(set_attr "type" "vecperm")]) + +(define_insn "xvcvbf16spn_bf" + [(set (match_operand:V4SF 0 "vsx_register_operand" "=wa") + (unspec:V4SF [(match_operand:V4SF 1 "vsx_register_operand" "wa")] + UNSPEC_CVT_FP16_TO_V4SF))] + "TARGET_BFLOAT16_HW" + "xvcvbf16spn %x0,%x1" + [(set_attr "type" "vecperm")]) + +;; Optimize __bfloat16 binary operations. Unlike _Float16 where we +;; have instructions to convert between HFmode and SFmode as scalar +;; values, with BFmode, we only have vector conversions. Thus to do: +;; +;; __bfloat16 a, b, c; +;; a = b + c; +;; +;; the GCC compiler would normally generate: +;; +;; lxsihzx 0,4,2 // load __bfloat16 value b +;; lxsihzx 12,5,2 // load __bfloat16 value c +;; xxsldwi 0,0,0,1 // shift b into bits 16..31 +;; xxsldwi 12,12,12,1 // shift c into bits 16..31 +;; xvcvbf16spn 0,0 // vector convert b into V4SFmode +;; xvcvbf16spn 12,12 // vector convert c into V4SFmode +;; xscvspdpn 0,0 // convert b into SFmode scalar +;; xscvspdpn 12,12 // convert c into SFmode scalar +;; fadds 0,0,12 // add b+c +;; xscvdpspn 0,0 // convert b+c into SFmode memory format +;; xvcvspbf16 0,0 // convert b+c into BFmode memory format +;; stxsihx 0,3,2 // store b+c +;; +;; Using the following combiner patterns, the code generated would now +;; be: +;; +;; lxsihzx 12,4,2 // load __bfloat16 value b +;; lxsihzx 0,5,2 // load __bfloat16 value c +;; xxspltw 12,12,1 // shift b into bits 16..31 +;; xxspltw 0,0,1 // shift c into bits 16..31 +;; xvcvbf16spn 12,12 // vector convert b into V4SFmode +;; xvcvbf16spn 0,0 // vector convert c into V4SFmode +;; xvaddsp 0,0,12 // vector b+c in V4SFmode +;; xvcvspbf16 0,0 // convert b+c into BFmode memory format +;; stxsihx 0,3,2 // store b+c +;; +;; We cannot just define insns like 'addbf3' to keep the operation as +;; BFmode because GCC will not generate these patterns unless the user +;; uses -Ofast. Without -Ofast, it will always convert BFmode into +;; SFmode. + +(define_insn_and_split "*bfloat16_as_v4sf_1" + [(set (match_operand:BF 0 "vsx_register_operand") + (match_operand:BF 1 "bfloat16_to_v4sf_operand"))] + "TARGET_BF_V4SF + && TARGET_BFLOAT16_HW + && can_create_pseudo_p ()" + "#" + "&& 1" + [(pc)] +{ + bfloat16_to_v4sf_split (operands[0], operands[1]); + DONE; +}) + +(define_insn_and_split "*bfloat16_as_v4sf_2" + [(set (match_operand:BF 0 "vsx_register_operand") + (float_truncate:BF + (match_operand:SF 1 "bfloat16_to_v4sf_operand")))] + "TARGET_BF_V4SF + && TARGET_BFLOAT16_HW + && can_create_pseudo_p ()" + "#" + "&& 1" + [(pc)] +{ + bfloat16_to_v4sf_split (operands[0], operands[1]); + DONE; +}) + +(define_insn_and_split "*bfloat16_as_v4sf_3" + [(set (match_operand:SF 0 "vsx_register_operand") + (match_operand:SF 1 "bfloat16_to_v4sf_operand"))] + "TARGET_BF_V4SF + && TARGET_BFLOAT16_HW + && can_create_pseudo_p ()" + "#" + "&& 1" + [(pc)] +{ + bfloat16_to_v4sf_split (operands[0], operands[1]); + DONE; +}) + diff --git a/gcc/config/rs6000/predicates.md b/gcc/config/rs6000/predicates.md index 9d58175602d9..643769376606 100644 --- a/gcc/config/rs6000/predicates.md +++ b/gcc/config/rs6000/predicates.md @@ -2291,3 +2291,18 @@ return constant_generates_xxspltiw (&vsx_const); }) + +;; Match bfloat16/float operands that can be optimized to do the +;; operation in V4SFmode. +(define_predicate "bfloat16_to_v4sf_operand" + (match_code "reg,subreg,const_double,float_extend,plus,minus,mult,fma,neg,abs") +{ + if (mode != BFmode && mode != SFmode) + return false; + + if (mode != GET_MODE (op)) + return false; + + return bfloat16_to_v4sf_p (op, mode); +}) + diff --git a/gcc/config/rs6000/rs6000-protos.h b/gcc/config/rs6000/rs6000-protos.h index 2a66adc7b305..6a2a28ec37b0 100644 --- a/gcc/config/rs6000/rs6000-protos.h +++ b/gcc/config/rs6000/rs6000-protos.h @@ -271,6 +271,10 @@ enum fp16_operation { extern void fp16_vectorization (enum rtx_code, rtx, rtx, rtx, rtx, enum fp16_operation); +extern void bfloat16_operation_as_v4sf (enum rtx_code, rtx, rtx, rtx, rtx, + enum fp16_operation); +extern bool bfloat16_to_v4sf_p (rtx, machine_mode); +extern void bfloat16_to_v4sf_split (rtx, rtx); #endif /* RTX_CODE */ #ifdef TREE_CODE diff --git a/gcc/config/rs6000/rs6000.opt b/gcc/config/rs6000/rs6000.opt index 28f679ff69c7..f39d6f0a3107 100644 --- a/gcc/config/rs6000/rs6000.opt +++ b/gcc/config/rs6000/rs6000.opt @@ -646,6 +646,18 @@ mfloat16 Target Mask(FLOAT16) Var(rs6000_isa_flags) Enable or disable 16-bit floating point. +mfloat16-bf-v4sf +Target Undocumented Var(TARGET_BF_V4SF) Init(1) Save +Optimize bfloat16 to use float vector instructions. + +mfloat16-debug1 +Target Undocumented Var(TARGET_FLOAT16_DEBUG1) Init(0) Save +Turn on float16 debugging #1. + +mfloat16-debug2 +Target Undocumented Var(TARGET_FLOAT16_DEBUG2) Init(0) Save +Turn on float16 debugging #2. + ; Documented parameters -param=rs6000-vect-unroll-limit=
