https://gcc.gnu.org/g:0e4b8c50f14151cd5661d69d6eecf308d74f0835
commit 0e4b8c50f14151cd5661d69d6eecf308d74f0835 Author: Michael Meissner <[email protected]> Date: Tue Oct 20 15:51:43 2026 -0400 Rework BF to V4SF optimizaton. 2026-10-20 Michael Meissner <[email protected]> * config/rs6000/float16.cc: Rework BF to V4SF optimization. * config/rs6000/float16.md (bfloat16_*_op_internal*): Delete. * config/rs6000/rs6000.h (enum fp16_operation): Delete FP16_UNARY. * config/rs6000/rs6000.opt (-mfloat16-debug1): New debug switch. (-mfloat16-debug2): Likewise. Diff: --- gcc/config/rs6000/float16.cc | 770 ++++++++++++++++---------------------- gcc/config/rs6000/float16.md | 241 ------------ gcc/config/rs6000/rs6000-protos.h | 1 - gcc/config/rs6000/rs6000.opt | 8 + 4 files changed, 329 insertions(+), 691 deletions(-) diff --git a/gcc/config/rs6000/float16.cc b/gcc/config/rs6000/float16.cc index 2876c1885de1..cc7183a3026c 100644 --- a/gcc/config/rs6000/float16.cc +++ b/gcc/config/rs6000/float16.cc @@ -72,10 +72,6 @@ fp16_vectorization (enum rtx_code icode, switch (subtype) { - case FP16_UNARY: - n_opts = 1; - break; - case FP16_BINARY: n_opts = 2; break; @@ -127,16 +123,6 @@ fp16_vectorization (enum rtx_code icode, /* Do 2 sets of V4SFmode operations. */ switch (subtype) { - case FP16_UNARY: - emit_insn (gen_rtx_SET (result_hi, - gen_rtx_fmt_e (icode, V4SFmode, - op_hi[0]))); - - emit_insn (gen_rtx_SET (result_lo, - gen_rtx_fmt_e (icode, V4SFmode, - op_lo[0]))); - break; - case FP16_BINARY: emit_insn (gen_rtx_SET (result_hi, gen_rtx_fmt_ee (icode, V4SFmode, @@ -199,13 +185,275 @@ fp16_vectorization (enum rtx_code icode, return; } -/* Expand a bfloat16 scalar floating point operation: +/* Decide whether an expression can optimize BFmode to use V4SFmode. - ICODE: Operation to perform. - OP1: Input operand1. - OP2: Input operand2. - OP3: Input operand3 or NULL_RTX. - SUBTYPE: Describe the operation. + On Power10, we do not have scalar conversion functions between BFmode and + SFmode/DFmode. + + To convert from BFmode to the DFmode scalar format we have to do a shift + left by 48 bits to convert the value into the SFmode format in the top 32 + bits of a VSX register, and then do a XSCVSPDPN instruction to convert it to + the DFmode used for scalar instructions. + + To convert from DFmode to BFmode, we have to convert the DFmode value into + the SFmode memory format via the XSCVDPSP instruction, and then use the + vector XVCVSPBF16 instruction to convert the V4SFmode vector to a V8BFmode + vector and use the appropriate element. + + This code determines whether a given RTX can be converted to use the V4SF + vector instructions. + + We allow multiple operations (a + b + c) to be folded together to try to do + as much possible in the vector unit without having to convert back to scalar + mode. + + We also fold in loads from memory to simplify the conversion. + + The expression that is combined is split before register allocation. + + For example: + + __bf16 *p, *q, *r; + *p = *q + *r; + + the old vs. new code would be: + + old code new code + ======== ======== + + lhz 9,0(4) lxsihzx 0,0,4 + lhz 2,0(5) lxsihzx 12,0,5 + sldi 9,9,48 xxspltw 0,0,1 + sldi 2,2,48 xxspltw 12,12,1 + mtvsrd 12,9 xvcvbf16spn 0,0 + mtvsrd 11,2 xvcvbf16spn 12,12 + xscvspdpn 0,12 xvaddsp 0,0,12 + xscvspdpn 12,11 xvcvspbf16 0,0 + fadds 0,0,12 stxsihx 0,0,3 + xscvdpsp 0,0 + xvcvspbf16 0,0 + stxsihx 0,0,3 + + We can combine more operations to do as much of the calculation using + V4SFmode arithmetic, for example: + + __bf16 *p, *q, *r, *s; + *p = *q + *r + *s; + + The old vs. new code would be: + + old code new code + ======== ======== + + lhz 10,0(5) lxsihzx 0,0,5 + lhz 9,0(6) lxsihzx 11,0,4 + lhz 2,0(4) lxsihzx 12,0,6 + sldi 10,10,48 xxspltw 0,0,1 + sldi 9,9,48 xxspltw 11,11,1 + sldi 2,2,48 xxspltw 12,12,1 + mtvsrd 12,10 xvcvbf16spn 0,0 + mtvsrd 10,2 xvcvbf16spn 11,11 + xscvspdpn 0,12 xvcvbf16spn 12,12 + mtvsrd 12,9 xvaddsp 0,0,11 + xscvspdpn 11,12 xvaddsp 0,0,12 + xscvspdpn 12,10 xvcvspbf 16 0,0 + fadds 0,0,11 stxsihx 0,0,3 + fadds 0,0,12 + xscvdpsp 0,0 + xvcvspbf16 0,0 + stxsihx 0,0,3 + + */ + +/* Stats for whole expression. */ + +struct bfloat16_to_v4sf_stats +{ + unsigned num_bf_regs; /* # BF registers used. */ + unsigned num_bf_loads; /* # BF values loaded from memory. */ + unsigned num_sf_regs; /* # SF registers used. */ + unsigned num_const; /* # constants used. */ + unsigned num_binary; /* # binary operations done. */ + unsigned num_fma; /* # FMA (fused multiply-adds) done. */ + unsigned num_unary; /* # unary ops not folded into FMA/ABS. */ + unsigned num_neg_folded; /* # negates folded into a FMA/ABS insn. */ + unsigned num_extend; /* # FLOAT_EXTENDs. */ +}; + +/* Recursively evaluate whether a RTX expression can be optimized to use + V4SFmode operations. Return true if the expression might be able to be + optimized, and update the stats. */ + +static bool +bfloat16_to_v4sf_inner (rtx op, struct bfloat16_to_v4sf_stats *stats) +{ + machine_mode mode = GET_MODE (op); + rtx arg; + rtx_code code; + + if (mode != BFmode && mode != SFmode) + return false; + + switch (GET_CODE (op)) + { + default: + break; + + // Update register counts. + case REG: + case SUBREG: + if (mode == BFmode) + stats->num_bf_regs++; + else + stats->num_sf_regs++; + + return true; + + // Update memory counts. + case MEM: + if (mode != BFmode) + return false; + + stats->num_bf_loads++; + return true; + + // Update constants + case CONST_DOUBLE: + stats->num_const++; + return true; + + // Process absolute value + case ABS: + stats->num_unary++; + return bfloat16_to_v4sf_inner (XEXP (op, 0), stats); + + // Process negation, deal with negate being folded into FMA or ABS. + case NEG: + arg = XEXP (op, 0); + code = GET_CODE (arg); + + if (code == FMA || code == ABS) + stats->num_neg_folded++; + else + stats->num_unary++; + + return bfloat16_to_v4sf_inner (arg, stats); + + // Process binary operations + case PLUS: + case MINUS: + case MULT: + case SMIN: + case SMAX: + stats->num_binary++; + return (bfloat16_to_v4sf_inner (XEXP (op, 0), stats) + && bfloat16_to_v4sf_inner (XEXP (op, 1), stats)); + + // Process fused multiply-adds + case FMA: + stats->num_fma++; + return (bfloat16_to_v4sf_inner (XEXP (op, 0), stats) + && bfloat16_to_v4sf_inner (XEXP (op, 1), stats) + && bfloat16_to_v4sf_inner (XEXP (op, 2), stats)); + + // Process extend BF -> SF + case FLOAT_EXTEND: + stats->num_extend++; + arg = XEXP (op, 0); + if (mode != SFmode || GET_MODE (arg) != BFmode) + return false; + + return bfloat16_to_v4sf_inner (arg, stats); + } + + return false; +} + +bool +bfloat16_to_v4sf_p (rtx op, machine_mode mode) +{ + static struct bfloat16_to_v4sf_stats zero_bfloat16_to_v4sf_stats; + + if (mode != BFmode && mode != SFmode) + return false; + + rtx orig_op = op; + struct bfloat16_to_v4sf_stats stats = zero_bfloat16_to_v4sf_stats; + + // Handle top level FLOAT_TRUNCATE + if (GET_CODE (op) == FLOAT_TRUNCATE + && mode == BFmode + && GET_MODE (XEXP (op, 0)) == SFmode) + op = XEXP (op, 0); + + bool ret = bfloat16_to_v4sf_inner (op, &stats); + + // Make sure we had enough BFmode operations to do the optimization + if (ret) + { + unsigned num_bf = stats.num_bf_regs + stats.num_bf_loads; + + // If we had no BFmode values, don't do the optimization. + if (!num_bf) + ret = false; + + // If we had no binary or fma opertors, don't do the optimization. + else if (!stats.num_binary && !stats.num_fma) + ret = false; + + // If the expression has much more SFmode elements than BFmode, prefer + // converting BFmode to SFmode instead of using V4SFmode. + else if (stats.num_sf_regs > (num_bf + 2)) + ret = false; + } + + // Debugging. + if (TARGET_FLOAT16_DEBUG1) + { + if (ret) + { + fprintf (stderr, "\n========== bfloat16_to_v4sf_p, return = true"); + + if (stats.num_bf_regs) + fprintf (stderr, ", # BF regs = %u", stats.num_bf_regs); + + if (stats.num_bf_loads) + fprintf (stderr, ", # BF values loaded = %u", stats.num_bf_loads); + + if (stats.num_sf_regs) + fprintf (stderr, ", # SF regs = %u", stats.num_sf_regs); + + if (stats.num_const) + fprintf (stderr, ", # constants = %u", stats.num_const); + + if (stats.num_extend) + fprintf (stderr, ", # extends = %u", stats.num_extend); + + if (stats.num_unary) + fprintf (stderr, ", # unary = %u", stats.num_unary); + + if (stats.num_binary) + fprintf (stderr, ", # binary = %u", stats.num_binary); + + if (stats.num_fma) + fprintf (stderr, ", # fma = %u", stats.num_fma); + + if (stats.num_neg_folded) + fprintf (stderr, ", # neg fold = %u", stats.num_neg_folded); + + fprintf (stderr, "\n"); + } + else + fprintf (stderr, "\n========== bfloat16_to_v4sf_p, return = false\n"); + + debug_rtx (orig_op); + } + + return ret; +} + + +/* Expand a bfloat16 scalar floating point operation. The operation is done as a V4SFmode vector operation. This is because converting BFmode from a scalar BFmode to SFmode to do the operation and @@ -218,37 +466,34 @@ fp16_vectorization (enum rtx_code icode, with the results. */ static rtx -bfloat16_operation_as_v4sf_inner (enum rtx_code icode, - rtx op1, - rtx op2, - rtx op3, - enum fp16_operation subtype) +bfloat16_to_v4sf_split_inner (rtx op) { gcc_assert (can_create_pseudo_p ()); rtx result_v4sf = gen_reg_rtx (V4SFmode); - rtx ops_orig[3] = { op1, op2, op3 }; + enum rtx_code icode = GET_CODE (op); + size_t n_opts = GET_RTX_LENGTH (icode); + rtx ops_orig[3]; rtx ops_v4sf[3]; - size_t n_opts; - switch (subtype) + switch (n_opts) { - case FP16_UNARY: - n_opts = 1; - gcc_assert (op2 == NULL_RTX && op3 == NULL_RTX); + case 1: + ops_orig[0] = XEXP (op, 0); + ops_orig[1] = NULL_RTX; + ops_orig[2] = NULL_RTX; break; - case FP16_BINARY: - n_opts = 2; - gcc_assert (op3 == NULL_RTX); + case 2: + ops_orig[0] = XEXP (op, 0); + ops_orig[1] = XEXP (op, 1); + ops_orig[2] = NULL_RTX; break; - case FP16_FMA: - case FP16_FMS: - case FP16_NFMA: - case FP16_NFMS: - gcc_assert (icode == FMA); - n_opts = 3; + case 3: + ops_orig[0] = XEXP (op, 0); + ops_orig[1] = XEXP (op, 1); + ops_orig[2] = XEXP (op, 2); break; default: @@ -263,24 +508,16 @@ bfloat16_operation_as_v4sf_inner (enum rtx_code icode, switch (GET_CODE (op)) { + /* Handle chained operations. */ case NEG: case ABS: - ops_v4sf[i] = bfloat16_operation_as_v4sf_inner (GET_CODE (op), - XEXP (op, 0), - NULL_RTX, - NULL_RTX, - FP16_UNARY); - continue; - - /* Handle chained operations. */ case PLUS: case MINUS: case MULT: - ops_v4sf[i] = bfloat16_operation_as_v4sf_inner (GET_CODE (op), - XEXP (op, 0), - XEXP (op, 1), - NULL_RTX, - FP16_BINARY); + case SMIN: + case SMAX: + case FMA: + ops_v4sf[i] = bfloat16_to_v4sf_split_inner (op); continue; /* Remove truncation/extend added. */ @@ -343,366 +580,51 @@ bfloat16_operation_as_v4sf_inner (enum rtx_code icode, } /* Do the operation in V4SFmode. */ - switch (subtype) + switch (n_opts) { - case FP16_UNARY: + case 1: emit_insn (gen_rtx_SET (result_v4sf, gen_rtx_fmt_e (icode, V4SFmode, ops_v4sf[0]))); break; - case FP16_BINARY: + case 2: emit_insn (gen_rtx_SET (result_v4sf, gen_rtx_fmt_ee (icode, V4SFmode, ops_v4sf[0], ops_v4sf[1]))); break; - case FP16_FMA: - case FP16_FMS: - case FP16_NFMA: - case FP16_NFMS: - { - rtx op1 = ops_v4sf[0]; - rtx op2 = ops_v4sf[1]; - rtx op3 = ops_v4sf[2]; - - if (subtype == FP16_FMS || subtype == FP16_NFMS) - op3 = gen_rtx_NEG (V4SFmode, op3); - - rtx op_fma = gen_rtx_FMA (V4SFmode, op1, op2, op3); - - if (subtype == FP16_NFMA || subtype == FP16_NFMS) - op_fma = gen_rtx_NEG (V4SFmode, op_fma); - - emit_insn (gen_rtx_SET (result_v4sf, op_fma)); - } + case 3: + emit_insn (gen_rtx_SET (result_v4sf, + gen_rtx_fmt_eee (icode, V4SFmode, + ops_v4sf[0], + ops_v4sf[1], + ops_v4sf[2]))); break; default: - gcc_unreachable (); + fatal_insn_not_found (op); } return result_v4sf; } -/* Expand a bfloat16 scalar floating point operation: - - ICODE: Operation to perform. - RESULT: Result of the operation. - OP1: Input operand1. - OP2: Input operand2. - OP3: Input operand3 or NULL_RTX. - SUBTYPE: Describe the operation. - - The operation is done as a V4SFmode vector operation. This is because - converting BFmode from a scalar BFmode to SFmode to do the operation and - back again takes quite a bit of time. GCC will only generate the native - operation if -Ofast is used. The float16.md code that calls this function - adds various combine operations to do the operation in V4SFmode instead of - SFmode. - - External part that calls bfloat16_operation_as_v4sf_inner to do the - calculation in V4SFmode, and then converts the result to SFmode or - BFmode. */ - -void -bfloat16_operation_as_v4sf (enum rtx_code icode, - rtx result, - rtx op1, - rtx op2, - rtx op3, - enum fp16_operation subtype) -{ - rtx result_v4sf - = bfloat16_operation_as_v4sf_inner (icode, op1, op2, op3, subtype); - - /* Convert V4SF result back to scalar mode. */ - if (GET_MODE (result) == BFmode) - emit_insn (gen_xvcvspbf16_bf (result, result_v4sf)); - - else if (GET_MODE (result) == SFmode) - { - rtx element = GEN_INT (WORDS_BIG_ENDIAN ? 2 : 3); - emit_insn (gen_vsx_extract_v4sf (result, result_v4sf, element)); - } - - else - gcc_unreachable (); -} - - -/* - * Recognize if can BFmode operations in V4SFmode. If the operation mostly - * involves BFmode values with some SFmode values. it is faster to convert the - * scalar code to vector and do the operation using one element of the vector, - * due to the lack of instructions to convert between BFmode and SFmode/DFmode - * scalars on power10. - * - * We allow multiple operations (a + b + c) to be folded together to try to do - * as much possible in the vector unit without having to convert back to scalar - * mode. - * - * We also fold in loads from memory to simplify the conversion. - * - * The expression that is combined is split before register allocation. - */ - -/* - * Stats for whole expression. - */ - -struct bfloat16_to_v4sf_stats -{ - unsigned num_bf_regs; // # of BF registers used - unsigned num_bf_loads; // # of BF values loaded from memory - unsigned num_sf_regs; // # of SF registers used - unsigned num_const; // # of constants used - unsigned num_binary; // # of binary operations done - unsigned num_fma; // # of FMA (fused multiply-adds) done - unsigned num_unary; // # of unary ops not folded into FMA/ABS - unsigned num_neg_folded; // # of negates folded into a FMA/ABS insn - unsigned num_extend; // # of FLOAT_EXTENDs -}; - -/* Initialize the bfloat16_to_v4sf_stats structure. */ -static struct bfloat16_to_v4sf_stats zero_bfloat16_to_v4sf_stats; - -/* - * Recursively evaluate whether a RTX expression can be optimized to use - * V4SFmode operations. Return true if the expression might be able to be - * optimized, and update the stats. - */ - -static bool -bfloat16_to_v4sf_inner (rtx op, struct bfloat16_to_v4sf_stats *stats) -{ - machine_mode mode = GET_MODE (op); - rtx arg; - rtx_code code; - bool ret = true; - bool update_stats = false; - -#ifdef TARGET_DEBUG_EXTRA1 - if (TARGET_DEBUG_EXTRA1) - update_stats = true; -#endif - - if (mode != BFmode && mode != SFmode) - ret = false; - - else - { - switch (GET_CODE (op)) - { - default: - ret = false; - break; - - // Update register counts. - case REG: - case SUBREG: - if (mode == BFmode) - stats->num_bf_regs++; - else - stats->num_sf_regs++; - - break; - - // Update memory counts. - case MEM: - if (mode != BFmode) - ret = false; - else - stats->num_bf_loads++; - - break; - - // Update constants - case CONST_DOUBLE: - stats->num_const++; - break; - - // Process absolute value - case ABS: - stats->num_unary++; - if (!bfloat16_to_v4sf_inner (XEXP (op, 0), stats)) - ret = false; - - break; - - // Process negation, deal with negate being folded into FMA or ABS. - case NEG: - arg = XEXP (op, 0); - code = GET_CODE (arg); - - if (code == FMA || code == ABS) - stats->num_neg_folded++; - else - stats->num_unary++; - - if (!bfloat16_to_v4sf_inner (arg, stats)) - ret = false; - - break; - - // Process binary operations - case PLUS: - case MINUS: - case MULT: - case SMIN: - case SMAX: - stats->num_binary++; - - if (update_stats) - { - bool ret1 = bfloat16_to_v4sf_inner (XEXP (op, 0), stats); - bool ret2 = bfloat16_to_v4sf_inner (XEXP (op, 1), stats); - if (!ret1 || !ret2) - ret = false; - } - else if (!bfloat16_to_v4sf_inner (XEXP (op, 0), stats) - || !bfloat16_to_v4sf_inner (XEXP (op, 1), stats)) - ret = false; - - break; - - // Process fused multiply-adds - case FMA: - stats->num_fma++; - - if (update_stats) - { - bool ret1 = bfloat16_to_v4sf_inner (XEXP (op, 0), stats); - bool ret2 = bfloat16_to_v4sf_inner (XEXP (op, 1), stats); - bool ret3 = bfloat16_to_v4sf_inner (XEXP (op, 2), stats); - if (!ret1 || !ret2 || !ret3) - ret = false; - } - else - if (!bfloat16_to_v4sf_inner (XEXP (op, 0), stats) - || !bfloat16_to_v4sf_inner (XEXP (op, 1), stats) - || !bfloat16_to_v4sf_inner (XEXP (op, 2), stats)) - ret = false; - - break; - - // Process extend BF -> SF - case FLOAT_EXTEND: - stats->num_extend++; - arg = XEXP (op, 0); - if (mode != SFmode || GET_MODE (arg) != BFmode) - ret = false;; - - if (!bfloat16_to_v4sf_inner (arg, stats)) - ret = false; - } - } - - return ret; -} - -/* - * Decide whether an expression can optimize BFmode to use V4SFmode. - */ - -bool -bfloat16_to_v4sf_p (rtx op, machine_mode mode) -{ - if (mode != BFmode && mode != SFmode) - return false; - -#ifdef TARGET_DEBUG_EXTRA1 - rtx orig_op = op; -#endif - - struct bfloat16_to_v4sf_stats stats = zero_bfloat16_to_v4sf_stats; - - // Handle top level FLOAT_TRUNCATE - if (GET_CODE (op) == FLOAT_TRUNCATE - && mode == BFmode - && GET_MODE (XEXP (op, 0)) == SFmode) - op = XEXP (op, 0); - - bool ret = bfloat16_to_v4sf_inner (op, &stats); - - // Make sure we had enough BFmode operations to do the optimization - if (ret) - { - unsigned num_bf = stats.num_bf_regs + stats.num_bf_loads; - - // If we had no BFmode values, don't do the optimization. - if (!num_bf) - ret = false; - - // If we had no binary or fma opertors, don't do the optimization. - else if (!stats.num_binary && !stats.num_fma) - ret = false; - - // If the expression has much more SFmode elements than BFmode, prefer - // converting BFmode to SFmode instead of using V4SFmode. - else if (stats.num_sf_regs > (num_bf + 2)) - ret = false; - } - - // Debugging. -#ifdef TARGET_DEBUG_EXTRA1 - if (TARGET_DEBUG_EXTRA1) - { - fprintf (stderr, "\n========== bfloat16_to_v4sf_p, return = %s", - ret ? "true" : "false"); - - if (stats.num_bf_regs) - fprintf (stderr, ", # BF regs = %u", stats.num_bf_regs); - - if (stats.num_bf_loads) - fprintf (stderr, ", # BF values loaded = %u", stats.num_bf_loads); - - if (stats.num_sf_regs) - fprintf (stderr, ", # SF regs = %u", stats.num_sf_regs); - - if (stats.num_const) - fprintf (stderr, ", # constants = %u", stats.num_const); - - if (stats.num_extend) - fprintf (stderr, ", # extends = %u", stats.num_extend); - - if (stats.num_unary) - fprintf (stderr, ", # unary = %u", stats.num_unary); - - if (stats.num_binary) - fprintf (stderr, ", # binary = %u", stats.num_binary); - - if (stats.num_fma) - fprintf (stderr, ", # fma = %u", stats.num_fma); - - if (stats.num_neg_folded) - fprintf (stderr, ", # neg fold = %u", stats.num_neg_folded); - - fprintf (stderr, "\n"); - debug_rtx (orig_op); - } -#endif - - return ret; -} - -/* - * Split the isnsns tht optimize BFmode to use V4SFmode. - */ +/* Split the isnsns tht optimize BFmode to use V4SFmode. */ void bfloat16_to_v4sf_split (rtx dest, rtx src) { rtx orig_src = src; enum rtx_code code = GET_CODE (src); - rtx arg, op0, op1, op2; + rtx dest_v4sf; -#ifdef TARGET_DEBUG_EXTRA2 - if (TARGET_DEBUG_EXTRA2) - debug_rtx (gen_rtx_SET (dest, src)); -#endif + if (TARGET_FLOAT16_DEBUG2) + { + fprintf (stderr, "\n========== bfloat16_to_v4sf_split:\n"); + debug_rtx (gen_rtx_SET (dest, src)); + } if (code == FLOAT_EXTEND || code == FLOAT_TRUNCATE) { @@ -713,82 +635,32 @@ bfloat16_to_v4sf_split (rtx dest, rtx src) switch (code) { case ABS: - bfloat16_operation_as_v4sf (code, dest, - XEXP (src, 0), - NULL_RTX, - NULL_RTX, - FP16_UNARY); - break; - case NEG: - arg = XEXP (src, 0); - if (GET_CODE (arg) == FMA) - { - op0 = XEXP (arg, 0); - op1 = XEXP (arg, 1); - op2 = XEXP (arg, 2); - if (GET_CODE (op2) == NEG) - { - op2 = XEXP (op2, 0); - bfloat16_operation_as_v4sf (FMA, dest, - op0, - op1, - op2, - FP16_NFMS); - } - - else - bfloat16_operation_as_v4sf (FMA, dest, - op0, - op1, - op2, - FP16_NFMA); - } - - else - bfloat16_operation_as_v4sf (GET_CODE (arg), dest, - XEXP (arg, 0), - NULL_RTX, - NULL_RTX, - FP16_UNARY); - - break; - case PLUS: case MINUS: case MULT: case SMIN: case SMAX: - bfloat16_operation_as_v4sf (code, dest, - XEXP (src, 0), - XEXP (src, 1), - NULL_RTX, - FP16_BINARY); - break; - case FMA: - op0 = XEXP (src, 0); - op1 = XEXP (src, 1); - op2 = XEXP (src, 2); - if (GET_CODE (op2) == NEG) - bfloat16_operation_as_v4sf (FMA, dest, - op0, - op1, - XEXP (op2, 0), - FP16_FMS); - - else - bfloat16_operation_as_v4sf (FMA, dest, - op0, - op1, - op2, - FP16_FMA); - + dest_v4sf = bfloat16_to_v4sf_split_inner (src); break; default: fatal_insn_not_found (gen_rtx_SET (dest, orig_src)); } + /* Convert V4SF result back to scalar mode. */ + if (GET_MODE (dest) == BFmode) + emit_insn (gen_xvcvspbf16_bf (dest, dest_v4sf)); + + else if (GET_MODE (dest) == SFmode) + { + rtx element = GEN_INT (WORDS_BIG_ENDIAN ? 2 : 3); + emit_insn (gen_vsx_extract_v4sf (dest, dest_v4sf, element)); + } + + else + gcc_unreachable (); + return; } diff --git a/gcc/config/rs6000/float16.md b/gcc/config/rs6000/float16.md index 4f14862945f4..6485f4d2d62d 100644 --- a/gcc/config/rs6000/float16.md +++ b/gcc/config/rs6000/float16.md @@ -1326,244 +1326,3 @@ DONE; }) -(define_insn_and_split "*bfloat16_binary_op_internal1" - [(set (match_operand:SF 0 "vsx_register_operand") - (match_operator:SF 1 "fp16_binary_operator" - [(match_operand:SF 2 "bfloat16_v4sf_operand") - (match_operand:SF 3 "bfloat16_v4sf_operand")]))] - "!TARGET_BF_V4SF && TARGET_BFLOAT16_HW && can_create_pseudo_p () - && (bfloat16_bf_operand (operands[2], SFmode) - || bfloat16_bf_operand (operands[3], SFmode))" - "#" - "&& 1" - [(pc)] -{ - bfloat16_operation_as_v4sf (GET_CODE (operands[1]), operands[0], operands[2], - operands[3], NULL_RTX, FP16_BINARY); - DONE; -}) - -(define_insn_and_split "*bfloat16_binary_op_internal2" - [(set (match_operand:BF 0 "vsx_register_operand") - (float_truncate:BF - (match_operator:SF 1 "fp16_binary_operator" - [(match_operand:SF 2 "bfloat16_v4sf_operand") - (match_operand:SF 3 "bfloat16_v4sf_operand")])))] - "!TARGET_BF_V4SF && TARGET_BFLOAT16_HW && can_create_pseudo_p () - && (bfloat16_bf_operand (operands[2], SFmode) - || bfloat16_bf_operand (operands[3], SFmode))" - "#" - "&& 1" - [(pc)] -{ - bfloat16_operation_as_v4sf (GET_CODE (operands[1]), operands[0], operands[2], - operands[3], NULL_RTX, FP16_BINARY); - DONE; -}) - -(define_insn_and_split "*bfloat16_fma_internal1" - [(set (match_operand:SF 0 "vsx_register_operand") - (fma:SF - (match_operand:SF 1 "bfloat16_v4sf_operand") - (match_operand:SF 2 "bfloat16_v4sf_operand") - (match_operand:SF 3 "bfloat16_v4sf_operand")))] - "!TARGET_BF_V4SF && TARGET_BFLOAT16_HW && can_create_pseudo_p () - && (bfloat16_bf_operand (operands[1], SFmode) - + bfloat16_bf_operand (operands[2], SFmode) - + bfloat16_bf_operand (operands[3], SFmode) >= 2)" - "#" - "&& 1" - [(pc)] -{ - bfloat16_operation_as_v4sf (FMA, operands[0], operands[1], operands[2], - operands[3], FP16_FMA); - DONE; -}) - -(define_insn_and_split "*bfloat16_fma_internal2" - [(set (match_operand:BF 0 "vsx_register_operand" "=wa") - (float_truncate:BF - (fma:SF - (match_operand:SF 1 "bfloat16_v4sf_operand") - (match_operand:SF 2 "bfloat16_v4sf_operand") - (match_operand:SF 3 "bfloat16_v4sf_operand"))))] - "!TARGET_BF_V4SF && TARGET_BFLOAT16_HW && can_create_pseudo_p () - && (bfloat16_bf_operand (operands[1], SFmode) - + bfloat16_bf_operand (operands[2], SFmode) - + bfloat16_bf_operand (operands[3], SFmode) >= 2)" - "#" - "&& 1" - [(pc)] -{ - bfloat16_operation_as_v4sf (FMA, operands[0], operands[1], operands[2], - operands[3], FP16_FMA); - DONE; -}) - -(define_insn_and_split "*bfloat16_fms_internal1" - [(set (match_operand:SF 0 "vsx_register_operand") - (fma:SF - (match_operand:SF 1 "bfloat16_v4sf_operand") - (match_operand:SF 2 "bfloat16_v4sf_operand") - (neg:SF - (match_operand:SF 3 "bfloat16_v4sf_operand"))))] - "!TARGET_BF_V4SF && TARGET_BFLOAT16_HW && can_create_pseudo_p () - && (bfloat16_bf_operand (operands[1], SFmode) - + bfloat16_bf_operand (operands[2], SFmode) - + bfloat16_bf_operand (operands[3], SFmode) >= 2)" - "#" - "&& 1" - [(pc)] -{ - bfloat16_operation_as_v4sf (FMA, operands[0], operands[1], operands[2], - operands[3], FP16_FMS); - DONE; -}) - -(define_insn_and_split "*bfloat16_fms_internal2" - [(set (match_operand:BF 0 "vsx_register_operand") - (float_truncate:BF - (fma:SF - (match_operand:SF 1 "bfloat16_v4sf_operand") - (match_operand:SF 2 "bfloat16_v4sf_operand") - (neg:SF - (match_operand:SF 3 "bfloat16_v4sf_operand")))))] - "!TARGET_BF_V4SF && TARGET_BFLOAT16_HW && can_create_pseudo_p () - && (bfloat16_bf_operand (operands[1], SFmode) - + bfloat16_bf_operand (operands[2], SFmode) - + bfloat16_bf_operand (operands[3], SFmode) >= 2)" - "#" - "&& 1" - [(pc)] -{ - bfloat16_operation_as_v4sf (FMA, operands[0], operands[1], operands[2], - operands[3], FP16_FMS); - DONE; -}) - -(define_insn_and_split "*bfloat16_nfma_internal1" - [(set (match_operand:SF 0 "vsx_register_operand") - (neg:SF - (fma:SF - (match_operand:SF 1 "bfloat16_v4sf_operand") - (match_operand:SF 2 "bfloat16_v4sf_operand") - (match_operand:SF 3 "bfloat16_v4sf_operand"))))] - "!TARGET_BF_V4SF && TARGET_BFLOAT16_HW && can_create_pseudo_p () - && (bfloat16_bf_operand (operands[1], SFmode) - + bfloat16_bf_operand (operands[2], SFmode) - + bfloat16_bf_operand (operands[3], SFmode) >= 2)" - "#" - "&& 1" - [(pc)] -{ - bfloat16_operation_as_v4sf (FMA, operands[0], operands[1], operands[2], - operands[3], FP16_NFMA); - DONE; -}) - -(define_insn_and_split "*bfloat16_nfma_internal2" - [(set (match_operand:BF 0 "vsx_register_operand" "=wa") - (float_truncate:BF - (neg:SF - (fma:SF - (match_operand:SF 1 "bfloat16_v4sf_operand") - (match_operand:SF 2 "bfloat16_v4sf_operand") - (match_operand:SF 3 "bfloat16_v4sf_operand")))))] - "!TARGET_BF_V4SF && TARGET_BFLOAT16_HW && can_create_pseudo_p () - && (bfloat16_bf_operand (operands[1], SFmode) - + bfloat16_bf_operand (operands[2], SFmode) - + bfloat16_bf_operand (operands[3], SFmode) >= 2)" - "#" - "&& 1" - [(pc)] -{ - bfloat16_operation_as_v4sf (FMA, operands[0], operands[1], operands[2], - operands[3], FP16_NFMA); - DONE; -}) - -(define_insn_and_split "*bfloat16_nfma_internal3" - [(set (match_operand:BF 0 "vsx_register_operand" "=wa") - (neg:BF - (float_truncate:BF - (fma:SF - (match_operand:SF 1 "bfloat16_v4sf_operand") - (match_operand:SF 2 "bfloat16_v4sf_operand") - (match_operand:SF 3 "bfloat16_v4sf_operand")))))] - "!TARGET_BF_V4SF && TARGET_BFLOAT16_HW && can_create_pseudo_p () - && (bfloat16_bf_operand (operands[1], SFmode) - + bfloat16_bf_operand (operands[2], SFmode) - + bfloat16_bf_operand (operands[3], SFmode) >= 2)" - "#" - "&& 1" - [(pc)] -{ - bfloat16_operation_as_v4sf (FMA, operands[0], operands[1], operands[2], - operands[3], FP16_NFMA); - DONE; -}) - -(define_insn_and_split "*bfloat16_nfms_internal1" - [(set (match_operand:SF 0 "vsx_register_operand") - (neg:SF - (fma:SF - (match_operand:SF 1 "bfloat16_v4sf_operand") - (match_operand:SF 2 "bfloat16_v4sf_operand") - (neg:SF - (match_operand:SF 3 "bfloat16_v4sf_operand")))))] - "!TARGET_BF_V4SF && TARGET_BFLOAT16_HW && can_create_pseudo_p () - && (bfloat16_bf_operand (operands[1], SFmode) - + bfloat16_bf_operand (operands[2], SFmode) - + bfloat16_bf_operand (operands[3], SFmode) >= 2)" - "#" - "&& 1" - [(pc)] -{ - bfloat16_operation_as_v4sf (FMA, operands[0], operands[1], operands[2], - operands[3], FP16_NFMS); - DONE; -}) - -(define_insn_and_split "*bfloat16_nfms_internal2" - [(set (match_operand:BF 0 "vsx_register_operand") - (float_truncate:BF - (neg:SF - (fma:SF - (match_operand:SF 1 "bfloat16_v4sf_operand") - (match_operand:SF 2 "bfloat16_v4sf_operand") - (neg:SF - (match_operand:SF 3 "bfloat16_v4sf_operand"))))))] - "!TARGET_BF_V4SF && TARGET_BFLOAT16_HW && can_create_pseudo_p () - && (bfloat16_bf_operand (operands[1], SFmode) - + bfloat16_bf_operand (operands[2], SFmode) - + bfloat16_bf_operand (operands[3], SFmode) >= 2)" - "#" - "&& 1" - [(pc)] -{ - bfloat16_operation_as_v4sf (FMA, operands[0], operands[1], operands[2], - operands[3], FP16_NFMS); - DONE; -}) - -(define_insn_and_split "*bfloat16_nfms_internal3" - [(set (match_operand:BF 0 "vsx_register_operand") - (neg:BF - (float_truncate:BF - (fma:SF - (match_operand:SF 1 "bfloat16_v4sf_operand") - (match_operand:SF 2 "bfloat16_v4sf_operand") - (neg:SF - (match_operand:SF 3 "bfloat16_v4sf_operand"))))))] - "!TARGET_BF_V4SF && TARGET_BFLOAT16_HW && can_create_pseudo_p () - && (bfloat16_bf_operand (operands[1], SFmode) - + bfloat16_bf_operand (operands[2], SFmode) - + bfloat16_bf_operand (operands[3], SFmode) >= 2)" - "#" - "&& 1" - [(pc)] -{ - bfloat16_operation_as_v4sf (FMA, operands[0], operands[1], operands[2], - operands[3], FP16_NFMS); - DONE; -}) diff --git a/gcc/config/rs6000/rs6000-protos.h b/gcc/config/rs6000/rs6000-protos.h index 1f8f75201461..6a2a28ec37b0 100644 --- a/gcc/config/rs6000/rs6000-protos.h +++ b/gcc/config/rs6000/rs6000-protos.h @@ -262,7 +262,6 @@ extern unsigned constant_generates_xxspltidp (vec_const_128bit_type *); /* From float16.cc. */ /* Optimize bfloat16 and float16 operations. */ enum fp16_operation { - FP16_UNARY, /* Bfloat16/float16 unary op. */ FP16_BINARY, /* Bfloat16/float16 binary op. */ FP16_FMA, /* (a * b) + c. */ FP16_FMS, /* (a * b) - c. */ diff --git a/gcc/config/rs6000/rs6000.opt b/gcc/config/rs6000/rs6000.opt index 47a705a20dbc..9736c134cba4 100644 --- a/gcc/config/rs6000/rs6000.opt +++ b/gcc/config/rs6000/rs6000.opt @@ -654,6 +654,14 @@ mfloat16-bf-v4sf Target Undocumented Var(TARGET_BF_V4SF) Init(1) Save Optimize bfloat16 to use float vector instructions. +mfloat16-debug1 +Target Undocumented Var(TARGET_FLOAT16_DEBUG1) Init(0) Save +Turn on float16 debugging #1. + +mfloat16-debug2 +Target Undocumented Var(TARGET_FLOAT16_DEBUG2) Init(0) Save +Turn on float16 debugging #2. + ; Documented parameters -param=rs6000-vect-unroll-limit=
