================ @@ -0,0 +1,1683 @@ +//===-------------------- HexagonXQFloatGenerator.cpp --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This pass enables generation of XQFloat instructions. XQF instructions +// are more efficient, but can be less precise in comparison to IEEE ones. +// Based on the accuracy preservation of the generated code, we enabled four +// modes - Strict IEEE-754 compliant, IEEE-754 compliant, Lossy subnormals and +// legacy mode. +// +// Strict IEEE mode adheres to similar accuracy and precision as of IEEE-754. +// +// IEEE-754 compliant mode excludes IEEE-754 overflows and lower precision +// subnormals due to larger dynamic range than IEEE-754. +// All subnormals have extra precision. +// +// Lossy subnormals mode without normalization result in a loss of accuracy. +// This provides greater precision than a clamp of subnormals to 0. +// If dataset excludes subnormals, it behavas as IEEE-754 compliant mode. +// +// The direct mode has a loss of 1 bit of accuracy compared to IEEE-754. +// +// V79 replaces the prior internal HVX floating point format for floating-point +// arithmetic. The new internal HVX floating-point format yields results +// identical to IEEE-754 round-to-even mode. The new format contains more bits +// than IEEE-754, which optionally produces results with greater range and +// accuracy. Only the HVX vector registers use the HVX floating-point format. +// Memory maintains all floating-point data in IEEE-754 format, +// and all loads/stores use the IEEE-754 format. A subset of HVX floating-point +// operations transform IEEE-754 floating-point data to HVX floating-point data. +// Subsequent HVX floating-point instructions may consume operands in the HVX +// floating-point without conversion to IEEE-754, which allows for performant +// & energy efficient code. The program does not need to switch between formats +// continuously. The program must convert the HVX floating-point results to +// IEEE-754 prior to storing to memory. + +// HVX floating-point achieves IEEE-754 compliance through normalization. +// The program may skip normalization when faster calculation is desired, and +// IEEE-754 compliance isn’t required. HVX floating-point contains two input +// types: qf32, single precision floating point, and qf16, half precision +// floating point. In Hexagon, IEEE-754 contains two input types: sf, single +// precision floating point, and hf, half precision floating point. +// +// Only HVX floating-point source and destination instructions use HVX +// floating-point values. Instructions specify the HVX floating-point format +// with the qf16 and qf32 identifier. A source vector register will drop the +// extended state of a HVX floating-point value when an instruction reads the +// source vector register without the qf16 or qf32 identifier. A destination +// vector register will reset its extended state when an instruction writes to +// a vector register without the qf16 or qf32 identifier. When dropping the +// extended state, the floating-point value loses accuracy. The program may +// preserve the floating-point value by converting HVX floating-point values +// to IEEE-754 values. Compiler must convert HVX floating-point values to +// IEEE-754 values before using as an input to stores, permutes, shifts, and +// any other operations that do not source the HVX floating-point format. +// +// Depending on the desired results, HVX floating-point operations may have +// some requirements on the input sources. The HVX floating-point values +// require normalization to achieve IEEE-754 compliance, while faster operations +// may skip normalization. The program normalizes HVX floating-point values +// before subsequent HVX floating-point operations, so the floating-point value +// does not lose precision. The program also obtains results identical to +// IEEE-754 by converting all HVX floating-point results to IEEE-754 format +// before consumed in any subsequent operation. There are however cases where +// this conversion is redundant, or the differences between IEEE-754 and HVX +// floating-point may not be a concern. +// +// The conversion logic can be understood by the table below: +// +// ================================================================================================================================================ +// | | | | +// | Inputs to add/subtarct | Inputs to +// multiplication instuctions | Non-HVX floating +// point | | instructions | | instruction +// | | | | | +// ===============================================================================================================================================| +// Sources | IEEE- | HVX | HVX | sf | qf32 | qf32 | hf +// | qf16 | qf16 | IEE-754 | HVX | HVX | +// | 754 | floating | floating | | from | from | | +// from | from | | floating | floating | | | +// point | point | | mult | adder | | mult +// | adder | | point | point | | | from | +// from | | | | | | | | +// from | from | | | multi | adder | | +// | | | | | | mult | +// adder | | | | | | | | | | | +// | | | +// ===============================================================================================================================================| +// Strict | Direct | Convert | Convert | Normalize | Convert | Convert +// | widening | Convert | Convert | Direct | Convert | Convert | IEEE-754 +// | Use | to | to | | to IEEE | to IEEE | multiply +// | to IEEE, | to IEEE, | use | to | to | compliance | | +// IEEE | IEEE | | then | then | then | widening +// | widening | | IEEE | IEEE | +// | | | | | normalize | normalize +// | convert | multiply,| multiply,| | | | +// | | | | | | | to IEEE +// | convert | convert | | | | | | +// | | | | | | to +// IEEE | to IEEE | | | | +// -----------------------------------------------------------------------------------------------------------------------------------------------| +// IEEE-754 | Direct | Direct | Direct | Normalize | Direct | Normalize +// | Widening | Direct | Widening | Direct | Convert | Convert | compliance +// | Use | Use | Use | | use | | multiply +// | use | multiply | use | to IEEE | to IEEE | +// -----------------------------------------------------------------------------------------------------------------------------------------------| +// Lossy | Direct | Direct | Direct | Direct | Direct | Normalize +// | Direct | Direct | Widening | Direct | Convert | Convert | Subnormals +// | Use | Use | Use | Use | use | | use | +// use | multiply | use | to IEEE | to IEEE | +// -----------------------------------------------------------------------------------------------------------------------------------------------| +// Direct | Direct | Direct | Direct | Direct | Direct | Direct | +// Direct | Direct | Direct | Direct | Direct | Direct | Lossy | +// Use | Use | Use | Use | use | use | use | +// use | use | use | use | use | +// -----------------------------------------------------------------------------------------------------------------------------------------------| +// +// For v81, the normalization sequence changes. Instead of multiplying 0 +// and -0, a simple copy operation normalizes the unnormal value. Both +// qf and IEEE-754 value can be unnormal. +// Additionally for v81, we have two new vsub instructions which are handled. + +#define HEXAGON_XQFLOAT_GENERATOR "XQFloat Generator pass" + +#include "Hexagon.h" +#include "HexagonInstrInfo.h" +#include "HexagonSubtarget.h" +#include "HexagonTargetMachine.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/MachineOperand.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/IR/DebugLoc.h" +#include "llvm/Pass.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" +#include <vector> + +#define DEBUG_TYPE "hexagon-xqf-gen" + +using namespace llvm; + +extern cl::opt<QFloatMode> QFloatModeValue; + +// Master flag to enable XQF generations +cl::opt<bool> EnableHVXXQFloat("enable-xqf-gen", cl::init(false), + cl::desc("Enable XQFloat generations")); +// This vector contains the opcodes which generate qf32 from add/subtract +static constexpr unsigned XQFPAdd32[] = { + // vector add instructions + Hexagon::V6_vadd_sf, Hexagon::V6_vadd_qf32, Hexagon::V6_vadd_qf32_mix, + + // vector subtract instructions + Hexagon::V6_vsub_qf32, Hexagon::V6_vsub_qf32_mix, Hexagon::V6_vsub_sf, + Hexagon::V6_vsub_sf_mix}; + +// This vector contains the opcodes which generate qf16 from add/subtract +static constexpr unsigned XQFPAdd16[] = { + // vector add instructions + Hexagon::V6_vadd_hf, Hexagon::V6_vadd_qf16, Hexagon::V6_vadd_qf16_mix, + + // vector subtract intrutions + Hexagon::V6_vsub_hf, Hexagon::V6_vsub_qf16, Hexagon::V6_vsub_qf16_mix, + Hexagon::V6_vsub_hf_mix}; + +// This vector contains the opcodes which generate qf32 from multiplication +static constexpr unsigned XQFPMult32[] = { + Hexagon::V6_vmpy_qf32, Hexagon::V6_vmpy_qf32_qf16, Hexagon::V6_vmpy_qf32_hf, + Hexagon::V6_vmpy_qf32_sf, Hexagon::V6_vmpy_qf32_mix_hf}; +// This vector contains the opcodes which generate qf16 from multiplication +static constexpr unsigned XQFPMult16[] = {Hexagon::V6_vmpy_qf16, + Hexagon::V6_vmpy_qf16_hf, + Hexagon::V6_vmpy_qf16_mix_hf}; + +namespace llvm { +FunctionPass *createHexagonXQFloatGenerator(); +void initializeHexagonXQFloatGeneratorPass(PassRegistry &); +} // namespace llvm + +namespace { + +struct HexagonXQFloatGenerator : public MachineFunctionPass { +public: + static char ID; + HexagonXQFloatGenerator() : MachineFunctionPass(ID) {} + + bool runOnMachineFunction(MachineFunction &MF) override; + + StringRef getPassName() const override { return HEXAGON_XQFLOAT_GENERATOR; } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + MachineFunctionPass::getAnalysisUsage(AU); + } + +private: + // Handle each XQF optimization level + bool HandleStrictIEEE(MachineFunction &); + bool HandleCompliantIEEE(MachineFunction &); + bool HandleLossySubnormals(MachineFunction &); + bool HandleLossyLegacy(MachineFunction &); + + // Checkers functions for input operands + bool checkIfInputFromAdder32(Register Reg); + bool checkIfInputFromAdder16(Register Reg); + bool checkIfInputFromMult32(Register Reg); + bool checkIfInputFromMult16(Register Reg); + bool deleteList(); + + // Helper functions for conversion/normalization/widening + bool widenMultiplicationInputF16(MachineInstr &, Register &, Register &, + Register &, bool); + bool widenMultiplicationInputF16Rt(MachineInstr &, Register &, Register &, + Register &); + void widenMultiplyInputHF(MachineInstr &, Register &, Register &, Register &); + bool normalizeMultiplicationInputF32(MachineInstr &, Register &, Register &, + Register &, Register &, bool &); + void normalizeMultiplicationInputSF(MachineInstr &, Register &, Register &, + Register &, Register &, bool &); + bool convertNormalizeMultOp32(MachineInstr &, Register &, Register &, + Register &, Register &, bool &); + bool convertWidenMultOp16(MachineInstr &, Register &, Register &, Register &, + bool); + bool convertWidenMultOp32(MachineInstr &, Register &, Register &, Register &, + bool); + void createPrologInstructions(MachineInstr &, Register &); + bool convertAddOpToIEEE16(MachineInstr &, Register &, Register &, Register &, + bool, bool, bool); + bool convertAddOpToIEEE32(MachineInstr &, Register &, Register &, Register &, + bool, bool, bool); + void generateQF16FromQF32(MachineInstr &, Register &, Register &); + bool convertIfInputToNonHVX(MachineInstr &, bool); + void createConvertInstr(MachineInstr *, Register &, Register &, bool); + + // V81 specific normalization function + bool V81normalizeMultF32(MachineInstr &, Register &, Register &, Register &, + bool, bool, bool); + + const HexagonSubtarget *HST = nullptr; + const HexagonInstrInfo *HII = nullptr; + MachineRegisterInfo *MRI = nullptr; + + SmallVector<MachineInstr *, 16> + OriginalMI; // Hold the instructions to be deleted +}; + +// This class removes redundant vector convert instructions from qf to hf/sf. +// Additionally, it relaces use of sf/hf registers with qf types. +// The resulting code is complete without dangling instructions. +// FIXME: Liveness is not preserved. +char HexagonXQFloatGenerator::ID = 0; + +} // namespace + +INITIALIZE_PASS(HexagonXQFloatGenerator, "hexagon-xqfloat-generator", + HEXAGON_XQFLOAT_GENERATOR, false, false) + +FunctionPass *llvm::createHexagonXQFloatGenerator() { + return new HexagonXQFloatGenerator(); +} + +// Returns true if qf32 input is from an adder/subtract unit +bool HexagonXQFloatGenerator::checkIfInputFromAdder32(Register Reg) { + MachineInstr *Def = MRI->getVRegDef(Reg); + if (!Def) + return false; + + // If the definition is a copy, we need to analyze its def again + if (Def->getOpcode() == TargetOpcode::COPY) { + Register SrcReg = Def->getOperand(1).getReg(); + if (SrcReg.isValid()) + return checkIfInputFromAdder32(SrcReg); + return false; + } else if (Def->getOpcode() == TargetOpcode::REG_SEQUENCE) { + Register SrcReg1 = Def->getOperand(1).getReg(); + Register SrcReg2 = Def->getOperand(2).getReg(); + bool isTrue = false; + if (SrcReg1.isValid()) + isTrue = checkIfInputFromAdder32(SrcReg1); + if (SrcReg2.isValid()) + isTrue |= checkIfInputFromAdder32(SrcReg2); + return isTrue; + } else + return llvm::is_contained(XQFPAdd32, Def->getOpcode()); +} + +// Returns true if qf16 input is from an adder/subtract unit +bool HexagonXQFloatGenerator::checkIfInputFromAdder16(Register Reg) { + MachineInstr *Def = MRI->getVRegDef(Reg); + if (!Def) + return false; + + // if the definition is a copy, we need to analyze its def again + if (Def->getOpcode() == TargetOpcode::COPY) { + Register SrcReg = Def->getOperand(1).getReg(); + if (SrcReg.isValid()) + return checkIfInputFromAdder16(SrcReg); + return false; + } else + return llvm::is_contained(XQFPAdd16, Def->getOpcode()); +} + +// Returns true if qf32 input is from a multiplier unit +bool HexagonXQFloatGenerator::checkIfInputFromMult32(Register Reg) { + MachineInstr *Def = MRI->getVRegDef(Reg); + if (!Def) + return false; + + // if the definition is a copy, we need to analyze its def again + if (Def->getOpcode() == TargetOpcode::COPY) { + Register SrcReg = Def->getOperand(1).getReg(); + if (SrcReg.isValid()) + return checkIfInputFromMult32(SrcReg); + return false; + } else if (Def->getOpcode() == TargetOpcode::REG_SEQUENCE) { + Register SrcReg1 = Def->getOperand(1).getReg(); + Register SrcReg2 = Def->getOperand(2).getReg(); + bool isTrue = false; + if (SrcReg1.isValid()) + isTrue |= checkIfInputFromMult32(SrcReg1); + if (SrcReg2.isValid()) + isTrue |= checkIfInputFromMult32(SrcReg2); + return isTrue; + } else + return llvm::is_contained(XQFPMult32, Def->getOpcode()); +} + +// Returns true if qf16 input is from a multiplier unit +bool HexagonXQFloatGenerator::checkIfInputFromMult16(Register Reg) { + MachineInstr *Def = MRI->getVRegDef(Reg); + if (!Def) + return false; + + // if the definition is a copy, we need to analyze its def again + if (Def->getOpcode() == TargetOpcode::COPY) { + Register SrcReg = Def->getOperand(1).getReg(); + if (SrcReg.isValid()) + return checkIfInputFromMult16(SrcReg); + return false; + } else + return llvm::is_contained(XQFPMult16, Def->getOpcode()); +} + +// Generates sf = qf32 instruction or hf = qf16 intruction +void HexagonXQFloatGenerator::createConvertInstr(MachineInstr *UseMI, + Register &NewR, Register &OldR, + bool is32bit) { + const DebugLoc &DL = UseMI->getDebugLoc(); + MachineBasicBlock *MBB = UseMI->getParent(); + NewR = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass); + if (is32bit) + BuildMI(*MBB, *UseMI, DL, HII->get(Hexagon::V6_vconv_sf_qf32), NewR) + .addReg(OldR); + else + BuildMI(*MBB, *UseMI, DL, HII->get(Hexagon::V6_vconv_hf_qf16), NewR) + .addReg(OldR); +} + +// Generate HVX to IEEE conversion instruction for all non-HVX uses +bool HexagonXQFloatGenerator::convertIfInputToNonHVX(MachineInstr &MI, + bool is32bit) { + Register NewR; + bool Changed = false; + ; + Register Dest = MI.getOperand(0).getReg(); + + // Iterate over all uses of the Def we are analyzing + for (auto &MO : make_range(MRI->use_begin(Dest), MRI->use_end())) { + MachineInstr *UseMI = MO.getParent(); + // Omit if the use is a REG_SEQUENCE instruction, since the only + // use of REG_SEQUENCE in qf context is transforming to IEEE. + // Omit for use in DBG instructions. + // Omit for use in PHI instructions since PHI result can be used as a qf + // operand. + if (UseMI->getOpcode() == TargetOpcode::REG_SEQUENCE || + UseMI->getOpcode() == TargetOpcode::DBG_VALUE || + UseMI->getOpcode() == TargetOpcode::DBG_LABEL || + UseMI->getOpcode() == TargetOpcode::PHI) + continue; + + // If 32-bit operand + if (is32bit) { + // If it is a copy instruction, we need to analyze it uses + if (UseMI->getOpcode() == TargetOpcode::COPY) + return convertIfInputToNonHVX(*UseMI, /* 32 bit */ true); + if (!HII->usesQFOperand(UseMI)) { + createConvertInstr(UseMI, NewR, Dest, /*32 bit*/ true); + MO.setReg(NewR); + Changed = true; + } + // If 16-bit operand + } else { + // If it is a copy instruction, we need to analyze it uses + if (UseMI->getOpcode() == TargetOpcode::COPY) + return convertIfInputToNonHVX(*UseMI, /* 16 bit */ false); + if (!HII->usesQFOperand(UseMI)) { + createConvertInstr(UseMI, NewR, Dest, /*16 bit*/ false); + MO.setReg(NewR); + Changed = true; + } + } + } + return Changed; +} + +// generate qf16 = qf32 via: +// hf = qf32 +// V0 = #0 +// qf16 = vsub(hf,V0) +void HexagonXQFloatGenerator::generateQF16FromQF32(MachineInstr &MI, + Register &Dest, + Register &SrcReg) { + + MachineBasicBlock &MBB = *MI.getParent(); + const DebugLoc &DL = MI.getDebugLoc(); + + Register convertReg = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vconv_hf_qf32), convertReg) + .addReg(SrcReg); + Register VR0 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vd0), VR0); + + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vsub_hf), Dest) + .addReg(convertReg) + .addReg(VR0); +} + +// Widen qf16 = vmpy(hf, hf) result unconditionally +void HexagonXQFloatGenerator::widenMultiplyInputHF(MachineInstr &MI, + Register &Reg1, + Register &Reg2, + Register &Dest) { + Register output_mpy = MRI->createVirtualRegister(&Hexagon::HvxWRRegClass); + MachineBasicBlock &MBB = *MI.getParent(); + const DebugLoc &DL = MI.getDebugLoc(); + + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32_hf), output_mpy) + .addReg(Reg1) + .addReg(Reg2); + generateQF16FromQF32(MI, Dest, output_mpy); +} + +// Widen vmpy(qf16, qf16/hf) result conditionally +bool HexagonXQFloatGenerator::widenMultiplicationInputF16(MachineInstr &MI, + Register &Reg1, + Register &Reg2, + Register &Dest, + bool twoOps) { + bool firstconvert = false, secondconvert = false; + MachineBasicBlock &MBB = *MI.getParent(); + const DebugLoc &DL = MI.getDebugLoc(); + + // We widen only that operand which comes from add/subtract unit. + if (checkIfInputFromAdder16(Reg1)) + firstconvert = true; + // twoOps == true suggest 2nd operand is qf16, else it is hf + if (twoOps && checkIfInputFromAdder16(Reg2)) + secondconvert = true; + + Register widenReg; + // if either operands from add/subtract unit, we widen + if (twoOps) { + if (firstconvert || secondconvert) { + widenReg = MRI->createVirtualRegister(&Hexagon::HvxWRRegClass); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32_qf16), widenReg) + .addReg(Reg1) + .addReg(Reg2); + } else { + return false; + } + } else { + if (firstconvert) { + widenReg = MRI->createVirtualRegister(&Hexagon::HvxWRRegClass); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32_mix_hf), widenReg) + .addReg(Reg1) + .addReg(Reg2); + } else { + return false; + } + } + + // generate qf16 = qf32 + generateQF16FromQF32(MI, Dest, widenReg); + + return true; +} + +// Handle qf16 = vmpy(qf16, Rt) +// For strict IEEE mode, convert the qf16 to IEEE before widening +bool HexagonXQFloatGenerator::widenMultiplicationInputF16Rt(MachineInstr &MI, + Register &Reg1, + Register &Reg2, + Register &Dest) { + // If the first input is not from an adder, for strict-ieee check if + // input from mult, else return false. + if (!checkIfInputFromAdder16(Reg1)) { + if (QFloatModeValue == QFloatMode::StrictIEEE) { + if (!checkIfInputFromMult16(Reg1)) + return false; + } else + return false; + } + + MachineBasicBlock &MBB = *MI.getParent(); + const DebugLoc &DL = MI.getDebugLoc(); + + Register VSplatReg = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_lvsplatw), VSplatReg).addReg(Reg2); + + Register widenReg = MRI->createVirtualRegister(&Hexagon::HvxWRRegClass); + if (QFloatModeValue == QFloatMode::StrictIEEE) { + Register VHf = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vconv_hf_qf16), VHf).addReg(Reg1); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32_hf), widenReg) + .addReg(VHf) + .addReg(VSplatReg); + } else { + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32_mix_hf), widenReg) + .addReg(Reg1) + .addReg(VSplatReg); + } + + // generate qf16 = qf32 + generateQF16FromQF32(MI, Dest, widenReg); + return true; +} + +// Handle qf32 = vadd/vsub(qf32/sf, qf32/sf) +// Handle vadd/vsub instructions with qf32 operands conditionally +// isAdd: true if an add instruction is analyzed, false for subtract +// isFirstOpQf: true if 1st operand is qf32 type, false if sf type +// isSecOpQf: true if 2nd operand is qf32 type, false if sf type +bool HexagonXQFloatGenerator::convertAddOpToIEEE32( + MachineInstr &MI, Register &Reg1, Register &Reg2, Register &Dest, + bool isAdd, bool isFirstOpQf, bool isSecOpQf) { + + Register VR1; + Register VR2; + bool firstconvert = false, secondconvert = false; + MachineBasicBlock &MBB = *MI.getParent(); + const DebugLoc &DL = MI.getDebugLoc(); + + // If the first operand is qf32 type + if (isFirstOpQf) { + // If the first operand is from add/sub/mul unit, + // generate IEEE conversion instruction sf = qf32 + if (checkIfInputFromAdder32(Reg1) || checkIfInputFromMult32(Reg1)) { + VR1 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vconv_sf_qf32), VR1) + .addReg(Reg1); + firstconvert = true; + } + } + + // If 2nd operand is of qf32 type + if (isSecOpQf) { + // If the second operand is from add/sub/mul unit, + // generate IEEE conversion instruction + if (checkIfInputFromAdder32(Reg2) || checkIfInputFromMult32(Reg2)) { + VR2 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vconv_sf_qf32), VR2) + .addReg(Reg2); + secondconvert = true; + } + } + + // If both operands are qf32 type, use V6_v[add/sub]_sf instruction + // If one of them is of sf type, use V6_v[add/sub]_qf32_mix instruction + // Output is qf32 + if (isFirstOpQf && isSecOpQf) { + if (firstconvert && secondconvert) { + BuildMI(MBB, MI, DL, + HII->get(isAdd ? Hexagon::V6_vadd_sf : Hexagon::V6_vsub_sf), Dest) + .addReg(VR1) + .addReg(VR2); + } else if (firstconvert) { + if (isAdd) + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vadd_qf32_mix), Dest) + .addReg(Reg2) + .addReg(VR1); + // For vsub type, for v81 we use a different opcode, + // for v79, we convert the 2nd op to IEEE too. + else { + if (HST->useHVXV81Ops()) + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vsub_sf_mix), Dest) + .addReg(VR1) + .addReg(Reg2); + else { + VR2 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vconv_sf_qf32), VR2) + .addReg(Reg2); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vsub_sf), Dest) + .addReg(VR1) + .addReg(VR2); + } + } + } else if (secondconvert) { + BuildMI(MBB, MI, DL, + HII->get(isAdd ? Hexagon::V6_vadd_qf32_mix + : Hexagon::V6_vsub_qf32_mix), + Dest) + .addReg(Reg1) + .addReg(VR2); + } else { // none of the inputs is from an add/sub/mul unit + return false; + } + // handle vadd/vsub when the 1st op of original instruction is qf type + } else if (isFirstOpQf) { + if (firstconvert) + BuildMI(MBB, MI, DL, + HII->get(isAdd ? Hexagon::V6_vadd_sf : Hexagon::V6_vsub_sf), Dest) + .addReg(VR1) + .addReg(Reg2); + else + return false; + // handle vadd/vsub when the 2nd op of original instruction is qf type + } else if (isSecOpQf) { + if (secondconvert) + BuildMI(MBB, MI, DL, + HII->get(isAdd ? Hexagon::V6_vadd_sf : Hexagon::V6_vsub_sf), Dest) + .addReg(Reg1) + .addReg(VR2); + else + return false; + } else + return false; + return true; +} + +// Handle qf16 = vadd/vsub(qf16, qf16/hf) +// Handle vadd/vsub instructions with qf16 operands conditionally +// isAdd: true if an add instruction is analyzed, false for subtract +// isFirstOpQf: true if 1st operand is qf16 type, false if hf type +// isSecOpQf: true if 2nd operand is qf16 type, false if hf type +bool HexagonXQFloatGenerator::convertAddOpToIEEE16( + MachineInstr &MI, Register &Reg1, Register &Reg2, Register &Dest, + bool isAdd, bool isFirstOpQf, bool isSecOpQf) { + + MachineBasicBlock &MBB = *MI.getParent(); + const DebugLoc &DL = MI.getDebugLoc(); + Register VR1; + Register VR2; + bool firstconvert = false, secondconvert = false; + + // If the first qf16 operand is from add/sub/mul unit, + // generate IEEE conversion instruction + if (isFirstOpQf) { + if (checkIfInputFromAdder16(Reg1) || checkIfInputFromMult16(Reg1)) { + VR1 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vconv_hf_qf16), VR1) + .addReg(Reg1); + firstconvert = true; + } + } + if (isSecOpQf) { + // If the second operand is from add/sub/mul unit, + // generate IEEE conversion instruction + if (checkIfInputFromAdder16(Reg2) || checkIfInputFromMult16(Reg2)) { + VR2 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vconv_hf_qf16), VR2) + .addReg(Reg2); + secondconvert = true; + } + } + + // If both operands are qf16 type, use V6_v[add/sub]_hf instruction + // If one of them is of hf type, use V6_v[add/sub]_qf16_mix instruction + // Output is qf16 + if (isFirstOpQf && isSecOpQf) { + if (firstconvert && secondconvert) { + BuildMI(MBB, MI, DL, + HII->get(isAdd ? Hexagon::V6_vadd_hf : Hexagon::V6_vsub_hf), Dest) + .addReg(VR1) + .addReg(VR2); + } else if (firstconvert) { + if (isAdd) + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vadd_qf16_mix), Dest) + .addReg(Reg2) + .addReg(VR1); + // For vsub type, for v81 we use a different opcode, + // for v79, we convert the 2nd op to IEEE too. + else { + if (HST->useHVXV81Ops()) + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vsub_hf_mix), Dest) + .addReg(VR1) + .addReg(Reg2); + else { + VR2 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vconv_hf_qf16), VR2) + .addReg(Reg2); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vsub_hf), Dest) + .addReg(VR1) + .addReg(VR2); + } + } + } else if (secondconvert) { + BuildMI(MBB, MI, DL, + HII->get(isAdd ? Hexagon::V6_vadd_qf16_mix + : Hexagon::V6_vsub_qf16_mix), + Dest) + .addReg(Reg1) + .addReg(VR2); + } else { // none of the inputs is from an add/sub/mul unit + return false; + } + // handle vadd/vsub when the 1st op of original instruction is qf type + } else if (isFirstOpQf) { + if (firstconvert) + BuildMI(MBB, MI, DL, + HII->get(isAdd ? Hexagon::V6_vadd_hf : Hexagon::V6_vsub_hf), Dest) + .addReg(VR1) + .addReg(Reg2); + else + return false; + // handle vadd/vsub when the 2nd op of original instruction is qf type + } else if (isSecOpQf) { + if (secondconvert) + BuildMI(MBB, MI, DL, + HII->get(isAdd ? Hexagon::V6_vadd_hf : Hexagon::V6_vsub_hf), Dest) + .addReg(Reg1) + .addReg(VR2); + else + return false; + } else + return false; + return true; +} + +// Create the prolog +// v0 = #0 +// R1 = #0x80000000 +// v1.sf = vsplat(R1) +// v2.sf = vmpy(v0.sf, v1.sf) +void HexagonXQFloatGenerator::createPrologInstructions(MachineInstr &MI, + Register &R_mpy) { + + MachineBasicBlock &MBB = *MI.getParent(); + const DebugLoc &DL = MI.getDebugLoc(); + + Register VR0 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vd0), VR0); + + Register R_0 = MRI->createVirtualRegister(&Hexagon::IntRegsRegClass); + BuildMI(MBB, MI, DL, HII->get(Hexagon::A2_tfrsi), R_0).addImm(0x80000000); + + Register VR_0 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_lvsplatw), VR_0).addReg(R_0); + + R_mpy = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32_sf), R_mpy) + .addReg(VR0) + .addReg(VR_0); +} + +bool HexagonXQFloatGenerator::V81normalizeMultF32( + MachineInstr &MI, Register &Reg1, Register &Reg2, Register &Dest, + bool firstconvert, bool secondconvert, bool strictieee) { + MachineBasicBlock &MBB = *MI.getParent(); + const DebugLoc &DL = MI.getDebugLoc(); + Register input_mpy1, input_mpy2; + + auto Op = + strictieee ? Hexagon::V6_vconv_qf32_sf : Hexagon::V6_vconv_qf32_qf32; + + // Normalize both input operands + if (firstconvert && secondconvert) { + input_mpy1 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass); + input_mpy2 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass); + + BuildMI(MBB, MI, DL, HII->get(Op), input_mpy1).addReg(Reg1); + BuildMI(MBB, MI, DL, HII->get(Op), input_mpy2).addReg(Reg2); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32), Dest) + .addReg(input_mpy1) + .addReg(input_mpy2); + } + // Normalize only first operand + else if (firstconvert) { + input_mpy1 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass); + BuildMI(MBB, MI, DL, HII->get(Op), input_mpy1).addReg(Reg1); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32), Dest) + .addReg(input_mpy1) + .addReg(Reg2); + } + // Normalize only second operand + else if (secondconvert) { + input_mpy2 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass); + BuildMI(MBB, MI, DL, HII->get(Op), input_mpy2).addReg(Reg2); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32), Dest) + .addReg(Reg1) + .addReg(input_mpy2); + } else + // we do nothing if the inputs are not from adder/sub/mult unit + return false; + + return true; +} + +// Normalize qf32 = vmpy(sf, sf) instruction unconditionally +void HexagonXQFloatGenerator::normalizeMultiplicationInputSF( + MachineInstr &MI, Register &Src1, Register &Src2, Register &Dest, + Register &R_mpy, bool &PrologCreated) { + + MachineBasicBlock &MBB = *MI.getParent(); + const DebugLoc &DL = MI.getDebugLoc(); + + if (HST->useHVXV81Ops()) { + Register input_mpy1 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass); + Register input_mpy2 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass); + + // Normalize both inputs + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vconv_qf32_sf), input_mpy1) + .addReg(Src1); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vconv_qf32_sf), input_mpy2) + .addReg(Src2); + // Add the new vmpy + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32), Dest) + .addReg(input_mpy1) + .addReg(input_mpy2); + return; + } + + if (!PrologCreated) { + createPrologInstructions(MI, R_mpy); + PrologCreated = true; + } + + Register input_mpy1 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass); + Register input_mpy2 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass); + // Normalize both inputs + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vadd_qf32_mix), input_mpy1) + .addReg(R_mpy) + .addReg(Src1); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vadd_qf32_mix), input_mpy2) + .addReg(R_mpy) + .addReg(Src2); + // Add the new vmpy + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32), Dest) + .addReg(input_mpy1) + .addReg(input_mpy2); +} + +// Convert and normalize qf32 = vmpy(qf32, qf32) instructions conditionally +bool HexagonXQFloatGenerator::convertNormalizeMultOp32( + MachineInstr &MI, Register &Reg1, Register &Reg2, Register &Dest, + Register &R_mpy, bool &PrologCreated) { + + Register VR1, VR2; + bool firstconvert = false, secondconvert = false; + MachineBasicBlock &MBB = *MI.getParent(); + const DebugLoc &DL = MI.getDebugLoc(); + + // If the first operand is from add/subtract/multiply unit, generate IEEE + // conversion instruction + if (checkIfInputFromAdder32(Reg1) || checkIfInputFromMult32(Reg1)) { + VR1 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vconv_sf_qf32), VR1).addReg(Reg1); + firstconvert = true; + } + + if (checkIfInputFromAdder32(Reg2) || checkIfInputFromMult32(Reg2)) { + VR2 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vconv_sf_qf32), VR2).addReg(Reg2); + secondconvert = true; + } + + if (HST->useHVXV81Ops()) { + if (firstconvert && secondconvert) + return V81normalizeMultF32(MI, VR1, VR2, Dest, true, true, true); + else if (firstconvert) + return V81normalizeMultF32(MI, VR1, Reg2, Dest, true, false, true); + else if (secondconvert) + return V81normalizeMultF32(MI, Reg1, VR2, Dest, false, true, true); + else + return false; + } + + // create prolog if not already created + if (!PrologCreated && (firstconvert || secondconvert)) { + createPrologInstructions(MI, R_mpy); + PrologCreated = true; + } + + Register input_mpy1, input_mpy2; + + // Normalize both IEEE converts + if (firstconvert && secondconvert) { + input_mpy2 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass); + input_mpy1 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass); + + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vadd_qf32_mix), input_mpy1) + .addReg(R_mpy) + .addReg(VR1); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vadd_qf32_mix), input_mpy2) + .addReg(R_mpy) + .addReg(VR2); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32), Dest) + .addReg(input_mpy1) + .addReg(input_mpy2); + // Normalize only first operand + } else if (firstconvert) { + input_mpy1 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass); + + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vadd_qf32_mix), input_mpy1) + .addReg(R_mpy) + .addReg(VR1); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32), Dest) + .addReg(input_mpy1) + .addReg(Reg2); + // Normalize only second operand + } else if (secondconvert) { + input_mpy2 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass); + + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vadd_qf32_mix), input_mpy2) + .addReg(R_mpy) + .addReg(VR2); + BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32), Dest) + .addReg(input_mpy2) + .addReg(Reg2); ---------------- nikic wrote:
Was this supposed to be Reg1? https://github.com/llvm/llvm-project/pull/198902 _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
