================ @@ -0,0 +1,1664 @@ +//===-- lib/CodeGen/GlobalISel/PISAPreLegalizerCombiner.cpp ---------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "MCTargetDesc/PISAMCTargetDesc.h" +#include "PISA.h" +#include "PISALegalizerInfo.h" +#include "PISATargetMachine.h" +#include "PISAUtils.h" +#include "llvm/ADT/FloatingPointMode.h" +#include "llvm/ADT/bit.h" +#include "llvm/CodeGen/GlobalISel/CSEInfo.h" +#include "llvm/CodeGen/GlobalISel/Combiner.h" +#include "llvm/CodeGen/GlobalISel/CombinerHelper.h" +#include "llvm/CodeGen/GlobalISel/CombinerInfo.h" +#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h" +#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" +#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" +#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h" +#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" +#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" +#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" +#include "llvm/CodeGen/GlobalISel/Utils.h" +#include "llvm/CodeGen/MachineDominators.h" +#include "llvm/CodeGen/TargetPassConfig.h" +#include "llvm/IR/IntrinsicsPISA.h" +#include "llvm/Target/TargetMachine.h" + +#define GET_GICOMBINER_DEPS +#include "PISAGenPreLegalizeGICombiner.inc" +#undef GET_GICOMBINER_DEPS + +#define DEBUG_TYPE "pisa-prelegalizer-combiner" + +using namespace llvm; +using namespace llvm::MIPatternMatch; + +namespace { + +#define GET_GICOMBINER_TYPES +#include "PISAGenPreLegalizeGICombiner.inc" +#undef GET_GICOMBINER_TYPES + +class PISAPreLegalizerCombinerImpl : public Combiner { +protected: + const PISAPreLegalizerCombinerImplRuleConfig &RuleConfig; + const PISASubtarget &STI; + MachineDominatorTree *MDT; + + // TODO: Make CombinerHelper methods const. + mutable CombinerHelper Helper; + +public: + PISAPreLegalizerCombinerImpl( + MachineFunction &MF, CombinerInfo &CInfo, GISelValueTracking &KB, + GISelCSEInfo *CSEInfo, + const PISAPreLegalizerCombinerImplRuleConfig &RuleConfig, + const PISASubtarget &STI, MachineDominatorTree *MDT, + const LegalizerInfo *LI); + + static const char *getName() { return "PISAGenPreLegalizeGICombiner"; } + + bool tryCombineAllImpl(MachineInstr &MI) const; + bool tryCombineAll(MachineInstr &I) const override; + + void applyTruncatedLoad(MachineInstr &MI) const; + + bool matchTruncatedStore(MachineInstr &MI) const; + void applyTruncatedStore(MachineInstr &MI) const; + + bool matchExtendedLoad(MachineInstr &MI) const; + void applyExtendedLoad(MachineInstr &MI) const; + + bool matchSimplifyNonPowerOf2LoadStoreChain( + MachineInstr &MI, + SmallVector<std::pair<MachineInstr *, unsigned>, 8> &Loads, + MachineInstr *&SizeModificationOp) const; + void applySimplifyNonPowerOf2LoadStoreChain( + MachineInstr &MI, + SmallVector<std::pair<MachineInstr *, unsigned>, 8> &Loads, + MachineInstr *&SizeModificationOp) const; + + bool matchExpandNonPowerOf2LoadStore(MachineInstr &MI) const; + void applyExpandNonPowerOf2LoadStore(MachineInstr &MI) const; + + bool matchTruncatedShift(MachineInstr &MI) const; + void applyTruncatedShift(MachineInstr &MI) const; + + bool matchRedundantMovesPre(MachineInstr &MI) const; + void applyRedundantMovesPre(MachineInstr &MI) const; + + bool + matchRcpSqrtToRsqrt(MachineInstr &MI, + std::function<void(MachineIRBuilder &)> &MatchInfo) const; + bool + matchSubFloorToFrc(MachineInstr &MI, + std::function<void(MachineIRBuilder &)> &MatchInfo) const; + + bool matchLaneIdLeftShiftChain( + MachineInstr &MI, + std::function<void(MachineIRBuilder &)> &MatchInfo) const; + + bool matchZExtAndToAndZExt( + MachineInstr &MI, + std::function<void(MachineIRBuilder &)> &MatchInfo) const; + + bool matchExtractInsertToBitcast(MachineInstr &MI, Register &) const; + void applyExtractInsertToBitcast(MachineInstr &MI, Register) const; + + bool matchExtractBuildVectorToBitcast(MachineInstr &MI, Register &) const; + void applyExtractBuildVectorToBitcast(MachineInstr &MI, Register) const; + + bool matchReducePredicates( + MachineInstr &MI, + std::function<void(MachineIRBuilder &)> &MatchInfo) const; + + bool matchCmpInt1(MachineInstr &MI, + std::function<void(MachineIRBuilder &)> &MatchInfo) const; + + bool matchSelectTruncOneZero( + MachineInstr &MI, + std::function<void(MachineIRBuilder &)> &MatchInfo) const; + +private: +#define GET_GICOMBINER_CLASS_MEMBERS +#include "PISAGenPreLegalizeGICombiner.inc" +#undef GET_GICOMBINER_CLASS_MEMBERS +}; + +#define GET_GICOMBINER_IMPL +#include "PISAGenPreLegalizeGICombiner.inc" +#undef GET_GICOMBINER_IMPL + +PISAPreLegalizerCombinerImpl::PISAPreLegalizerCombinerImpl( + MachineFunction &MF, CombinerInfo &CInfo, GISelValueTracking &KB, + GISelCSEInfo *CSEInfo, + const PISAPreLegalizerCombinerImplRuleConfig &RuleConfig, + const PISASubtarget &STI, MachineDominatorTree *MDT, + const LegalizerInfo *LI) + : Combiner(MF, CInfo, &KB, CSEInfo), RuleConfig(RuleConfig), STI(STI), + MDT(MDT), Helper(Observer, B, /*IsPreLegalize=*/true, &KB, MDT, LI), +#define GET_GICOMBINER_CONSTRUCTOR_INITS +#include "PISAGenPreLegalizeGICombiner.inc" +#undef GET_GICOMBINER_CONSTRUCTOR_INITS +{ +} + +bool PISAPreLegalizerCombinerImpl::tryCombineAll(MachineInstr &MI) const { + // G_FPTRUNC to bf16 needs special constant folding because + // getFltSemanticForLLT() doesn't support BFloat16 yet. + if (MI.getOpcode() == TargetOpcode::G_FPTRUNC && + MRI.getType(MI.getOperand(0).getReg()).getScalarType().isBFloat16()) { + const ConstantFP *Cst = nullptr; + if (mi_match(MI.getOperand(1).getReg(), MRI, m_GFCst(Cst))) { + APFloat Result(Cst->getValue()); + bool Unused; + Result.convert(APFloat::BFloat(), APFloat::rmNearestTiesToEven, &Unused); + const ConstantFP *NewCst = ConstantFP::get(B.getContext(), Result); + MachineIRBuilder Builder(MI); + Builder.buildFConstant(MI.getOperand(0), *NewCst); + MI.eraseFromParent(); + return true; + } + } + + if (tryCombineAllImpl(MI)) + return true; + + return false; +} + +void PISAPreLegalizerCombinerImpl::applyTruncatedLoad(MachineInstr &MI) const { + auto *LoadMI = getDefIgnoringCopies(MI.getOperand(1).getReg(), MRI); + auto *MMO = LoadMI->memoperands()[0]; + auto Dst = MI.getOperand(0); + auto Addr = LoadMI->getOperand(1); + auto *NewMMO = MI.getMF()->getMachineMemOperand(MMO, MMO->getOffset(), + MRI.getType(Dst.getReg())); + B.buildLoad(Dst, Addr, *NewMMO); + MI.eraseFromParent(); +} + +bool PISAPreLegalizerCombinerImpl::matchExpandNonPowerOf2LoadStore( + MachineInstr &MI) const { + assert(MI.getOpcode() == TargetOpcode::G_LOAD || + MI.getOpcode() == TargetOpcode::G_STORE); + GLoadStore &LS = cast<GLoadStore>(MI); + + auto Size = LS.getMemSizeInBits().getValue(); + + if (isPowerOf2_32(Size)) + return false; + + if (LS.getMMO().getMemoryType().isVector()) + return false; + + // Check if this is a load and only has zext uses that are handled by the + // extended load pattern - if yes, we want to handle it via said pattern. + if (MI.getOpcode() == TargetOpcode::G_LOAD) { + for (auto &Use : MRI.use_operands(MI.getOperand(0).getReg())) { + auto *Inst = Use.getParent(); + if (Inst->getOpcode() != TargetOpcode::G_ZEXT || + !matchExtendedLoad(*Inst)) + return true; + } + return false; + } + + return true; +} +void PISAPreLegalizerCombinerImpl::applyExpandNonPowerOf2LoadStore( + MachineInstr &MI) const { + assert(MI.getOpcode() == TargetOpcode::G_LOAD || + MI.getOpcode() == TargetOpcode::G_STORE); + + GLoadStore &LS = cast<GLoadStore>(MI); + + auto PointerReg = LS.getPointerReg(); + auto ValueReg = LS.getOperand(0).getReg(); + auto &MMO = LS.getMMO(); + + /// The remaining size in Bits that still has to be loaded/stored + ssize_t Size = LS.getMemSizeInBits().getValue(); + // Support sizes that are not a multiple of 8 by "promoting" them to the next + // multiple of 8. If the size is already a multiple of 8, it is not modified + Size = (Size + 7) & ~7; + + unsigned Offset = 0; + const auto SizeTy = LLT::integer(Size); + + // If the size was changed to the next power of 8 and we are storing a value, + // we need to modify the register's type as well. + // A similar check is needed for loads, but this is done at the end + if (MI.getOpcode() == TargetOpcode::G_STORE && + SizeTy != MRI.getType(ValueReg)) { + auto NewValueReg = MRI.createGenericVirtualRegister(SizeTy); + B.buildZExt(NewValueReg, ValueReg); + ValueReg = NewValueReg; + } + + /// The register holding the loaded value at the end + Register LoadRes; + while (Size > 0) { + auto OpSize = bit_floor(static_cast<size_t>(Size)); + const auto ShiftAmount = Offset * 8; + + const LLT OpTy = LLT::integer(OpSize); + + auto *NewMMO = + MI.getMF()->getMachineMemOperand(&MMO, MMO.getOffset() + Offset, OpTy); + + /// Stores the (potentially modified) pointer register + auto AddrReg = PointerReg; + // We might need to change the store offset + if (Offset != 0) { + auto NewAddr = MRI.cloneVirtualRegister(AddrReg); + + // Get the pointer size from the pointer register type + const LLT PtrTy = MRI.getType(AddrReg); + const LLT IntTy = LLT::integer(PtrTy.getSizeInBits()); + auto Const = MRI.createGenericVirtualRegister(IntTy); + B.buildConstant(Const, Offset); + B.buildPtrAdd(NewAddr, AddrReg, Const); + + AddrReg = NewAddr; + } + + if (MI.getOpcode() == TargetOpcode::G_STORE) { + auto Res = ValueReg; + // If we're not storing from the start, we need to shift our value first + if (Offset != 0) { + auto ShrRes = MRI.createGenericVirtualRegister(SizeTy); + auto ShiftConst = MRI.createGenericVirtualRegister(SizeTy); + B.buildConstant(ShiftConst, ShiftAmount); + B.buildLShr(ShrRes, ValueReg, ShiftConst); + Res = ShrRes; + } + + // Next, truncate it to the correct size, if needed + if (SizeTy != OpTy) { + auto TruncRes = MRI.createGenericVirtualRegister(OpTy); + B.buildTrunc(TruncRes, Res); + Res = TruncRes; + } + + B.buildStore(Res, AddrReg, *NewMMO); + } else { + // When loading, do the same thing but basically in reverse + // First, load the value + auto LoadedValReg = MRI.createGenericVirtualRegister(OpTy); + B.buildLoad(LoadedValReg, AddrReg, *NewMMO); + auto Res = LoadedValReg; + + // Extend it to the correct size, if needed + if (SizeTy != OpTy) { + auto ZextRes = MRI.createGenericVirtualRegister(SizeTy); + B.buildZExt(ZextRes, LoadedValReg); + Res = ZextRes; + } + + // If we're not loading the first Bytes, then we need to shift it + if (Offset != 0) { + auto ShiftRes = MRI.createGenericVirtualRegister(SizeTy); + + auto ShiftConst = MRI.createGenericVirtualRegister(SizeTy); + B.buildConstant(ShiftConst, ShiftAmount); + + B.buildShl(ShiftRes, Res, ShiftConst); + Res = ShiftRes; + } + + if (!LoadRes.isValid()) { + LoadRes = MRI.createGenericVirtualRegister(SizeTy); + B.buildConstant(LoadRes, 0); + } + + auto NewLoadRes = MRI.createGenericVirtualRegister(SizeTy); + B.buildOr(NewLoadRes, LoadRes, Res); + LoadRes = NewLoadRes; + } + + Offset += OpSize / 8; + Size -= OpSize; + } + + if (MI.getOpcode() == TargetOpcode::G_LOAD) { + // If we modified the original size of the load (to support sizes that are + // not a multiple of 8), we need to truncate it to the correct size again, + // in order not to break any following instructions + if (SizeTy != MRI.getType(ValueReg)) { + auto TruncRes = MRI.createGenericVirtualRegister(MRI.getType(ValueReg)); + B.buildTrunc(TruncRes, LoadRes); + LoadRes = TruncRes; + } + MRI.replaceRegWith(ValueReg, LoadRes); + } + + MI.eraseFromParent(); +} + +/// If the value we store resulted from a G_LOAD that the rule above expanded, +/// then we can use the individual load values directly instead of merging into +/// one integer, and then splitting it again. Size modifications between the +/// loads and store (i.e. SEXT/ZEXT/TRUNC) are also supported by +/// truncating/extending the relevant load results (or, in the case of +/// truncation, ignoring some loads entirely) +/// +/// Below is a simple example where we simply load and store an i56: +/// +/// bb.1.entry: +/// %0:reg32b(p0) = functionParameter_32b 0 +/// -> %3:_(s56) = G_ZEXTLOAD %0:reg32b(p0) :: +/// (load (s32) from %ir.dst) +/// %4:_(s56) = G_CONSTANT i56 0 +/// %7:_(s32) = G_CONSTANT i32 4 +/// %6:reg32b(p0) = G_PTR_ADD %0:reg32b, %7:_(s32) +/// -> %9:_(s56) = G_ZEXTLOAD %6:reg32b(p0) :: +/// (load (s16) from %ir.dst + 4, align 4) +/// %11:_(s56) = G_CONSTANT i56 32 +/// %10:_(s56) = G_SHL %9:_, %11:_(s56) +/// %12:_(s56) = G_OR %3:_, %10:_ +/// %14:_(s32) = G_CONSTANT i32 6 +/// %13:reg32b(p0) = G_PTR_ADD %0:reg32b, %14:_(s32) +/// -> %16:_(s56) = G_ZEXTLOAD %13:reg32b(p0) :: +/// (load (s8) from %ir.dst + 6, align 2, basealign 4) +/// %18:_(s56) = G_CONSTANT i56 48 +/// %17:_(s56) = G_SHL %16:_, %18:_(s56) +/// %19:_(s56) = G_OR %12:_, %17:_ +/// G_STORE %19:_(s56), %0:reg32b(p0) :: +/// (store (s56) into %ir.dst, align 4) +/// ret +/// +/// @param Loads stores all the load instructions that we find along the way. +/// The second parameter in the pair is the offset from which the value was +/// loaded +/// @param SizeModificationOp stores the opcode to the instruction that modified +/// the size of the integer after loading and before storing, if it exists. +/// Possible opcode values are null (none), G_SEXT, G_SEXT_INREG, G_ZEXT, and +/// G_TRUNC +bool PISAPreLegalizerCombinerImpl::matchSimplifyNonPowerOf2LoadStoreChain( + MachineInstr &MI, + SmallVector<std::pair<MachineInstr *, unsigned>, 8> &Loads, + MachineInstr *&SizeModificationOp) const { + GLoadStore &StoreInst = cast<GLoadStore>(MI); + + auto ValueReg = StoreInst.getOperand(0).getReg(); + + /// This has the integer size at the beginning, and each individual load + /// decreases the value by its load size. At the end, this must be zero. + /// NB: We use the size of the store here, promoted to the next multiple + /// of 8. This size might be modified later if we find a G_*EXT/G_TRUNC. + unsigned ValueSize = StoreInst.getMemSize().getValue() * 8; + + /// This stores the size of the last load instruction in Bytes, s.t. we can + /// verify that the current load is larger than the previous one ---------------- v-klochkov wrote:
```suggestion // This has the integer size at the beginning, and each individual load // decreases the value by its load size. At the end, this must be zero. // NB: We use the size of the store here, promoted to the next multiple // of 8. This size might be modified later if we find a G_*EXT/G_TRUNC. unsigned ValueSize = StoreInst.getMemSize().getValue() * 8; // This stores the size of the last load instruction in Bytes, s.t. we can // verify that the current load is larger than the previous one ``` https://github.com/llvm/llvm-project/pull/214373 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
