================ @@ -0,0 +1,1664 @@ +//===-- lib/CodeGen/GlobalISel/PISAPreLegalizerCombiner.cpp ---------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "MCTargetDesc/PISAMCTargetDesc.h" +#include "PISA.h" +#include "PISALegalizerInfo.h" +#include "PISATargetMachine.h" +#include "PISAUtils.h" +#include "llvm/ADT/FloatingPointMode.h" +#include "llvm/ADT/bit.h" +#include "llvm/CodeGen/GlobalISel/CSEInfo.h" +#include "llvm/CodeGen/GlobalISel/Combiner.h" +#include "llvm/CodeGen/GlobalISel/CombinerHelper.h" +#include "llvm/CodeGen/GlobalISel/CombinerInfo.h" +#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h" +#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" +#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" +#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h" +#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" +#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" +#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" +#include "llvm/CodeGen/GlobalISel/Utils.h" +#include "llvm/CodeGen/MachineDominators.h" +#include "llvm/CodeGen/TargetPassConfig.h" +#include "llvm/IR/IntrinsicsPISA.h" +#include "llvm/Target/TargetMachine.h" + +#define GET_GICOMBINER_DEPS +#include "PISAGenPreLegalizeGICombiner.inc" +#undef GET_GICOMBINER_DEPS + +#define DEBUG_TYPE "pisa-prelegalizer-combiner" + +using namespace llvm; +using namespace llvm::MIPatternMatch; + +namespace { + +#define GET_GICOMBINER_TYPES +#include "PISAGenPreLegalizeGICombiner.inc" +#undef GET_GICOMBINER_TYPES + +class PISAPreLegalizerCombinerImpl : public Combiner { +protected: + const PISAPreLegalizerCombinerImplRuleConfig &RuleConfig; + const PISASubtarget &STI; + MachineDominatorTree *MDT; + + // TODO: Make CombinerHelper methods const. + mutable CombinerHelper Helper; + +public: + PISAPreLegalizerCombinerImpl( + MachineFunction &MF, CombinerInfo &CInfo, GISelValueTracking &KB, + GISelCSEInfo *CSEInfo, + const PISAPreLegalizerCombinerImplRuleConfig &RuleConfig, + const PISASubtarget &STI, MachineDominatorTree *MDT, + const LegalizerInfo *LI); + + static const char *getName() { return "PISAGenPreLegalizeGICombiner"; } + + bool tryCombineAllImpl(MachineInstr &MI) const; + bool tryCombineAll(MachineInstr &I) const override; + + void applyTruncatedLoad(MachineInstr &MI) const; + + bool matchTruncatedStore(MachineInstr &MI) const; + void applyTruncatedStore(MachineInstr &MI) const; + + bool matchExtendedLoad(MachineInstr &MI) const; + void applyExtendedLoad(MachineInstr &MI) const; + + bool matchSimplifyNonPowerOf2LoadStoreChain( + MachineInstr &MI, + SmallVector<std::pair<MachineInstr *, unsigned>, 8> &Loads, + MachineInstr *&SizeModificationOp) const; + void applySimplifyNonPowerOf2LoadStoreChain( + MachineInstr &MI, + SmallVector<std::pair<MachineInstr *, unsigned>, 8> &Loads, + MachineInstr *&SizeModificationOp) const; + + bool matchExpandNonPowerOf2LoadStore(MachineInstr &MI) const; + void applyExpandNonPowerOf2LoadStore(MachineInstr &MI) const; + + bool matchTruncatedShift(MachineInstr &MI) const; + void applyTruncatedShift(MachineInstr &MI) const; + + bool matchRedundantMovesPre(MachineInstr &MI) const; + void applyRedundantMovesPre(MachineInstr &MI) const; + + bool + matchRcpSqrtToRsqrt(MachineInstr &MI, + std::function<void(MachineIRBuilder &)> &MatchInfo) const; + bool + matchSubFloorToFrc(MachineInstr &MI, + std::function<void(MachineIRBuilder &)> &MatchInfo) const; + + bool matchLaneIdLeftShiftChain( + MachineInstr &MI, + std::function<void(MachineIRBuilder &)> &MatchInfo) const; + + bool matchZExtAndToAndZExt( + MachineInstr &MI, + std::function<void(MachineIRBuilder &)> &MatchInfo) const; + + bool matchExtractInsertToBitcast(MachineInstr &MI, Register &) const; + void applyExtractInsertToBitcast(MachineInstr &MI, Register) const; + + bool matchExtractBuildVectorToBitcast(MachineInstr &MI, Register &) const; + void applyExtractBuildVectorToBitcast(MachineInstr &MI, Register) const; + + bool matchReducePredicates( + MachineInstr &MI, + std::function<void(MachineIRBuilder &)> &MatchInfo) const; + + bool matchCmpInt1(MachineInstr &MI, + std::function<void(MachineIRBuilder &)> &MatchInfo) const; + + bool matchSelectTruncOneZero( + MachineInstr &MI, + std::function<void(MachineIRBuilder &)> &MatchInfo) const; + +private: +#define GET_GICOMBINER_CLASS_MEMBERS +#include "PISAGenPreLegalizeGICombiner.inc" +#undef GET_GICOMBINER_CLASS_MEMBERS +}; + +#define GET_GICOMBINER_IMPL +#include "PISAGenPreLegalizeGICombiner.inc" +#undef GET_GICOMBINER_IMPL + +PISAPreLegalizerCombinerImpl::PISAPreLegalizerCombinerImpl( + MachineFunction &MF, CombinerInfo &CInfo, GISelValueTracking &KB, + GISelCSEInfo *CSEInfo, + const PISAPreLegalizerCombinerImplRuleConfig &RuleConfig, + const PISASubtarget &STI, MachineDominatorTree *MDT, + const LegalizerInfo *LI) + : Combiner(MF, CInfo, &KB, CSEInfo), RuleConfig(RuleConfig), STI(STI), + MDT(MDT), Helper(Observer, B, /*IsPreLegalize=*/true, &KB, MDT, LI), +#define GET_GICOMBINER_CONSTRUCTOR_INITS +#include "PISAGenPreLegalizeGICombiner.inc" +#undef GET_GICOMBINER_CONSTRUCTOR_INITS +{ +} + +bool PISAPreLegalizerCombinerImpl::tryCombineAll(MachineInstr &MI) const { + // G_FPTRUNC to bf16 needs special constant folding because + // getFltSemanticForLLT() doesn't support BFloat16 yet. + if (MI.getOpcode() == TargetOpcode::G_FPTRUNC && + MRI.getType(MI.getOperand(0).getReg()).getScalarType().isBFloat16()) { + const ConstantFP *Cst = nullptr; + if (mi_match(MI.getOperand(1).getReg(), MRI, m_GFCst(Cst))) { + APFloat Result(Cst->getValue()); + bool Unused; + Result.convert(APFloat::BFloat(), APFloat::rmNearestTiesToEven, &Unused); + const ConstantFP *NewCst = ConstantFP::get(B.getContext(), Result); + MachineIRBuilder Builder(MI); + Builder.buildFConstant(MI.getOperand(0), *NewCst); + MI.eraseFromParent(); + return true; + } + } + + if (tryCombineAllImpl(MI)) + return true; + + return false; +} + +void PISAPreLegalizerCombinerImpl::applyTruncatedLoad(MachineInstr &MI) const { + auto *LoadMI = getDefIgnoringCopies(MI.getOperand(1).getReg(), MRI); + auto *MMO = LoadMI->memoperands()[0]; + auto Dst = MI.getOperand(0); + auto Addr = LoadMI->getOperand(1); + auto *NewMMO = MI.getMF()->getMachineMemOperand(MMO, MMO->getOffset(), + MRI.getType(Dst.getReg())); + B.buildLoad(Dst, Addr, *NewMMO); + MI.eraseFromParent(); +} + +bool PISAPreLegalizerCombinerImpl::matchExpandNonPowerOf2LoadStore( + MachineInstr &MI) const { + assert(MI.getOpcode() == TargetOpcode::G_LOAD || + MI.getOpcode() == TargetOpcode::G_STORE); + GLoadStore &LS = cast<GLoadStore>(MI); + + auto Size = LS.getMemSizeInBits().getValue(); + + if (isPowerOf2_32(Size)) + return false; + + if (LS.getMMO().getMemoryType().isVector()) + return false; + + // Check if this is a load and only has zext uses that are handled by the + // extended load pattern - if yes, we want to handle it via said pattern. + if (MI.getOpcode() == TargetOpcode::G_LOAD) { + for (auto &Use : MRI.use_operands(MI.getOperand(0).getReg())) { + auto *Inst = Use.getParent(); + if (Inst->getOpcode() != TargetOpcode::G_ZEXT || + !matchExtendedLoad(*Inst)) + return true; + } + return false; + } + + return true; +} +void PISAPreLegalizerCombinerImpl::applyExpandNonPowerOf2LoadStore( + MachineInstr &MI) const { + assert(MI.getOpcode() == TargetOpcode::G_LOAD || + MI.getOpcode() == TargetOpcode::G_STORE); + + GLoadStore &LS = cast<GLoadStore>(MI); + + auto PointerReg = LS.getPointerReg(); + auto ValueReg = LS.getOperand(0).getReg(); + auto &MMO = LS.getMMO(); + + /// The remaining size in Bits that still has to be loaded/stored ---------------- v-klochkov wrote:
Minor issue: I did not check all places, but some comments are `///` instead of `//`. https://github.com/llvm/llvm-project/pull/214373 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
