Author: Ryan Buchner Date: 2026-09-17T14:01:51-07:00 New Revision: a7ecfe57a587d0a5465503581bc0bfce170e96b5
URL: https://github.com/llvm/llvm-project/commit/a7ecfe57a587d0a5465503581bc0bfce170e96b5 DIFF: https://github.com/llvm/llvm-project/commit/a7ecfe57a587d0a5465503581bc0bfce170e96b5.diff LOG: Revert "[SLP] More accurately cost RISCV scalar splats (#213104)" This reverts commit bb9a57a70d35e36d4c8258cc9a15e5c29041d0d7. Added: Modified: llvm/include/llvm/Analysis/TargetTransformInfo.h llvm/include/llvm/Analysis/TargetTransformInfoImpl.h llvm/include/llvm/CodeGen/BasicTTIImpl.h llvm/lib/Analysis/TargetTransformInfo.cpp llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp llvm/lib/Target/ARM/ARMTargetTransformInfo.h llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h llvm/lib/Target/X86/X86TargetTransformInfo.cpp llvm/lib/Target/X86/X86TargetTransformInfo.h llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.cpp llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.h llvm/test/Transforms/SLPVectorizer/RISCV/check-node-without-vector-user.ll llvm/test/Transforms/SLPVectorizer/RISCV/folded-broadcast-cost.ll llvm/test/Transforms/SLPVectorizer/RISCV/remark-zext-incoming-for-neg-icmp.ll llvm/test/Transforms/SLPVectorizer/RISCV/reordered-buildvector-scalars.ll Removed: ################################################################################ diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h index 0b183954198dd..f1731bc364555 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -24,7 +24,6 @@ #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/BitmaskEnum.h" -#include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/ADT/Uniformity.h" #include "llvm/Analysis/IVDescriptors.h" #include "llvm/Analysis/InterestingMemoryOperand.h" @@ -191,8 +190,7 @@ enum class VectorInstrContext : uint8_t { None, ///< The instruction is not folded. Load, ///< The value being inserted comes from a load (InsertElement only). Store, ///< The extracted value is stored (ExtractElement only). - BinaryOp, ///< One of the operands is a binary op. - SplatOpFolded, ///< All of the value's users support splatting the value. + BinaryOp, ///< One of the operands is a binary op. }; class IntrinsicCostAttributes { @@ -1074,34 +1072,10 @@ class TargetTransformInfo { using VectorInstrContext = llvm::VectorInstrContext; - /// Combines 2 context hints into a single value. If both are equal, keep the - /// shared context, otherwise fall back to no specific context. - LLVM_ABI static TargetTransformInfo::VectorInstrContext - combineVectorInstrContexts(TargetTransformInfo::VectorInstrContext Ctx1, - TargetTransformInfo::VectorInstrContext Ctx2); - - /// Stores information about the uses of a build vector - struct BuildVectorUseOp { - unsigned Opcode; - int OperandIndex; - BuildVectorUseOp(unsigned Opcode, int OperandIndex) - : Opcode(Opcode), OperandIndex(OperandIndex) {} - }; - /// Calculates a VectorInstrContext from \p I. LLVM_ABI static VectorInstrContext getVectorInstrContextHint(const Instruction *I); - /// Calculates a VectorInstrContext for buildvector-like gather sequences. - /// - /// \p GatherUserOps must collect all users of \p Scalars relevant for - /// determining whether a splat can be folded as a scalar operand. It returns - /// false if those users cannot be gathered in the required form. - LLVM_ABI VectorInstrContext getBuildVectorContextHint( - ArrayRef<int> Mask, ArrayRef<Value *> Scalars, - function_ref<bool(SmallVectorImpl<BuildVectorUseOp> &)> GatherUseOps) - const; - /// Estimate the overhead of scalarizing an instruction. Insert and Extract /// are set if the demanded result elements need to be inserted and/or /// extracted from vectors. The involved values may be passed in VL if @@ -1588,8 +1562,7 @@ class TargetTransformInfo { ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, TTI::TargetCostKind CostKind, ArrayRef<int> Mask = {}, int Index = 0, VectorType *SubTp = nullptr, ArrayRef<const Value *> Args = {}, - const Instruction *CxtI = nullptr, - TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None) const; + const Instruction *CxtI = nullptr) const; /// Represents a hint about the context in which a cast is used. /// diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h index 5103d3dce25b2..84cb3a6e664b9 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -773,12 +773,11 @@ class LLVM_ABI TargetTransformInfoImplBase { return InstructionCost::getInvalid(); } - virtual InstructionCost getShuffleCost( - TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, - TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index, - VectorType *SubTp, ArrayRef<const Value *> Args = {}, - const Instruction *CxtI = nullptr, - TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None) const { + virtual InstructionCost + getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, + TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index, + VectorType *SubTp, ArrayRef<const Value *> Args = {}, + const Instruction *CxtI = nullptr) const { return 1; } @@ -1141,14 +1140,6 @@ class LLVM_ABI TargetTransformInfoImplBase { return true; } - virtual TargetTransformInfo::VectorInstrContext getBuildVectorContextHint( - ArrayRef<int> Mask, ArrayRef<Value *> Scalars, - function_ref< - bool(SmallVectorImpl<TargetTransformInfo::BuildVectorUseOp> &)> - GatherUseOps) const { - return TargetTransformInfo::VectorInstrContext::None; - } - virtual bool isElementTypeLegalForScalableVector(Type *Ty) const { return true; } diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h index 9895bc94478b4..9723016ca23b4 100644 --- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -1226,9 +1226,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> { getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index, VectorType *SubTp, ArrayRef<const Value *> Args = {}, - const Instruction *CxtI = nullptr, - TTI::VectorInstrContext VIC = - TTI::VectorInstrContext::None) const override { + const Instruction *CxtI = nullptr) const override { switch (improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTp)) { case TTI::SK_Broadcast: if (auto *FVT = dyn_cast<FixedVectorType>(SrcTy)) diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp index afd6f895039e0..a9f76a735a48c 100644 --- a/llvm/lib/Analysis/TargetTransformInfo.cpp +++ b/llvm/lib/Analysis/TargetTransformInfo.cpp @@ -643,13 +643,6 @@ bool TargetTransformInfo::isTargetIntrinsicWithStructReturnOverloadAtField( return TTIImpl->isTargetIntrinsicWithStructReturnOverloadAtField(ID, RetIdx); } -TargetTransformInfo::VectorInstrContext -TargetTransformInfo::combineVectorInstrContexts( - TargetTransformInfo::VectorInstrContext Ctx1, - TargetTransformInfo::VectorInstrContext Ctx2) { - return Ctx1 == Ctx2 ? Ctx1 : TargetTransformInfo::VectorInstrContext::None; -} - TargetTransformInfo::VectorInstrContext TargetTransformInfo::getVectorInstrContextHint(const Instruction *I) { if (!I) @@ -669,14 +662,6 @@ TargetTransformInfo::getVectorInstrContextHint(const Instruction *I) { return VectorInstrContext::None; } -TargetTransformInfo::VectorInstrContext -TargetTransformInfo::getBuildVectorContextHint( - ArrayRef<int> Mask, ArrayRef<Value *> Scalars, - function_ref<bool(SmallVectorImpl<BuildVectorUseOp> &)> GatherUseOps) - const { - return TTIImpl->getBuildVectorContextHint(Mask, Scalars, GatherUseOps); -} - InstructionCost TargetTransformInfo::getScalarizationOverhead( VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract, TTI::TargetCostKind CostKind, bool ForPoisonSrc, ArrayRef<Value *> VL, @@ -1052,15 +1037,15 @@ InstructionCost TargetTransformInfo::getAltInstrCost( InstructionCost TargetTransformInfo::getShuffleCost( ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index, - VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI, - TTI::VectorInstrContext VIC) const { + VectorType *SubTp, ArrayRef<const Value *> Args, + const Instruction *CxtI) const { assert((Mask.empty() || DstTy->isScalableTy() || Mask.size() == DstTy->getElementCount().getKnownMinValue()) && "Expected the Mask to match the return size if given"); assert(SrcTy->getScalarType() == DstTy->getScalarType() && "Expected the same scalar types"); InstructionCost Cost = TTIImpl->getShuffleCost( - Kind, DstTy, SrcTy, CostKind, Mask, Index, SubTp, Args, CxtI, VIC); + Kind, DstTy, SrcTy, CostKind, Mask, Index, SubTp, Args, CxtI); assert(Cost >= 0 && "TTI should not produce negative costs!"); return Cost; } diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index 272bd8b0af64d..8e0b88dbad0eb 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -6948,11 +6948,12 @@ InstructionCost AArch64TTIImpl::getPartialReductionCost( BinOp, CostKind, FMF); } -InstructionCost AArch64TTIImpl::getShuffleCost( - TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, - TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index, - VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI, - TTI::VectorInstrContext VIC) const { +InstructionCost +AArch64TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, + VectorType *SrcTy, TTI::TargetCostKind CostKind, + ArrayRef<int> Mask, int Index, VectorType *SubTp, + ArrayRef<const Value *> Args, + const Instruction *CxtI) const { assert((Mask.empty() || DstTy->isScalableTy() || Mask.size() == DstTy->getElementCount().getKnownMinValue()) && "Expected the Mask to match the return size if given"); diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h index f8f771525150e..dc840f5c7565f 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h @@ -516,9 +516,7 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> { getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index, VectorType *SubTp, ArrayRef<const Value *> Args = {}, - const Instruction *CxtI = nullptr, - TTI::VectorInstrContext VIC = - TTI::VectorInstrContext::None) const override; + const Instruction *CxtI = nullptr) const override; InstructionCost getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts, diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp index 740042db01a51..a7556278b7e0d 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp @@ -1379,11 +1379,13 @@ Value *GCNTTIImpl::rewriteIntrinsicWithAddressSpace(IntrinsicInst *II, } } -InstructionCost GCNTTIImpl::getShuffleCost( - TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, - TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index, - VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI, - TTI::VectorInstrContext VIC) const { +InstructionCost GCNTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, + VectorType *DstTy, VectorType *SrcTy, + TTI::TargetCostKind CostKind, + ArrayRef<int> Mask, int Index, + VectorType *SubTp, + ArrayRef<const Value *> Args, + const Instruction *CxtI) const { if (!isa<FixedVectorType>(SrcTy)) return BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask, Index, SubTp); diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h index 4d9ff8d2d767f..887802c950982 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h @@ -249,9 +249,7 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> { getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index, VectorType *SubTp, ArrayRef<const Value *> Args = {}, - const Instruction *CxtI = nullptr, - TTI::VectorInstrContext VIC = - TTI::VectorInstrContext::None) const override; + const Instruction *CxtI = nullptr) const override; bool isProfitableToSinkOperands(Instruction *I, SmallVectorImpl<Use *> &Ops) const override; diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp index d9b83ea76a20e..d8ba99a4da023 100644 --- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp +++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp @@ -1224,11 +1224,13 @@ InstructionCost ARMTTIImpl::getMemcpyCost(const Instruction *I) const { return NumOps; } -InstructionCost ARMTTIImpl::getShuffleCost( - TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, - TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index, - VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI, - TTI::VectorInstrContext VIC) const { +InstructionCost ARMTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, + VectorType *DstTy, VectorType *SrcTy, + TTI::TargetCostKind CostKind, + ArrayRef<int> Mask, int Index, + VectorType *SubTp, + ArrayRef<const Value *> Args, + const Instruction *CxtI) const { assert((Mask.empty() || DstTy->isScalableTy() || Mask.size() == DstTy->getElementCount().getKnownMinValue()) && "Expected the Mask to match the return size if given"); diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h index 0ad782320ecc0..3e6ad7f13b4e7 100644 --- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h +++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h @@ -195,9 +195,7 @@ class ARMTTIImpl final : public BasicTTIImplBase<ARMTTIImpl> { getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index, VectorType *SubTp, ArrayRef<const Value *> Args = {}, - const Instruction *CxtI = nullptr, - TTI::VectorInstrContext VIC = - TTI::VectorInstrContext::None) const override; + const Instruction *CxtI = nullptr) const override; bool preferInLoopReduction(RecurKind Kind, Type *Ty) const override; diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp index fad70ff87cc75..466b5ab3a0aed 100644 --- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp +++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp @@ -231,11 +231,12 @@ InstructionCost HexagonTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, OpInfo, I); } -InstructionCost HexagonTTIImpl::getShuffleCost( - TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, - TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index, - VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI, - TTI::VectorInstrContext VIC) const { +InstructionCost +HexagonTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, + VectorType *SrcTy, TTI::TargetCostKind CostKind, + ArrayRef<int> Mask, int Index, VectorType *SubTp, + ArrayRef<const Value *> Args, + const Instruction *CxtI) const { return 1; } diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h index 59642d6cf2d97..1189b9bc9e065 100644 --- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h +++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h @@ -128,9 +128,7 @@ class HexagonTTIImpl final : public BasicTTIImplBase<HexagonTTIImpl> { getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index, VectorType *SubTp, ArrayRef<const Value *> Args = {}, - const Instruction *CxtI = nullptr, - TTI::VectorInstrContext VIC = - TTI::VectorInstrContext::None) const override; + const Instruction *CxtI = nullptr) const override; InstructionCost getInterleavedMemoryOpCost( unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp index 9d46853e5d583..3b2c5b06e0cfa 100644 --- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp +++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp @@ -617,11 +617,13 @@ InstructionCost PPCTTIImpl::getArithmeticInstrCost( return Cost * CostFactor; } -InstructionCost PPCTTIImpl::getShuffleCost( - TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, - TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index, - VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI, - TTI::VectorInstrContext VIC) const { +InstructionCost PPCTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, + VectorType *DstTy, VectorType *SrcTy, + TTI::TargetCostKind CostKind, + ArrayRef<int> Mask, int Index, + VectorType *SubTp, + ArrayRef<const Value *> Args, + const Instruction *CxtI) const { InstructionCost CostFactor = vectorCostAdjustmentFactor(Instruction::ShuffleVector, SrcTy, nullptr); diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h index a16463275cb1e..b8e7277f2ff51 100644 --- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h +++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h @@ -115,9 +115,7 @@ class PPCTTIImpl final : public BasicTTIImplBase<PPCTTIImpl> { getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index, VectorType *SubTp, ArrayRef<const Value *> Args = {}, - const Instruction *CxtI = nullptr, - TTI::VectorInstrContext VIC = - TTI::VectorInstrContext::None) const override; + const Instruction *CxtI = nullptr) const override; InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src, TTI::CastContextHint CCH, TTI::TargetCostKind CostKind, diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index 54ffa1504823a..43b4791998d39 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -733,11 +733,12 @@ InstructionCost RISCVTTIImpl::getSlideCost(FixedVectorType *Tp, return FirstSlideCost + SecondSlideCost + MaskCost; } -InstructionCost RISCVTTIImpl::getShuffleCost( - TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, - TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index, - VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI, - TTI::VectorInstrContext VIC) const { +InstructionCost +RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, + VectorType *SrcTy, TTI::TargetCostKind CostKind, + ArrayRef<int> Mask, int Index, VectorType *SubTp, + ArrayRef<const Value *> Args, + const Instruction *CxtI) const { assert((Mask.empty() || DstTy->isScalableTy() || Mask.size() == DstTy->getElementCount().getKnownMinValue()) && "Expected the Mask to match the return size if given"); @@ -745,9 +746,6 @@ InstructionCost RISCVTTIImpl::getShuffleCost( "Expected the same scalar types"); Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTp); - if (VIC == TTI::VectorInstrContext::SplatOpFolded && - ST->sinkSplatOperands() && Kind == TTI::SK_Broadcast) - return TTI::TCC_Free; // TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16) // For now, skip all fixed vector cost analysis when P extension is available @@ -2652,13 +2650,6 @@ InstructionCost RISCVTTIImpl::getVectorInstrCost( return BaseT::getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1, VIC); - // Scalar splat operand can be folded for vector ops that support splatting - // the scalar operand, so the explicit insertelement is free in this context. - if (Opcode == Instruction::InsertElement && - VIC == TTI::VectorInstrContext::SplatOpFolded && - ST->sinkSplatOperands() && Index == 0) - return TTI::TCC_Free; - // Legalize the type. std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Val); @@ -3631,33 +3622,6 @@ bool RISCVTTIImpl::canSplatOperand(Instruction *I, int Operand) const { } } -TargetTransformInfo::VectorInstrContext RISCVTTIImpl::getBuildVectorContextHint( - ArrayRef<int> Mask, ArrayRef<Value *> Scalars, - function_ref<bool(SmallVectorImpl<TargetTransformInfo::BuildVectorUseOp> &)> - GatherUseOps) const { - if (Scalars.empty() || !ST->hasVInstructions() || !ST->sinkSplatOperands() || - !ShuffleVectorInst::isZeroEltSplatMask(Mask, Mask.size())) - return VectorInstrContext::None; - - const auto *SplatIt = find_if_not(Scalars, IsaPred<UndefValue>); - if (SplatIt == Scalars.end() || (*SplatIt)->getType()->isIntegerTy(1) || - isa<VectorType>((*SplatIt)->getType()) || - isa<ExtractElementInst>(*SplatIt)) - return VectorInstrContext::None; - - SmallVector<TargetTransformInfo::BuildVectorUseOp, 4> UserOps; - if (!GatherUseOps(UserOps) || UserOps.empty()) - return VectorInstrContext::None; - - if (all_of(UserOps, - [this](const TargetTransformInfo::BuildVectorUseOp &UserOp) { - return canSplatOperand(UserOp.Opcode, UserOp.OperandIndex); - })) - return VectorInstrContext::SplatOpFolded; - - return VectorInstrContext::None; -} - /// Check if sinking \p I's operands to I's basic block is profitable, because /// the operands can be folded into a target instruction, e.g. /// splats of scalars can fold into vector instructions. diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h index d6e904c53dc70..4523ebd6ae892 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h @@ -178,9 +178,7 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> { getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index, VectorType *SubTp, ArrayRef<const Value *> Args = {}, - const Instruction *CxtI = nullptr, - TTI::VectorInstrContext VIC = - TTI::VectorInstrContext::None) const override; + const Instruction *CxtI = nullptr) const override; InstructionCost getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts, @@ -548,12 +546,6 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> { /// able to splat the given operand. bool canSplatOperand(unsigned Opcode, int Operand) const; - TargetTransformInfo::VectorInstrContext getBuildVectorContextHint( - ArrayRef<int> Mask, ArrayRef<Value *> Scalars, - function_ref< - bool(SmallVectorImpl<TargetTransformInfo::BuildVectorUseOp> &)> - GatherUseOps) const override; - bool isProfitableToSinkOperands(Instruction *I, SmallVectorImpl<Use *> &Ops) const override; diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp index e3c0aa2755175..8d6dee7c03361 100644 --- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp +++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp @@ -794,11 +794,12 @@ InstructionCost SystemZTTIImpl::getArithmeticInstrCost( Args, CxtI); } -InstructionCost SystemZTTIImpl::getShuffleCost( - TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, - TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index, - VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI, - TTI::VectorInstrContext VIC) const { +InstructionCost +SystemZTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, + VectorType *SrcTy, TTI::TargetCostKind CostKind, + ArrayRef<int> Mask, int Index, VectorType *SubTp, + ArrayRef<const Value *> Args, + const Instruction *CxtI) const { Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTp); if (ST->hasVector()) { unsigned NumVectors = getNumVectorRegs(SrcTy); diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h index 36a2bc90bdcbb..e022414014352 100644 --- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h +++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h @@ -118,9 +118,7 @@ class SystemZTTIImpl final : public BasicTTIImplBase<SystemZTTIImpl> { getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index, VectorType *SubTp, ArrayRef<const Value *> Args = {}, - const Instruction *CxtI = nullptr, - TTI::VectorInstrContext VIC = - TTI::VectorInstrContext::None) const override; + const Instruction *CxtI = nullptr) const override; unsigned getVectorTruncCost(Type *SrcTy, Type *DstTy) const; unsigned getVectorBitmaskConversionCost(Type *SrcTy, Type *DstTy) const; unsigned getBoolVecToIntConversionCost(unsigned Opcode, Type *Dst, diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp index 4b392a7c802dc..d790dfae1d0be 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp @@ -302,8 +302,8 @@ InstructionCost WebAssemblyTTIImpl::getMemoryOpCost( InstructionCost WebAssemblyTTIImpl::getShuffleCost( TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index, - VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI, - TTI::VectorInstrContext VIC) const { + VectorType *SubTp, ArrayRef<const Value *> Args, + const Instruction *CxtI) const { // Canonicalize the ShuffleKind in case optimizations didn't. // Otherwise, we might end up with the wrong ShuffleKind to match against. diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h index 83c1736fcee4d..957813ab54d79 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h +++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h @@ -90,9 +90,7 @@ class WebAssemblyTTIImpl final : public BasicTTIImplBase<WebAssemblyTTIImpl> { getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index, VectorType *SubTp, ArrayRef<const Value *> Args = {}, - const Instruction *CxtI = nullptr, - TTI::VectorInstrContext VIC = - TTI::VectorInstrContext::None) const override; + const Instruction *CxtI = nullptr) const override; using BaseT::getVectorInstrCost; InstructionCost diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp index d529e1e0d8f91..773a9cee55b54 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -1945,11 +1945,13 @@ X86TTIImpl::getAltInstrCost(VectorType *VecTy, unsigned Opcode0, return InstructionCost::getInvalid(); } -InstructionCost X86TTIImpl::getShuffleCost( - TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, - TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index, - VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI, - TTI::VectorInstrContext VIC) const { +InstructionCost X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, + VectorType *DstTy, VectorType *SrcTy, + TTI::TargetCostKind CostKind, + ArrayRef<int> Mask, int Index, + VectorType *SubTp, + ArrayRef<const Value *> Args, + const Instruction *CxtI) const { assert((Mask.empty() || DstTy->isScalableTy() || Mask.size() == DstTy->getElementCount().getKnownMinValue()) && "Expected the Mask to match the return size if given"); diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h index f4197c260ba81..dd413277cdc86 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.h +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h @@ -81,9 +81,7 @@ class X86TTIImpl final : public BasicTTIImplBase<X86TTIImpl> { getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index, VectorType *SubTp, ArrayRef<const Value *> Args = {}, - const Instruction *CxtI = nullptr, - TTI::VectorInstrContext VIC = - TTI::VectorInstrContext::None) const override; + const Instruction *CxtI = nullptr) const override; InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src, TTI::CastContextHint CCH, TTI::TargetCostKind CostKind, diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 40952c66faf1a..10b27edb97362 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -14685,22 +14685,7 @@ void BoUpSLP::transformNodes() { class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis { bool IsFinalized = false; SmallVector<int> CommonMask; - - // Each input vector can be tagged with extra context information - struct InVector { - PointerUnion<Value *, const TreeEntry *> Vec = nullptr; - TargetTransformInfo::VectorInstrContext Ctx = - TargetTransformInfo::VectorInstrContext::None; - - InVector(Value *Vec, TargetTransformInfo::VectorInstrContext Ctx = - TargetTransformInfo::VectorInstrContext::None) - : Vec(Vec), Ctx(Ctx) {} - InVector(const TreeEntry *Vec, - TargetTransformInfo::VectorInstrContext Ctx = - TargetTransformInfo::VectorInstrContext::None) - : Vec(Vec), Ctx(Ctx) {} - }; - SmallVector<InVector, 2> InVectors; + SmallVector<PointerUnion<Value *, const TreeEntry *>, 2> InVectors; /// Captures the original scalar VL of a single, "clean" gather() call so /// the values can be forwarded as the Args operand to getShuffleCost() for /// the final permutation in finalize(). This lets the target cost model @@ -14740,8 +14725,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis { return Constant::getAllOnesValue(Ty); } - InstructionCost getBuildVectorCost(ArrayRef<Value *> VL, Value *Root, - TTI::VectorInstrContext ContextHint) { + InstructionCost getBuildVectorCost(ArrayRef<Value *> VL, Value *Root) { if ((!Root && allConstant(VL)) || all_of(VL, IsaPred<UndefValue>)) return TTI::TCC_Free; auto *VecTy = cast<VectorType>(getWidenedType(ScalarTy, VL.size())); @@ -14765,8 +14749,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis { } return TTI.getVectorInstrCost(Instruction::InsertElement, VecTy, CostKind, std::distance(VL.begin(), It), - PoisonValue::get(VecTy), *It, - ContextHint); + PoisonValue::get(VecTy), *It); } SmallVector<int> ShuffleMask(VL.size(), PoisonMaskElem); @@ -14775,14 +14758,12 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis { }); InstructionCost InsertCost = TTI.getVectorInstrCost(Instruction::InsertElement, VecTy, CostKind, 0, - PoisonValue::get(VecTy), *It, ContextHint); + PoisonValue::get(VecTy), *It); return InsertCost + getShuffleCost(TTI, TargetTransformInfo::SK_Broadcast, VecTy, CostKind, ShuffleMask, /*Index=*/0, /*SubTp=*/nullptr, - /*Args=*/*It, ContextHint); + /*Args=*/*It); } - assert(ContextHint != TTI::VectorInstrContext::SplatOpFolded && - "Expected SplatOpFolded to be a splat"); return GatherCost + (all_of(VL, IsaPred<UndefValue>) ? TTI::TCC_Free : R.getGatherCost(VL, !Root, ScalarTy)); @@ -14933,9 +14914,9 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis { // sub-Mask into the CommonMask to estimate it later and avoid double cost // estimation. if ((InVectors.size() == 2 && - cast<const TreeEntry *>(InVectors.front().Vec) == &E1 && - cast<const TreeEntry *>(InVectors.back().Vec) == E2) || - (!E2 && cast<const TreeEntry *>(InVectors.front().Vec) == &E1)) { + cast<const TreeEntry *>(InVectors.front()) == &E1 && + cast<const TreeEntry *>(InVectors.back()) == E2) || + (!E2 && cast<const TreeEntry *>(InVectors.front()) == &E1)) { unsigned Limit = getNumElems(Mask.size(), SliceSize, Part); assert(all_of(ArrayRef(CommonMask).slice(Part * SliceSize, Limit), [](int Idx) { return Idx == PoisonMaskElem; }) && @@ -14946,51 +14927,43 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis { } // Found non-matching nodes - need to estimate the cost for the matched // and transform mask. - if (InVectors.size() == 1) - Cost += createShuffle(InVectors.front().Vec, nullptr, CommonMask, {}, - InVectors.front().Ctx); - else - Cost += createShuffle(InVectors.front().Vec, InVectors.back().Vec, - CommonMask, {}, InVectors.front().Ctx, - InVectors.back().Ctx); + Cost += createShuffle(InVectors.front(), + InVectors.size() == 1 ? nullptr : InVectors.back(), + CommonMask); transformMaskAfterShuffle(CommonMask, CommonMask); } else if (InVectors.size() == 2) { - Cost += - createShuffle(InVectors.front().Vec, InVectors.back().Vec, CommonMask, - {}, InVectors.front().Ctx, InVectors.back().Ctx); + Cost += createShuffle(InVectors.front(), InVectors.back(), CommonMask); transformMaskAfterShuffle(CommonMask, CommonMask); } SameNodesEstimated = false; if (!E2 && InVectors.size() == 1) { unsigned VF = E1.getVectorFactor(); - if (Value *V1 = InVectors.front().Vec.dyn_cast<Value *>()) { + if (Value *V1 = dyn_cast<Value *>(InVectors.front())) { VF = std::max(VF, getVF(V1)); } else { - const auto *E = cast<const TreeEntry *>(InVectors.front().Vec); + const auto *E = cast<const TreeEntry *>(InVectors.front()); VF = std::max(VF, E->getVectorFactor()); } for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx) if (Mask[Idx] != PoisonMaskElem && CommonMask[Idx] == PoisonMaskElem) CommonMask[Idx] = Mask[Idx] + VF; - Cost += createShuffle(InVectors.front().Vec, &E1, CommonMask, {}, - InVectors.front().Ctx); + Cost += createShuffle(InVectors.front(), &E1, CommonMask); transformMaskAfterShuffle(CommonMask, CommonMask); } else { auto P = InVectors.front(); - Cost += createShuffle(&E1, E2, Mask, {}, TTI::VectorInstrContext::None); + Cost += createShuffle(&E1, E2, Mask); unsigned VF = Mask.size(); - if (Value *V1 = dyn_cast<Value *>(P.Vec)) { + if (Value *V1 = dyn_cast<Value *>(P)) { VF = std::max(VF, getNumElements(V1->getType())); } else { - const auto *E = cast<const TreeEntry *>(P.Vec); + const auto *E = cast<const TreeEntry *>(P); VF = std::max(VF, E->getVectorFactor()); } for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx) if (Mask[Idx] != PoisonMaskElem) CommonMask[Idx] = Idx + (InVectors.empty() ? 0 : VF); - Cost += createShuffle(P.Vec, InVectors.front().Vec, CommonMask, {}, P.Ctx, - InVectors.front().Ctx); + Cost += createShuffle(P, InVectors.front(), CommonMask); transformMaskAfterShuffle(CommonMask, CommonMask); } } @@ -15023,10 +14996,8 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis { return getShuffleCost(TTI, TTI::SK_PermuteTwoSrc, cast<VectorType>(V1->getType()), CostKind, Mask); } - InstructionCost - createShuffleVector(Value *V1, ArrayRef<int> Mask, ArrayRef<Value *> VL, - TTI::VectorInstrContext ContextHint = - TTI::VectorInstrContext::None) const { + InstructionCost createShuffleVector(Value *V1, ArrayRef<int> Mask, + ArrayRef<Value *> VL) const { // Empty mask or identity mask are free. unsigned VF = cast<VectorType>(V1->getType())->getElementCount().getKnownMinValue(); @@ -15034,7 +15005,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis { return TTI::TCC_Free; return getShuffleCost(TTI, TTI::SK_PermuteSingleSrc, cast<VectorType>(V1->getType()), CostKind, Mask, - /*Index=*/0, /*SubTp=*/nullptr, VL, ContextHint); + /*Index=*/0, /*SubTp=*/nullptr, VL); } InstructionCost createIdentity(Value *) const { return TTI::TCC_Free; } InstructionCost createPoison(Type *Ty, unsigned VF) const { @@ -15049,9 +15020,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis { InstructionCost createShuffle(const PointerUnion<Value *, const TreeEntry *> &P1, const PointerUnion<Value *, const TreeEntry *> &P2, - ArrayRef<int> Mask, ArrayRef<Value *> VL = {}, - TTI::VectorInstrContext Ctx1 = TTI::VectorInstrContext::None, - TTI::VectorInstrContext Ctx2 = TTI::VectorInstrContext::None) { + ArrayRef<int> Mask, ArrayRef<Value *> VL = {}) { ShuffleCostBuilder Builder(TTI, CostKind); SmallVector<int> CommonMask(Mask); Value *V1 = P1.dyn_cast<Value *>(), *V2 = P2.dyn_cast<Value *>(); @@ -15258,11 +15227,8 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis { Constant::getNullValue(getWidenedType(ScalarTy, CommonMask.size())); if (InVectors.size() == 2) InVectors.pop_back(); - TTI::VectorInstrContext Ctx = - P2.isNull() ? Ctx1 : TTI::combineVectorInstrContexts(Ctx1, Ctx2); - return ExtraCost + - BaseShuffleAnalysis::createShuffle<InstructionCost>( - V1, V2, CommonMask, Builder, ScalarTy, SLPReVec, VL, Ctx); + return ExtraCost + BaseShuffleAnalysis::createShuffle<InstructionCost>( + V1, V2, CommonMask, Builder, ScalarTy, SLPReVec, VL); } public: @@ -15464,7 +15430,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis { if (P.value() == PoisonMaskElem) return Mask[P.index()] == PoisonMaskElem; auto *EI = cast<ExtractElementInst>( - cast<const TreeEntry *>(InVectors.front().Vec) + cast<const TreeEntry *>(InVectors.front()) ->getOrdered(P.index())); return EI->getVectorOperand() == V1 || EI->getVectorOperand() == V2; @@ -15472,25 +15438,23 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis { "Expected extractelement vectors."); } /// Adds another one input vector and the mask for the shuffling. - void - add(Value *V1, ArrayRef<int> Mask, bool ForExtracts = false, - TTI::VectorInstrContext ContextHint = TTI::VectorInstrContext::None) { + void add(Value *V1, ArrayRef<int> Mask, bool ForExtracts = false) { if (BVValues && !isa<Constant>(V1)) BVValues.reset(); if (InVectors.empty()) { assert(CommonMask.empty() && !ForExtracts && "Expected empty input mask/vectors."); CommonMask.assign(Mask.begin(), Mask.end()); - InVectors.assign(1, InVector(V1, ContextHint)); + InVectors.assign(1, V1); return; } if (ForExtracts) { // No need to add vectors here, already handled them in adjustExtracts. - assert(InVectors.size() == 1 && - isa<const TreeEntry *>(InVectors[0].Vec) && !CommonMask.empty() && + assert(InVectors.size() == 1 && isa<const TreeEntry *>(InVectors[0]) && + !CommonMask.empty() && all_of(enumerate(CommonMask), [&](auto P) { - Value *Scalar = cast<const TreeEntry *>(InVectors[0].Vec) + Value *Scalar = cast<const TreeEntry *>(InVectors[0]) ->getOrdered(P.index()); if (P.value() == PoisonMaskElem) return P.value() == Mask[P.index()] || @@ -15507,28 +15471,25 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis { "Expected only tree entries from extracts/reused buildvectors."); unsigned VF = getVF(V1); if (InVectors.size() == 2) { - Cost += - createShuffle(InVectors.front().Vec, InVectors.back().Vec, CommonMask, - {}, InVectors.front().Ctx, InVectors.back().Ctx); + Cost += createShuffle(InVectors.front(), InVectors.back(), CommonMask); transformMaskAfterShuffle(CommonMask, CommonMask); VF = std::max<unsigned>(VF, CommonMask.size()); } else if (const auto *InTE = - InVectors.front().Vec.dyn_cast<const TreeEntry *>()) { + InVectors.front().dyn_cast<const TreeEntry *>()) { VF = std::max(VF, InTE->getVectorFactor()); } else { - VF = std::max(VF, cast<FixedVectorType>( - cast<Value *>(InVectors.front().Vec)->getType()) - ->getNumElements()); + VF = std::max( + VF, cast<FixedVectorType>(cast<Value *>(InVectors.front())->getType()) + ->getNumElements()); } - InVectors.emplace_back(V1, ContextHint); + InVectors.push_back(V1); for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx) if (Mask[Idx] != PoisonMaskElem && CommonMask[Idx] == PoisonMaskElem) CommonMask[Idx] = Mask[Idx] + VF; } - Value * - gather(ArrayRef<Value *> VL, unsigned MaskVF = 0, Value *Root = nullptr, - TTI::VectorInstrContext ContextHint = TTI::VectorInstrContext::None) { - Cost += getBuildVectorCost(VL, Root, ContextHint); + Value *gather(ArrayRef<Value *> VL, unsigned MaskVF = 0, + Value *Root = nullptr) { + Cost += getBuildVectorCost(VL, Root); if (BVValues) { if (BVValues->empty() && InVectors.empty()) BVValues->assign(VL.begin(), VL.end()); @@ -15578,14 +15539,11 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis { Action = {}) { IsFinalized = true; if (Action) { - const PointerUnion<Value *, const TreeEntry *> &Vec = - InVectors.front().Vec; + const PointerUnion<Value *, const TreeEntry *> &Vec = InVectors.front(); if (InVectors.size() == 2) - Cost += createShuffle(Vec, InVectors.back().Vec, CommonMask, {}, - InVectors.front().Ctx, InVectors.back().Ctx); + Cost += createShuffle(Vec, InVectors.back(), CommonMask); else - Cost += - createShuffle(Vec, nullptr, CommonMask, {}, InVectors.front().Ctx); + Cost += createShuffle(Vec, nullptr, CommonMask); transformMaskAfterShuffle(CommonMask, CommonMask); assert(VF > 0 && "Expected vector length for the final value before action."); @@ -15597,14 +15555,11 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis { InVectors.front() = V; } if (!SubVectors.empty()) { - const PointerUnion<Value *, const TreeEntry *> &Vec = - InVectors.front().Vec; + const PointerUnion<Value *, const TreeEntry *> &Vec = InVectors.front(); if (InVectors.size() == 2) - Cost += createShuffle(Vec, InVectors.back().Vec, CommonMask, {}, - InVectors.front().Ctx, InVectors.back().Ctx); + Cost += createShuffle(Vec, InVectors.back(), CommonMask); else - Cost += - createShuffle(Vec, nullptr, CommonMask, {}, InVectors.front().Ctx); + Cost += createShuffle(Vec, nullptr, CommonMask); transformMaskAfterShuffle(CommonMask, CommonMask); // Add subvectors permutation cost. if (!SubVectorsMask.empty()) { @@ -15675,12 +15630,10 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis { ArrayRef<Value *> VL; if (BVValues) VL = *BVValues; - if (InVectors.size() == 2) - return Cost + createShuffle(InVectors.front().Vec, InVectors.back().Vec, - CommonMask, VL, InVectors.front().Ctx, - InVectors.back().Ctx); - return Cost + createShuffle(InVectors.front().Vec, nullptr, CommonMask, VL, - InVectors.front().Ctx); + return Cost + + createShuffle(InVectors.front(), + InVectors.size() == 2 ? InVectors.back() : nullptr, + CommonMask, VL); } ~ShuffleCostEstimator() { @@ -22242,8 +22195,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis { InVectors.push_back(V1); } /// Adds another one input vector and the mask for the shuffling. - void add(Value *V1, ArrayRef<int> Mask, bool = false, - TTI::VectorInstrContext = TTI::VectorInstrContext::None) { + void add(Value *V1, ArrayRef<int> Mask, bool = false) { assert(isa<FixedVectorType>(V1->getType()) && "castToScalarTyElem expects V1 to be FixedVectorType"); V1 = castToScalarTyElem(V1); @@ -22302,8 +22254,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis { add(V1, NewMask); } Value *gather(ArrayRef<Value *> VL, unsigned MaskVF = 0, - Value *Root = nullptr, - TTI::VectorInstrContext = TTI::VectorInstrContext::None) { + Value *Root = nullptr) { return R.gather(VL, Root, ScalarTy, [&](Value *V1, Value *V2, ArrayRef<int> Mask) { return createShuffle(V1, V2, Mask); @@ -22989,39 +22940,10 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy, // Gather unique scalars and all constants. SmallVector<int> ReuseMask(GatheredScalars.size(), PoisonMaskElem); TryPackScalars(GatheredScalars, ReuseMask, /*IsRootPoison=*/true); - TTI::VectorInstrContext ContextHint = TTI::VectorInstrContext::None; - if constexpr (std::is_same_v<ResTy, InstructionCost>) { - auto GatherUserOps = - [&](SmallVectorImpl<TTI::BuildVectorUseOp> &UserOps) { - UserOps.clear(); - if (NeedFreeze) - return false; - for (const auto &TE : VectorizableTree) { - if (DeletedNodes.contains(TE.get())) - continue; - if (!(TE->isGather() || - TransformedToGatherNodes.contains(TE.get())) || - !E->isSame(TE->Scalars)) - continue; - auto *UserTE = TE->UserTreeIndex.UserTE; - if (!UserTE || !UserTE->hasState() || UserTE->isAltShuffle() || - TransformedToGatherNodes.contains(UserTE)) - return false; - UserOps.emplace_back(UserTE->getOpcode(), - TE->UserTreeIndex.EdgeIdx); - } - assert(UserOps.size() && - "Ought to at least match with current entry"); - return true; - }; - ContextHint = - TTI->getBuildVectorContextHint(ReuseMask, E->Scalars, GatherUserOps); - } - Value *BV = ShuffleBuilder.gather(GatheredScalars, ReuseMask.size(), - /*Root*/ nullptr, ContextHint); - ShuffleBuilder.add(BV, ReuseMask, /*ForExtract*/ false, ContextHint); + Value *BV = ShuffleBuilder.gather(GatheredScalars, ReuseMask.size()); + ShuffleBuilder.add(BV, ReuseMask); Res = ShuffleBuilder.finalize(E->ReuseShuffleIndices, SubVectors, - SubVectorsMask, /*VF*/ 0, /*Action*/ {}); + SubVectorsMask); } else { // Gather all constants. SmallVector<int> Mask(GatheredScalars.size(), PoisonMaskElem); diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.cpp index 71cde2765d02b..a8259d158cf19 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.cpp @@ -38,15 +38,14 @@ InstructionCost getShuffleCost(const TargetTransformInfo &TTI, TTI::ShuffleKind Kind, VectorType *Tp, const TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index, VectorType *SubTp, - ArrayRef<const Value *> Args, - TTI::VectorInstrContext VIC) { + ArrayRef<const Value *> Args) { VectorType *DstTy = Tp; if (!Mask.empty()) DstTy = FixedVectorType::get(Tp->getScalarType(), Mask.size()); if (Kind != TTI::SK_PermuteTwoSrc) return TTI.getShuffleCost(Kind, DstTy, Tp, CostKind, Mask, Index, SubTp, - Args, /*CxtI=*/nullptr, VIC); + Args); int NumSrcElts = Tp->getElementCount().getKnownMinValue(); int NumSubElts; if (Mask.size() > 2 && ShuffleVectorInst::isInsertSubvectorMask( @@ -56,8 +55,8 @@ InstructionCost getShuffleCost(const TargetTransformInfo &TTI, return TTI.getShuffleCost(TTI::SK_InsertSubvector, DstTy, Tp, CostKind, Mask, Index, Tp); } - return TTI.getShuffleCost(Kind, DstTy, Tp, CostKind, Mask, Index, SubTp, Args, - /*CxtI=*/nullptr, VIC); + return TTI.getShuffleCost(Kind, DstTy, Tp, CostKind, Mask, Index, SubTp, + Args); } std::pair<InstructionCost, InstructionCost> @@ -215,8 +214,8 @@ getScalarizationOverhead(const TargetTransformInfo &TTI, bool ReVec, InstructionCost getVectorInstrCost( const TargetTransformInfo &TTI, bool ReVec, Type *ScalarTy, unsigned Opcode, Type *Val, const TTI::TargetCostKind CostKind, unsigned Index, - Value *Scalar, ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx, - TTI::VectorInstrContext VIC) { + Value *Scalar, + ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx) { if (Opcode == Instruction::ExtractElement) { if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) { assert(ReVec && "Only supported by REVEC."); @@ -227,7 +226,7 @@ InstructionCost getVectorInstrCost( } } return TTI.getVectorInstrCost(Opcode, Val, CostKind, Index, Scalar, - ScalarUserAndIdx, VIC); + ScalarUserAndIdx); } InstructionCost getExtractWithExtendCost(const TargetTransformInfo &TTI, diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.h index 321ff5b087229..a2956043070de 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.h +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.h @@ -47,9 +47,7 @@ getShuffleCost(const TargetTransformInfo &TTI, TargetTransformInfo::ShuffleKind Kind, VectorType *Tp, const TargetTransformInfo::TargetCostKind CostKind, ArrayRef<int> Mask = {}, int Index = 0, - VectorType *SubTp = nullptr, ArrayRef<const Value *> Args = {}, - TargetTransformInfo::VectorInstrContext VIC = - TargetTransformInfo::VectorInstrContext::None); + VectorType *SubTp = nullptr, ArrayRef<const Value *> Args = {}); /// Calculate the scalar and the vector costs from vectorizing set of GEPs. std::pair<InstructionCost, InstructionCost> @@ -114,8 +112,7 @@ getVectorInstrCost(const TargetTransformInfo &TTI, bool ReVec, Type *ScalarTy, unsigned Opcode, Type *Val, const TargetTransformInfo::TargetCostKind CostKind, unsigned Index, Value *Scalar, - ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx, - TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None); + ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx); /// This is similar to TargetTransformInfo::getExtractWithExtendCost, but if Dst /// is a FixedVectorType, a vector will be extracted instead of a scalar. diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/check-node-without-vector-user.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/check-node-without-vector-user.ll index b5dbe1b089df8..8f738c3978029 100644 --- a/llvm/test/Transforms/SLPVectorizer/RISCV/check-node-without-vector-user.ll +++ b/llvm/test/Transforms/SLPVectorizer/RISCV/check-node-without-vector-user.ll @@ -6,17 +6,37 @@ define void @test(i64 %0, ptr %1) { ; CHECK-LABEL: define void @test( ; CHECK-SAME: i64 [[TMP0:%.*]], ptr [[TMP1:%.*]]) #[[ATTR0:[0-9]+]] { -; CHECK-NEXT: [[TMP3:%.*]] = call <4 x i8> @llvm.experimental.vp.strided.load.v4i8.p0.i64(ptr align 1 getelementptr (i8, ptr @r, i64 -24147), i64 8049, <4 x i1> splat (i1 true), i32 4) -; CHECK-NEXT: [[TMP4:%.*]] = trunc <4 x i8> [[TMP3]] to <4 x i1> -; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i64> poison, i64 [[TMP0]], i64 0 -; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i64> [[TMP5]], <4 x i64> poison, <4 x i32> zeroinitializer -; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP4]], <4 x i64> [[TMP6]], <4 x i64> zeroinitializer -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, <4 x ptr> <ptr @r, ptr @r, ptr @r, ptr @r>, <4 x i64> [[TMP7]] -; CHECK-NEXT: [[TMP9:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 [[TMP8]], <4 x i1> splat (i1 true), <4 x i8> poison) -; CHECK-NEXT: [[TMP10:%.*]] = icmp ule <4 x i8> [[TMP3]], [[TMP9]] -; CHECK-NEXT: [[TMP11:%.*]] = sext <4 x i1> [[TMP10]] to <4 x i8> -; CHECK-NEXT: [[TMP12:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP11]]) -; CHECK-NEXT: [[TMP33:%.*]] = sext i8 [[TMP12]] to i32 +; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr @r, align 1 +; CHECK-NEXT: [[TMP4:%.*]] = trunc i8 [[TMP3]] to i1 +; CHECK-NEXT: [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP0]], i64 0 +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr @r, i64 [[TMP5]] +; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[TMP6]], align 1 +; CHECK-NEXT: [[TMP8:%.*]] = icmp ule i8 [[TMP3]], [[TMP7]] +; CHECK-NEXT: [[TMP9:%.*]] = sext i1 [[TMP8]] to i32 +; CHECK-NEXT: [[TMP10:%.*]] = load i8, ptr getelementptr (i8, ptr @r, i64 -8049), align 1 +; CHECK-NEXT: [[TMP11:%.*]] = trunc i8 [[TMP10]] to i1 +; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP11]], i64 [[TMP0]], i64 0 +; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr @r, i64 [[TMP12]] +; CHECK-NEXT: [[TMP14:%.*]] = load i8, ptr [[TMP13]], align 1 +; CHECK-NEXT: [[TMP15:%.*]] = icmp ule i8 [[TMP10]], [[TMP14]] +; CHECK-NEXT: [[TMP16:%.*]] = sext i1 [[TMP15]] to i32 +; CHECK-NEXT: [[TMP17:%.*]] = add i32 [[TMP9]], [[TMP16]] +; CHECK-NEXT: [[TMP18:%.*]] = load i8, ptr getelementptr (i8, ptr @r, i64 -16098), align 1 +; CHECK-NEXT: [[TMP19:%.*]] = trunc i8 [[TMP18]] to i1 +; CHECK-NEXT: [[TMP20:%.*]] = select i1 [[TMP19]], i64 [[TMP0]], i64 0 +; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr @r, i64 [[TMP20]] +; CHECK-NEXT: [[TMP22:%.*]] = load i8, ptr [[TMP21]], align 1 +; CHECK-NEXT: [[TMP23:%.*]] = icmp ule i8 [[TMP18]], [[TMP22]] +; CHECK-NEXT: [[TMP24:%.*]] = sext i1 [[TMP23]] to i32 +; CHECK-NEXT: [[TMP25:%.*]] = add i32 [[TMP17]], [[TMP24]] +; CHECK-NEXT: [[TMP26:%.*]] = load i8, ptr getelementptr (i8, ptr @r, i64 -24147), align 1 +; CHECK-NEXT: [[TMP27:%.*]] = trunc i8 [[TMP26]] to i1 +; CHECK-NEXT: [[TMP28:%.*]] = select i1 [[TMP27]], i64 [[TMP0]], i64 0 +; CHECK-NEXT: [[TMP29:%.*]] = getelementptr i8, ptr @r, i64 [[TMP28]] +; CHECK-NEXT: [[TMP30:%.*]] = load i8, ptr [[TMP29]], align 1 +; CHECK-NEXT: [[TMP31:%.*]] = icmp ule i8 [[TMP26]], [[TMP30]] +; CHECK-NEXT: [[TMP32:%.*]] = sext i1 [[TMP31]] to i32 +; CHECK-NEXT: [[TMP33:%.*]] = add i32 [[TMP25]], [[TMP32]] ; CHECK-NEXT: store i32 [[TMP33]], ptr [[TMP1]], align 4 ; CHECK-NEXT: ret void ; diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/folded-broadcast-cost.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/folded-broadcast-cost.ll index 861ec175a6585..a0d3d2b136833 100644 --- a/llvm/test/Transforms/SLPVectorizer/RISCV/folded-broadcast-cost.ll +++ b/llvm/test/Transforms/SLPVectorizer/RISCV/folded-broadcast-cost.ll @@ -8,7 +8,7 @@ ; YAML-NEXT: Function: fmul_splat ; YAML-NEXT: Args: ; YAML-NEXT: - String: 'Stores SLP vectorized with cost ' -; YAML-NEXT: - Cost: '-9' +; YAML-NEXT: - Cost: '-7' ; YAML-NEXT: - String: ' and with tree size ' ; YAML-NEXT: - TreeSize: '4' ; YAML-NEXT: ... @@ -18,7 +18,7 @@ ; YAML-NEXT: Function: fmul_splat_nested ; YAML-NEXT: Args: ; YAML-NEXT: - String: 'Stores SLP vectorized with cost ' -; YAML-NEXT: - Cost: '-15' +; YAML-NEXT: - Cost: '-13' ; YAML-NEXT: - String: ' and with tree size ' ; YAML-NEXT: - TreeSize: '8' ; YAML-NEXT: ... diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/remark-zext-incoming-for-neg-icmp.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/remark-zext-incoming-for-neg-icmp.ll index 8c640da2fa718..5ef40a3f6d2f7 100644 --- a/llvm/test/Transforms/SLPVectorizer/RISCV/remark-zext-incoming-for-neg-icmp.ll +++ b/llvm/test/Transforms/SLPVectorizer/RISCV/remark-zext-incoming-for-neg-icmp.ll @@ -8,7 +8,7 @@ ; YAML-NEXT: Function: test ; YAML-NEXT: Args: ; YAML-NEXT: - String: 'Vectorized horizontal reduction with cost ' -; YAML-NEXT: - Cost: '-12' +; YAML-NEXT: - Cost: '-10' ; YAML-NEXT: - String: ' and with tree size ' ; YAML-NEXT: - TreeSize: '8' ; YAML-NEXT:... diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/reordered-buildvector-scalars.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/reordered-buildvector-scalars.ll index 6501394690d97..9a9fcd1b259c3 100644 --- a/llvm/test/Transforms/SLPVectorizer/RISCV/reordered-buildvector-scalars.ll +++ b/llvm/test/Transforms/SLPVectorizer/RISCV/reordered-buildvector-scalars.ll @@ -54,10 +54,10 @@ define fastcc i32 @test(i32 %0, i32 %add111.i.i, <4 x i32> %PredPel.i.sroa.86.72 ; THRESH-SAME: i32 [[TMP0:%.*]], i32 [[ADD111_I_I:%.*]], <4 x i32> [[PREDPEL_I_SROA_86_72_VEC_EXTRACT:%.*]], <4 x i32> [[TMP1:%.*]]) #[[ATTR0:[0-9]+]] { ; THRESH-NEXT: [[ENTRY:.*:]] ; THRESH-NEXT: [[MUL1445_I:%.*]] = shl i32 [[TMP0]], 1 +; THRESH-NEXT: [[ADD2235_I16:%.*]] = or i32 [[TMP0]], 1 +; THRESH-NEXT: [[ADD2323_I:%.*]] = add i32 [[TMP0]], 1 ; THRESH-NEXT: [[TMP30:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0 ; THRESH-NEXT: [[TMP31:%.*]] = shufflevector <2 x i32> [[TMP30]], <2 x i32> poison, <2 x i32> zeroinitializer -; THRESH-NEXT: [[TMP11:%.*]] = or <2 x i32> [[TMP31]], <i32 1, i32 0> -; THRESH-NEXT: [[ADD2323_I:%.*]] = add i32 [[TMP0]], 1 ; THRESH-NEXT: [[TMP32:%.*]] = add <2 x i32> [[TMP31]], <i32 0, i32 1> ; THRESH-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP32]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 0> ; THRESH-NEXT: [[TMP21:%.*]] = ashr <4 x i32> [[TMP5]], <i32 1, i32 0, i32 0, i32 1> @@ -71,8 +71,8 @@ define fastcc i32 @test(i32 %0, i32 %add111.i.i, <4 x i32> %PredPel.i.sroa.86.72 ; THRESH-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[PREDPEL_I_SROA_86_72_VEC_EXTRACT]], <4 x i32> [[TMP1]], <8 x i32> <i32 0, i32 poison, i32 poison, i32 4, i32 poison, i32 poison, i32 poison, i32 poison> ; THRESH-NEXT: [[TMP9:%.*]] = shufflevector <3 x i32> [[TMP4]], <3 x i32> poison, <8 x i32> <i32 poison, i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 0, i32 poison> ; THRESH-NEXT: [[TMP10:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> [[TMP9]], <8 x i32> <i32 0, i32 poison, i32 10, i32 3, i32 poison, i32 poison, i32 14, i32 poison> -; THRESH-NEXT: [[TMP35:%.*]] = shufflevector <2 x i32> [[TMP11]], <2 x i32> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison> -; THRESH-NEXT: [[TMP36:%.*]] = shufflevector <8 x i32> [[TMP10]], <8 x i32> [[TMP35]], <8 x i32> <i32 0, i32 8, i32 2, i32 3, i32 11, i32 5, i32 6, i32 7> +; THRESH-NEXT: [[TMP35:%.*]] = insertelement <8 x i32> [[TMP10]], i32 [[TMP0]], i64 4 +; THRESH-NEXT: [[TMP36:%.*]] = insertelement <8 x i32> [[TMP35]], i32 [[ADD2235_I16]], i64 1 ; THRESH-NEXT: [[TMP13:%.*]] = insertelement <8 x i32> [[TMP36]], i32 [[MUL1445_I]], i64 5 ; THRESH-NEXT: [[TMP20:%.*]] = shufflevector <4 x i32> [[TMP21]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison> ; THRESH-NEXT: [[TMP14:%.*]] = shufflevector <8 x i32> [[TMP13]], <8 x i32> [[TMP20]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 11> _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
