https://github.com/madhur13490 updated https://github.com/llvm/llvm-project/pull/214673
>From 064ca37550367b4d4a77d232a192a860cc4838b1 Mon Sep 17 00:00:00 2001 From: Madhur Amilkanthwar <[email protected]> Date: Wed, 5 Aug 2026 23:51:35 -0700 Subject: [PATCH 1/3] [SLP][modularisation][NFC] Move full-vector width helpers to SLPTypeUtils (2/2) Move the following BoUpSLP-independent helpers out of SLPVectorizer.cpp into SLPVectorizer/SLPTypeUtils.{h,cpp}: getFullVectorNumberOfElements getFloorFullVectorNumberOfElements getMaskedDivRemType hasFullVectorsOrPowerOf2 They build on the type helpers moved in (1/2). Behavior is unchanged. Part of the SLPVectorizer.cpp modularization effort: https://discourse.llvm.org/t/modularizing-slpvectorizer-cpp/90922 --- .../Transforms/Vectorize/SLPVectorizer.cpp | 66 ------------------- .../Vectorize/SLPVectorizer/SLPTypeUtils.cpp | 56 ++++++++++++++++ .../Vectorize/SLPVectorizer/SLPTypeUtils.h | 28 ++++++++ 3 files changed, 84 insertions(+), 66 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 3fd0934be1653..2069efe4f3b28 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -351,54 +351,6 @@ static const int MinScheduleRegionSize = 16; /// Maximum allowed number of operands in the PHI nodes. static const unsigned MaxPHINumOperands = 128; -/// Returns the number of elements of the given type \p Ty, not less than \p Sz, -/// which forms type, which splits by \p TTI into whole vector types during -/// legalization. -static unsigned getFullVectorNumberOfElements(const TargetTransformInfo &TTI, - Type *Ty, unsigned Sz) { - if (!isValidElementType(Ty, SLPReVec) || isa<StructType>(Ty)) - return bit_ceil(Sz); - // Find the number of elements, which forms full vectors. - const unsigned NumParts = TTI.getNumberOfParts(getWidenedType(Ty, Sz)); - if (NumParts == 0 || NumParts >= Sz) - return bit_ceil(Sz); - return bit_ceil(divideCeil(Sz, NumParts)) * NumParts; -} - -/// Returns the number of elements of the given type \p Ty, not greater than \p -/// Sz, which forms type, which splits by \p TTI into whole vector types during -/// legalization. -static unsigned -getFloorFullVectorNumberOfElements(const TargetTransformInfo &TTI, Type *Ty, - unsigned Sz) { - if (!isValidElementType(Ty, SLPReVec) || isa<StructType>(Ty)) - return bit_floor(Sz); - // Find the number of elements, which forms full vectors. - unsigned NumParts = TTI.getNumberOfParts(getWidenedType(Ty, Sz)); - if (NumParts == 0 || NumParts >= Sz) - return bit_floor(Sz); - unsigned RegVF = bit_ceil(divideCeil(Sz, NumParts)); - if (RegVF > Sz) - return bit_floor(Sz); - return (Sz / RegVF) * RegVF; -} - -/// For a non-power-of-2 \p NumElts-wide integer div/rem \p Opcode, returns the -/// padded full-register vector type if padding is structurally possible, or -/// nullptr if the vector already fills a register or the opcode is not -/// div/rem. Does not check profitability; see getMaskedDivRemCost for that. -static FixedVectorType *getMaskedDivRemType(const TargetTransformInfo &TTI, - unsigned Opcode, Type *ScalarTy, - unsigned NumElts) { - if (!Instruction::isIntDivRem(Opcode) || has_single_bit(NumElts)) - return nullptr; - unsigned PaddedNumElts = - getFullVectorNumberOfElements(TTI, ScalarTy, NumElts); - if (PaddedNumElts == NumElts) - return nullptr; - return cast<FixedVectorType>(getWidenedType(ScalarTy, PaddedNumElts)); -} - /// For a non-power-of-2 \p NumElts-wide integer div/rem \p Opcode, checks if /// padding to a full register and using the masked div/rem intrinsic is /// cheaper than the direct vector op. Returns the cost of the masked @@ -540,24 +492,6 @@ isFixedVectorShuffle(ArrayRef<Value *> VL, SmallVectorImpl<int> &Mask, : TargetTransformInfo::SK_PermuteSingleSrc; } -/// Returns true if widened type of \p Ty elements with size \p Sz represents -/// full vector type, i.e. adding extra element results in extra parts upon type -/// legalization. -static bool hasFullVectorsOrPowerOf2(const TargetTransformInfo &TTI, Type *Ty, - unsigned Sz) { - if (Sz <= 1) - return false; - if (!isValidElementType(Ty, SLPReVec) && !isa<FixedVectorType>(Ty)) - return false; - if (has_single_bit(Sz)) - return true; - if (isa<StructType>(Ty)) - return false; - const unsigned NumParts = TTI.getNumberOfParts(getWidenedType(Ty, Sz)); - return NumParts > 0 && NumParts < Sz && has_single_bit(Sz / NumParts) && - Sz % NumParts == 0; -} - /// Returns number of parts, the type \p VecTy will be split at the codegen /// phase. If the type is going to be scalarized or does not uses whole /// registers, returns 1. diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.cpp index 24a4115533887..eef5012a65956 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.cpp @@ -11,12 +11,16 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVectorExtras.h" +#include "llvm/ADT/bit.h" +#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/VectorUtils.h" #include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Type.h" #include "llvm/IR/Value.h" #include "llvm/Support/Casting.h" +#include "llvm/Support/MathExtras.h" #include <cassert> @@ -69,4 +73,56 @@ Type *getWidenedType(Type *ScalarTy, unsigned VF) { ElementCount::getFixed(VF * getNumElements(ScalarTy))); } +unsigned getFullVectorNumberOfElements(const TargetTransformInfo &TTI, Type *Ty, + unsigned Sz) { + if (!isValidElementType(Ty) || isa<StructType>(Ty)) + return bit_ceil(Sz); + // Find the number of elements, which forms full vectors. + const unsigned NumParts = TTI.getNumberOfParts(getWidenedType(Ty, Sz)); + if (NumParts == 0 || NumParts >= Sz) + return bit_ceil(Sz); + return bit_ceil(divideCeil(Sz, NumParts)) * NumParts; +} + +unsigned getFloorFullVectorNumberOfElements(const TargetTransformInfo &TTI, + Type *Ty, unsigned Sz) { + if (!isValidElementType(Ty) || isa<StructType>(Ty)) + return bit_floor(Sz); + // Find the number of elements, which forms full vectors. + unsigned NumParts = TTI.getNumberOfParts(getWidenedType(Ty, Sz)); + if (NumParts == 0 || NumParts >= Sz) + return bit_floor(Sz); + unsigned RegVF = bit_ceil(divideCeil(Sz, NumParts)); + if (RegVF > Sz) + return bit_floor(Sz); + return (Sz / RegVF) * RegVF; +} + +FixedVectorType *getMaskedDivRemType(const TargetTransformInfo &TTI, + unsigned Opcode, Type *ScalarTy, + unsigned NumElts) { + if (!Instruction::isIntDivRem(Opcode) || has_single_bit(NumElts)) + return nullptr; + unsigned PaddedNumElts = + getFullVectorNumberOfElements(TTI, ScalarTy, NumElts); + if (PaddedNumElts == NumElts) + return nullptr; + return cast<FixedVectorType>(getWidenedType(ScalarTy, PaddedNumElts)); +} + +bool hasFullVectorsOrPowerOf2(const TargetTransformInfo &TTI, Type *Ty, + unsigned Sz) { + if (Sz <= 1) + return false; + if (!isValidElementType(Ty) && !isa<FixedVectorType>(Ty)) + return false; + if (has_single_bit(Sz)) + return true; + if (isa<StructType>(Ty)) + return false; + const unsigned NumParts = TTI.getNumberOfParts(getWidenedType(Ty, Sz)); + return NumParts > 0 && NumParts < Sz && has_single_bit(Sz / NumParts) && + Sz % NumParts == 0; +} + } // namespace llvm::slpvectorizer diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.h index f92c83ba660ee..dc9b0c6c0644f 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.h +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.h @@ -15,6 +15,8 @@ #define LLVM_LIB_TRANSFORMS_VECTORIZE_SLPVECTORIZER_SLPTYPEUTILS_H namespace llvm { +class FixedVectorType; +class TargetTransformInfo; class Type; class Value; } // namespace llvm @@ -42,6 +44,32 @@ Type *getValueType(Value *V, bool ReVec, bool LookThroughCmp = false); /// \returns the vector type of ScalarTy based on vectorization factor. Type *getWidenedType(Type *ScalarTy, unsigned VF); +/// Returns the number of elements of the given type \p Ty, not less than \p Sz, +/// which forms type, which splits by \p TTI into whole vector types during +/// legalization. +unsigned getFullVectorNumberOfElements(const TargetTransformInfo &TTI, Type *Ty, + unsigned Sz); + +/// Returns the number of elements of the given type \p Ty, not greater than \p +/// Sz, which forms type, which splits by \p TTI into whole vector types during +/// legalization. +unsigned getFloorFullVectorNumberOfElements(const TargetTransformInfo &TTI, + Type *Ty, unsigned Sz); + +/// For a non-power-of-2 \p NumElts-wide integer div/rem \p Opcode, returns the +/// padded full-register vector type if padding is structurally possible, or +/// nullptr if the vector already fills a register or the opcode is not +/// div/rem. Does not check profitability; see getMaskedDivRemCost for that. +FixedVectorType *getMaskedDivRemType(const TargetTransformInfo &TTI, + unsigned Opcode, Type *ScalarTy, + unsigned NumElts); + +/// Returns true if widened type of \p Ty elements with size \p Sz represents +/// full vector type, i.e. adding extra element results in extra parts upon type +/// legalization. +bool hasFullVectorsOrPowerOf2(const TargetTransformInfo &TTI, Type *Ty, + unsigned Sz); + } // namespace llvm::slpvectorizer #endif // LLVM_LIB_TRANSFORMS_VECTORIZE_SLPVECTORIZER_SLPTYPEUTILS_H >From 783b01b8d5297891037f7316a49d9e33bda22f81 Mon Sep 17 00:00:00 2001 From: Madhur Amilkanthwar <[email protected]> Date: Sun, 30 Aug 2026 21:36:13 -0700 Subject: [PATCH 2/3] fixup! [SLP][modularisation][NFC] Move full-vector width helpers to SLPTypeUtils (2/2) --- llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.h index dc9b0c6c0644f..30c6f059d36e8 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.h +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.h @@ -59,7 +59,7 @@ unsigned getFloorFullVectorNumberOfElements(const TargetTransformInfo &TTI, /// For a non-power-of-2 \p NumElts-wide integer div/rem \p Opcode, returns the /// padded full-register vector type if padding is structurally possible, or /// nullptr if the vector already fills a register or the opcode is not -/// div/rem. Does not check profitability; see getMaskedDivRemCost for that. +/// div/rem. Does not check profitability. FixedVectorType *getMaskedDivRemType(const TargetTransformInfo &TTI, unsigned Opcode, Type *ScalarTy, unsigned NumElts); >From 22192230d3e1e2576a4bab599769b1b08fa95be4 Mon Sep 17 00:00:00 2001 From: Madhur Amilkanthwar <[email protected]> Date: Sun, 30 Aug 2026 21:55:10 -0700 Subject: [PATCH 3/3] fixup! [SLP][modularisation][NFC] Move full-vector width helpers to SLPTypeUtils (2/2) --- .../Transforms/Vectorize/SLPVectorizer.cpp | 93 ++++++++++--------- .../Vectorize/SLPVectorizer/SLPTypeUtils.cpp | 16 ++-- .../Vectorize/SLPVectorizer/SLPTypeUtils.h | 8 +- 3 files changed, 62 insertions(+), 55 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 2069efe4f3b28..76b69eb767a26 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -361,7 +361,7 @@ getMaskedDivRemCost(const TargetTransformInfo &TTI, unsigned Opcode, const TTI::TargetCostKind CostKind, FixedVectorType **PaddedTy = nullptr) { FixedVectorType *PaddedVecTy = - getMaskedDivRemType(TTI, Opcode, ScalarTy, NumElts); + getMaskedDivRemType(TTI, Opcode, ScalarTy, NumElts, SLPReVec); if (!PaddedVecTy) return InstructionCost::getInvalid(); // One mask bit per element of the padded vector, not per padded lane. @@ -506,10 +506,10 @@ getNumberOfParts(const TargetTransformInfo &TTI, Type *VecTy, Type *ScalarTy, unsigned Sz = getNumElements(VecTy); unsigned ScalarSz = getNumElements(ScalarTy); Type *ElementTy = toScalarizedTy(VecTy); - unsigned PWSz = getFullVectorNumberOfElements(TTI, ElementTy, Sz); + unsigned PWSz = getFullVectorNumberOfElements(TTI, ElementTy, Sz, SLPReVec); if (NumParts >= Sz || PWSz % NumParts != 0 || (PWSz / NumParts) % ScalarSz != 0 || - !hasFullVectorsOrPowerOf2(TTI, ElementTy, PWSz / NumParts)) + !hasFullVectorsOrPowerOf2(TTI, ElementTy, PWSz / NumParts, SLPReVec)) return 1; const unsigned NumElts = PWSz / NumParts; if (divideCeil(Sz, NumElts) != NumParts) @@ -6295,7 +6295,8 @@ static bool isMaskedLoadCompress( if (IsStrided && !IsMasked && Order.empty()) { // Check for potential segmented(interleaved) loads. VectorType *AlignedLoadVecTy = cast<VectorType>(getWidenedType( - ScalarTy, getFullVectorNumberOfElements(TTI, ScalarTy, *Diff + 1))); + ScalarTy, + getFullVectorNumberOfElements(TTI, ScalarTy, *Diff + 1, SLPReVec))); SimplifyQuery SQ(DL, &TLI, &DT, &AC, cast<LoadInst>(VL.back())); if (!isSafeToLoadUnconditionally(Ptr0, AlignedLoadVecTy, CommonAlignment, SQ)) @@ -6941,10 +6942,10 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads( DemandedElts.clearAllBits(); // Iterate through possible vectorization factors and check if vectorized + // shuffles is better than just gather. - for (unsigned VF = - getFloorFullVectorNumberOfElements(TTI, ScalarTy, VL.size() - 1); - VF >= MinVF; - VF = getFloorFullVectorNumberOfElements(TTI, ScalarTy, VF - 1)) { + for (unsigned VF = getFloorFullVectorNumberOfElements( + TTI, ScalarTy, VL.size() - 1, SLPReVec); + VF >= MinVF; VF = getFloorFullVectorNumberOfElements( + TTI, ScalarTy, VF - 1, SLPReVec)) { SmallVector<std::pair<unsigned, LoadsState>> States; for (unsigned Cnt = 0, End = VL.size(); Cnt < End; Cnt += VF) { const unsigned SliceVF = std::min(VF, End - Cnt); @@ -9236,9 +9237,10 @@ void BoUpSLP::tryToVectorizeGatheredLoads( if (isAllowedNonPowerOf2VF(MaxVF)) CandidateVFs.push_back(MaxVF); for (int NumElts = getFloorFullVectorNumberOfElements( - *TTI, Loads.front()->getType(), MaxVF); - NumElts > 1; NumElts = getFloorFullVectorNumberOfElements( - *TTI, Loads.front()->getType(), NumElts - 1)) { + *TTI, Loads.front()->getType(), MaxVF, SLPReVec); + NumElts > 1; + NumElts = getFloorFullVectorNumberOfElements( + *TTI, Loads.front()->getType(), NumElts - 1, SLPReVec)) { CandidateVFs.push_back(NumElts); if (VectorizeNonPowerOf2 && NumElts > 2) CandidateVFs.push_back(NumElts - 1); @@ -10894,8 +10896,8 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL, S.getMainOp()->getDataLayout().getTypeSizeInBits(ScalarTy); unsigned MinVF = R.getMinVF(EltBits); auto RegWidth = [&](unsigned N) { - return std::max(getFullVectorNumberOfElements(TTI, ScalarTy, N), - MinVF); + return std::max( + getFullVectorNumberOfElements(TTI, ScalarTy, N, SLPReVec), MinVF); }; // Keeping the originals just moves the reshuffle to the operand // columns with duplicates; keep them only if at most one column has @@ -11044,11 +11046,12 @@ bool BoUpSLP::canBuildSplitNode(ArrayRef<Value *> VL, ReorderIndices.clear(); // When VL fills a power-of-2 register but the split halves do not, the // reorder shuffle makes the split unprofitable - reject. - else if (hasFullVectorsOrPowerOf2(*TTI, Op1.front()->getType(), VL.size()) && - (!hasFullVectorsOrPowerOf2(*TTI, Op1.front()->getType(), - Op1.size()) || - !hasFullVectorsOrPowerOf2(*TTI, Op2.front()->getType(), - Op2.size()))) + else if (hasFullVectorsOrPowerOf2(*TTI, Op1.front()->getType(), VL.size(), + SLPReVec) && + (!hasFullVectorsOrPowerOf2(*TTI, Op1.front()->getType(), Op1.size(), + SLPReVec) || + !hasFullVectorsOrPowerOf2(*TTI, Op2.front()->getType(), Op2.size(), + SLPReVec))) return false; SmallVector<int> Mask; if (!ReorderIndices.empty()) @@ -14366,7 +14369,7 @@ void BoUpSLP::reorderGatherNode(TreeEntry &TE) { } if (Sz > 1 && isa<Instruction>(P.second.front())) { const unsigned SubVF = getFloorFullVectorNumberOfElements( - *TTI, TE.Scalars.front()->getType(), Sz); + *TTI, TE.Scalars.front()->getType(), Sz, SLPReVec); SubVectors.emplace_back(Cnt - Sz, SubVF); for (unsigned I : seq<unsigned>(Cnt - Sz, Cnt - Sz + SubVF)) DemandedElts.clearBit(I); @@ -14979,9 +14982,9 @@ void BoUpSLP::transformNodes() { unsigned End = VL.size(); SmallBitVector Processed(End); for (unsigned VF = getFloorFullVectorNumberOfElements( - *TTI, VL.front()->getType(), VL.size() - 1); + *TTI, VL.front()->getType(), VL.size() - 1, SLPReVec); VF >= MinVF; VF = getFloorFullVectorNumberOfElements( - *TTI, VL.front()->getType(), VF - 1)) { + *TTI, VL.front()->getType(), VF - 1, SLPReVec)) { if (StartIdx + VF > End) continue; SmallVector<std::pair<unsigned, unsigned>> Slices; @@ -15667,7 +15670,8 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis { SubMask); } const unsigned BaseVF = getFullVectorNumberOfElements( - *R.TTI, VL.front()->getType(), alignTo(NumElts, EltsPerVector)); + *R.TTI, VL.front()->getType(), alignTo(NumElts, EltsPerVector), + SLPReVec); for (const auto [Idx, SubVecSize] : zip(Indices, SubVecSizes)) { assert((Idx + SubVecSize) <= BaseVF && "SK_ExtractSubvector index out of range"); @@ -21341,7 +21345,7 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry( unsigned MinIdx = MinElement % VF; if (MinIdx > 1) { unsigned RegFloor = getFloorFullVectorNumberOfElements( - *TTI, VL.front()->getType(), MinIdx); + *TTI, VL.front()->getType(), MinIdx, SLPReVec); auto *RegFloorTy = getWidenedType(VL.front()->getType(), RegFloor); unsigned RegFloorParts = getNumberOfParts(RegFloorTy, VL.front()->getType(), RegFloor); @@ -21487,8 +21491,8 @@ BoUpSLP::isGatherShuffledEntry( Mask.assign(VL.size(), PoisonMaskElem); assert((TE->UserTreeIndex || TE == &getRootNode()) && "Expected only single user of the gather node."); - unsigned PWSz = - getFullVectorNumberOfElements(*TTI, VL.front()->getType(), VL.size()); + unsigned PWSz = getFullVectorNumberOfElements(*TTI, VL.front()->getType(), + VL.size(), SLPReVec); if (TE->UserTreeIndex && TE->UserTreeIndex.UserTE->isGather() && TE->UserTreeIndex.EdgeIdx == UINT_MAX && (TE->Idx == 0 || @@ -28838,7 +28842,7 @@ SLPVectorizerPass::vectorizeStoreChainImpl(ArrayRef<Value *> Chain, BoUpSLP &R, if (!has_single_bit(Sz) || !hasFullVectorsOrPowerOf2( *TTI, cast<StoreInst>(Chain.front())->getValueOperand()->getType(), - VF) || + VF, SLPReVec) || VF < 2 || VF < MinVF) { // Check if vectorizing with a non-power-of-2 VF should be considered; see // isAllowedNonPowerOf2VF for supported widths. @@ -28858,9 +28862,10 @@ SLPVectorizerPass::vectorizeStoreChainImpl(ArrayRef<Value *> Chain, BoUpSLP &R, Analysis.buildInstructionsState(ValOps.getArrayRef(), R); if (all_of(ValOps, IsaPred<Instruction>) && ValOps.size() > 1) { DenseSet<Value *> Stores(Chain.begin(), Chain.end()); - bool IsAllowedSize = hasFullVectorsOrPowerOf2( - *TTI, ValOps.front()->getType(), ValOps.size()) || - isAllowedNonPowerOf2VF(ValOps.size()); + bool IsAllowedSize = + hasFullVectorsOrPowerOf2(*TTI, ValOps.front()->getType(), ValOps.size(), + SLPReVec) || + isAllowedNonPowerOf2VF(ValOps.size()); if ((!IsAllowedSize && S && S.getOpcode() != Instruction::Load && (!S.getMainOp()->isSafeToRemove() || any_of(ValOps.getArrayRef(), @@ -29305,7 +29310,7 @@ bool StoreChainContext::updateCandidateVFs(const TargetTransformInfo &TTI) { // Attempt again to vectorize even larger chains if all previous // attempts were unsuccessful because of the cost issues. unsigned Limit = - getFloorFullVectorNumberOfElements(TTI, StoreTy, MaxTotalNum); + getFloorFullVectorNumberOfElements(TTI, StoreTy, MaxTotalNum, SLPReVec); if (bit_floor(Limit) == VF && Limit != VF) CandidateVFs.push(Limit); CandidateVFs.push(VF); @@ -29879,11 +29884,12 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R, Type *ScalarTy = getValueType(VL[0], SLPReVec, /*LookThroughCmp=*/true); unsigned Sz = R.getVectorElementSize(I0); unsigned MinVF = R.getMinVF(Sz); - unsigned MaxVF = std::max<unsigned>( - isAllowedNonPowerOf2VF(VL.size()) - ? VL.size() - : getFloorFullVectorNumberOfElements(*TTI, ScalarTy, VL.size()), - MinVF); + unsigned MaxVF = + std::max<unsigned>(isAllowedNonPowerOf2VF(VL.size()) + ? VL.size() + : getFloorFullVectorNumberOfElements( + *TTI, ScalarTy, VL.size(), SLPReVec), + MinVF); MaxVF = std::min(R.getMaximumVF(Sz, S.getOpcode()), MaxVF); // Standalone seeds only need one register worth of lanes. if (StandaloneSeeds && Sz != 0) @@ -29903,7 +29909,8 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R, unsigned NextInst = 0, MaxInst = VL.size(); for (unsigned VF = MaxVF; NextInst + 1 < MaxInst && VF >= MinVF; - VF = getFloorFullVectorNumberOfElements(*TTI, I0->getType(), VF - 1)) { + VF = getFloorFullVectorNumberOfElements(*TTI, I0->getType(), VF - 1, + SLPReVec)) { // No actual vectorization should happen, if number of parts is the same as // provided vectorization factor (i.e. the scalar type is used for vector // code during codegen). @@ -29913,7 +29920,7 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R, for (unsigned I = NextInst; I < MaxInst; ++I) { unsigned ActualVF = std::min(MaxInst - I, VF); - if (!hasFullVectorsOrPowerOf2(*TTI, ScalarTy, ActualVF) && + if (!hasFullVectorsOrPowerOf2(*TTI, ScalarTy, ActualVF, SLPReVec) && (ActualVF != VL.size() || !isAllowedNonPowerOf2VF(ActualVF))) continue; @@ -31251,8 +31258,8 @@ class HorizontalReduction { auto GetVectorFactor = [&, &TTI = *TTI](unsigned ReduxWidth) { unsigned NumParts, NumRegs; Type *ScalarTy = Candidates.front()->getType(); - ReduxWidth = - getFloorFullVectorNumberOfElements(TTI, ScalarTy, ReduxWidth); + ReduxWidth = getFloorFullVectorNumberOfElements(TTI, ScalarTy, + ReduxWidth, SLPReVec); VectorType *Tp = cast<VectorType>(getWidenedType(ScalarTy, ReduxWidth)); NumParts = ::getNumberOfParts(TTI, Tp, ScalarTy); NumRegs = @@ -31446,10 +31453,10 @@ class HorizontalReduction { if (ReduxWidth > ReductionLimit && V.isTreeNotExtendable()) { // Add subvectors of VL to the list of the analyzed values. for (unsigned VF = getFloorFullVectorNumberOfElements( - *TTI, VL.front()->getType(), ReduxWidth - 1); + *TTI, VL.front()->getType(), ReduxWidth - 1, SLPReVec); VF >= ReductionLimit; VF = getFloorFullVectorNumberOfElements( - *TTI, VL.front()->getType(), VF - 1)) { + *TTI, VL.front()->getType(), VF - 1, SLPReVec)) { if (has_single_bit(VF) && V.getCanonicalGraphSize() != V.getTreeSize()) continue; @@ -31819,8 +31826,8 @@ class HorizontalReduction { unsigned ReduxWidth = 0; auto GetVectorFactor = [&, &TTI = *TTI](unsigned ReduxWidth) { Type *ScalarTy = Candidates.front()->getType(); - ReduxWidth = - getFloorFullVectorNumberOfElements(TTI, ScalarTy, ReduxWidth); + ReduxWidth = getFloorFullVectorNumberOfElements(TTI, ScalarTy, ReduxWidth, + SLPReVec); Type *Tp = getWidenedType(ScalarTy, ReduxWidth); unsigned NumParts = ::getNumberOfParts(TTI, Tp, ScalarTy); unsigned NumRegs = diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.cpp index eef5012a65956..3c3a9fb002901 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.cpp @@ -74,8 +74,8 @@ Type *getWidenedType(Type *ScalarTy, unsigned VF) { } unsigned getFullVectorNumberOfElements(const TargetTransformInfo &TTI, Type *Ty, - unsigned Sz) { - if (!isValidElementType(Ty) || isa<StructType>(Ty)) + unsigned Sz, bool ReVec) { + if (!isValidElementType(Ty, ReVec) || isa<StructType>(Ty)) return bit_ceil(Sz); // Find the number of elements, which forms full vectors. const unsigned NumParts = TTI.getNumberOfParts(getWidenedType(Ty, Sz)); @@ -85,8 +85,8 @@ unsigned getFullVectorNumberOfElements(const TargetTransformInfo &TTI, Type *Ty, } unsigned getFloorFullVectorNumberOfElements(const TargetTransformInfo &TTI, - Type *Ty, unsigned Sz) { - if (!isValidElementType(Ty) || isa<StructType>(Ty)) + Type *Ty, unsigned Sz, bool ReVec) { + if (!isValidElementType(Ty, ReVec) || isa<StructType>(Ty)) return bit_floor(Sz); // Find the number of elements, which forms full vectors. unsigned NumParts = TTI.getNumberOfParts(getWidenedType(Ty, Sz)); @@ -100,21 +100,21 @@ unsigned getFloorFullVectorNumberOfElements(const TargetTransformInfo &TTI, FixedVectorType *getMaskedDivRemType(const TargetTransformInfo &TTI, unsigned Opcode, Type *ScalarTy, - unsigned NumElts) { + unsigned NumElts, bool ReVec) { if (!Instruction::isIntDivRem(Opcode) || has_single_bit(NumElts)) return nullptr; unsigned PaddedNumElts = - getFullVectorNumberOfElements(TTI, ScalarTy, NumElts); + getFullVectorNumberOfElements(TTI, ScalarTy, NumElts, ReVec); if (PaddedNumElts == NumElts) return nullptr; return cast<FixedVectorType>(getWidenedType(ScalarTy, PaddedNumElts)); } bool hasFullVectorsOrPowerOf2(const TargetTransformInfo &TTI, Type *Ty, - unsigned Sz) { + unsigned Sz, bool ReVec) { if (Sz <= 1) return false; - if (!isValidElementType(Ty) && !isa<FixedVectorType>(Ty)) + if (!isValidElementType(Ty, ReVec) && !isa<FixedVectorType>(Ty)) return false; if (has_single_bit(Sz)) return true; diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.h index 30c6f059d36e8..6e0ed47b6b966 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.h +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.h @@ -48,13 +48,13 @@ Type *getWidenedType(Type *ScalarTy, unsigned VF); /// which forms type, which splits by \p TTI into whole vector types during /// legalization. unsigned getFullVectorNumberOfElements(const TargetTransformInfo &TTI, Type *Ty, - unsigned Sz); + unsigned Sz, bool ReVec); /// Returns the number of elements of the given type \p Ty, not greater than \p /// Sz, which forms type, which splits by \p TTI into whole vector types during /// legalization. unsigned getFloorFullVectorNumberOfElements(const TargetTransformInfo &TTI, - Type *Ty, unsigned Sz); + Type *Ty, unsigned Sz, bool ReVec); /// For a non-power-of-2 \p NumElts-wide integer div/rem \p Opcode, returns the /// padded full-register vector type if padding is structurally possible, or @@ -62,13 +62,13 @@ unsigned getFloorFullVectorNumberOfElements(const TargetTransformInfo &TTI, /// div/rem. Does not check profitability. FixedVectorType *getMaskedDivRemType(const TargetTransformInfo &TTI, unsigned Opcode, Type *ScalarTy, - unsigned NumElts); + unsigned NumElts, bool ReVec); /// Returns true if widened type of \p Ty elements with size \p Sz represents /// full vector type, i.e. adding extra element results in extra parts upon type /// legalization. bool hasFullVectorsOrPowerOf2(const TargetTransformInfo &TTI, Type *Ty, - unsigned Sz); + unsigned Sz, bool ReVec); } // namespace llvm::slpvectorizer _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
