https://github.com/MacDue updated https://github.com/llvm/llvm-project/pull/214491
>From b51765d92c1f5708981a3c5bae9236f15b7e6b6a Mon Sep 17 00:00:00 2001 From: Sergey Kachkov <[email protected]> Date: Wed, 5 Aug 2026 18:04:38 +0000 Subject: [PATCH 1/3] [LoopVectorize] Support vectorization of compressing patterns in VPlan RFC link: https://discourse.llvm.org/t/rfc-loop-vectorization-of-compress-store-expand-load-patterns/86442 This adds loop vectorizer support for "compressing" patterns, for example: ``` int dst_idx = 0; for (int i = 0; i < n; i++) { if (cond[i]) dst[dst_idx++] = src[i]; } ``` Can be vectorized with a `llvm.masked.compressstore` as: ``` int dst_idx = 0; for (int i = 0; i < n; i++) { %cond = load(%cond) != 0 %src = masked.load(%src[i], %cond) masked.compressstore(%src, %dst[dst_idx], %cond) dst_idx += num.active.lanes(%cond) } ``` and: ``` int src_idx = 0; for (int i = 0; i < n; i++) { if (cond[i]) dst[i] = src[src_idx++]; } ``` Can be vectorized with a `llvm.masked.expandload` as: ``` int src_idx = 0; for (int i = 0; i < n; i++) { %cond = load(%cond) != 0 %src = masked.expandload(%src[src_idx], %cond) masked.store(%src, %dst[i], %cond) src_idx += num.active.lanes(%cond) } ``` This uses the new `MonotonicDescriptor` to recognize monotonic/compressing patterns. The phis are mapped to a new `VPMonotonicPHIRecipe`, this will map to a scalar phi. We only allow uniform uses of monotonic phis in the loop (e.g., as the pointer to a compressed load/store). Compressed loads/stores are recognized with `LoopVectorizationLegality::isCompressedPtr`. Currently, we only allow cases where: - The (monotonic) pointer has a stride equal to the access size - The memory operation is predicated with the same condition as the increment This is a continuation Sergey Kachkov's patch (#140723). There are a number of changes from the initial patch: - Expandloads/compresstores directly use `VPWidenMemIntrinsic` - ComputeMonotonicResult is replaced with existing VP instructions - AArch64, VPlan, and target-agnostic tests have been added - This style of vectorization if off by default - The switch can be flipped soon after this patch lands --- .../Vectorize/LoopVectorizationLegality.h | 15 + llvm/lib/Analysis/VectorUtils.cpp | 6 + .../Vectorize/LoopVectorizationLegality.cpp | 34 ++ .../Vectorize/LoopVectorizationPlanner.cpp | 7 + .../Vectorize/LoopVectorizationPlanner.h | 6 + .../Transforms/Vectorize/LoopVectorize.cpp | 125 +++++- llvm/lib/Transforms/Vectorize/VPlan.cpp | 6 +- llvm/lib/Transforms/Vectorize/VPlan.h | 64 ++- .../Vectorize/VPlanConstruction.cpp | 6 + llvm/lib/Transforms/Vectorize/VPlanHelpers.h | 4 + .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 74 ++- .../Transforms/Vectorize/VPlanTransforms.cpp | 49 ++ .../Transforms/Vectorize/VPlanTransforms.h | 8 + llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 4 +- .../LoopVectorize/AArch64/compress-idioms.ll | 132 ++++++ .../LoopVectorize/VPlan/compress-idioms.ll | 157 +++++++ .../VPlan/vplan-print-before-after-all.ll | 1 + .../LoopVectorize/compress-idioms.ll | 424 ++++++++++++++++++ .../Transforms/Vectorize/VPlanTestBase.h | 1 + 19 files changed, 1096 insertions(+), 27 deletions(-) create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/compress-idioms.ll create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/compress-idioms.ll create mode 100644 llvm/test/Transforms/LoopVectorize/compress-idioms.ll diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h index 3e8db73fd79d2..c4a35ffaa0ee7 100644 --- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h +++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h @@ -293,6 +293,10 @@ class LoopVectorizationLegality { /// induction descriptor. using InductionList = MapVector<PHINode *, InductionDescriptor>; + /// MonotonicPHIList saves monotonic phi variables and maps them to the + /// monotonic phi descriptor. + using MonotonicPHIList = MapVector<PHINode *, MonotonicDescriptor>; + /// RecurrenceSet contains the phi nodes that are recurrences other than /// inductions and reductions. using RecurrenceSet = SmallPtrSet<const PHINode *, 8>; @@ -336,6 +340,11 @@ class LoopVectorizationLegality { /// Returns the induction variables found in the loop. const InductionList &getInductionVars() const { return Inductions; } + /// Returns the monotonic phi variables found in the loop. + const MonotonicPHIList &getMonotonicPHIs() const { return MonotonicPHIs; } + + bool hasMonotonicPHIs() const { return !MonotonicPHIs.empty(); } + /// Return the fixed-order recurrences found in the loop. RecurrenceSet &getFixedOrderRecurrences() { return FixedOrderRecurrences; } @@ -389,6 +398,9 @@ class LoopVectorizationLegality { /// loop. Do not use after invoking 'createVectorizedLoopSkeleton' (PR34965). LLVM_ABI int isConsecutivePtr(Type *AccessTy, Value *Ptr) const; + /// Check if memory access is compressed when vectorizing. + bool isCompressedPtr(Type *AccessTy, Value *Ptr, BasicBlock *BB) const; + /// Returns true if \p V is invariant across all loop iterations according to /// SCEV. LLVM_ABI bool isInvariant(Value *V) const; @@ -695,6 +707,9 @@ class LoopVectorizationLegality { /// variables can be pointers. InductionList Inductions; + /// Holds all of the monotonic phi variables that we found in the loop. + MonotonicPHIList MonotonicPHIs; + /// Holds all the casts that participate in the update chain of the induction /// variables, and that have been proven to be redundant (possibly under a /// runtime guard). These casts can be ignored when creating the vectorized diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp index 193fb6720cf60..f501c4ee058fa 100644 --- a/llvm/lib/Analysis/VectorUtils.cpp +++ b/llvm/lib/Analysis/VectorUtils.cpp @@ -159,6 +159,7 @@ bool llvm::isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, case Intrinsic::vp_is_fpclass: case Intrinsic::powi: case Intrinsic::vector_extract: + case Intrinsic::masked_compressstore: return (ScalarOpdIdx == 1); case Intrinsic::smul_fix: case Intrinsic::smul_fix_sat: @@ -171,6 +172,8 @@ bool llvm::isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, return ScalarOpdIdx == 2 || ScalarOpdIdx == 4; case Intrinsic::experimental_vp_strided_load: return ScalarOpdIdx == 0 || ScalarOpdIdx == 1; + case Intrinsic::masked_expandload: + return ScalarOpdIdx == 0; case Intrinsic::loop_dependence_war_mask: return true; default: @@ -201,6 +204,7 @@ bool llvm::isVectorIntrinsicWithOverloadTypeAtArg( case Intrinsic::scmp: case Intrinsic::vector_extract: case Intrinsic::loop_dependence_war_mask: + case Intrinsic::masked_expandload: return OpdIdx == -1 || OpdIdx == 0; case Intrinsic::modf: case Intrinsic::sincos: @@ -213,6 +217,8 @@ bool llvm::isVectorIntrinsicWithOverloadTypeAtArg( return OpdIdx == -1 || OpdIdx == 1; case Intrinsic::experimental_vp_strided_load: return OpdIdx == -1 || OpdIdx == 0 || OpdIdx == 1; + case Intrinsic::masked_compressstore: + return OpdIdx == 0 || OpdIdx == 1; default: return OpdIdx == -1; } diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp index 9086880599231..0d7f22cea0b03 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp @@ -48,6 +48,10 @@ AllowStridedPointerIVs("lv-strided-pointer-ivs", cl::init(false), cl::Hidden, cl::desc("Enable recognition of non-constant strided " "pointer induction variables.")); +static cl::opt<bool> EnableMonotonicPatterns( + "lv-monotonic-patterns", cl::init(false), cl::Hidden, + cl::desc("Enable recognition of monotonic patterns.")); + static cl::opt<bool> HintsAllowReordering("hints-allow-reordering", cl::init(true), cl::Hidden, cl::desc("Allow enabling loop hints to reorder " @@ -469,6 +473,29 @@ int LoopVectorizationLegality::isConsecutivePtr(Type *AccessTy, return 0; } +bool LoopVectorizationLegality::isCompressedPtr(Type *AccessTy, Value *Ptr, + BasicBlock *BB) const { + if (!EnableMonotonicPatterns) + return false; + + MonotonicDescriptor Desc; + if (!MonotonicDescriptor::isMonotonicVal(Ptr, TheLoop, Desc, *PSE.getSE())) + return false; + + // Check that the memory operation has the same predicate as the step. + // TODO: Relax these restrictions. + if (Desc.getPredicateEdge() != + MonotonicDescriptor::Edge(BB, BB->getUniqueSuccessor())) + return false; + + // Check if pointer step equals access size. + auto *Step = + dyn_cast<SCEVConstant>(Desc.getExpr()->getStepRecurrence(*PSE.getSE())); + if (!Step) + return false; + return Step->getAPInt() == BB->getDataLayout().getTypeAllocSize(AccessTy); +} + bool LoopVectorizationLegality::isInvariant(Value *V) const { return LAI->isInvariant(V); } @@ -883,6 +910,13 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) { return true; } + MonotonicDescriptor MD; + if (EnableMonotonicPatterns && + MonotonicDescriptor::isMonotonicPHI(Phi, TheLoop, MD, *PSE.getSE())) { + MonotonicPHIs[Phi] = MD; + return true; + } + if (RecurrenceDescriptor::isFixedOrderRecurrence(Phi, TheLoop, DT)) { FixedOrderRecurrences.insert(Phi); return true; diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp index fd7fd2e011a83..d5c711f1fab42 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp @@ -157,6 +157,13 @@ bool VFSelectionContext::isLegalGatherOrScatter(Value *V, (SI && TTI.isLegalMaskedScatter(Ty, Align)); } +bool VFSelectionContext::isLegalExpandLoadOrCompressStore( + bool IsLoad, Type *ScalarTy, Align Alignment) const { + return ForceTargetSupportsMaskedMemoryOps || + (IsLoad ? TTI.isLegalMaskedExpandLoad(ScalarTy, Alignment) + : TTI.isLegalMaskedCompressStore(ScalarTy, Alignment)); +} + bool VFSelectionContext::supportsScalableVectors() const { return TTI.supportsScalableVectors() || ForceTargetSupportsScalableVectors || VectorizerParams::VectorizationFactor.isScalable(); diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h index fa317f290022e..c9a06b4b2117e 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -791,6 +791,12 @@ class VFSelectionContext { /// or scatter operation. bool isLegalGatherOrScatter(Value *V, ElementCount VF) const; + /// Returns true if the target machine supports a masked expand load (if \p + /// IsLoad) or masked compress store of scalar type \p ScalarTy with \p + /// Alignment. + bool isLegalExpandLoadOrCompressStore(bool IsLoad, Type *ScalarTy, + Align Alignment) const; + /// Split reductions into those that happen in the loop, and those that /// happen outside. In-loop reductions are collected into InLoopReductions. /// InLoopReductionImmediateChains is filled with each in-loop reduction diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 3eb6d0530da6d..24174ae275d2e 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -878,6 +878,7 @@ class LoopVectorizationCostModel { CM_Widen_Reverse, // For consecutive accesses with stride -1. CM_Interleave, CM_GatherScatter, + CM_Compressed, CM_Scalarize, /// A widening decision that has been invalidated after replacing the /// corresponding recipe during VPlan transforms. @@ -1063,6 +1064,10 @@ class LoopVectorizationCostModel { /// consecutive or part of an interleave group. bool isLegalMaskedLoadOrStore(Instruction *I, ElementCount VF) const; + /// Returns true if the target machine supports a masked expand load or masked + /// compress store for \p I's data type and alignment. + bool isLegalExpandLoadOrCompressStore(Instruction *I) const; + /// Check if \p Instr belongs to any interleaved access group. bool isAccessInterleaved(Instruction *Instr) const { return InterleaveInfo.isInterleaved(Instr); @@ -2400,6 +2405,13 @@ bool LoopVectorizationCostModel::isLegalMaskedLoadOrStore( getLoadStoreAddressSpace(I)); } +bool LoopVectorizationCostModel::isLegalExpandLoadOrCompressStore( + Instruction *I) const { + assert(isa<LoadInst>(I) || isa<StoreInst>(I)); + return Config.isLegalExpandLoadOrCompressStore( + isa<LoadInst>(I), getLoadStoreType(I), getLoadStoreAlignment(I)); +} + bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I, ElementCount VF) { if (!isPredicatedInst(I)) @@ -2420,9 +2432,13 @@ bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I, } case Instruction::Load: case Instruction::Store: { - bool IsConsecutive = Legal->isConsecutivePtr(getLoadStoreType(I), - getLoadStorePointerOperand(I)); + Type *ScalarTy = getLoadStoreType(I); + Value *Ptr = getLoadStorePointerOperand(I); + bool IsConsecutive = Legal->isConsecutivePtr(ScalarTy, Ptr); + bool IsCompressed = + !IsConsecutive && Legal->isCompressedPtr(ScalarTy, Ptr, I->getParent()); return !(IsConsecutive && isLegalMaskedLoadOrStore(I, VF)) && + !(IsCompressed && isLegalExpandLoadOrCompressStore(I)) && !Config.isLegalGatherOrScatter(I, VF); } case Instruction::UDiv: @@ -2661,8 +2677,9 @@ LoopVectorizationCostModel::memoryInstructionCanBeWidened(Instruction *I, auto *Ptr = getLoadStorePointerOperand(I); auto *ScalarTy = getLoadStoreType(I); - // In order to be widened, the pointer should be consecutive, first of all. - int Stride = Legal->isConsecutivePtr(ScalarTy, Ptr); + // In order to be widened, the pointer should be consecutive or compressed. + bool Compressed = Legal->isCompressedPtr(ScalarTy, Ptr, I->getParent()); + int Stride = Compressed ? 1 : Legal->isConsecutivePtr(ScalarTy, Ptr); if (!Stride) return std::nullopt; @@ -2677,6 +2694,8 @@ LoopVectorizationCostModel::memoryInstructionCanBeWidened(Instruction *I, if (hasIrregularType(ScalarTy, DL)) return std::nullopt; + if (Compressed) + return CM_Compressed; return Stride == 1 ? CM_Widen : CM_Widen_Reverse; } @@ -2770,9 +2789,9 @@ void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) { if (IsUniformMemOpUse(I)) return true; - return (WideningDecision == CM_Widen || - WideningDecision == CM_Widen_Reverse || - WideningDecision == CM_Interleave); + return ( + WideningDecision == CM_Widen || WideningDecision == CM_Widen_Reverse || + WideningDecision == CM_Interleave || WideningDecision == CM_Compressed); }; // Returns true if Ptr is the pointer operand of a memory access instruction @@ -2917,6 +2936,38 @@ void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) { AddToWorklistIfAllowed(IndUpdate); } + // Handle monotonic phis (similarly to induction vars). + for (const auto &MonotonicPHI : Legal->getMonotonicPHIs()) { + auto *Phi = MonotonicPHI.first; + auto *PhiUpdate = cast<Instruction>(Phi->getIncomingValueForBlock(Latch)); + const auto &Desc = MonotonicPHI.second; + + auto UniformPhi = all_of(Phi->users(), [&](User *U) -> bool { + auto *I = cast<Instruction>(U); + if (I == Desc.getStepInst()) + return true; + if (auto *PN = dyn_cast<PHINode>(I); PN && Desc.getChain().contains(PN)) + return true; + return !TheLoop->contains(I) || Worklist.count(I) || + IsVectorizedMemAccessUse(I, Phi); + }); + if (!UniformPhi) + continue; + + auto UniformPhiUpdate = all_of(PhiUpdate->users(), [&](User *U) -> bool { + auto *I = cast<Instruction>(U); + if (I == Phi) + return true; + return !TheLoop->contains(I) || Worklist.count(I) || + IsVectorizedMemAccessUse(I, Phi); + }); + if (!UniformPhiUpdate) + continue; + + AddToWorklistIfAllowed(Phi); + AddToWorklistIfAllowed(PhiUpdate); + } + Uniforms[VF].insert_range(Worklist); } @@ -3291,6 +3342,7 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF, case VPRecipeBase::VPExpandSCEVSC: case VPRecipeBase::VPPredInstPHISC: case VPRecipeBase::VPBranchOnMaskSC: + case VPRecipeBase::VPMonotonicPHISC: continue; case VPRecipeBase::VPReductionSC: case VPRecipeBase::VPActiveLaneMaskPHISC: @@ -3676,6 +3728,10 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF, if (Plan.hasEarlyExit()) return 1; + // Monotonic vars don't support interleaving. + if (Legal->hasMonotonicPHIs()) + return 1; + const bool HasReductions = any_of(Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(), IsaPred<VPReductionPHIRecipe>); @@ -4036,8 +4092,10 @@ void LoopVectorizationCostModel::collectInstsToScalarize(ElementCount VF) { // of the instruction. // 2. Scalable VF, as that would lead to invalid scalarization costs. // 3. Emulated masked memrefs, if a hacked cost is needed. + // 4. Compressed loads/stores (which do not support scalarization) if (!isScalarAfterVectorization(&I, VF) && !VF.isScalable() && !useEmulatedMaskMemRefHack(&I, VF) && + getWideningDecision(&I, VF) != CM_Compressed && computePredInstDiscount(&I, ScalarCosts, VF) >= 0) { for (const auto &[I, IC] : ScalarCosts) ScalarCostsVF.insert({I, IC}); @@ -4287,8 +4345,9 @@ LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I, InstructionCost LoopVectorizationCostModel::getConsecutiveMemOpCost( Instruction *I, ElementCount VF, InstWidening Kind) { - assert((Kind == CM_Widen || Kind == CM_Widen_Reverse) && - "Expected a consecutive widening decision"); + assert( + (Kind == CM_Widen || Kind == CM_Widen_Reverse || Kind == CM_Compressed) && + "Expected a consecutive widening decision"); Type *ValTy = getLoadStoreType(I); auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF)); unsigned AS = getLoadStoreAddressSpace(I); @@ -4296,9 +4355,14 @@ InstructionCost LoopVectorizationCostModel::getConsecutiveMemOpCost( const Align Alignment = getLoadStoreAlignment(I); InstructionCost Cost = 0; if (isMaskRequired(I)) { - unsigned IID = I->getOpcode() == Instruction::Load - ? Intrinsic::masked_load - : Intrinsic::masked_store; + Intrinsic::ID LoadIID = Intrinsic::masked_load; + Intrinsic::ID StoreIID = Intrinsic::masked_store; + if (Kind == CM_Compressed) { + LoadIID = Intrinsic::masked_expandload; + StoreIID = Intrinsic::masked_compressstore; + } + + unsigned IID = I->getOpcode() == Instruction::Load ? LoadIID : StoreIID; Cost += TTI.getMemIntrinsicInstrCost( MemIntrinsicCostAttributes(IID, VectorTy, Alignment, AS), Config.CostKind); @@ -5276,6 +5340,8 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I, return TTI::CastContextHint::Reversed; case LoopVectorizationCostModel::CM_Unknown: llvm_unreachable("Instr did not go through cost modelling?"); + case LoopVectorizationCostModel::CM_Compressed: + // TODO: Add Compressed hint (not needed for any targets yet). case LoopVectorizationCostModel::CM_InvalidatedDecision: return TTI::CastContextHint::None; } @@ -5636,6 +5702,11 @@ bool VPCostContext::willBeScalarized(Instruction *I, ElementCount VF) const { (VF.isVector() && CM.isProfitableToScalarize(I, VF)); } +bool VPCostContext::isUniformAfterVectorization(Instruction *I, + ElementCount VF) const { + return CM.isUniformAfterVectorization(I, VF); +} + bool VPCostContext::isMaskRequired(Instruction *I) const { return CM.isMaskRequired(I); } @@ -6225,8 +6296,9 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI, LoopVectorizationCostModel::InstWidening Decision = CM.getWideningDecision(I, Range.Start); bool Reverse = Decision == LoopVectorizationCostModel::CM_Widen_Reverse; + bool Compressed = Decision == LoopVectorizationCostModel::CM_Compressed; bool Consecutive = - Reverse || Decision == LoopVectorizationCostModel::CM_Widen; + Reverse || Compressed || Decision == LoopVectorizationCostModel::CM_Widen; VPValue *Ptr = VPI->getOpcode() == Instruction::Load ? VPI->getOperand(0) : VPI->getOperand(1); @@ -6241,6 +6313,13 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI, if (VPI->getOpcode() == Instruction::Load) { auto *Load = cast<LoadInst>(I); + Type *LoadTy = Load->getType(); + + if (Compressed) + return Builder.createWidenMemIntrinsic( + Intrinsic::masked_expandload, {Ptr, Mask, Plan.getPoison(LoadTy)}, + LoadTy, Load->getAlign(), *VPI, Load->getDebugLoc()); + auto *LoadR = Builder.createWidenLoad(*Load, Ptr, Mask, Consecutive, *VPI, Load->getDebugLoc()); if (Reverse) @@ -6251,6 +6330,12 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI, StoreInst *Store = cast<StoreInst>(I); VPValue *StoredVal = VPI->getOperand(0); + if (Compressed) + return Builder.createWidenMemIntrinsic( + Intrinsic::masked_compressstore, {StoredVal, Ptr, Mask}, + StoredVal->getScalarType(), Store->getAlign(), *VPI, + Store->getDebugLoc()); + if (Reverse) StoredVal = Builder.createNaryOp(VPInstruction::Reverse, StoredVal, Store->getDebugLoc()); @@ -6565,7 +6650,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() { // and in-loop reductions are empty since legality doesn't detect them. if (!RUN_VPLAN_PASS(VPlanTransforms::createHeaderPhiRecipes, *VPlan0, PSE, *OrigLoop, VPDT, Legal->getInductionVars(), - Legal->getReductionVars(), + Legal->getReductionVars(), Legal->getMonotonicPHIs(), Legal->getFixedOrderRecurrences(), Config.getInLoopReductions(), Hints.allowReordering())) { return nullptr; @@ -6834,6 +6919,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan, // bring the VPlan to its final state. // --------------------------------------------------------------------------- + RUN_VPLAN_PASS(VPlanTransforms::adjustMonotonicPhiBackedgeUsers, *Plan, + HeaderVPBB, PSE); + addReductionResultComputation(Plan, RecipeBuilder, Range.Start); // Optimize FindIV reductions to use sentinel-based approach when possible. @@ -8120,6 +8208,15 @@ bool LoopVectorizePass::processLoop(Loop *L) { IC = LVP.selectInterleaveCount(*BestPlanPtr, VF.Width, VF.Cost); unsigned SelectedIC = std::max(IC, UserIC); + + if (LVL.hasMonotonicPHIs() && SelectedIC > 1) { + reportVectorizationFailure( + "Interleaving of loop with monotonic vars", + "Interleaving of loops with monotonic vars is not supported", + "CantInterleaveWithMonotonicVars", ORE, L); + return false; + } + // Optimistically generate runtime checks if they are needed. Drop them if // they turn out to not be profitable. if (VF.Width.isVector() || SelectedIC > 1) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 37060b15a7b97..6b5a852453a48 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -403,9 +403,9 @@ void VPTransformState::fixupHeaderPhis() { for (VPRecipeBase &R : Header->phis()) { auto *PhiR = cast<VPSingleDefRecipe>(&R); - bool NeedsScalar = - isa<VPPhi>(PhiR) || (isa<VPReductionPHIRecipe>(PhiR) && - cast<VPReductionPHIRecipe>(PhiR)->isInLoop()); + bool NeedsScalar = isa<VPPhi>(PhiR) || isa<VPMonotonicPHIRecipe>(PhiR) || + (isa<VPReductionPHIRecipe>(PhiR) && + cast<VPReductionPHIRecipe>(PhiR)->isInLoop()); Value *Phi = get(PhiR, NeedsScalar); Value *Val = get(PhiR->getOperand(1), NeedsScalar); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 814b77a96e825..42a63dc0bfd3f 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -462,12 +462,13 @@ class LLVM_ABI_FOR_TEST VPRecipeBase VPWidenIntOrFpInductionSC, VPWidenPointerInductionSC, VPReductionPHISC, + VPMonotonicPHISC, // END: SubclassID for recipes that inherit VPHeaderPHIRecipe // END: Phi-like recipes VPFirstPHISC = VPWidenPHISC, VPFirstHeaderPHISC = VPCurrentIterationPHISC, - VPLastHeaderPHISC = VPReductionPHISC, - VPLastPHISC = VPReductionPHISC, + VPLastHeaderPHISC = VPMonotonicPHISC, + VPLastPHISC = VPMonotonicPHISC, }; VPRecipeBase(VPRecipeTy SC, ArrayRef<VPValue *> Operands, @@ -660,6 +661,7 @@ class LLVM_ABI_FOR_TEST VPSingleDefRecipe : public VPRecipeBase, case VPRecipeBase::VPReductionPHISC: case VPRecipeBase::VPWidenLoadEVLSC: case VPRecipeBase::VPWidenLoadSC: + case VPRecipeBase::VPMonotonicPHISC: return true; case VPRecipeBase::VPBranchOnMaskSC: case VPRecipeBase::VPInterleaveEVLSC: @@ -2066,7 +2068,9 @@ class VPWidenMemIntrinsicRecipe final : public VPWidenIntrinsicRecipe { VectorIntrinsicID, CallArguments, Ty, {}, MD, DL), Alignment(Alignment) { - assert(VectorIntrinsicID == Intrinsic::experimental_vp_strided_load && + assert((VectorIntrinsicID == Intrinsic::experimental_vp_strided_load || + VectorIntrinsicID == Intrinsic::masked_compressstore || + VectorIntrinsicID == Intrinsic::masked_expandload) && "Unexpected intrinsic"); } @@ -2944,6 +2948,57 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe, public VPIRFlags { #endif }; +/// A recipe for handling monotonic phis. The start value is the first operand +/// of the recipe and the incoming value from the backedge is the second +/// operand. +class VPMonotonicPHIRecipe : public VPHeaderPHIRecipe { + MonotonicDescriptor Desc; + +public: + VPMonotonicPHIRecipe(PHINode *Phi, const MonotonicDescriptor &Desc, + VPValue &Start, VPValue &BackedgeValue) + : VPHeaderPHIRecipe(VPRecipeBase::VPMonotonicPHISC, Phi, &Start), + Desc(Desc) { + + addOperand(&BackedgeValue); + } + + ~VPMonotonicPHIRecipe() override = default; + + VPMonotonicPHIRecipe *clone() override { + auto *R = + new VPMonotonicPHIRecipe(cast<PHINode>(getUnderlyingInstr()), Desc, + *getStartValue(), *getBackedgeValue()); + return R; + } + + VP_CLASSOF_IMPL(VPRecipeBase::VPMonotonicPHISC) + + static inline bool classof(const VPHeaderPHIRecipe *R) { + return R->getVPRecipeID() == VPRecipeBase::VPMonotonicPHISC; + } + + void execute(VPTransformState &State) override; + + InstructionCost computeCost(ElementCount VF, + VPCostContext &Ctx) const override; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + /// Print the recipe. + void printRecipe(raw_ostream &O, const Twine &Indent, + VPSlotTracker &SlotTracker) const override; +#endif + + const MonotonicDescriptor &getDescriptor() const { return Desc; } + + /// Returns true if the recipe only uses the first lane of operand \p Op. + bool usesFirstLaneOnly(const VPValue *Op) const override { + assert(is_contained(operands(), Op) && + "Op must be an operand of the recipe"); + return true; + } +}; + /// A recipe for vectorizing a phi-node as a sequence of mask-based select /// instructions. class LLVM_ABI_FOR_TEST VPBlendRecipe : public VPRecipeWithIRFlags { @@ -4332,7 +4387,8 @@ struct CastInfoMixinImpl template <> struct CastInfo<VPPhiAccessors, VPRecipeBase *> : vpdetail::CastInfoMixinImpl<VPPhiAccessors, VPPhi, VPIRPhi, - VPWidenPHIRecipe, VPHeaderPHIRecipe> {}; + VPWidenPHIRecipe, VPHeaderPHIRecipe, + VPMonotonicPHIRecipe> {}; template <> struct CastInfo<VPPhiAccessors, const VPRecipeBase *> diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp index 421478c85c188..b31212172c6eb 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp @@ -919,6 +919,7 @@ bool VPlanTransforms::createHeaderPhiRecipes( const VPDominatorTree &VPDT, const MapVector<PHINode *, InductionDescriptor> &Inductions, const MapVector<PHINode *, RecurrenceDescriptor> &Reductions, + const MapVector<PHINode *, MonotonicDescriptor> &MonotonicPHIs, const SmallPtrSetImpl<const PHINode *> &FixedOrderRecurrences, const SmallPtrSetImpl<PHINode *> &InLoopReductions, bool AllowReordering) { // Retrieve the header manually from the intial plain-CFG VPlan. @@ -951,6 +952,11 @@ bool VPlanTransforms::createHeaderPhiRecipes( Plan, PSE, OrigLoop, PhiR->getDebugLoc()); + auto MonotonicIt = MonotonicPHIs.find(Phi); + if (MonotonicIt != MonotonicPHIs.end()) + return new VPMonotonicPHIRecipe(Phi, MonotonicIt->second, *Start, + *BackedgeValue); + assert(Reductions.contains(Phi) && "only reductions are expected now"); const RecurrenceDescriptor &RdxDesc = Reductions.lookup(Phi); assert(RdxDesc.getRecurrenceStartValue() == diff --git a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h index 1c987abc649c8..06ff7d4c3cc63 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h +++ b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h @@ -360,6 +360,10 @@ struct VPCostContext { /// Forwards to LoopVectorizationCostModel::getPredBlockCostDivisor. uint64_t getPredBlockCostDivisor(BasicBlock *BB) const; + /// Returns true if \p I is known to be uniform after vectorization. + /// Forwards to LoopVectorizationCostModel::isUniformAfterVectorization. + bool isUniformAfterVectorization(Instruction *I, ElementCount VF) const; + /// Returns true if \p I is known to be scalarized at \p VF. bool willBeScalarized(Instruction *I, ElementCount VF) const; diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index ca63d1498316b..56746ca46267d 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -1209,6 +1209,7 @@ InstructionCost VPRecipeWithIRFlags::getCostForRecipeWithOpcode( return ReplicateRecipe->isPredicated() ? TTI::CastContextHint::Masked : TTI::CastContextHint::Normal; } + const auto *WidenMemoryRecipe = dyn_cast<VPWidenMemoryRecipe>(R); if (WidenMemoryRecipe == nullptr) return TTI::CastContextHint::None; @@ -1445,6 +1446,12 @@ InstructionCost VPInstruction::computeCost(ElementCount VF, VectorTy, /*Mask=*/{}, Ctx.CostKind, /*Index=*/0); } + case VPInstruction::NumActiveLanes: { + Type *ElementTy = getOperand(0)->getScalarType(); + auto *VectorTy = cast<VectorType>(toVectorTy(ElementTy, VF)); + return Ctx.TTI.getArithmeticReductionCost(Instruction::Add, VectorTy, + std::nullopt, Ctx.CostKind); + } case VPInstruction::ExtractLastLane: { // Add on the cost of extracting the element. auto *VecTy = toVectorTy(getOperand(0)->getScalarType(), VF); @@ -2379,9 +2386,31 @@ void VPWidenIntrinsicRecipe::printRecipe(raw_ostream &O, const Twine &Indent, void VPWidenMemIntrinsicRecipe::execute(VPTransformState &State) { CallInst *MemI = createVectorCall(State); + + bool IsLoad; + unsigned PointerOpIdx; + switch (getVectorIntrinsicID()) { + case Intrinsic::experimental_vp_strided_load: + IsLoad = true; + PointerOpIdx = 0; + break; + case Intrinsic::masked_compressstore: + IsLoad = false; + PointerOpIdx = 1; + break; + case Intrinsic::masked_expandload: + IsLoad = true; + PointerOpIdx = 0; + break; + default: + llvm_unreachable("Unexpected IID"); + } + MemI->addParamAttr( - 0, Attribute::getWithAlignment(MemI->getContext(), Alignment)); - State.set(this, MemI); + PointerOpIdx, Attribute::getWithAlignment(MemI->getContext(), Alignment)); + + if (IsLoad) + State.set(this, MemI); } InstructionCost VPWidenMemIntrinsicRecipe::computeMemIntrinsicCost( @@ -2395,10 +2424,23 @@ InstructionCost VPWidenMemIntrinsicRecipe::computeMemIntrinsicCost( InstructionCost VPWidenMemIntrinsicRecipe::computeCost(ElementCount VF, VPCostContext &Ctx) const { + unsigned MaskOpIdx; + switch (getVectorIntrinsicID()) { + case Intrinsic::masked_compressstore: + case Intrinsic::experimental_vp_strided_load: + MaskOpIdx = 2; + break; + case Intrinsic::masked_expandload: + MaskOpIdx = 1; + break; + default: + llvm_unreachable("Unexpected IID"); + } + Type *Ty = toVectorTy(getScalarType(), VF); return computeMemIntrinsicCost(getVectorIntrinsicID(), Ty, - !match(getOperand(2), m_True()), Alignment, - Ctx); + !match(getOperand(MaskOpIdx), m_True()), + Alignment, Ctx); } void VPHistogramRecipe::execute(VPTransformState &State) { @@ -4969,6 +5011,30 @@ bool VPBlendRecipe::usesFirstLaneOnly(const VPValue *Op) const { return vputils::onlyFirstLaneUsed(this); } +void VPMonotonicPHIRecipe::execute(VPTransformState &State) { + executePhiRecipe(this, *this, State, /*IsScalar=*/true, "monotonic.iv"); +} + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +void VPMonotonicPHIRecipe::printRecipe(raw_ostream &O, const Twine &Indent, + VPSlotTracker &SlotTracker) const { + O << Indent << "MONOTONIC-PHI "; + + printAsOperand(O, SlotTracker); + O << " = phi "; + printOperands(O, SlotTracker); +} +#endif + +InstructionCost VPMonotonicPHIRecipe::computeCost(ElementCount VF, + VPCostContext &Ctx) const { + auto *Phi = cast<PHINode>(getUnderlyingValue()); + // The value of a monotonic phi must be uniform across the VF. + if (!Ctx.isUniformAfterVectorization(Phi, VF)) + return InstructionCost::getInvalid(); + return VPHeaderPHIRecipe::computeCost(VF, Ctx); +} + void VPWidenPHIRecipe::execute(VPTransformState &State) { executePhiRecipe(this, *this, State, /*IsScalar=*/false, Name); } diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 541a3da855992..33aab5e26a657 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -4412,6 +4412,55 @@ void VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers(VPlan &Plan, } } +void VPlanTransforms::adjustMonotonicPhiBackedgeUsers( + VPlan &Plan, VPBasicBlock *HeaderVPBB, PredicatedScalarEvolution &PSE) { + for (VPRecipeBase &R : HeaderVPBB->phis()) { + auto *MonotonicPhi = dyn_cast<VPMonotonicPHIRecipe>(&R); + if (!MonotonicPhi) + continue; + + // Obtain mask value for the predicate edge from the last VPBlendRecipe in + // chain. + VPValue *Chain = MonotonicPhi->getBackedgeValue(); + VPValue *Mask = nullptr; + while (auto *BlendR = dyn_cast<VPBlendRecipe>(Chain)) + for (unsigned I = 0, E = BlendR->getNumIncomingValues(); I != E; ++I) + if (auto *IncomingVal = BlendR->getIncomingValue(I); + IncomingVal != MonotonicPhi) { + Chain = IncomingVal; + Mask = BlendR->getMask(I); + break; + } + assert(Mask); + + auto &Desc = MonotonicPhi->getDescriptor(); + auto &SE = *PSE.getSE(); + auto *Step = vputils::getOrCreateVPValueForSCEVExpr( + Plan, Desc.getExpr()->getStepRecurrence(SE)); + + auto *BackedgeVal = MonotonicPhi->getIncomingValue(1); + auto *InsertBlock = BackedgeVal->getDefiningRecipe()->getParent(); + VPBuilder Builder(InsertBlock, InsertBlock->getFirstNonPhi()); + + Type *UpdateType = MonotonicPhi->getScalarType(); + if (UpdateType->isPointerTy()) + UpdateType = Plan.getDataLayout().getIndexType(UpdateType); + + auto *HandledLanes = Builder.createNaryOp( + VPInstruction::NumActiveLanes, {Mask}, nullptr, {}, {}, + DebugLoc::getUnknown(), "handled.lanes", UpdateType); + VPValue *Offset = + Builder.createOverflowingOp(Instruction::Mul, {Step, HandledLanes}); + VPValue *Update; + if (MonotonicPhi->getScalarType()->isPointerTy()) + Update = Builder.createPtrAdd(MonotonicPhi, Offset); + else + Update = Builder.createAdd(MonotonicPhi, Offset, {}, "monotonic.add"); + + BackedgeVal->replaceAllUsesWith(Update); + } +} + /// Check if \p V is a binary expression of a widened IV and a loop-invariant /// value. Returns the widened IV if found, nullptr otherwise. static VPWidenIntOrFpInductionRecipe *getExpressionIV(VPValue *V) { diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h index af8f6de26e0f9..2bce8d6ce0579 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -156,6 +156,7 @@ struct VPlanTransforms { const VPDominatorTree &VPDT, const MapVector<PHINode *, InductionDescriptor> &Inductions, const MapVector<PHINode *, RecurrenceDescriptor> &Reductions, + const MapVector<PHINode *, MonotonicDescriptor> &MonotonicPHIs, const SmallPtrSetImpl<const PHINode *> &FixedOrderRecurrences, const SmallPtrSetImpl<PHINode *> &InLoopReductions, bool AllowReordering); @@ -568,6 +569,13 @@ struct VPlanTransforms { static void adjustFirstOrderRecurrenceMiddleUsers(VPlan &Plan, VFRange &Range); + /// Adjust the backedge value for monotonic PHIs. Changes the update to be + /// number of active lanes of the predicated edge between the step increment + /// and the loop (multiplied by the step size). + static void adjustMonotonicPhiBackedgeUsers(VPlan &Plan, + VPBasicBlock *HeaderVPBB, + PredicatedScalarEvolution &PSE); + /// Optimize FindLast reductions selecting IVs (or expressions of IVs) by /// converting them to FindIV reductions, if their IV range excludes a /// suitable sentinel value. For expressions of IVs, the expression is sunk diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp index 93b18b31e9e7d..965e4c9ac8c8c 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp @@ -445,8 +445,8 @@ bool vputils::isSingleScalar(const VPValue *VPV) { all_of(VPI->operands(), isSingleScalar)); if (auto *RR = dyn_cast<VPReductionRecipe>(VPV)) return !RR->isPartialReduction(); - if (isa<VPVectorPointerRecipe, VPVectorEndPointerRecipe, VPDerivedIVRecipe>( - VPV)) + if (isa<VPVectorPointerRecipe, VPVectorEndPointerRecipe, VPDerivedIVRecipe, + VPMonotonicPHIRecipe>(VPV)) return true; if (auto *Expr = dyn_cast<VPExpressionRecipe>(VPV)) return Expr->isVectorToScalar(); diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/compress-idioms.ll b/llvm/test/Transforms/LoopVectorize/AArch64/compress-idioms.ll new file mode 100644 index 0000000000000..d887180de6552 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/AArch64/compress-idioms.ll @@ -0,0 +1,132 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 5 +; RUN: opt < %s -lv-monotonic-patterns=true -mtriple=aarch64 -mattr=+sve2p2 -passes=loop-vectorize -S 2>&1 | FileCheck %s + +; SVE compresstore/expandload vectorization (requires +sve2p2 for expandload and +sve for compresstore). + +define void @compress_store(ptr writeonly noalias %dst, ptr readonly %src, i32 %c, i64 %n) { +; CHECK-LABEL: define void @compress_store( +; CHECK-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: [[VECTOR_PH:.*:]] +; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]] +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]] +; CHECK: [[VECTOR_PH1]]: +; CHECK-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP0]], 2 +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP10]] +; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[C]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[MONOTONIC_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH1]] ], [ [[MONOTONIC_ADD:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP3]], align 4 +; CHECK-NEXT: [[TMP4:%.*]] = icmp slt <vscale x 4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: [[TMP5:%.*]] = sext i32 [[MONOTONIC_IV]] to i64 +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP5]] +; CHECK-NEXT: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> [[WIDE_LOAD]], ptr align 4 [[TMP6]], <vscale x 4 x i1> [[TMP4]]) +; CHECK-NEXT: [[TMP7:%.*]] = zext <vscale x 4 x i1> [[TMP4]] to <vscale x 4 x i32> +; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP7]]) +; CHECK-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP8]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]] +; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP9]], label %[[IF_THEN:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK: [[IF_THEN]]: +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] +; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] +; CHECK: [[SCALAR_PH]]: +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] + %idx = phi i32 [ 0, %entry ], [ %idx.1, %for.inc ] + %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv + %load.src = load i32, ptr %src.ptr, align 4 + %cmp = icmp slt i32 %load.src, %c + br i1 %cmp, label %if.then, label %for.inc + +if.then: + %dst.idx = sext i32 %idx to i64 + %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %dst.idx + store i32 %load.src, ptr %dst.ptr, align 4 + %idx.next = add nsw i32 %idx, 1 + br label %for.inc + +for.inc: + %idx.1 = phi i32 [ %idx.next, %if.then ], [ %idx, %for.body ] + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %n + br i1 %exitcond.not, label %exit, label %for.body + +exit: + ret void +} + +define void @expand_load(ptr noalias %dst, ptr readonly %src, i32 %c, i64 %n) { +; CHECK-LABEL: define void @expand_load( +; CHECK-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[VECTOR_PH:.*:]] +; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]] +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]] +; CHECK: [[VECTOR_PH1]]: +; CHECK-NEXT: [[TMP11:%.*]] = shl nuw i64 [[TMP0]], 2 +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP11]] +; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[C]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[MONOTONIC_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH1]] ], [ [[MONOTONIC_ADD:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP3]], align 4 +; CHECK-NEXT: [[TMP4:%.*]] = icmp slt <vscale x 4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: [[TMP5:%.*]] = sext i32 [[MONOTONIC_IV]] to i64 +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP5]] +; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.masked.expandload.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> [[TMP4]], <vscale x 4 x i32> poison) +; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP7]], ptr align 4 [[TMP3]], <vscale x 4 x i1> [[TMP4]]) +; CHECK-NEXT: [[TMP8:%.*]] = zext <vscale x 4 x i1> [[TMP4]] to <vscale x 4 x i32> +; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP8]]) +; CHECK-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP9]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] +; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP10]], label %[[IF_THEN:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; CHECK: [[IF_THEN]]: +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] +; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] +; CHECK: [[SCALAR_PH]]: +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] + %idx = phi i32 [ 0, %entry ], [ %idx.1, %for.inc ] + %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %iv + %load.dst = load i32, ptr %dst.ptr, align 4 + %cmp = icmp slt i32 %load.dst, %c + br i1 %cmp, label %if.then, label %for.inc + +if.then: + %src.idx = sext i32 %idx to i64 + %src.ptr = getelementptr inbounds i32, ptr %src, i64 %src.idx + %load.src = load i32, ptr %src.ptr, align 4 + store i32 %load.src, ptr %dst.ptr, align 4 + %idx.next = add nsw i32 %idx, 1 + br label %for.inc + +for.inc: + %idx.1 = phi i32 [ %idx.next, %if.then ], [ %idx, %for.body ] + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %n + br i1 %exitcond.not, label %exit, label %for.body + +exit: + ret void +} diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/compress-idioms.ll b/llvm/test/Transforms/LoopVectorize/VPlan/compress-idioms.ll new file mode 100644 index 0000000000000..470e1bbf85ae3 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/VPlan/compress-idioms.ll @@ -0,0 +1,157 @@ +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "^scalar.ph:" --version 6 +; RUN: opt -lv-monotonic-patterns=true -force-target-supports-masked-memory-ops -force-vector-width=4 -mtriple=aarch64 -mattr=+sve2p2 -passes=loop-vectorize -vplan-print-after=printOptimizedVPlan -disable-output %s -S 2>&1 | FileCheck %s + +define void @compress_store(ptr writeonly noalias %dst, ptr readonly %src, i32 %c, i64 %n) { +; CHECK-LABEL: VPlan for loop in 'compress_store' +; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF +; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF +; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count +; CHECK-NEXT: Live-in ir<%n> = original trip-count +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb<entry>: +; CHECK-NEXT: Successor(s): scalar.ph, vector.ph +; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: MONOTONIC-PHI ir<%idx> = phi ir<0>, vp<%monotonic.add> +; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]> +; CHECK-NEXT: CLONE ir<%src.ptr> = getelementptr inbounds ir<%src>, vp<[[VP4]]> +; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds i32, ir<%src.ptr>, ir<1> +; CHECK-NEXT: WIDEN ir<%load.src> = load vp<[[VP5]]> +; CHECK-NEXT: WIDEN ir<%cmp> = icmp slt ir<%load.src>, ir<%c> +; CHECK-NEXT: EMIT-SCALAR ir<%dst.idx> = sext ir<%idx> to i64 +; CHECK-NEXT: CLONE ir<%dst.ptr> = getelementptr inbounds ir<%dst>, ir<%dst.idx> +; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds i32, ir<%dst.ptr>, ir<1> +; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP7:%[0-9]+]]> = call llvm.masked.compressstore(ir<%load.src>, vp<[[VP6]]>, ir<%cmp>) +; CHECK-NEXT: EMIT vp<%handled.lanes> = num-active-lanes ir<%cmp> +; CHECK-NEXT: EMIT vp<%monotonic.add> = add ir<%idx>, vp<%handled.lanes> +; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = extract-last-part vp<%monotonic.add> +; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = extract-last-lane vp<[[VP9]]> +; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]> +; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n> +; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb<exit>: +; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: scalar.ph: +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] + %idx = phi i32 [ 0, %entry ], [ %idx.1, %for.inc ] + %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv + %load.src = load i32, ptr %src.ptr, align 4 + %cmp = icmp slt i32 %load.src, %c + br i1 %cmp, label %if.then, label %for.inc + +if.then: + %dst.idx = sext i32 %idx to i64 + %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %dst.idx + store i32 %load.src, ptr %dst.ptr, align 4 + %idx.next = add nsw i32 %idx, 1 + br label %for.inc + +for.inc: + %idx.1 = phi i32 [ %idx.next, %if.then ], [ %idx, %for.body ] + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %n + br i1 %exitcond.not, label %exit, label %for.body + +exit: + ret void +} + +define void @expand_load(ptr noalias %dst, ptr readonly %src, i32 %c, i64 %n) { +; CHECK-LABEL: VPlan for loop in 'expand_load' +; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF +; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF +; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count +; CHECK-NEXT: Live-in ir<%n> = original trip-count +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb<entry>: +; CHECK-NEXT: Successor(s): scalar.ph, vector.ph +; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: MONOTONIC-PHI ir<%idx> = phi ir<0>, vp<%monotonic.add> +; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]> +; CHECK-NEXT: CLONE ir<%dst.ptr> = getelementptr ir<%dst>, vp<[[VP4]]> +; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds i32, ir<%dst.ptr>, ir<1> +; CHECK-NEXT: WIDEN ir<%load.dst> = load vp<[[VP5]]> +; CHECK-NEXT: WIDEN ir<%cmp> = icmp slt ir<%load.dst>, ir<%c> +; CHECK-NEXT: EMIT-SCALAR ir<%src.idx> = sext ir<%idx> to i64 +; CHECK-NEXT: CLONE ir<%src.ptr> = getelementptr inbounds ir<%src>, ir<%src.idx> +; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds i32, ir<%src.ptr>, ir<1> +; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP7:%[0-9]+]]> = call llvm.masked.expandload(vp<[[VP6]]>, ir<%cmp>, ir<poison>) +; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i32, ir<%dst.ptr>, ir<1> +; CHECK-NEXT: WIDEN store vp<[[VP8]]>, vp<[[VP7]]>, ir<%cmp> +; CHECK-NEXT: EMIT vp<%handled.lanes> = num-active-lanes ir<%cmp> +; CHECK-NEXT: EMIT vp<%monotonic.add> = add ir<%idx>, vp<%handled.lanes> +; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = extract-last-part vp<%monotonic.add> +; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extract-last-lane vp<[[VP10]]> +; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]> +; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n> +; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb<exit>: +; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: scalar.ph: +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] + %idx = phi i32 [ 0, %entry ], [ %idx.1, %for.inc ] + %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %iv + %load.dst = load i32, ptr %dst.ptr, align 4 + %cmp = icmp slt i32 %load.dst, %c + br i1 %cmp, label %if.then, label %for.inc + +if.then: + %src.idx = sext i32 %idx to i64 + %src.ptr = getelementptr inbounds i32, ptr %src, i64 %src.idx + %load.src = load i32, ptr %src.ptr, align 4 + store i32 %load.src, ptr %dst.ptr, align 4 + %idx.next = add nsw i32 %idx, 1 + br label %for.inc + +for.inc: + %idx.1 = phi i32 [ %idx.next, %if.then ], [ %idx, %for.body ] + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %n + br i1 %exitcond.not, label %exit, label %for.body + +exit: + ret void +} diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll index a7271524f0191..abdf0754ea4af 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll @@ -26,6 +26,7 @@ ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::makeScalarizationDecisions ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::makeCallWideningDecisions ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers +; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::adjustMonotonicPhiBackedgeUsers ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::clearReductionWrapFlags ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::optimizeFindIVReductions ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::optimizeInductionLiveOutUsers diff --git a/llvm/test/Transforms/LoopVectorize/compress-idioms.ll b/llvm/test/Transforms/LoopVectorize/compress-idioms.ll new file mode 100644 index 0000000000000..da7a1f913f6cd --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/compress-idioms.ll @@ -0,0 +1,424 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^for.body:" --version 5 +; RUN: opt < %s -lv-monotonic-patterns=true -force-target-supports-masked-memory-ops -force-vector-width=4 -passes=loop-vectorize -S 2>&1 | FileCheck %s -check-prefixes=CHECK,CHECK-IC1 +; RUN: opt < %s -lv-monotonic-patterns=true -force-target-supports-masked-memory-ops -force-vector-width=4 -tail-folding-policy=must-fold-tail -passes=loop-vectorize -S 2>&1 | FileCheck %s -check-prefixes=CHECK,CHECK-TF +; RUN: opt < %s -lv-monotonic-patterns=true -force-target-supports-masked-memory-ops -force-vector-width=4 -force-vector-interleave=2 -passes=loop-vectorize -disable-output -pass-remarks-analysis=loop-vectorize 2>&1 | FileCheck %s --check-prefix=IC2 + +; IC2: loop not vectorized: Interleaving of loops with monotonic vars is not supported + +define void @test_compress_store_with_index(ptr writeonly noalias %dst, ptr readonly %src, i32 %c, i64 %n) { +; CHECK-IC1-LABEL: define void @test_compress_store_with_index( +; CHECK-IC1-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) { +; CHECK-IC1-NEXT: [[ENTRY:.*]]: +; CHECK-IC1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 +; CHECK-IC1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK-IC1: [[VECTOR_PH]]: +; CHECK-IC1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4 +; CHECK-IC1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-IC1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0 +; CHECK-IC1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer +; CHECK-IC1-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK-IC1: [[VECTOR_BODY]]: +; CHECK-IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-IC1-NEXT: [[MONOTONIC_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[MONOTONIC_ADD:%.*]], %[[VECTOR_BODY]] ] +; CHECK-IC1-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]] +; CHECK-IC1-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4 +; CHECK-IC1-NEXT: [[TMP1:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]] +; CHECK-IC1-NEXT: [[TMP2:%.*]] = sext i32 [[MONOTONIC_IV]] to i64 +; CHECK-IC1-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP2]] +; CHECK-IC1-NEXT: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> [[WIDE_LOAD]], ptr align 4 [[TMP3]], <4 x i1> [[TMP1]]) +; CHECK-IC1-NEXT: [[TMP4:%.*]] = zext <4 x i1> [[TMP1]] to <4 x i32> +; CHECK-IC1-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP4]]) +; CHECK-IC1-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP5]] +; CHECK-IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-IC1-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-IC1-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK-IC1: [[MIDDLE_BLOCK]]: +; CHECK-IC1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] +; CHECK-IC1-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] +; CHECK-IC1: [[SCALAR_PH]]: +; CHECK-IC1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] +; CHECK-IC1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[MONOTONIC_ADD]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] +; CHECK-IC1-NEXT: br label %[[FOR_INC:.*]] +; CHECK-IC1: [[FOR_INC]]: +; +; CHECK-TF-LABEL: define void @test_compress_store_with_index( +; CHECK-TF-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) { +; CHECK-TF-NEXT: [[ENTRY:.*:]] +; CHECK-TF-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK-TF: [[VECTOR_PH]]: +; CHECK-TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3 +; CHECK-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4 +; CHECK-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1 +; CHECK-TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0 +; CHECK-TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer +; CHECK-TF-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0 +; CHECK-TF-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer +; CHECK-TF-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK-TF: [[VECTOR_BODY]]: +; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-TF-NEXT: [[MONOTONIC_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[MONOTONIC_ADD:%.*]], %[[VECTOR_BODY]] ] +; CHECK-TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-TF-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]] +; CHECK-TF-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]] +; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP1]], <4 x i1> [[TMP0]], <4 x i32> poison) +; CHECK-TF-NEXT: [[TMP2:%.*]] = icmp slt <4 x i32> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT2]] +; CHECK-TF-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer +; CHECK-TF-NEXT: [[TMP4:%.*]] = sext i32 [[MONOTONIC_IV]] to i64 +; CHECK-TF-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP4]] +; CHECK-TF-NEXT: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> [[WIDE_MASKED_LOAD]], ptr align 4 [[TMP5]], <4 x i1> [[TMP3]]) +; CHECK-TF-NEXT: [[TMP6:%.*]] = zext <4 x i1> [[TMP3]] to <4 x i32> +; CHECK-TF-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP6]]) +; CHECK-TF-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP7]] +; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 +; CHECK-TF-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4) +; CHECK-TF-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-TF-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK-TF: [[MIDDLE_BLOCK]]: +; CHECK-TF-NEXT: br label %[[EXIT:.*]] +; CHECK-TF: [[EXIT]]: +; CHECK-TF-NEXT: ret void +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] + %idx = phi i32 [ 0, %entry ], [ %idx.1, %for.inc ] + %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv + %load.src = load i32, ptr %src.ptr, align 4 + %cmp = icmp slt i32 %load.src, %c + br i1 %cmp, label %if.then, label %for.inc + +if.then: + %dst.idx = sext i32 %idx to i64 + %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %dst.idx + store i32 %load.src, ptr %dst.ptr, align 4 + %idx.next = add nsw i32 %idx, 1 + br label %for.inc + +for.inc: + %idx.1 = phi i32 [ %idx.next, %if.then ], [ %idx, %for.body ] + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %n + br i1 %exitcond.not, label %exit, label %for.body + +exit: + ret void +} + +; IC2: loop not vectorized: Interleaving of loops with monotonic vars is not supported + +define void @test_expand_load_with_index(ptr noalias %dst, ptr readonly %src, i32 %c, i64 %n) { +; CHECK-IC1-LABEL: define void @test_expand_load_with_index( +; CHECK-IC1-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) { +; CHECK-IC1-NEXT: [[ENTRY:.*]]: +; CHECK-IC1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 +; CHECK-IC1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK-IC1: [[VECTOR_PH]]: +; CHECK-IC1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4 +; CHECK-IC1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-IC1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0 +; CHECK-IC1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer +; CHECK-IC1-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK-IC1: [[VECTOR_BODY]]: +; CHECK-IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-IC1-NEXT: [[MONOTONIC_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[MONOTONIC_ADD:%.*]], %[[VECTOR_BODY]] ] +; CHECK-IC1-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX]] +; CHECK-IC1-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4 +; CHECK-IC1-NEXT: [[TMP1:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]] +; CHECK-IC1-NEXT: [[TMP2:%.*]] = sext i32 [[MONOTONIC_IV]] to i64 +; CHECK-IC1-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP2]] +; CHECK-IC1-NEXT: [[TMP4:%.*]] = call <4 x i32> @llvm.masked.expandload.v4i32.p0(ptr align 4 [[TMP3]], <4 x i1> [[TMP1]], <4 x i32> poison) +; CHECK-IC1-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP4]], ptr align 4 [[TMP0]], <4 x i1> [[TMP1]]) +; CHECK-IC1-NEXT: [[TMP5:%.*]] = zext <4 x i1> [[TMP1]] to <4 x i32> +; CHECK-IC1-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP5]]) +; CHECK-IC1-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP6]] +; CHECK-IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-IC1-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-IC1-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; CHECK-IC1: [[MIDDLE_BLOCK]]: +; CHECK-IC1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] +; CHECK-IC1-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] +; CHECK-IC1: [[SCALAR_PH]]: +; CHECK-IC1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] +; CHECK-IC1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[MONOTONIC_ADD]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] +; CHECK-IC1-NEXT: br label %[[FOR_INC:.*]] +; CHECK-IC1: [[FOR_INC]]: +; +; CHECK-TF-LABEL: define void @test_expand_load_with_index( +; CHECK-TF-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) { +; CHECK-TF-NEXT: [[ENTRY:.*:]] +; CHECK-TF-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK-TF: [[VECTOR_PH]]: +; CHECK-TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3 +; CHECK-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4 +; CHECK-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1 +; CHECK-TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0 +; CHECK-TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer +; CHECK-TF-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0 +; CHECK-TF-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer +; CHECK-TF-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK-TF: [[VECTOR_BODY]]: +; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-TF-NEXT: [[MONOTONIC_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[MONOTONIC_ADD:%.*]], %[[VECTOR_BODY]] ] +; CHECK-TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-TF-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]] +; CHECK-TF-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX]] +; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP1]], <4 x i1> [[TMP0]], <4 x i32> poison) +; CHECK-TF-NEXT: [[TMP2:%.*]] = icmp slt <4 x i32> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT2]] +; CHECK-TF-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer +; CHECK-TF-NEXT: [[TMP4:%.*]] = sext i32 [[MONOTONIC_IV]] to i64 +; CHECK-TF-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP4]] +; CHECK-TF-NEXT: [[TMP6:%.*]] = call <4 x i32> @llvm.masked.expandload.v4i32.p0(ptr align 4 [[TMP5]], <4 x i1> [[TMP3]], <4 x i32> poison) +; CHECK-TF-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP6]], ptr align 4 [[TMP1]], <4 x i1> [[TMP3]]) +; CHECK-TF-NEXT: [[TMP7:%.*]] = zext <4 x i1> [[TMP3]] to <4 x i32> +; CHECK-TF-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP7]]) +; CHECK-TF-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP8]] +; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 +; CHECK-TF-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4) +; CHECK-TF-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-TF-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] +; CHECK-TF: [[MIDDLE_BLOCK]]: +; CHECK-TF-NEXT: br label %[[EXIT:.*]] +; CHECK-TF: [[EXIT]]: +; CHECK-TF-NEXT: ret void +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] + %idx = phi i32 [ 0, %entry ], [ %idx.1, %for.inc ] + %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %iv + %load.dst = load i32, ptr %dst.ptr, align 4 + %cmp = icmp slt i32 %load.dst, %c + br i1 %cmp, label %if.then, label %for.inc + +if.then: + %src.idx = sext i32 %idx to i64 + %src.ptr = getelementptr inbounds i32, ptr %src, i64 %src.idx + %load.src = load i32, ptr %src.ptr, align 4 + store i32 %load.src, ptr %dst.ptr, align 4 + %idx.next = add nsw i32 %idx, 1 + br label %for.inc + +for.inc: + %idx.1 = phi i32 [ %idx.next, %if.then ], [ %idx, %for.body ] + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %n + br i1 %exitcond.not, label %exit, label %for.body + +exit: + ret void +} + +; IC2: loop not vectorized: Interleaving of loops with monotonic vars is not supported + +define i32 @test_conditionally_incremented_phi_liveout(ptr writeonly noalias %dst, ptr readonly %src, i32 %c, i64 %n) { +; CHECK-IC1-LABEL: define i32 @test_conditionally_incremented_phi_liveout( +; CHECK-IC1-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) { +; CHECK-IC1-NEXT: [[ENTRY:.*]]: +; CHECK-IC1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 +; CHECK-IC1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK-IC1: [[VECTOR_PH]]: +; CHECK-IC1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4 +; CHECK-IC1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-IC1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0 +; CHECK-IC1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer +; CHECK-IC1-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK-IC1: [[VECTOR_BODY]]: +; CHECK-IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-IC1-NEXT: [[MONOTONIC_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[MONOTONIC_ADD:%.*]], %[[VECTOR_BODY]] ] +; CHECK-IC1-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]] +; CHECK-IC1-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4 +; CHECK-IC1-NEXT: [[TMP1:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]] +; CHECK-IC1-NEXT: [[TMP2:%.*]] = sext i32 [[MONOTONIC_IV]] to i64 +; CHECK-IC1-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP2]] +; CHECK-IC1-NEXT: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> [[WIDE_LOAD]], ptr align 4 [[TMP3]], <4 x i1> [[TMP1]]) +; CHECK-IC1-NEXT: [[TMP4:%.*]] = zext <4 x i1> [[TMP1]] to <4 x i32> +; CHECK-IC1-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP4]]) +; CHECK-IC1-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP5]] +; CHECK-IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-IC1-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-IC1-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] +; CHECK-IC1: [[MIDDLE_BLOCK]]: +; CHECK-IC1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] +; CHECK-IC1-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] +; CHECK-IC1: [[SCALAR_PH]]: +; CHECK-IC1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] +; CHECK-IC1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[MONOTONIC_ADD]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] +; CHECK-IC1-NEXT: br label %[[FOR_INC:.*]] +; CHECK-IC1: [[FOR_INC]]: +; +; CHECK-TF-LABEL: define i32 @test_conditionally_incremented_phi_liveout( +; CHECK-TF-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) { +; CHECK-TF-NEXT: [[ENTRY:.*:]] +; CHECK-TF-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK-TF: [[VECTOR_PH]]: +; CHECK-TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3 +; CHECK-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4 +; CHECK-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1 +; CHECK-TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0 +; CHECK-TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer +; CHECK-TF-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0 +; CHECK-TF-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer +; CHECK-TF-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK-TF: [[VECTOR_BODY]]: +; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-TF-NEXT: [[MONOTONIC_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[MONOTONIC_ADD:%.*]], %[[VECTOR_BODY]] ] +; CHECK-TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-TF-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]] +; CHECK-TF-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]] +; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP1]], <4 x i1> [[TMP0]], <4 x i32> poison) +; CHECK-TF-NEXT: [[TMP2:%.*]] = icmp slt <4 x i32> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT2]] +; CHECK-TF-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer +; CHECK-TF-NEXT: [[TMP4:%.*]] = sext i32 [[MONOTONIC_IV]] to i64 +; CHECK-TF-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP4]] +; CHECK-TF-NEXT: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> [[WIDE_MASKED_LOAD]], ptr align 4 [[TMP5]], <4 x i1> [[TMP3]]) +; CHECK-TF-NEXT: [[TMP6:%.*]] = zext <4 x i1> [[TMP3]] to <4 x i32> +; CHECK-TF-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP6]]) +; CHECK-TF-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP7]] +; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 +; CHECK-TF-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4) +; CHECK-TF-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-TF-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; CHECK-TF: [[MIDDLE_BLOCK]]: +; CHECK-TF-NEXT: br label %[[EXIT:.*]] +; CHECK-TF: [[EXIT]]: +; CHECK-TF-NEXT: ret i32 [[MONOTONIC_ADD]] +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] + %idx = phi i32 [ 0, %entry ], [ %idx.1, %for.inc ] + %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv + %load.src = load i32, ptr %src.ptr, align 4 + %cmp = icmp slt i32 %load.src, %c + br i1 %cmp, label %if.then, label %for.inc + +if.then: + %dst.idx = sext i32 %idx to i64 + %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %dst.idx + store i32 %load.src, ptr %dst.ptr, align 4 + %idx.next = add nsw i32 %idx, 1 + br label %for.inc + +for.inc: + %idx.1 = phi i32 [ %idx.next, %if.then ], [ %idx, %for.body ] + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %n + br i1 %exitcond.not, label %exit, label %for.body + +exit: + ret i32 %idx.1 +} + +; Negative test: Conditional pointer (rather than index) increments are not supported yet (needs LAA support). +define void @test_compress_store_with_pointer(ptr writeonly noalias %init.dst, ptr readonly %src, i32 %c, i64 %n) { +; CHECK-LABEL: define void @test_compress_store_with_pointer( +; CHECK-SAME: ptr noalias writeonly [[INIT_DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) { +; CHECK-NEXT: [[IF_THEN:.*:]] +; CHECK-NEXT: br label %[[FOR_INC:.*]] +; CHECK: [[FOR_INC]]: +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] + %dst = phi ptr [ %init.dst, %entry ], [ %dst.1, %for.inc ] + %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv + %load.src = load i32, ptr %src.ptr, align 4 + %cmp = icmp slt i32 %load.src, %c + br i1 %cmp, label %if.then, label %for.inc + +if.then: + %dst.inc = getelementptr inbounds i8, ptr %dst, i64 4 + store i32 %load.src, ptr %dst, align 4 + br label %for.inc + +for.inc: + %dst.1 = phi ptr [ %dst.inc, %if.then ], [ %dst, %for.body ] + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %n + br i1 %exitcond.not, label %exit, label %for.body + +exit: + ret void +} + +; Negative test: Storing the conditionally incremented phi is invalid (as all uses must be uniform). +define void @test_store_conditionally_incremented_value(ptr writeonly noalias %dst, ptr readonly %src, i32 %c, i64 %n) { +; CHECK-LABEL: define void @test_store_conditionally_incremented_value( +; CHECK-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) { +; CHECK-NEXT: [[IF_THEN:.*:]] +; CHECK-NEXT: br label %[[FOR_INC:.*]] +; CHECK: [[FOR_INC]]: +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] + %idx = phi i32 [ 0, %entry ], [ %idx.1, %for.inc ] + %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv + %load.src = load i32, ptr %src.ptr, align 4 + %cmp = icmp slt i32 %load.src, %c + br i1 %cmp, label %if.then, label %for.inc + +if.then: + %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %iv + store i32 %idx, ptr %dst.ptr, align 4 + %idx.next = add nsw i32 %idx, 1 + br label %for.inc + +for.inc: + %idx.1 = phi i32 [ %idx.next, %if.then ], [ %idx, %for.body ] + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %n + br i1 %exitcond.not, label %exit, label %for.body + +exit: + ret void +} + +; Pre-increment is currently not matched as we require one use of the step instruction. +define i32 @test_pre_increment_compress_store(ptr writeonly noalias %dst, ptr readonly %src, i32 %c, i64 %n) { +; CHECK-LABEL: define i32 @test_pre_increment_compress_store( +; CHECK-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) { +; CHECK-NEXT: [[IF_THEN:.*:]] +; CHECK-NEXT: br label %[[FOR_INC:.*]] +; CHECK: [[FOR_INC]]: +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] + %idx = phi i32 [ 0, %entry ], [ %idx.1, %for.inc ] + %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv + %load.src = load i32, ptr %src.ptr, align 4 + %cmp = icmp slt i32 %load.src, %c + br i1 %cmp, label %if.then, label %for.inc + +if.then: + %idx.next = add nsw i32 %idx, 1 + %dst.idx = sext i32 %idx.next to i64 + %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %dst.idx + store i32 %load.src, ptr %dst.ptr, align 4 + br label %for.inc + +for.inc: + %idx.1 = phi i32 [ %idx.next, %if.then ], [ %idx, %for.body ] + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %n + br i1 %exitcond.not, label %exit, label %for.body + +exit: + ret i32 %idx.1 +} diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h index 3bcbd3ca6c937..6dd891c5e5567 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h +++ b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h @@ -93,6 +93,7 @@ class VPlanTestIRBase : public testing::Test { VPlanTransforms::createHeaderPhiRecipes( *Plan, PSE, *L, VPDT, Inductions, MapVector<PHINode *, RecurrenceDescriptor>(), + MapVector<PHINode *, MonotonicDescriptor>(), SmallPtrSet<const PHINode *, 1>(), SmallPtrSet<PHINode *, 1>(), /*AllowReordering=*/false); } >From a7cdf805a403233c6b375f21cd775ce3c51dd7f5 Mon Sep 17 00:00:00 2001 From: Benjamin Maxwell <[email protected]> Date: Thu, 6 Aug 2026 14:50:46 +0000 Subject: [PATCH 2/3] Fix epilogue resume handling --- .../Transforms/Vectorize/LoopVectorize.cpp | 3 +- llvm/lib/Transforms/Vectorize/VPlan.h | 10 +- .../compress-store-vec-epilogue.ll | 98 +++++++++++++++++++ 3 files changed, 105 insertions(+), 6 deletions(-) create mode 100644 llvm/test/Transforms/LoopVectorize/compress-store-vec-epilogue.ll diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 24174ae275d2e..1068315b6b46f 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7802,7 +7802,8 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop( } } else { // Retrieve the induction resume value via ResumeForEpilogue. - PHINode *IndPhi = cast<VPWidenInductionRecipe>(&R)->getPHINode(); + assert(isa<VPWidenInductionRecipe>(&R) || isa<VPMonotonicPHIRecipe>(&R)); + PHINode *IndPhi = cast<VPHeaderPHIRecipe>(&R)->getPHINode(); ResumeV = IRPhiToResumeForEpi.at(IndPhi)->getUnderlyingValue(); } assert(ResumeV && "Must have a resume value"); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 42a63dc0bfd3f..52e8ed53c7f7f 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -2501,6 +2501,11 @@ class LLVM_ABI_FOR_TEST VPHeaderPHIRecipe : public VPSingleDefRecipe, VPUser::addOperand(V); } + /// Returns the underlying PHINode if one exists, or null otherwise. + PHINode *getPHINode() const { + return cast_if_present<PHINode>(getUnderlyingValue()); + } + protected: #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) /// Print the recipe. @@ -2577,11 +2582,6 @@ class VPWidenInductionRecipe : public VPHeaderPHIRecipe { /// incoming value, its start value. unsigned getNumIncoming() const override { return 1; } - /// Returns the underlying PHINode if one exists, or null otherwise. - PHINode *getPHINode() const { - return cast_if_present<PHINode>(getUnderlyingValue()); - } - /// Returns the induction descriptor for the recipe. const InductionDescriptor &getInductionDescriptor() const { return IndDesc; } diff --git a/llvm/test/Transforms/LoopVectorize/compress-store-vec-epilogue.ll b/llvm/test/Transforms/LoopVectorize/compress-store-vec-epilogue.ll new file mode 100644 index 0000000000000..37c605c2b3b00 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/compress-store-vec-epilogue.ll @@ -0,0 +1,98 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^for.body:" --version 5 +; RUN: opt < %s -lv-monotonic-patterns=true -force-target-supports-masked-memory-ops -force-vector-width=16 -epilogue-vectorization-force-VF=4 -passes=loop-vectorize -S 2>&1 | FileCheck %s -check-prefixes=CHECK + +define void @compress_store(ptr writeonly noalias %dst, ptr readonly %src, i32 %c, i64 %n) { +; CHECK-LABEL: define void @compress_store( +; CHECK-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) { +; CHECK-NEXT: [[ITER_CHECK:.*]]: +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] +; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: +; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 16 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16 +; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[C]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[MONOTONIC_IV:%.*]] = phi i32 [ 42, %[[VECTOR_PH]] ], [ [[MONOTONIC_ADD:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr [[TMP0]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: [[TMP2:%.*]] = sext i32 [[MONOTONIC_IV]] to i64 +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP2]] +; CHECK-NEXT: call void @llvm.masked.compressstore.v16i32.p0(<16 x i32> [[WIDE_LOAD]], ptr align 4 [[TMP3]], <16 x i1> [[TMP1]]) +; CHECK-NEXT: [[TMP4:%.*]] = zext <16 x i1> [[TMP1]] to <16 x i32> +; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP4]]) +; CHECK-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP5]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 +; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] +; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] +; CHECK: [[VEC_EPILOG_ITER_CHECK]]: +; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4 +; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]] +; CHECK: [[VEC_EPILOG_PH]]: +; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] +; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[MONOTONIC_ADD]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 42, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] +; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], 4 +; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]] +; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT4]], <4 x i32> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] +; CHECK: [[VEC_EPILOG_VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX6:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT10:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] +; CHECK-NEXT: [[MONOTONIC_IV7:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[VEC_EPILOG_PH]] ], [ [[MONOTONIC_ADD9:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX6]] +; CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load <4 x i32>, ptr [[TMP7]], align 4 +; CHECK-NEXT: [[TMP8:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD8]], [[BROADCAST_SPLAT5]] +; CHECK-NEXT: [[TMP9:%.*]] = sext i32 [[MONOTONIC_IV7]] to i64 +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP9]] +; CHECK-NEXT: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> [[WIDE_LOAD8]], ptr align 4 [[TMP10]], <4 x i1> [[TMP8]]) +; CHECK-NEXT: [[TMP11:%.*]] = zext <4 x i1> [[TMP8]] to <4 x i32> +; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP11]]) +; CHECK-NEXT: [[MONOTONIC_ADD9]] = add i32 [[MONOTONIC_IV7]], [[TMP12]] +; CHECK-NEXT: [[INDEX_NEXT10]] = add nuw i64 [[INDEX6]], 4 +; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT10]], [[N_VEC3]] +; CHECK-NEXT: br i1 [[TMP13]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]: +; CHECK-NEXT: [[CMP_N11:%.*]] = icmp eq i64 [[N]], [[N_VEC3]] +; CHECK-NEXT: br i1 [[CMP_N11]], [[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]] +; CHECK: [[VEC_EPILOG_SCALAR_PH]]: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] +; CHECK-NEXT: [[BC_MERGE_RDX12:%.*]] = phi i32 [ [[MONOTONIC_ADD9]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[MONOTONIC_ADD]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 42, %[[ITER_CHECK]] ] +; CHECK-NEXT: br label %[[FOR_BODY:.*]] +; CHECK: [[FOR_BODY]]: +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] + %idx = phi i32 [ 42, %entry ], [ %idx.1, %for.inc ] + %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv + %load.src = load i32, ptr %src.ptr, align 4 + %cmp = icmp slt i32 %load.src, %c + br i1 %cmp, label %if.then, label %for.inc + +if.then: + %dst.idx = sext i32 %idx to i64 + %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %dst.idx + store i32 %load.src, ptr %dst.ptr, align 4 + %idx.next = add nsw i32 %idx, 1 + br label %for.inc + +for.inc: + %idx.1 = phi i32 [ %idx.next, %if.then ], [ %idx, %for.body ] + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %n + br i1 %exitcond.not, label %exit, label %for.body + +exit: + ret void +} >From 84b4348eab2fa19e7c69bc97e0a759d27225e3ef Mon Sep 17 00:00:00 2001 From: Benjamin Maxwell <[email protected]> Date: Thu, 6 Aug 2026 15:44:58 +0000 Subject: [PATCH 3/3] Don't allow null phi --- llvm/lib/Transforms/Vectorize/VPlan.h | 10 ++++------ llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 52e8ed53c7f7f..64c06b0625306 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -2955,9 +2955,9 @@ class VPMonotonicPHIRecipe : public VPHeaderPHIRecipe { MonotonicDescriptor Desc; public: - VPMonotonicPHIRecipe(PHINode *Phi, const MonotonicDescriptor &Desc, + VPMonotonicPHIRecipe(PHINode &Phi, const MonotonicDescriptor &Desc, VPValue &Start, VPValue &BackedgeValue) - : VPHeaderPHIRecipe(VPRecipeBase::VPMonotonicPHISC, Phi, &Start), + : VPHeaderPHIRecipe(VPRecipeBase::VPMonotonicPHISC, &Phi, &Start), Desc(Desc) { addOperand(&BackedgeValue); @@ -2966,10 +2966,8 @@ class VPMonotonicPHIRecipe : public VPHeaderPHIRecipe { ~VPMonotonicPHIRecipe() override = default; VPMonotonicPHIRecipe *clone() override { - auto *R = - new VPMonotonicPHIRecipe(cast<PHINode>(getUnderlyingInstr()), Desc, - *getStartValue(), *getBackedgeValue()); - return R; + return new VPMonotonicPHIRecipe(*getPHINode(), Desc, *getStartValue(), + *getBackedgeValue()); } VP_CLASSOF_IMPL(VPRecipeBase::VPMonotonicPHISC) diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp index b31212172c6eb..ddfed244998e0 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp @@ -954,7 +954,7 @@ bool VPlanTransforms::createHeaderPhiRecipes( auto MonotonicIt = MonotonicPHIs.find(Phi); if (MonotonicIt != MonotonicPHIs.end()) - return new VPMonotonicPHIRecipe(Phi, MonotonicIt->second, *Start, + return new VPMonotonicPHIRecipe(*Phi, MonotonicIt->second, *Start, *BackedgeValue); assert(Reductions.contains(Phi) && "only reductions are expected now"); _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
