https://github.com/SamTebbs33 created https://github.com/llvm/llvm-project/pull/165543
This PR re-introduces the assert that the cost of a partial reduction is valid during VPExpressionRecipe creation. This is a stacked PR: 1. https://github.com/llvm/llvm-project/pull/165536 >From 0aa3c2d73ee5546b2d45b8ccf7c33ca39e7ec4d2 Mon Sep 17 00:00:00 2001 From: Samuel Tebbs <[email protected]> Date: Wed, 29 Oct 2025 10:10:00 +0000 Subject: [PATCH] [LV] Use assertion in VPExpressionRecipe creation --- .../Transforms/Vectorize/VPlanTransforms.cpp | 70 ++++++++++--------- 1 file changed, 38 insertions(+), 32 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index d9ac26bba7507..e75c99c35938e 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -3532,24 +3532,28 @@ tryToMatchAndCreateExtendedReduction(VPReductionRecipe *Red, VPCostContext &Ctx, auto *SrcVecTy = cast<VectorType>(toVectorTy(SrcTy, VF)); TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput; - InstructionCost ExtRedCost; - InstructionCost ExtCost = - cast<VPWidenCastRecipe>(VecOp)->computeCost(VF, Ctx); - InstructionCost RedCost = Red->computeCost(VF, Ctx); - if (isa<VPPartialReductionRecipe>(Red)) { TargetTransformInfo::PartialReductionExtendKind ExtKind = TargetTransformInfo::getPartialReductionExtendKind(ExtOpc); // FIXME: Move partial reduction creation, costing and clamping // here from LoopVectorize.cpp. - ExtRedCost = Ctx.TTI.getPartialReductionCost( - Opcode, SrcTy, nullptr, RedTy, VF, ExtKind, - llvm::TargetTransformInfo::PR_None, std::nullopt, Ctx.CostKind); - } else { - ExtRedCost = Ctx.TTI.getExtendedReductionCost( - Opcode, ExtOpc == Instruction::CastOps::ZExt, RedTy, SrcVecTy, - Red->getFastMathFlags(), CostKind); + InstructionCost PartialReductionCost = + Ctx.TTI.getPartialReductionCost( + Opcode, SrcTy, nullptr, RedTy, VF, ExtKind, + llvm::TargetTransformInfo::PR_None, std::nullopt, + Ctx.CostKind); + assert(PartialReductionCost.isValid() && + "A partial reduction should have a valid cost"); + return true; } + + InstructionCost ExtCost = + cast<VPWidenCastRecipe>(VecOp)->computeCost(VF, Ctx); + InstructionCost RedCost = Red->computeCost(VF, Ctx); + + InstructionCost ExtRedCost = Ctx.TTI.getExtendedReductionCost( + Opcode, ExtOpc == Instruction::CastOps::ZExt, RedTy, SrcVecTy, + Red->getFastMathFlags(), CostKind); return ExtRedCost.isValid() && ExtRedCost < ExtCost + RedCost; }, Range); @@ -3595,33 +3599,35 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red, TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput; Type *SrcTy = Ext0 ? Ctx.Types.inferScalarType(Ext0->getOperand(0)) : RedTy; - InstructionCost MulAccCost; if (IsPartialReduction) { Type *SrcTy2 = Ext1 ? Ctx.Types.inferScalarType(Ext1->getOperand(0)) : nullptr; // FIXME: Move partial reduction creation, costing and clamping // here from LoopVectorize.cpp. - MulAccCost = Ctx.TTI.getPartialReductionCost( - Opcode, SrcTy, SrcTy2, RedTy, VF, - Ext0 ? TargetTransformInfo::getPartialReductionExtendKind( - Ext0->getOpcode()) - : TargetTransformInfo::PR_None, - Ext1 ? TargetTransformInfo::getPartialReductionExtendKind( - Ext1->getOpcode()) - : TargetTransformInfo::PR_None, - Mul->getOpcode(), CostKind); - } else { - // Only partial reductions support mixed extends at the moment. - if (Ext0 && Ext1 && Ext0->getOpcode() != Ext1->getOpcode()) - return false; - - bool IsZExt = - !Ext0 || Ext0->getOpcode() == Instruction::CastOps::ZExt; - auto *SrcVecTy = cast<VectorType>(toVectorTy(SrcTy, VF)); - MulAccCost = Ctx.TTI.getMulAccReductionCost(IsZExt, Opcode, RedTy, - SrcVecTy, CostKind); + InstructionCost PartialReductionCost = + Ctx.TTI.getPartialReductionCost( + Opcode, SrcTy, SrcTy2, RedTy, VF, + Ext0 ? TargetTransformInfo::getPartialReductionExtendKind( + Ext0->getOpcode()) + : TargetTransformInfo::PR_None, + Ext1 ? TargetTransformInfo::getPartialReductionExtendKind( + Ext1->getOpcode()) + : TargetTransformInfo::PR_None, + Mul->getOpcode(), CostKind); + assert(PartialReductionCost.isValid() && + "A partial reduction should have a valid cost"); + return true; } + // Only partial reductions support mixed extends at the moment. + if (Ext0 && Ext1 && Ext0->getOpcode() != Ext1->getOpcode()) + return false; + + bool IsZExt = + !Ext0 || Ext0->getOpcode() == Instruction::CastOps::ZExt; + auto *SrcVecTy = cast<VectorType>(toVectorTy(SrcTy, VF)); + InstructionCost MulAccCost = Ctx.TTI.getMulAccReductionCost( + IsZExt, Opcode, RedTy, SrcVecTy, CostKind); InstructionCost MulCost = Mul->computeCost(VF, Ctx); InstructionCost RedCost = Red->computeCost(VF, Ctx); _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
