================
@@ -3773,19 +3775,76 @@ static void
tryToCreateAbstractReductionRecipe(VPReductionRecipe *Red,
AbstractR = ExtRed;
// Cannot create abstract inloop reduction recipes.
if (!AbstractR)
- return;
+ return false;
AbstractR->insertBefore(*VPBB, IP);
Red->replaceAllUsesWith(AbstractR);
+ return true;
+}
+
+/// Lower a partial reduction back to a regular reduction, by
+/// changing the in-loop partial reduction to a binop and removing
+/// the scale factor from the PHI node.
+static void lowerPartialReduction(VPlan &Plan, VPPartialReductionRecipe *Red,
+ VPCostContext &Ctx) {
+ VPRecipeBase *Acc = Red->getChainOp()->getDefiningRecipe();
+ if (auto *PhiR = dyn_cast<VPReductionPHIRecipe>(Acc)) {
+ PhiR->setVFScaleFactor(1);
+
+ // We also need to update the scale factor of the reduction-start-vector
+ // operand.
+ VPValue *StartV, *IdentityV;
+ if (!match(PhiR->getOperand(0),
+ m_VPInstruction<VPInstruction::ReductionStartVector>(
+ m_VPValue(StartV), m_VPValue(IdentityV), m_VPValue())))
+ llvm_unreachable("Unexpected operand for a partial reduction");
+ Type *I32Ty = IntegerType::getInt32Ty(Plan.getContext());
+ auto *ScaleFactorVPV = Plan.getOrAddLiveIn(ConstantInt::get(I32Ty, 1));
+ cast<VPInstruction>(PhiR->getOperand(0))->setOperand(2, ScaleFactorVPV);
+ }
+
+ if (auto *R = dyn_cast<VPPartialReductionRecipe>(Acc))
+ if (R->getVFScaleFactor() != 1)
+ lowerPartialReduction(Plan, R, Ctx);
+
+ LLVM_DEBUG(
+ dbgs() << "LV: Lowering " << *Red
+ << " back to regular reduction, because it is not profitable\n");
+
+ // Lower the partial reduction to a regular binop.
+ VPBuilder Builder(Red);
+ VPInstruction *Add = Builder.createNaryOp(
+ RecurrenceDescriptor::getOpcode(Red->getRecurrenceKind()),
+ {Red->getChainOp(), Red->getVecOp()});
+ if (Red->isConditional())
+ Add = Builder.createSelect(Red->getCondOp(), Add, Red->getChainOp());
+
+ Red->replaceAllUsesWith(Add);
+ Red->eraseFromParent();
}
void VPlanTransforms::convertToAbstractRecipes(VPlan &Plan, VPCostContext &Ctx,
VFRange &Range) {
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_deep(Plan.getVectorLoopRegion()))) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
- if (auto *Red = dyn_cast<VPReductionRecipe>(&R))
- tryToCreateAbstractReductionRecipe(Red, Ctx, Range);
+ auto *Red = dyn_cast<VPReductionRecipe>(&R);
+ if (!Red)
+ continue;
+
+ if (!tryToCreateAbstractReductionRecipe(Red, Ctx, Range) &&
----------------
sushgokh wrote:
If 'Red' is converted to AbstractRecipe, Red should be null. Can you check for
`Red` instead of returning bool from the `tryToCreateAbstractReductionRecipe` ?
https://github.com/llvm/llvm-project/pull/166138
_______________________________________________
llvm-branch-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
