================
@@ -3773,19 +3775,76 @@ static void
tryToCreateAbstractReductionRecipe(VPReductionRecipe *Red,
AbstractR = ExtRed;
// Cannot create abstract inloop reduction recipes.
if (!AbstractR)
- return;
+ return false;
AbstractR->insertBefore(*VPBB, IP);
Red->replaceAllUsesWith(AbstractR);
+ return true;
+}
+
+/// Lower a partial reduction back to a regular reduction, by
+/// changing the in-loop partial reduction to a binop and removing
+/// the scale factor from the PHI node.
+static void lowerPartialReduction(VPlan &Plan, VPPartialReductionRecipe *Red,
+ VPCostContext &Ctx) {
+ VPRecipeBase *Acc = Red->getChainOp()->getDefiningRecipe();
+ if (auto *PhiR = dyn_cast<VPReductionPHIRecipe>(Acc)) {
+ PhiR->setVFScaleFactor(1);
+
+ // We also need to update the scale factor of the reduction-start-vector
+ // operand.
+ VPValue *StartV, *IdentityV;
+ if (!match(PhiR->getOperand(0),
+ m_VPInstruction<VPInstruction::ReductionStartVector>(
+ m_VPValue(StartV), m_VPValue(IdentityV), m_VPValue())))
+ llvm_unreachable("Unexpected operand for a partial reduction");
+ Type *I32Ty = IntegerType::getInt32Ty(Plan.getContext());
+ auto *ScaleFactorVPV = Plan.getOrAddLiveIn(ConstantInt::get(I32Ty, 1));
+ cast<VPInstruction>(PhiR->getOperand(0))->setOperand(2, ScaleFactorVPV);
+ }
+
+ if (auto *R = dyn_cast<VPPartialReductionRecipe>(Acc))
+ if (R->getVFScaleFactor() != 1)
+ lowerPartialReduction(Plan, R, Ctx);
+
+ LLVM_DEBUG(
+ dbgs() << "LV: Lowering " << *Red
+ << " back to regular reduction, because it is not profitable\n");
----------------
SamTebbs33 wrote:
I think we want an `a` before `regular`.
https://github.com/llvm/llvm-project/pull/166138
_______________________________________________
llvm-branch-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
