================
@@ -21932,6 +21932,26 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry(
getScalarizationOverhead(*TTI, Opts.slp_revec, VL.front()->getType(),
MaskVecTy, DemandedElts, /*Insert=*/true,
/*Extract=*/false, CostKind);
+ // A lane taken from another vectorized entry needs a shuffle, which a zero
+ // insert price misses.
+ APInt VectorizedElts = APInt::getZero(SubMask.size());
+ SmallVector<int> VectorizedMask(SubMask.size(), PoisonMaskElem);
+ for (auto [I, Idx] : enumerate(SubMask)) {
+ if (Idx == PoisonMaskElem || !isVectorized(VL[I]))
+ continue;
+ VectorizedElts.setBit(I);
+ VectorizedMask[I] = Idx;
+ }
+ if (!VectorizedElts.isZero() &&
+ getScalarizationOverhead(*TTI, Opts.slp_revec, VL.front()->getType(),
+ MaskVecTy, VectorizedElts, /*Insert=*/true,
+ /*Extract=*/false,
+ CostKind) == TTI::TCC_Free) {
+ InstructionCost VectorizedShuffleCost =
+ GetShuffleCost(VectorizedMask, Entries, VecTy);
----------------
MrSidims wrote:
> Plus, I think you are double-counting reuses.
I don't see a double count. ExternalUses has no extract for these lanes, and
the gather cost prices only the inserts. The closest case I found is a repeated
scalar, where without a target the patch emits one more shuffle. Is that what
you mean?
> Vectorized mask should be applied to the actual vectorized entries, not ones
> selected here
Indeed, I'll fix this, thanks!
https://github.com/llvm/llvm-project/pull/240136
_______________________________________________
llvm-branch-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits