llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-llvm-transforms Author: Alexey Bataev (alexey-bataev) <details> <summary>Changes</summary> sext/sitofp of an unsigned demoted operand must be zext/uitofp, and zext/uitofp cannot absorb the sign extension of a signed demoted operand, so the operand must be extended back first. Fixes https://github.com/llvm/llvm-project/issues/234923#issuecomment-6373744225 Assisted-by: Cursor --- Full diff: https://github.com/llvm/llvm-project/pull/235280.diff 2 Files Affected: - (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (+98-26) - (modified) llvm/test/Transforms/SLPVectorizer/X86/minbw-alt-cast-demoted-operand.ll (+8-10) ``````````diff diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index ae9ea4c56698f2..e23d0749656781 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -16700,6 +16700,30 @@ getVectorInstrContextHint(ArrayRef<Value *> VL, const APInt &DemandedElts) { return VIC; } +/// \returns the cast opcode, which gives the same result for the operand, +/// demoted with the signedness \p IsSigned, as \p Opcode gives for the original +/// operand, or std::nullopt if the operand must be extended back first. +static std::optional<unsigned> getCastOpcodeForDemotedOperand(unsigned Opcode, + bool IsSigned) { + switch (Opcode) { + case Instruction::SExt: + case Instruction::SIToFP: + // The zero extended operand is non-negative. + if (IsSigned) + return Opcode; + return Opcode == Instruction::SExt ? Instruction::ZExt + : Instruction::UIToFP; + case Instruction::ZExt: + case Instruction::UIToFP: + // Cannot absorb the sign extension of the operand. + if (IsSigned) + return std::nullopt; + return Opcode; + default: + return std::nullopt; + } +} + InstructionCost BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals, SmallPtrSetImpl<Value *> &CheckedExtracts) { @@ -18195,23 +18219,39 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals, cast<CmpInst>(E->getAltOp())->getPredicate(), CostKind, Op0Info, Op1Info, E->getAltOp()); } else { - Type *SrcSclTy = E->getMainOp()->getOperand(0)->getType(); - auto *SrcTy = getWidenedType(SrcSclTy, VL.size()); - if (SrcSclTy->isIntegerTy() && ScalarTy->isIntegerTy()) { - auto SrcIt = MinBWs.find(getOperandEntry(E, 0)); - unsigned BWSz = DL->getTypeSizeInBits(ScalarTy); - unsigned SrcBWSz = - DL->getTypeSizeInBits(E->getMainOp()->getOperand(0)->getType()); - if (SrcIt != MinBWs.end()) { - SrcBWSz = SrcIt->second.first; - SrcSclTy = IntegerType::get(SrcSclTy->getContext(), SrcBWSz); - SrcTy = getWidenedType(SrcSclTy, VL.size()); + Type *SrcTy = getWidenedType(E->getMainOp()->getOperand(0)->getType(), + VL.size()); + TTI::CastContextHint CCH = + GetCastContextHint(E->getMainOp()->getOperand(0)); + unsigned Opcode0 = E->getOpcode(); + unsigned Opcode1 = E->getAltOpcode(); + // The casts of the demoted operand must take its signedness into + // account. If any of them cannot do it, the operand is extended back + // first. + if (auto SrcIt = MinBWs.find(getOperandEntry(E, 0)); + SrcIt != MinBWs.end()) { + Type *DemotedSrcTy = SrcTy->getWithNewBitWidth(SrcIt->second.first); + std::optional<unsigned> DemotedOpcode0 = + getCastOpcodeForDemotedOperand(Opcode0, SrcIt->second.second); + std::optional<unsigned> DemotedOpcode1 = + getCastOpcodeForDemotedOperand(Opcode1, SrcIt->second.second); + if (DemotedOpcode0 && DemotedOpcode1) { + Opcode0 = *DemotedOpcode0; + Opcode1 = *DemotedOpcode1; + SrcTy = DemotedSrcTy; + } else { + VecCost += TTIRef.getCastInstrCost( + SrcIt->second.second ? Instruction::SExt : Instruction::ZExt, + SrcTy, DemotedSrcTy, CCH, CostKind); } + } + if (SrcTy->isIntOrIntVectorTy() && ScalarTy->isIntegerTy()) { + unsigned BWSz = DL->getTypeSizeInBits(ScalarTy); + unsigned SrcBWSz = DL->getTypeSizeInBits(SrcTy->getScalarType()); if (BWSz <= SrcBWSz) { if (BWSz < SrcBWSz) - VecCost = TTIRef.getCastInstrCost( - Instruction::Trunc, VecTy, SrcTy, - GetCastContextHint(E->getMainOp()->getOperand(0)), CostKind); + VecCost += TTIRef.getCastInstrCost(Instruction::Trunc, VecTy, + SrcTy, CCH, CostKind); LLVM_DEBUG({ dbgs() << "SLP: alternate extension, which should be truncated.\n"; @@ -18220,14 +18260,14 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals, return VecCost; } } - VecCost = TTIRef.getCastInstrCost( - E->getOpcode(), VecTy, SrcTy, - GetCastContextHint(E->getMainOp()->getOperand(0)), CostKind, - E->getMainOp()); - VecCost += TTIRef.getCastInstrCost( - E->getAltOpcode(), VecTy, SrcTy, - GetCastContextHint(E->getMainOp()->getOperand(0)), CostKind, - E->getAltOp()); + // Same casts of the demoted operand, no need to shuffle. + if (Opcode0 == Opcode1) + return VecCost + TTIRef.getCastInstrCost(Opcode0, VecTy, SrcTy, CCH, + CostKind); + VecCost += TTIRef.getCastInstrCost(Opcode0, VecTy, SrcTy, CCH, + CostKind, E->getMainOp()); + VecCost += TTIRef.getCastInstrCost(Opcode1, VecTy, SrcTy, CCH, + CostKind, E->getAltOp()); } SmallVector<int> Mask; E->buildAltOpShuffleMask( @@ -25697,6 +25737,28 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) { CmpInst::Predicate AltPred = AltCI->getPredicate(); V1 = Builder.CreateCmp(AltPred, LHS, RHS); } else { + unsigned Opcode0 = E->getOpcode(); + unsigned Opcode1 = E->getAltOpcode(); + // The casts of the demoted operand must take its signedness into + // account. If any of them cannot do it, the operand is extended back + // first. + if (auto SrcIt = MinBWs.find(getOperandEntry(E, 0)); + SrcIt != MinBWs.end()) { + std::optional<unsigned> DemotedOpcode0 = + getCastOpcodeForDemotedOperand(Opcode0, SrcIt->second.second); + std::optional<unsigned> DemotedOpcode1 = + getCastOpcodeForDemotedOperand(Opcode1, SrcIt->second.second); + if (DemotedOpcode0 && DemotedOpcode1) { + Opcode0 = *DemotedOpcode0; + Opcode1 = *DemotedOpcode1; + } else { + LHS = Builder.CreateIntCast( + LHS, + getWidenedType(E->getMainOp()->getOperand(0)->getType(), + E->Scalars.size()), + SrcIt->second.second); + } + } if (LHS->getType()->isIntOrIntVectorTy() && ScalarTy->isIntegerTy()) { unsigned SrcBWSz = DL->getTypeSizeInBits( cast<VectorType>(LHS->getType())->getElementType()); @@ -25713,10 +25775,20 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) { return LHS; } } - V0 = Builder.CreateCast( - static_cast<Instruction::CastOps>(E->getOpcode()), LHS, VecTy); - V1 = Builder.CreateCast( - static_cast<Instruction::CastOps>(E->getAltOpcode()), LHS, VecTy); + // Same casts of the demoted operand, no need to shuffle. + if (Opcode0 == Opcode1) { + V = Builder.CreateCast(static_cast<Instruction::CastOps>(Opcode0), + LHS, VecTy); + V = FinalShuffle(V, E); + E->VectorizedValue = V; + ++NumVectorInstructions; + return V; + } + V0 = Builder.CreateCast(static_cast<Instruction::CastOps>(Opcode0), + LHS, VecTy); + V1 = Builder.CreateCast(static_cast<Instruction::CastOps>(Opcode1), + LHS, VecTy); + NumVectorInstructions += 2; } // Add V0 and V1 to later analysis to try to find and remove matching // instruction, if any. diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minbw-alt-cast-demoted-operand.ll b/llvm/test/Transforms/SLPVectorizer/X86/minbw-alt-cast-demoted-operand.ll index b45037e1be67a1..4c257a43e19307 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/minbw-alt-cast-demoted-operand.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/minbw-alt-cast-demoted-operand.ll @@ -9,11 +9,9 @@ define void @alt_zext_sext_unsigned_demoted_operand(ptr noalias %p, ptr noalias ; CHECK-NEXT: [[PQ0:%.*]] = getelementptr inbounds i32, ptr [[Q]], i64 0 ; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[PA0]], align 1 ; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i32> -; CHECK-NEXT: [[TMP2:%.*]] = sext <4 x i8> [[TMP0]] to <4 x i32> -; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 5, i32 2, i32 7> ; CHECK-NEXT: [[TMP4:%.*]] = icmp ult <4 x i8> [[TMP0]], splat (i8 -6) ; CHECK-NEXT: [[TMP5:%.*]] = sext <4 x i1> [[TMP4]] to <4 x i32> -; CHECK-NEXT: [[TMP6:%.*]] = and <4 x i32> [[TMP3]], [[TMP5]] +; CHECK-NEXT: [[TMP6:%.*]] = and <4 x i32> [[TMP1]], [[TMP5]] ; CHECK-NEXT: [[TMP7:%.*]] = mul <4 x i32> [[TMP6]], splat (i32 16843009) ; CHECK-NEXT: store <4 x i32> [[TMP7]], ptr [[PQ0]], align 4 ; CHECK-NEXT: ret void @@ -69,8 +67,9 @@ define void @alt_sext_zext_signed_demoted_operand(ptr noalias %p, ptr noalias %q ; CHECK-NEXT: [[PA0:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 0 ; CHECK-NEXT: [[PQ0:%.*]] = getelementptr inbounds i32, ptr [[Q]], i64 0 ; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[PA0]], align 1 -; CHECK-NEXT: [[TMP1:%.*]] = sext <4 x i8> [[TMP0]] to <4 x i32> -; CHECK-NEXT: [[TMP2:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i32> +; CHECK-NEXT: [[TMP8:%.*]] = sext <4 x i8> [[TMP0]] to <4 x i16> +; CHECK-NEXT: [[TMP1:%.*]] = sext <4 x i16> [[TMP8]] to <4 x i32> +; CHECK-NEXT: [[TMP2:%.*]] = zext <4 x i16> [[TMP8]] to <4 x i32> ; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 5, i32 2, i32 7> ; CHECK-NEXT: [[TMP4:%.*]] = icmp slt <4 x i8> [[TMP0]], splat (i8 100) ; CHECK-NEXT: [[TMP5:%.*]] = sext <4 x i1> [[TMP4]] to <4 x i32> @@ -131,9 +130,7 @@ define void @alt_uitofp_sitofp_unsigned_demoted_operand(ptr noalias %p, ptr noal ; CHECK-NEXT: [[PQ0:%.*]] = getelementptr inbounds i32, ptr [[Q]], i64 0 ; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[PA0]], align 1 ; CHECK-NEXT: [[TMP1:%.*]] = uitofp <4 x i8> [[TMP0]] to <4 x float> -; CHECK-NEXT: [[TMP2:%.*]] = sitofp <4 x i8> [[TMP0]] to <4 x float> -; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 5, i32 2, i32 7> -; CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x float> [[TMP3]] to <4 x i32> +; CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32> ; CHECK-NEXT: [[TMP5:%.*]] = icmp ult <4 x i8> [[TMP0]], splat (i8 -6) ; CHECK-NEXT: [[TMP6:%.*]] = sext <4 x i1> [[TMP5]] to <4 x i32> ; CHECK-NEXT: [[TMP7:%.*]] = and <4 x i32> [[TMP4]], [[TMP6]] @@ -191,8 +188,9 @@ define void @alt_sitofp_uitofp_signed_demoted_operand(ptr noalias %p, ptr noalia ; CHECK-NEXT: [[PA0:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 0 ; CHECK-NEXT: [[PQ0:%.*]] = getelementptr inbounds i32, ptr [[Q]], i64 0 ; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[PA0]], align 1 -; CHECK-NEXT: [[TMP1:%.*]] = sitofp <4 x i8> [[TMP0]] to <4 x float> -; CHECK-NEXT: [[TMP2:%.*]] = uitofp <4 x i8> [[TMP0]] to <4 x float> +; CHECK-NEXT: [[TMP8:%.*]] = sext <4 x i8> [[TMP0]] to <4 x i16> +; CHECK-NEXT: [[TMP1:%.*]] = sitofp <4 x i16> [[TMP8]] to <4 x float> +; CHECK-NEXT: [[TMP2:%.*]] = uitofp <4 x i16> [[TMP8]] to <4 x float> ; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 5, i32 2, i32 7> ; CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x float> [[TMP3]] to <4 x i32> ; CHECK-NEXT: [[TMP5:%.*]] = icmp slt <4 x i8> [[TMP0]], splat (i8 100) `````````` </details> https://github.com/llvm/llvm-project/pull/235280 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
