Author: Oscar Priego Date: 2026-09-07T08:07:18+02:00 New Revision: 5f27b704b2fc7fe09aaabe4fcae964f432ba81db
URL: https://github.com/llvm/llvm-project/commit/5f27b704b2fc7fe09aaabe4fcae964f432ba81db DIFF: https://github.com/llvm/llvm-project/commit/5f27b704b2fc7fe09aaabe4fcae964f432ba81db.diff LOG: [SelectionDAG] Avoid irregular INSERT_SUBVECTOR when widening CTTZ_ELTS (#218019) Fixes #217985 Non-poison `CTTZ_ELTS` pads widened operands with active lanes using `INSERT_SUBVECTOR`. For irregular fixed-length vectors whose widened type subsequently requires splitting, this can reach the `SplitVecRes_INSERT_SUBVECTOR` stack-spill fallback. That fallback calls `getVectorSubVecPointer`, which requires byte-addressable elements. Use `VECTOR_SHUFFLE` for this specific widening case. Original lanes are selected from the normally widened source, while padding lanes are selected from an all-ones vector. This avoids the irregular `INSERT_SUBVECTOR` and prevents poison/undef widened padding from being observed. Other widening cases retain their existing behavior. AI-assisted: OpenAI Codex was used during investigation and implementation. I manually reviewed the final patch, validated the SelectionDAG type-action behavior, and ran the X86, AArch64, and RISC-V regression checks described above. (cherry picked from commit 01ff7258c2374b3cca02bc8631a445ac32f9b930) Added: Modified: llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll Removed: ################################################################################ diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp index 3817a77f8bd44..c0fab69cb518a 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -8738,8 +8738,8 @@ SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) { SDValue DAGTypeLegalizer::WidenVecOp_CttzElements(SDNode *N) { SDLoc DL(N); SDValue Source = N->getOperand(0); - EVT WideVT = - TLI.getTypeToTransformTo(*DAG.getContext(), Source.getValueType()); + EVT SourceVT = Source.getValueType(); + EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), SourceVT); SDValue WideSource; if (N->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON) { @@ -8748,7 +8748,18 @@ SDValue DAGTypeLegalizer::WidenVecOp_CttzElements(SDNode *N) { // Pad the widened portion with all-ones so the extra lanes appear as // active (non-zero) elements and do not contribute trailing zeros. SDValue AllOnes = DAG.getAllOnesConstant(DL, WideVT); - WideSource = DAG.getInsertSubvector(DL, AllOnes, Source, 0); + if (WideVT.isFixedLengthVector() && + getTypeAction(WideVT) == TargetLowering::TypeSplitVector) { + WideSource = GetWidenedVector(Source); + unsigned WideElts = WideVT.getVectorNumElements(); + SmallVector<int> Mask(WideElts); + std::iota(Mask.begin(), Mask.end(), 0); + for (unsigned I = SourceVT.getVectorNumElements(); I != WideElts; ++I) + Mask[I] += WideElts; + WideSource = DAG.getVectorShuffle(WideVT, DL, WideSource, AllOnes, Mask); + } else { + WideSource = DAG.getInsertSubvector(DL, AllOnes, Source, 0); + } } return DAG.getNode(N->getOpcode(), DL, N->getValueType(0), WideSource, diff --git a/llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll b/llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll index f8cc45c1b5bda..a2f2ffdfd3325 100644 --- a/llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll +++ b/llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll @@ -196,5 +196,72 @@ define i32 @ctz_zero_v8i1() { ret i32 %res } +; Irregular predicate vectors widen to a power-of-two number of lanes. The +; widened lanes must be active so an all-zero input returns the original lane +; count, without requiring an irregular INSERT_SUBVECTOR during legalization. +define i32 @ctz_zero_v17i1() { +; CHECK-LABEL: ctz_zero_v17i1: +; CHECK: # %bb.0: +; CHECK-NEXT: movdqa {{.*#+}} xmm0 = [84281096,16909060,84281096,16909060] +; CHECK-NEXT: pmaxub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1] +; CHECK-NEXT: pmaxub %xmm0, %xmm1 +; CHECK-NEXT: movdqa %xmm1, %xmm0 +; CHECK-NEXT: psrld $16, %xmm0 +; CHECK-NEXT: pmaxub %xmm1, %xmm0 +; CHECK-NEXT: movdqa %xmm0, %xmm1 +; CHECK-NEXT: psrlw $8, %xmm1 +; CHECK-NEXT: pmaxub %xmm0, %xmm1 +; CHECK-NEXT: movd %xmm1, %eax +; CHECK-NEXT: movb $16, %cl +; CHECK-NEXT: subb %al, %cl +; CHECK-NEXT: movzbl %cl, %eax +; CHECK-NEXT: addl $16, %eax +; CHECK-NEXT: retq +; +; AVX512-LABEL: ctz_zero_v17i1: +; AVX512: # %bb.0: +; AVX512-NEXT: vpmovzxbw {{.*#+}} xmm0 = [240,242,244,246,248,250,252,254] +; AVX512-NEXT: vpminub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX512-NEXT: vphminposuw %xmm0, %xmm0 +; AVX512-NEXT: vmovd %xmm0, %eax +; AVX512-NEXT: addb $17, %al +; AVX512-NEXT: movzbl %al, %eax +; AVX512-NEXT: addl $16, %eax +; AVX512-NEXT: retq + %res = call i32 @llvm.experimental.cttz.elts.i32.v17i1(<17 x i1> zeroinitializer, i1 false) + ret i32 %res +} + +define i32 @ctz_zero_v31i1() { +; CHECK-LABEL: ctz_zero_v31i1: +; CHECK: # %bb.0: +; CHECK-NEXT: movdqa {{.*#+}} xmm0 = [16777216,16777216,16777216,16777216] +; CHECK-NEXT: pmaxub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; CHECK-NEXT: movdqa %xmm0, %xmm1 +; CHECK-NEXT: psrld $24, %xmm1 +; CHECK-NEXT: psrld $16, %xmm0 +; CHECK-NEXT: pmaxub %xmm1, %xmm0 +; CHECK-NEXT: movd %xmm0, %eax +; CHECK-NEXT: movb $16, %cl +; CHECK-NEXT: subb %al, %cl +; CHECK-NEXT: movzbl %cl, %eax +; CHECK-NEXT: addl $16, %eax +; CHECK-NEXT: retq +; +; AVX512-LABEL: ctz_zero_v31i1: +; AVX512: # %bb.0: +; AVX512-NEXT: vpmovzxbw {{.*#+}} xmm0 = [255,255,255,255,255,255,255,254] +; AVX512-NEXT: vpminub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX512-NEXT: vphminposuw %xmm0, %xmm0 +; AVX512-NEXT: vmovd %xmm0, %eax +; AVX512-NEXT: addb $17, %al +; AVX512-NEXT: movzbl %al, %eax +; AVX512-NEXT: addl $16, %eax +; AVX512-NEXT: retq + %res = call i32 @llvm.experimental.cttz.elts.i32.v31i1(<31 x i1> zeroinitializer, i1 false) + ret i32 %res +} + declare i8 @llvm.experimental.cttz.elts.i8.v8i16(<8 x i16>, i1) declare i16 @llvm.experimental.cttz.elts.i16.v4i32(<4 x i32>, i1) _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
