llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-vectorizers Author: Dmitry Sidorov (MrSidims) <details> <summary>Changes</summary> On a target whose consecutive scalar loads coalesce, cancel the load saving of a bundle feeding a cast that widens its lanes, in store chains, value lists and integer reductions. Without it a load of <4 x i8> through sitofp into a store chain vectorizes into longer code. A bundle keeps its saving when the loaded or the widened lanes pack two to a register, since the scalar form unpacks every lane. Floating point reductions keep their own rule. --- <sub>Stack created with <a href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub> --- Patch is 62.63 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/228904.diff 5 Files Affected: - (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (+99-14) - (modified) llvm/test/CodeGen/AMDGPU/slp-coalesced-loads.ll (+48-175) - (modified) llvm/test/CodeGen/AMDGPU/slp-int-to-fp.ll (+10-10) - (modified) llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-coalesced-loads.ll (+153-24) - (modified) llvm/test/Transforms/SLPVectorizer/AMDGPU/int-reduction-coalesced-loads.ll (+160-36) ``````````diff diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 26a6119308360..3c5e2df4a6ecb 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -481,6 +481,10 @@ class slpvectorizer::BoUpSLP { Keep, /// Every clean bundle loses its saving. Cancel, + /// Only the clean bundles feeding a cast whose lanes get wider than the + /// loaded type before they leave the tree lose their saving, unless the + /// loaded or the widened lanes are of a packed type. + CancelWidened, }; /// Calculates the cost of the subtrees, trims non-profitable ones and returns @@ -2491,14 +2495,29 @@ class slpvectorizer::BoUpSLP { const SmallDenseSet<unsigned, 8> &NodesToKeepBWs, unsigned &MaxDepthLevel, bool &IsProfitableToDemote, bool IsTruncRoot) const; + /// \returns the widest type the lanes of \p TE reach before they leave the + /// tree through a reduction, an external use or a user that is neither a + /// cast nor narrowed. With \p ApplyMinBWs the lanes are first narrowed to + /// their minimum bit widths, as the vector code is. + /// \p HasExternalUses tells whether a node has scalars used outside of the + /// tree. + Type *getWidestLaneType( + const TreeEntry &TE, bool ApplyMinBWs, + function_ref<bool(const TreeEntry &)> HasExternalUses) const; + + /// \returns true if two lanes of \p Ty fit a scalar register as a legal + /// vector type, so the target has packed arithmetic on them. + bool isPackedLaneType(Type *Ty) const; + /// \returns the load saving credited to the vectorized load bundle \p TE /// that never materializes on a target whose consecutive scalar loads /// coalesce into the same wide access, or zero if \p LoadSavings keeps it. /// Only clean bundles with no reordering, reuse or bit-width reduction lose - /// the saving. - InstructionCost - getCoalescedLoadPhantomSaving(const TreeEntry &TE, - CoalescedLoadSavings LoadSavings) const; + /// the saving. \p HasExternalUses tells whether a node has scalars used + /// outside of the tree. + InstructionCost getCoalescedLoadPhantomSaving( + const TreeEntry &TE, CoalescedLoadSavings LoadSavings, + function_ref<bool(const TreeEntry &)> HasExternalUses) const; /// Builds the list of reorderable operands on the edges \p Edges of the \p /// UserTE, which allow reordering (i.e. the operands can be reordered because @@ -13431,12 +13450,16 @@ InstructionCost BoUpSLP::getUnfusedFMulsPenalty(const TreeEntry &TE) const { /// \returns which load bundles of a reduction over \p VL lose the saving they /// are credited for although their scalar loads coalesce anyway. A reduction -/// that would lose an fma has a stake in every load bundle, and any other -/// reduction keeps every saving. A reassociable reduction loses no fma because -/// its vector fmuls fuse into the reduction. +/// that would lose an fma has a stake in every load bundle, an integer +/// reduction only in the bundles whose lanes get widened, the same as an +/// elementwise tree, and any other reduction keeps every saving. A +/// reassociable reduction loses no fma because its vector fmuls fuse into the +/// reduction. static BoUpSLP::CoalescedLoadSavings getReductionCoalescedLoadSavings(RecurKind RdxKind, FastMathFlags RdxFMF, ArrayRef<Value *> VL) { + if (RecurrenceDescriptor::isIntegerRecurrenceKind(RdxKind)) + return BoUpSLP::CoalescedLoadSavings::CancelWidened; if (RdxKind != RecurKind::FAdd || !RdxFMF.allowContract() || RdxFMF.allowReassoc() || none_of(VL, [](Value *V) { return match(V, @@ -18594,9 +18617,51 @@ bool BoUpSLP::isTreeNotExtendable() const { return Res; } -InstructionCost -BoUpSLP::getCoalescedLoadPhantomSaving(const TreeEntry &TE, - CoalescedLoadSavings LoadSavings) const { +Type *BoUpSLP::getWidestLaneType( + const TreeEntry &TE, bool ApplyMinBWs, + function_ref<bool(const TreeEntry &)> HasExternalUses) const { + Type *WidestTy = nullptr; + auto Widen = [&](Type *Ty) { + if (!WidestTy || + DL->getTypeSizeInBits(Ty) > DL->getTypeSizeInBits(WidestTy)) + WidestTy = Ty; + }; + for (const TreeEntry *E = &TE;;) { + Type *OrigTy = E->Scalars.front()->getType()->getScalarType(); + Type *Ty = OrigTy; + if (auto It = MinBWs.find(E); + ApplyMinBWs && It != MinBWs.end() && + It->second.first < DL->getTypeSizeInBits(Ty) && !HasExternalUses(*E)) + Ty = IntegerType::get(Ty->getContext(), It->second.first); + Widen(Ty); + const TreeEntry *UserTE = E->UserTreeIndex.UserTE; + if (!UserTE) { + if (ApplyMinBWs && UserIgnoreList && E == &getRootNode()) + Widen(getReductionType()->getScalarType()); + else + Widen(OrigTy); + return WidestTy; + } + if (!UserTE->hasState() || UserTE->isGather() || + (!Instruction::isCast(UserTE->getOpcode()) && + !MinBWs.contains(UserTE))) { + Widen(OrigTy); + return WidestTy; + } + E = UserTE; + } +} + +bool BoUpSLP::isPackedLaneType(Type *Ty) const { + unsigned Bits = DL->getTypeSizeInBits(Ty); + unsigned RegBits = + TTI->getRegisterBitWidth(TargetTransformInfo::RGK_Scalar).getFixedValue(); + return 2 * Bits <= RegBits && TTI->isTypeLegal(FixedVectorType::get(Ty, 2)); +} + +InstructionCost BoUpSLP::getCoalescedLoadPhantomSaving( + const TreeEntry &TE, CoalescedLoadSavings LoadSavings, + function_ref<bool(const TreeEntry &)> HasExternalUses) const { if (LoadSavings == CoalescedLoadSavings::Keep) return 0; if (!TE.hasState() || TE.isGather() || TE.getOpcode() != Instruction::Load || @@ -18616,6 +18681,20 @@ BoUpSLP::getCoalescedLoadPhantomSaving(const TreeEntry &TE, if (!TTI->consecutiveLoadsCoalesce(LI0->getType(), TE.Scalars.size(), BestAlign, LI0->getPointerAddressSpace())) return 0; + if (LoadSavings == CoalescedLoadSavings::CancelWidened) { + const TreeEntry *UserTE = TE.UserTreeIndex.UserTE; + if (!UserTE || UserTE->isGather() || !UserTE->hasState() || + !Instruction::isCast(UserTE->getOpcode())) + return 0; + Type *LoadedTy = LI0->getType()->getScalarType(); + Type *WidestTy = + getWidestLaneType(*UserTE, /*ApplyMinBWs=*/true, HasExternalUses); + if (DL->getTypeSizeInBits(WidestTy) <= DL->getTypeSizeInBits(LoadedTy) || + isPackedLaneType(LoadedTy) || + isPackedLaneType( + getWidestLaneType(*UserTE, /*ApplyMinBWs=*/false, HasExternalUses))) + return 0; + } InstructionCost ScalarLdCost = 0; for (Value *V : TE.Scalars) { auto *LI = cast<LoadInst>(V); @@ -19486,6 +19565,12 @@ InstructionCost BoUpSLP::calculateTreeCostAndTrimNonProfitable( return false; return IsExternallyUsedV(V); }; + auto HasExternalUses = [&](const TreeEntry &TE) { + return any_of(TE.Scalars, [&](Value *V) { + return !(TE.hasCopyableElements() && TE.isCopyableElement(V)) && + IsExternallyUsedV(V); + }); + }; InstructionCost Cost = 0; SmallDenseMap<const TreeEntry *, uint64_t> EntryToScale; uint64_t PrevScale = 0; @@ -19523,7 +19608,7 @@ InstructionCost BoUpSLP::calculateTreeCostAndTrimNonProfitable( "Expected gather nodes with users only."); InstructionCost C = getEntryCost(&TE, VectorizedVals, CheckedExtracts); - C += getCoalescedLoadPhantomSaving(TE, LoadSavings); + C += getCoalescedLoadPhantomSaving(TE, LoadSavings, HasExternalUses); uint64_t Scale = 0; bool CostIsFree = C == 0; // For gather/buildvector (and split-vectorize) entries, prefer the @@ -19726,7 +19811,7 @@ InstructionCost BoUpSLP::calculateTreeCostAndTrimNonProfitable( }; auto RecostEntry = [&](const TreeEntry *TE) { InstructionCost C = getEntryCost(TE, VectorizedVals, CheckedExtracts); - C += getCoalescedLoadPhantomSaving(*TE, LoadSavings); + C += getCoalescedLoadPhantomSaving(*TE, LoadSavings, HasExternalUses); if (!C.isValid() || C == 0) return C; uint64_t Scale = EntryToScale.lookup(TE); @@ -29719,7 +29804,7 @@ SLPVectorizerPass::vectorizeStoreChainImpl(ArrayRef<Value *> Chain, BoUpSLP &R, InstructionCost TreeCost = R.calculateTreeCostAndTrimNonProfitable( /*VectorizedVals=*/{}, /*RdxRoot=*/nullptr, - BoUpSLP::CoalescedLoadSavings::Keep); + BoUpSLP::CoalescedLoadSavings::CancelWidened); R.buildExternalUses(); Size = R.getCanonicalGraphSize() - R.getNumSplatSubtreeEntries(); @@ -30803,7 +30888,7 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R, R.computeMinimumValueSizes(); InstructionCost TreeCost = R.calculateTreeCostAndTrimNonProfitable( /*VectorizedVals=*/{}, /*RdxRoot=*/nullptr, - BoUpSLP::CoalescedLoadSavings::Keep); + BoUpSLP::CoalescedLoadSavings::CancelWidened); R.buildExternalUses(); InstructionCost Cost = R.getTreeCost(TreeCost); diff --git a/llvm/test/CodeGen/AMDGPU/slp-coalesced-loads.ll b/llvm/test/CodeGen/AMDGPU/slp-coalesced-loads.ll index 281221e8181fa..0385e52fbdc32 100644 --- a/llvm/test/CodeGen/AMDGPU/slp-coalesced-loads.ll +++ b/llvm/test/CodeGen/AMDGPU/slp-coalesced-loads.ll @@ -327,11 +327,10 @@ define void @store4_sitofp(ptr addrspace(1) %in, ptr addrspace(1) %out) { ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX90A-NEXT: global_load_dword v0, v[0:1], off ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: v_ashrrev_i16_e32 v1, 8, v0 -; GFX90A-NEXT: v_cvt_f32_i32_sdwa v7, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 -; GFX90A-NEXT: v_cvt_f32_i32_sdwa v6, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 ; GFX90A-NEXT: v_cvt_f32_i32_sdwa v4, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 -; GFX90A-NEXT: v_cvt_f32_i32_sdwa v5, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 +; GFX90A-NEXT: v_cvt_f32_i32_sdwa v5, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 +; GFX90A-NEXT: v_cvt_f32_i32_sdwa v6, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 +; GFX90A-NEXT: v_cvt_f32_i32_sdwa v7, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 ; GFX90A-NEXT: global_store_dwordx4 v[2:3], v[4:7], off ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] @@ -392,10 +391,12 @@ define void @zext4_bv_loop(ptr addrspace(1) %a, ptr addrspace(1) %out) { ; GFX90A-LABEL: zext4_bv_loop: ; GFX90A: ; %bb.0: ; %entry ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0 +; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GFX90A-NEXT: s_mov_b64 s[6:7], 0 ; GFX90A-NEXT: .LBB4_1: ; %loop ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX90A-NEXT: global_load_dword v6, v[0:1], off +; GFX90A-NEXT: global_load_dword v6, v[0:1], off offset:-1 ; GFX90A-NEXT: s_add_u32 s6, s6, 1 ; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 4, v0 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc @@ -417,12 +418,14 @@ define void @zext4_bv_loop(ptr addrspace(1) %a, ptr addrspace(1) %out) { ; GFX1030-LABEL: zext4_bv_loop: ; GFX1030: ; %bb.0: ; %entry ; GFX1030-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1030-NEXT: v_add_co_u32 v0, vcc_lo, v0, 1 +; GFX1030-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo ; GFX1030-NEXT: v_mov_b32_e32 v4, 0xffff ; GFX1030-NEXT: s_mov_b64 s[6:7], 0 ; GFX1030-NEXT: .p2align 6 ; GFX1030-NEXT: .LBB4_1: ; %loop ; GFX1030-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX1030-NEXT: global_load_dword v7, v[0:1], off +; GFX1030-NEXT: global_load_dword v7, v[0:1], off offset:-1 ; GFX1030-NEXT: s_add_u32 s6, s6, 1 ; GFX1030-NEXT: v_add_co_u32 v0, vcc_lo, v0, 4 ; GFX1030-NEXT: s_addc_u32 s7, s7, 0 @@ -444,10 +447,11 @@ define void @zext4_bv_loop(ptr addrspace(1) %a, ptr addrspace(1) %out) { ; GFX1250: ; %bb.0: ; %entry ; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], 1, v[0:1] ; GFX1250-NEXT: s_mov_b64 s[0:1], 0 ; GFX1250-NEXT: .LBB4_1: ; %loop ; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX1250-NEXT: global_load_b32 v6, v[0:1], off +; GFX1250-NEXT: global_load_b32 v6, v[0:1], off offset:-1 ; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[0:1], 1 ; GFX1250-NEXT: s_wait_xcnt 0x0 ; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], 4, v[0:1] @@ -501,51 +505,21 @@ define i32 @dot8_sext_i8(ptr addrspace(1) %a, ptr addrspace(1) %b) { ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX90A-NEXT: global_load_dwordx2 v[4:5], v[0:1], off ; GFX90A-NEXT: global_load_dwordx2 v[6:7], v[2:3], off -; GFX90A-NEXT: s_waitcnt vmcnt(1) -; GFX90A-NEXT: v_lshrrev_b16_e32 v0, 8, v4 -; GFX90A-NEXT: v_lshrrev_b16_e32 v1, 8, v5 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: v_lshrrev_b16_e32 v2, 8, v6 -; GFX90A-NEXT: v_lshrrev_b16_e32 v3, 8, v7 -; GFX90A-NEXT: v_mul_i32_i24_sdwa v8, sext(v4), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX90A-NEXT: v_mul_i32_i24_sdwa v9, sext(v4), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 -; GFX90A-NEXT: v_mul_i32_i24_sdwa v4, sext(v4), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 -; GFX90A-NEXT: v_mul_i32_i24_sdwa v6, sext(v5), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX90A-NEXT: v_mul_i32_i24_sdwa v10, sext(v5), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 -; GFX90A-NEXT: v_mul_i32_i24_sdwa v5, sext(v5), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 -; GFX90A-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX90A-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX90A-NEXT: v_add_u32_e32 v4, v4, v5 -; GFX90A-NEXT: v_add_u32_e32 v2, v9, v10 -; GFX90A-NEXT: v_add_u32_e32 v3, v8, v6 -; GFX90A-NEXT: v_add3_u32 v0, v0, v1, v4 -; GFX90A-NEXT: v_add3_u32 v0, v3, v2, v0 +; GFX90A-NEXT: v_dot4c_i32_i8_e32 v0, v4, v6 +; GFX90A-NEXT: v_dot4c_i32_i8_e32 v0, v5, v7 ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; ; GFX1030-LABEL: dot8_sext_i8: ; GFX1030: ; %bb.0: ; %entry ; GFX1030-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX1030-NEXT: global_load_dwordx2 v[0:1], v[0:1], off -; GFX1030-NEXT: global_load_dwordx2 v[2:3], v[2:3], off -; GFX1030-NEXT: s_waitcnt vmcnt(1) -; GFX1030-NEXT: v_lshrrev_b16 v4, 8, v0 -; GFX1030-NEXT: v_lshrrev_b16 v5, 8, v1 +; GFX1030-NEXT: global_load_dwordx2 v[4:5], v[0:1], off +; GFX1030-NEXT: global_load_dwordx2 v[1:2], v[2:3], off +; GFX1030-NEXT: v_mov_b32_e32 v0, 0 ; GFX1030-NEXT: s_waitcnt vmcnt(0) -; GFX1030-NEXT: v_lshrrev_b16 v6, 8, v2 -; GFX1030-NEXT: v_lshrrev_b16 v7, 8, v3 -; GFX1030-NEXT: v_mul_i32_i24_sdwa v8, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX1030-NEXT: v_mul_i32_i24_sdwa v9, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 -; GFX1030-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 -; GFX1030-NEXT: v_mul_i32_i24_sdwa v2, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 -; GFX1030-NEXT: v_mul_i32_i24_sdwa v10, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX1030-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 -; GFX1030-NEXT: v_mul_i32_i24_sdwa v3, sext(v4), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX1030-NEXT: v_mul_i32_i24_sdwa v4, sext(v5), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX1030-NEXT: v_add_nc_u32_e32 v0, v0, v2 -; GFX1030-NEXT: v_add_nc_u32_e32 v2, v8, v10 -; GFX1030-NEXT: v_add_nc_u32_e32 v1, v9, v1 -; GFX1030-NEXT: v_add3_u32 v0, v3, v4, v0 -; GFX1030-NEXT: v_add3_u32 v0, v2, v1, v0 +; GFX1030-NEXT: v_dot4c_i32_i8 v0, v4, v1 +; GFX1030-NEXT: v_dot4c_i32_i8 v0, v5, v2 ; GFX1030-NEXT: s_setpc_b64 s[30:31] ; ; GFX1250-LABEL: dot8_sext_i8: @@ -554,39 +528,11 @@ define i32 @dot8_sext_i8(ptr addrspace(1) %a, ptr addrspace(1) %b) { ; GFX1250-NEXT: s_wait_kmcnt 0x0 ; GFX1250-NEXT: global_load_b64 v[4:5], v[0:1], off ; GFX1250-NEXT: global_load_b64 v[6:7], v[2:3], off -; GFX1250-NEXT: s_wait_loadcnt 0x1 -; GFX1250-NEXT: s_wait_xcnt 0x1 -; GFX1250-NEXT: v_lshrrev_b16 v0.l, 8, v4.l -; GFX1250-NEXT: v_lshrrev_b16 v1.l, 8, v5.l ; GFX1250-NEXT: s_wait_loadcnt 0x0 -; GFX1250-NEXT: v_lshrrev_b16 v8.l, 8, v6.l -; GFX1250-NEXT: v_lshrrev_b16 v9.l, 8, v7.l -; GFX1250-NEXT: v_dual_ashrrev_i32 v12, 24, v5 :: v_dual_ashrrev_i32 v14, 24, v7 -; GFX1250-NEXT: s_wait_xcnt 0x0 -; GFX1250-NEXT: v_dual_ashrrev_i32 v2, 24, v4 :: v_dual_ashrrev_i32 v10, 24, v6 -; GFX1250-NEXT: v_bfe_i32 v13, v5, 16, 8 -; GFX1250-NEXT: v_bfe_i32 v5, v5, 0, 8 -; GFX1250-NEXT: v_bfe_i32 v15, v7, 16, 8 -; GFX1250-NEXT: v_bfe_i32 v7, v7, 0, 8 -; GFX1250-NEXT: v_bfe_i32 v0, v0, 0, 8 -; GFX1250-NEXT: v_bfe_i32 v8, v8, 0, 8 -; GFX1250-NEXT: v_bfe_i32 v1, v1, 0, 8 -; GFX1250-NEXT: v_bfe_i32 v9, v9, 0, 8 -; GFX1250-NEXT: v_mul_i32_i24_e32 v12, v12, v14 -; GFX1250-NEXT: v_bfe_i32 v3, v4, 0, 8 -; GFX1250-NEXT: v_bfe_i32 v4, v4, 16, 8 -; GFX1250-NEXT: v_bfe_i32 v11, v6, 0, 8 -; GFX1250-NEXT: v_bfe_i32 v6, v6, 16, 8 -; GFX1250-NEXT: v_mul_i32_i24_e32 v5, v5, v7 -; GFX1250-NEXT: v_mul_i32_i24_e32 v7, v13, v15 -; GFX1250-NEXT: v_mul_i32_i24_e32 v0, v0, v8 -; GFX1250-NEXT: v_mul_i32_i24_e32 v1, v1, v9 -; GFX1250-NEXT: v_mad_i32_i24 v2, v2, v10, v12 -; GFX1250-NEXT: v_mad_i32_i24 v3, v3, v11, v5 -; GFX1250-NEXT: v_mad_i32_i24 v4, v4, v6, v7 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250-NEXT: v_add3_u32 v0, v0, v1, v2 -; GFX1250-NEXT: v_add3_u32 v0, v3, v4, v0 +; GFX1250-NEXT: s_wait_xcnt 0x1 +; GFX1250-NEXT: v_dot4_i32_iu8 v0, v4, v6, 0 neg_lo:[1,1,0] +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-NEXT: v_dot4_i32_iu8 v0, v5, v7, v0 neg_lo:[1,1,0] ; GFX1250-NEXT: s_set_pc_i64 s[30:31] entry: %pa0 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 0 @@ -660,35 +606,22 @@ define i32 @sum4_zext_i8(ptr addrspace(1) %a) { ; GFX90A: ; %bb.0: ; %entry ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX90A-NEXT: global_load_dword v0, v[0:1], off -; GFX90A-NEXT: s_movk_i32 s4, 0xff -; GFX90A-NEXT: s_mov_b32 s5, 0x5040100 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: v_lshrrev_b32_e32 v1, 24, v0 -; GFX90A-NEXT: v_lshrrev_b16_e32 v2, 8, v0 -; GFX90A-NEXT: v_and_b32_sdwa v3, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD -; GFX90A-NEXT: v_and_b32_e32 v0, 0xff, v0 -; GFX90A-NEXT: v_perm_b32 v0, v2, v0, s5 -; GFX90A-NEXT: v_perm_b32 v1, v1, v3, s5 -; GFX90A-NEXT: v_pk_add_u16 v0, v0, v1 -; GFX90A-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 +; GFX90A-NEXT: v_add_u32_sdwa v2, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_1 +; GFX90A-NEXT: v_bfe_u32 v0, v0, 16, 8 +; GFX90A-NEXT: v_add3_u32 v0, v2, v0, v1 ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; ; GFX1030-LABEL: sum4_zext_i8: ; GFX1030: ; %bb.0: ; %entry ; GFX1030-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX1030-NEXT: global_load_dword v0, v[0:1], off -; GFX1030-NEXT: v_mov_b32_e32 v1, 0xff ; GFX1030-NEXT: s_waitcnt vmcnt(0) -; GFX1030-NEXT: v_lshrrev_b32_e32 v2, 24, v0 -; GFX1030-NEXT: v_lshrrev_b16 v3, 8, v0 -; GFX1030-NEXT: v_and_b32_e32 v4, 0xff, v0 -; GFX103... [truncated] `````````` </details> https://github.com/llvm/llvm-project/pull/228904 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
