https://github.com/MrSidims created https://github.com/llvm/llvm-project/pull/228904
On a target whose consecutive scalar loads coalesce, cancel the load saving of a bundle feeding a cast that widens its lanes, in store chains, value lists and integer reductions. Without it a load of <4 x i8> through sitofp into a store chain vectorizes into longer code. A bundle keeps its saving when the loaded or the widened lanes pack two to a register, since the scalar form unpacks every lane. Floating point reductions keep their own rule. --- <sub>Stack created with <a href="https://github.com/github/gh-stack";>GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback";>Give Feedback 💬</a></sub> >From 51b090d8cdd944d8b3ba4d88926ca6734b664ad4 Mon Sep 17 00:00:00 2001 From: Dmitry Sidorov <[email protected]> Date: Sat, 3 Oct 2026 21:50:00 +0200 Subject: [PATCH] [SLP] Cancel the phantom load saving on widened loads On a target whose consecutive scalar loads coalesce, cancel the load saving of a bundle feeding a cast that widens its lanes, in store chains, value lists and integer reductions. Without it a load of <4 x i8> through sitofp into a store chain vectorizes into longer code. A bundle keeps its saving when the loaded or the widened lanes pack two to a register, since the scalar form unpacks every lane. Floating point reductions keep their own rule. --- .../Transforms/Vectorize/SLPVectorizer.cpp | 113 +++++++-- .../CodeGen/AMDGPU/slp-coalesced-loads.ll | 223 ++++-------------- llvm/test/CodeGen/AMDGPU/slp-int-to-fp.ll | 20 +- .../AMDGPU/elementwise-coalesced-loads.ll | 177 ++++++++++++-- .../AMDGPU/int-reduction-coalesced-loads.ll | 196 ++++++++++++--- 5 files changed, 470 insertions(+), 259 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 26a6119308360..3c5e2df4a6ecb 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -481,6 +481,10 @@ class slpvectorizer::BoUpSLP { Keep, /// Every clean bundle loses its saving. Cancel, + /// Only the clean bundles feeding a cast whose lanes get wider than the + /// loaded type before they leave the tree lose their saving, unless the + /// loaded or the widened lanes are of a packed type. + CancelWidened, }; /// Calculates the cost of the subtrees, trims non-profitable ones and returns @@ -2491,14 +2495,29 @@ class slpvectorizer::BoUpSLP { const SmallDenseSet<unsigned, 8> &NodesToKeepBWs, unsigned &MaxDepthLevel, bool &IsProfitableToDemote, bool IsTruncRoot) const; + /// \returns the widest type the lanes of \p TE reach before they leave the + /// tree through a reduction, an external use or a user that is neither a + /// cast nor narrowed. With \p ApplyMinBWs the lanes are first narrowed to + /// their minimum bit widths, as the vector code is. + /// \p HasExternalUses tells whether a node has scalars used outside of the + /// tree. + Type *getWidestLaneType( + const TreeEntry &TE, bool ApplyMinBWs, + function_ref<bool(const TreeEntry &)> HasExternalUses) const; + + /// \returns true if two lanes of \p Ty fit a scalar register as a legal + /// vector type, so the target has packed arithmetic on them. + bool isPackedLaneType(Type *Ty) const; + /// \returns the load saving credited to the vectorized load bundle \p TE /// that never materializes on a target whose consecutive scalar loads /// coalesce into the same wide access, or zero if \p LoadSavings keeps it. /// Only clean bundles with no reordering, reuse or bit-width reduction lose - /// the saving. - InstructionCost - getCoalescedLoadPhantomSaving(const TreeEntry &TE, - CoalescedLoadSavings LoadSavings) const; + /// the saving. \p HasExternalUses tells whether a node has scalars used + /// outside of the tree. + InstructionCost getCoalescedLoadPhantomSaving( + const TreeEntry &TE, CoalescedLoadSavings LoadSavings, + function_ref<bool(const TreeEntry &)> HasExternalUses) const; /// Builds the list of reorderable operands on the edges \p Edges of the \p /// UserTE, which allow reordering (i.e. the operands can be reordered because @@ -13431,12 +13450,16 @@ InstructionCost BoUpSLP::getUnfusedFMulsPenalty(const TreeEntry &TE) const { /// \returns which load bundles of a reduction over \p VL lose the saving they /// are credited for although their scalar loads coalesce anyway. A reduction -/// that would lose an fma has a stake in every load bundle, and any other -/// reduction keeps every saving. A reassociable reduction loses no fma because -/// its vector fmuls fuse into the reduction. +/// that would lose an fma has a stake in every load bundle, an integer +/// reduction only in the bundles whose lanes get widened, the same as an +/// elementwise tree, and any other reduction keeps every saving. A +/// reassociable reduction loses no fma because its vector fmuls fuse into the +/// reduction. static BoUpSLP::CoalescedLoadSavings getReductionCoalescedLoadSavings(RecurKind RdxKind, FastMathFlags RdxFMF, ArrayRef<Value *> VL) { + if (RecurrenceDescriptor::isIntegerRecurrenceKind(RdxKind)) + return BoUpSLP::CoalescedLoadSavings::CancelWidened; if (RdxKind != RecurKind::FAdd || !RdxFMF.allowContract() || RdxFMF.allowReassoc() || none_of(VL, [](Value *V) { return match(V, @@ -18594,9 +18617,51 @@ bool BoUpSLP::isTreeNotExtendable() const { return Res; } -InstructionCost -BoUpSLP::getCoalescedLoadPhantomSaving(const TreeEntry &TE, - CoalescedLoadSavings LoadSavings) const { +Type *BoUpSLP::getWidestLaneType( + const TreeEntry &TE, bool ApplyMinBWs, + function_ref<bool(const TreeEntry &)> HasExternalUses) const { + Type *WidestTy = nullptr; + auto Widen = [&](Type *Ty) { + if (!WidestTy || + DL->getTypeSizeInBits(Ty) > DL->getTypeSizeInBits(WidestTy)) + WidestTy = Ty; + }; + for (const TreeEntry *E = &TE;;) { + Type *OrigTy = E->Scalars.front()->getType()->getScalarType(); + Type *Ty = OrigTy; + if (auto It = MinBWs.find(E); + ApplyMinBWs && It != MinBWs.end() && + It->second.first < DL->getTypeSizeInBits(Ty) && !HasExternalUses(*E)) + Ty = IntegerType::get(Ty->getContext(), It->second.first); + Widen(Ty); + const TreeEntry *UserTE = E->UserTreeIndex.UserTE; + if (!UserTE) { + if (ApplyMinBWs && UserIgnoreList && E == &getRootNode()) + Widen(getReductionType()->getScalarType()); + else + Widen(OrigTy); + return WidestTy; + } + if (!UserTE->hasState() || UserTE->isGather() || + (!Instruction::isCast(UserTE->getOpcode()) && + !MinBWs.contains(UserTE))) { + Widen(OrigTy); + return WidestTy; + } + E = UserTE; + } +} + +bool BoUpSLP::isPackedLaneType(Type *Ty) const { + unsigned Bits = DL->getTypeSizeInBits(Ty); + unsigned RegBits = + TTI->getRegisterBitWidth(TargetTransformInfo::RGK_Scalar).getFixedValue(); + return 2 * Bits <= RegBits && TTI->isTypeLegal(FixedVectorType::get(Ty, 2)); +} + +InstructionCost BoUpSLP::getCoalescedLoadPhantomSaving( + const TreeEntry &TE, CoalescedLoadSavings LoadSavings, + function_ref<bool(const TreeEntry &)> HasExternalUses) const { if (LoadSavings == CoalescedLoadSavings::Keep) return 0; if (!TE.hasState() || TE.isGather() || TE.getOpcode() != Instruction::Load || @@ -18616,6 +18681,20 @@ BoUpSLP::getCoalescedLoadPhantomSaving(const TreeEntry &TE, if (!TTI->consecutiveLoadsCoalesce(LI0->getType(), TE.Scalars.size(), BestAlign, LI0->getPointerAddressSpace())) return 0; + if (LoadSavings == CoalescedLoadSavings::CancelWidened) { + const TreeEntry *UserTE = TE.UserTreeIndex.UserTE; + if (!UserTE || UserTE->isGather() || !UserTE->hasState() || + !Instruction::isCast(UserTE->getOpcode())) + return 0; + Type *LoadedTy = LI0->getType()->getScalarType(); + Type *WidestTy = + getWidestLaneType(*UserTE, /*ApplyMinBWs=*/true, HasExternalUses); + if (DL->getTypeSizeInBits(WidestTy) <= DL->getTypeSizeInBits(LoadedTy) || + isPackedLaneType(LoadedTy) || + isPackedLaneType( + getWidestLaneType(*UserTE, /*ApplyMinBWs=*/false, HasExternalUses))) + return 0; + } InstructionCost ScalarLdCost = 0; for (Value *V : TE.Scalars) { auto *LI = cast<LoadInst>(V); @@ -19486,6 +19565,12 @@ InstructionCost BoUpSLP::calculateTreeCostAndTrimNonProfitable( return false; return IsExternallyUsedV(V); }; + auto HasExternalUses = [&](const TreeEntry &TE) { + return any_of(TE.Scalars, [&](Value *V) { + return !(TE.hasCopyableElements() && TE.isCopyableElement(V)) && + IsExternallyUsedV(V); + }); + }; InstructionCost Cost = 0; SmallDenseMap<const TreeEntry *, uint64_t> EntryToScale; uint64_t PrevScale = 0; @@ -19523,7 +19608,7 @@ InstructionCost BoUpSLP::calculateTreeCostAndTrimNonProfitable( "Expected gather nodes with users only."); InstructionCost C = getEntryCost(&TE, VectorizedVals, CheckedExtracts); - C += getCoalescedLoadPhantomSaving(TE, LoadSavings); + C += getCoalescedLoadPhantomSaving(TE, LoadSavings, HasExternalUses); uint64_t Scale = 0; bool CostIsFree = C == 0; // For gather/buildvector (and split-vectorize) entries, prefer the @@ -19726,7 +19811,7 @@ InstructionCost BoUpSLP::calculateTreeCostAndTrimNonProfitable( }; auto RecostEntry = [&](const TreeEntry *TE) { InstructionCost C = getEntryCost(TE, VectorizedVals, CheckedExtracts); - C += getCoalescedLoadPhantomSaving(*TE, LoadSavings); + C += getCoalescedLoadPhantomSaving(*TE, LoadSavings, HasExternalUses); if (!C.isValid() || C == 0) return C; uint64_t Scale = EntryToScale.lookup(TE); @@ -29719,7 +29804,7 @@ SLPVectorizerPass::vectorizeStoreChainImpl(ArrayRef<Value *> Chain, BoUpSLP &R, InstructionCost TreeCost = R.calculateTreeCostAndTrimNonProfitable( /*VectorizedVals=*/{}, /*RdxRoot=*/nullptr, - BoUpSLP::CoalescedLoadSavings::Keep); + BoUpSLP::CoalescedLoadSavings::CancelWidened); R.buildExternalUses(); Size = R.getCanonicalGraphSize() - R.getNumSplatSubtreeEntries(); @@ -30803,7 +30888,7 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R, R.computeMinimumValueSizes(); InstructionCost TreeCost = R.calculateTreeCostAndTrimNonProfitable( /*VectorizedVals=*/{}, /*RdxRoot=*/nullptr, - BoUpSLP::CoalescedLoadSavings::Keep); + BoUpSLP::CoalescedLoadSavings::CancelWidened); R.buildExternalUses(); InstructionCost Cost = R.getTreeCost(TreeCost); diff --git a/llvm/test/CodeGen/AMDGPU/slp-coalesced-loads.ll b/llvm/test/CodeGen/AMDGPU/slp-coalesced-loads.ll index 281221e8181fa..0385e52fbdc32 100644 --- a/llvm/test/CodeGen/AMDGPU/slp-coalesced-loads.ll +++ b/llvm/test/CodeGen/AMDGPU/slp-coalesced-loads.ll @@ -327,11 +327,10 @@ define void @store4_sitofp(ptr addrspace(1) %in, ptr addrspace(1) %out) { ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX90A-NEXT: global_load_dword v0, v[0:1], off ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: v_ashrrev_i16_e32 v1, 8, v0 -; GFX90A-NEXT: v_cvt_f32_i32_sdwa v7, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 -; GFX90A-NEXT: v_cvt_f32_i32_sdwa v6, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 ; GFX90A-NEXT: v_cvt_f32_i32_sdwa v4, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 -; GFX90A-NEXT: v_cvt_f32_i32_sdwa v5, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 +; GFX90A-NEXT: v_cvt_f32_i32_sdwa v5, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 +; GFX90A-NEXT: v_cvt_f32_i32_sdwa v6, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 +; GFX90A-NEXT: v_cvt_f32_i32_sdwa v7, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 ; GFX90A-NEXT: global_store_dwordx4 v[2:3], v[4:7], off ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] @@ -392,10 +391,12 @@ define void @zext4_bv_loop(ptr addrspace(1) %a, ptr addrspace(1) %out) { ; GFX90A-LABEL: zext4_bv_loop: ; GFX90A: ; %bb.0: ; %entry ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0 +; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GFX90A-NEXT: s_mov_b64 s[6:7], 0 ; GFX90A-NEXT: .LBB4_1: ; %loop ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX90A-NEXT: global_load_dword v6, v[0:1], off +; GFX90A-NEXT: global_load_dword v6, v[0:1], off offset:-1 ; GFX90A-NEXT: s_add_u32 s6, s6, 1 ; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 4, v0 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc @@ -417,12 +418,14 @@ define void @zext4_bv_loop(ptr addrspace(1) %a, ptr addrspace(1) %out) { ; GFX1030-LABEL: zext4_bv_loop: ; GFX1030: ; %bb.0: ; %entry ; GFX1030-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1030-NEXT: v_add_co_u32 v0, vcc_lo, v0, 1 +; GFX1030-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo ; GFX1030-NEXT: v_mov_b32_e32 v4, 0xffff ; GFX1030-NEXT: s_mov_b64 s[6:7], 0 ; GFX1030-NEXT: .p2align 6 ; GFX1030-NEXT: .LBB4_1: ; %loop ; GFX1030-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX1030-NEXT: global_load_dword v7, v[0:1], off +; GFX1030-NEXT: global_load_dword v7, v[0:1], off offset:-1 ; GFX1030-NEXT: s_add_u32 s6, s6, 1 ; GFX1030-NEXT: v_add_co_u32 v0, vcc_lo, v0, 4 ; GFX1030-NEXT: s_addc_u32 s7, s7, 0 @@ -444,10 +447,11 @@ define void @zext4_bv_loop(ptr addrspace(1) %a, ptr addrspace(1) %out) { ; GFX1250: ; %bb.0: ; %entry ; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], 1, v[0:1] ; GFX1250-NEXT: s_mov_b64 s[0:1], 0 ; GFX1250-NEXT: .LBB4_1: ; %loop ; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX1250-NEXT: global_load_b32 v6, v[0:1], off +; GFX1250-NEXT: global_load_b32 v6, v[0:1], off offset:-1 ; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[0:1], 1 ; GFX1250-NEXT: s_wait_xcnt 0x0 ; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], 4, v[0:1] @@ -501,51 +505,21 @@ define i32 @dot8_sext_i8(ptr addrspace(1) %a, ptr addrspace(1) %b) { ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX90A-NEXT: global_load_dwordx2 v[4:5], v[0:1], off ; GFX90A-NEXT: global_load_dwordx2 v[6:7], v[2:3], off -; GFX90A-NEXT: s_waitcnt vmcnt(1) -; GFX90A-NEXT: v_lshrrev_b16_e32 v0, 8, v4 -; GFX90A-NEXT: v_lshrrev_b16_e32 v1, 8, v5 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: v_lshrrev_b16_e32 v2, 8, v6 -; GFX90A-NEXT: v_lshrrev_b16_e32 v3, 8, v7 -; GFX90A-NEXT: v_mul_i32_i24_sdwa v8, sext(v4), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX90A-NEXT: v_mul_i32_i24_sdwa v9, sext(v4), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 -; GFX90A-NEXT: v_mul_i32_i24_sdwa v4, sext(v4), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 -; GFX90A-NEXT: v_mul_i32_i24_sdwa v6, sext(v5), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX90A-NEXT: v_mul_i32_i24_sdwa v10, sext(v5), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 -; GFX90A-NEXT: v_mul_i32_i24_sdwa v5, sext(v5), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 -; GFX90A-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX90A-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX90A-NEXT: v_add_u32_e32 v4, v4, v5 -; GFX90A-NEXT: v_add_u32_e32 v2, v9, v10 -; GFX90A-NEXT: v_add_u32_e32 v3, v8, v6 -; GFX90A-NEXT: v_add3_u32 v0, v0, v1, v4 -; GFX90A-NEXT: v_add3_u32 v0, v3, v2, v0 +; GFX90A-NEXT: v_dot4c_i32_i8_e32 v0, v4, v6 +; GFX90A-NEXT: v_dot4c_i32_i8_e32 v0, v5, v7 ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; ; GFX1030-LABEL: dot8_sext_i8: ; GFX1030: ; %bb.0: ; %entry ; GFX1030-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX1030-NEXT: global_load_dwordx2 v[0:1], v[0:1], off -; GFX1030-NEXT: global_load_dwordx2 v[2:3], v[2:3], off -; GFX1030-NEXT: s_waitcnt vmcnt(1) -; GFX1030-NEXT: v_lshrrev_b16 v4, 8, v0 -; GFX1030-NEXT: v_lshrrev_b16 v5, 8, v1 +; GFX1030-NEXT: global_load_dwordx2 v[4:5], v[0:1], off +; GFX1030-NEXT: global_load_dwordx2 v[1:2], v[2:3], off +; GFX1030-NEXT: v_mov_b32_e32 v0, 0 ; GFX1030-NEXT: s_waitcnt vmcnt(0) -; GFX1030-NEXT: v_lshrrev_b16 v6, 8, v2 -; GFX1030-NEXT: v_lshrrev_b16 v7, 8, v3 -; GFX1030-NEXT: v_mul_i32_i24_sdwa v8, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX1030-NEXT: v_mul_i32_i24_sdwa v9, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 -; GFX1030-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 -; GFX1030-NEXT: v_mul_i32_i24_sdwa v2, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 -; GFX1030-NEXT: v_mul_i32_i24_sdwa v10, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX1030-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 -; GFX1030-NEXT: v_mul_i32_i24_sdwa v3, sext(v4), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX1030-NEXT: v_mul_i32_i24_sdwa v4, sext(v5), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX1030-NEXT: v_add_nc_u32_e32 v0, v0, v2 -; GFX1030-NEXT: v_add_nc_u32_e32 v2, v8, v10 -; GFX1030-NEXT: v_add_nc_u32_e32 v1, v9, v1 -; GFX1030-NEXT: v_add3_u32 v0, v3, v4, v0 -; GFX1030-NEXT: v_add3_u32 v0, v2, v1, v0 +; GFX1030-NEXT: v_dot4c_i32_i8 v0, v4, v1 +; GFX1030-NEXT: v_dot4c_i32_i8 v0, v5, v2 ; GFX1030-NEXT: s_setpc_b64 s[30:31] ; ; GFX1250-LABEL: dot8_sext_i8: @@ -554,39 +528,11 @@ define i32 @dot8_sext_i8(ptr addrspace(1) %a, ptr addrspace(1) %b) { ; GFX1250-NEXT: s_wait_kmcnt 0x0 ; GFX1250-NEXT: global_load_b64 v[4:5], v[0:1], off ; GFX1250-NEXT: global_load_b64 v[6:7], v[2:3], off -; GFX1250-NEXT: s_wait_loadcnt 0x1 -; GFX1250-NEXT: s_wait_xcnt 0x1 -; GFX1250-NEXT: v_lshrrev_b16 v0.l, 8, v4.l -; GFX1250-NEXT: v_lshrrev_b16 v1.l, 8, v5.l ; GFX1250-NEXT: s_wait_loadcnt 0x0 -; GFX1250-NEXT: v_lshrrev_b16 v8.l, 8, v6.l -; GFX1250-NEXT: v_lshrrev_b16 v9.l, 8, v7.l -; GFX1250-NEXT: v_dual_ashrrev_i32 v12, 24, v5 :: v_dual_ashrrev_i32 v14, 24, v7 -; GFX1250-NEXT: s_wait_xcnt 0x0 -; GFX1250-NEXT: v_dual_ashrrev_i32 v2, 24, v4 :: v_dual_ashrrev_i32 v10, 24, v6 -; GFX1250-NEXT: v_bfe_i32 v13, v5, 16, 8 -; GFX1250-NEXT: v_bfe_i32 v5, v5, 0, 8 -; GFX1250-NEXT: v_bfe_i32 v15, v7, 16, 8 -; GFX1250-NEXT: v_bfe_i32 v7, v7, 0, 8 -; GFX1250-NEXT: v_bfe_i32 v0, v0, 0, 8 -; GFX1250-NEXT: v_bfe_i32 v8, v8, 0, 8 -; GFX1250-NEXT: v_bfe_i32 v1, v1, 0, 8 -; GFX1250-NEXT: v_bfe_i32 v9, v9, 0, 8 -; GFX1250-NEXT: v_mul_i32_i24_e32 v12, v12, v14 -; GFX1250-NEXT: v_bfe_i32 v3, v4, 0, 8 -; GFX1250-NEXT: v_bfe_i32 v4, v4, 16, 8 -; GFX1250-NEXT: v_bfe_i32 v11, v6, 0, 8 -; GFX1250-NEXT: v_bfe_i32 v6, v6, 16, 8 -; GFX1250-NEXT: v_mul_i32_i24_e32 v5, v5, v7 -; GFX1250-NEXT: v_mul_i32_i24_e32 v7, v13, v15 -; GFX1250-NEXT: v_mul_i32_i24_e32 v0, v0, v8 -; GFX1250-NEXT: v_mul_i32_i24_e32 v1, v1, v9 -; GFX1250-NEXT: v_mad_i32_i24 v2, v2, v10, v12 -; GFX1250-NEXT: v_mad_i32_i24 v3, v3, v11, v5 -; GFX1250-NEXT: v_mad_i32_i24 v4, v4, v6, v7 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250-NEXT: v_add3_u32 v0, v0, v1, v2 -; GFX1250-NEXT: v_add3_u32 v0, v3, v4, v0 +; GFX1250-NEXT: s_wait_xcnt 0x1 +; GFX1250-NEXT: v_dot4_i32_iu8 v0, v4, v6, 0 neg_lo:[1,1,0] +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-NEXT: v_dot4_i32_iu8 v0, v5, v7, v0 neg_lo:[1,1,0] ; GFX1250-NEXT: s_set_pc_i64 s[30:31] entry: %pa0 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 0 @@ -660,35 +606,22 @@ define i32 @sum4_zext_i8(ptr addrspace(1) %a) { ; GFX90A: ; %bb.0: ; %entry ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX90A-NEXT: global_load_dword v0, v[0:1], off -; GFX90A-NEXT: s_movk_i32 s4, 0xff -; GFX90A-NEXT: s_mov_b32 s5, 0x5040100 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: v_lshrrev_b32_e32 v1, 24, v0 -; GFX90A-NEXT: v_lshrrev_b16_e32 v2, 8, v0 -; GFX90A-NEXT: v_and_b32_sdwa v3, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD -; GFX90A-NEXT: v_and_b32_e32 v0, 0xff, v0 -; GFX90A-NEXT: v_perm_b32 v0, v2, v0, s5 -; GFX90A-NEXT: v_perm_b32 v1, v1, v3, s5 -; GFX90A-NEXT: v_pk_add_u16 v0, v0, v1 -; GFX90A-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 +; GFX90A-NEXT: v_add_u32_sdwa v2, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_1 +; GFX90A-NEXT: v_bfe_u32 v0, v0, 16, 8 +; GFX90A-NEXT: v_add3_u32 v0, v2, v0, v1 ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; ; GFX1030-LABEL: sum4_zext_i8: ; GFX1030: ; %bb.0: ; %entry ; GFX1030-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX1030-NEXT: global_load_dword v0, v[0:1], off -; GFX1030-NEXT: v_mov_b32_e32 v1, 0xff ; GFX1030-NEXT: s_waitcnt vmcnt(0) -; GFX1030-NEXT: v_lshrrev_b32_e32 v2, 24, v0 -; GFX1030-NEXT: v_lshrrev_b16 v3, 8, v0 -; GFX1030-NEXT: v_and_b32_e32 v4, 0xff, v0 -; GFX1030-NEXT: v_and_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD -; GFX1030-NEXT: v_perm_b32 v1, v3, v4, 0x5040100 -; GFX1030-NEXT: v_perm_b32 v0, v2, v0, 0x5040100 -; GFX1030-NEXT: v_pk_add_u16 v0, v1, v0 -; GFX1030-NEXT: v_lshrrev_b32_e32 v1, 16, v0 -; GFX1030-NEXT: v_add_nc_u16 v0, v0, v1 -; GFX1030-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX1030-NEXT: v_lshrrev_b32_e32 v1, 24, v0 +; GFX1030-NEXT: v_add_nc_u32_sdwa v2, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_1 +; GFX1030-NEXT: v_bfe_u32 v0, v0, 16, 8 +; GFX1030-NEXT: v_add3_u32 v0, v2, v0, v1 ; GFX1030-NEXT: s_setpc_b64 s[30:31] ; ; GFX1250-LABEL: sum4_zext_i8: @@ -696,18 +629,15 @@ define i32 @sum4_zext_i8(ptr addrspace(1) %a) { ; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX1250-NEXT: s_wait_kmcnt 0x0 ; GFX1250-NEXT: global_load_b32 v0, v[0:1], off +; GFX1250-NEXT: s_mov_b32 s0, 0xff ; GFX1250-NEXT: s_wait_loadcnt 0x0 ; GFX1250-NEXT: s_wait_xcnt 0x0 -; GFX1250-NEXT: v_lshrrev_b32_e32 v1, 24, v0 -; GFX1250-NEXT: v_lshrrev_b16 v2.h, 8, v0.l -; GFX1250-NEXT: v_and_b16 v3.l, 0xff, v0.h -; GFX1250-NEXT: v_and_b16 v2.l, 0xff, v0.l -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250-NEXT: v_mov_b16_e32 v3.h, v1.l -; GFX1250-NEXT: v_pk_add_u16 v0, v2, v3 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h -; GFX1250-NEXT: v_cvt_u32_u16_e32 v0, v0.l +; GFX1250-NEXT: v_dual_lshrrev_b32 v3, 24, v0 :: v_dual_bitop2_b32 v1, s0, v0 bitop3:0x40 +; GFX1250-NEXT: v_bfe_u32 v2, v0, 8, 8 +; GFX1250-NEXT: v_bfe_u32 v0, v0, 16, 8 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-NEXT: v_add_nc_u32_e32 v1, v1, v2 +; GFX1250-NEXT: v_add3_u32 v0, v1, v0, v3 ; GFX1250-NEXT: s_set_pc_i64 s[30:31] entry: %pa0 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 0 @@ -931,52 +861,22 @@ define i32 @dot8_sext_i8_lds_a16(ptr addrspace(3) %a, ptr addrspace(3) %b) { ; GFX90A: ; %bb.0: ; %entry ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX90A-NEXT: ds_read_b64 v[2:3], v0 -; GFX90A-NEXT: ds_read_b64 v[0:1], v1 -; GFX90A-NEXT: s_waitcnt lgkmcnt(1) -; GFX90A-NEXT: v_lshrrev_b16_e32 v4, 8, v2 -; GFX90A-NEXT: v_lshrrev_b16_e32 v5, 8, v3 +; GFX90A-NEXT: ds_read_b64 v[4:5], v1 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_lshrrev_b16_e32 v6, 8, v0 -; GFX90A-NEXT: v_lshrrev_b16_e32 v7, 8, v1 -; GFX90A-NEXT: v_mul_i32_i24_sdwa v8, sext(v2), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX90A-NEXT: v_mul_i32_i24_sdwa v4, sext(v4), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX90A-NEXT: v_mul_i32_i24_sdwa v6, sext(v2), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 -; GFX90A-NEXT: v_mul_i32_i24_sdwa v0, sext(v2), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 -; GFX90A-NEXT: v_mul_i32_i24_sdwa v2, sext(v3), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX90A-NEXT: v_mul_i32_i24_sdwa v5, sext(v5), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX90A-NEXT: v_mul_i32_i24_sdwa v7, sext(v3), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 -; GFX90A-NEXT: v_mul_i32_i24_sdwa v1, sext(v3), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 -; GFX90A-NEXT: v_add_u32_e32 v0, v0, v1 -; GFX90A-NEXT: v_add_u32_e32 v1, v6, v7 -; GFX90A-NEXT: v_add_u32_e32 v2, v8, v2 -; GFX90A-NEXT: v_add3_u32 v0, v4, v5, v0 -; GFX90A-NEXT: v_add3_u32 v0, v2, v1, v0 +; GFX90A-NEXT: v_dot4c_i32_i8_e32 v0, v2, v4 +; GFX90A-NEXT: v_dot4c_i32_i8_e32 v0, v3, v5 ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; ; GFX1030-LABEL: dot8_sext_i8_lds_a16: ; GFX1030: ; %bb.0: ; %entry ; GFX1030-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX1030-NEXT: ds_read_b64 v[2:3], v0 -; GFX1030-NEXT: ds_read_b64 v[0:1], v1 -; GFX1030-NEXT: s_waitcnt lgkmcnt(1) -; GFX1030-NEXT: v_lshrrev_b16 v4, 8, v2 -; GFX1030-NEXT: v_lshrrev_b16 v5, 8, v3 +; GFX1030-NEXT: ds_read_b64 v[4:5], v1 +; GFX1030-NEXT: v_mov_b32_e32 v0, 0 ; GFX1030-NEXT: s_waitcnt lgkmcnt(0) -; GFX1030-NEXT: v_lshrrev_b16 v6, 8, v0 -; GFX1030-NEXT: v_lshrrev_b16 v7, 8, v1 -; GFX1030-NEXT: v_mul_i32_i24_sdwa v8, sext(v2), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX1030-NEXT: v_mul_i32_i24_sdwa v9, sext(v2), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 -; GFX1030-NEXT: v_mul_i32_i24_sdwa v0, sext(v2), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 -; GFX1030-NEXT: v_mul_i32_i24_sdwa v2, sext(v3), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 -; GFX1030-NEXT: v_mul_i32_i24_sdwa v10, sext(v3), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX1030-NEXT: v_mul_i32_i24_sdwa v1, sext(v3), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 -; GFX1030-NEXT: v_mul_i32_i24_sdwa v3, sext(v4), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX1030-NEXT: v_mul_i32_i24_sdwa v4, sext(v5), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX1030-NEXT: v_add_nc_u32_e32 v0, v0, v2 -; GFX1030-NEXT: v_add_nc_u32_e32 v2, v8, v10 -; GFX1030-NEXT: v_add_nc_u32_e32 v1, v9, v1 -; GFX1030-NEXT: v_add3_u32 v0, v3, v4, v0 -; GFX1030-NEXT: v_add3_u32 v0, v2, v1, v0 +; GFX1030-NEXT: v_dot4c_i32_i8 v0, v2, v4 +; GFX1030-NEXT: v_dot4c_i32_i8 v0, v3, v5 ; GFX1030-NEXT: s_setpc_b64 s[30:31] ; ; GFX1250-LABEL: dot8_sext_i8_lds_a16: @@ -985,37 +885,10 @@ define i32 @dot8_sext_i8_lds_a16(ptr addrspace(3) %a, ptr addrspace(3) %b) { ; GFX1250-NEXT: s_wait_kmcnt 0x0 ; GFX1250-NEXT: ds_load_b64 v[2:3], v0 ; GFX1250-NEXT: ds_load_b64 v[0:1], v1 -; GFX1250-NEXT: s_wait_dscnt 0x1 -; GFX1250-NEXT: v_lshrrev_b16 v4.l, 8, v2.l -; GFX1250-NEXT: v_lshrrev_b16 v5.l, 8, v3.l ; GFX1250-NEXT: s_wait_dscnt 0x0 -; GFX1250-NEXT: v_lshrrev_b16 v8.l, 8, v0.l -; GFX1250-NEXT: v_lshrrev_b16 v9.l, 8, v1.l -; GFX1250-NEXT: v_dual_ashrrev_i32 v12, 24, v3 :: v_dual_ashrrev_i32 v14, 24, v1 -; GFX1250-NEXT: v_dual_ashrrev_i32 v6, 24, v2 :: v_dual_ashrrev_i32 v10, 24, v0 -; GFX1250-NEXT: v_bfe_i32 v13, v3, 16, 8 -; GFX1250-NEXT: v_bfe_i32 v3, v3, 0, 8 -; GFX1250-NEXT: v_bfe_i32 v15, v1, 16, 8 -; GFX1250-NEXT: v_bfe_i32 v1, v1, 0, 8 -; GFX1250-NEXT: v_bfe_i32 v4, v4, 0, 8 -; GFX1250-NEXT: v_bfe_i32 v8, v8, 0, 8 -; GFX1250-NEXT: v_bfe_i32 v5, v5, 0, 8 -; GFX1250-NEXT: v_bfe_i32 v9, v9, 0, 8 -; GFX1250-NEXT: v_mul_i32_i24_e32 v12, v12, v14 -; GFX1250-NEXT: v_bfe_i32 v7, v2, 0, 8 -; GFX1250-NEXT: v_bfe_i32 v2, v2, 16, 8 -; GFX1250-NEXT: v_bfe_i32 v11, v0, 0, 8 -; GFX1250-NEXT: v_bfe_i32 v0, v0, 16, 8 -; GFX1250-NEXT: v_mul_i32_i24_e32 v1, v3, v1 -; GFX1250-NEXT: v_mul_i32_i24_e32 v3, v13, v15 -; GFX1250-NEXT: v_mul_i32_i24_e32 v4, v4, v8 -; GFX1250-NEXT: v_mul_i32_i24_e32 v5, v5, v9 -; GFX1250-NEXT: v_mad_i32_i24 v6, v6, v10, v12 -; GFX1250-NEXT: v_mad_i32_i24 v1, v7, v11, v1 -; GFX1250-NEXT: v_mad_i32_i24 v0, v2, v0, v3 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250-NEXT: v_add3_u32 v2, v4, v5, v6 -; GFX1250-NEXT: v_add3_u32 v0, v1, v0, v2 +; GFX1250-NEXT: v_dot4_i32_iu8 v0, v2, v0, 0 neg_lo:[1,1,0] +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-NEXT: v_dot4_i32_iu8 v0, v3, v1, v0 neg_lo:[1,1,0] ; GFX1250-NEXT: s_set_pc_i64 s[30:31] entry: %pa0 = getelementptr inbounds i8, ptr addrspace(3) %a, i32 0 diff --git a/llvm/test/CodeGen/AMDGPU/slp-int-to-fp.ll b/llvm/test/CodeGen/AMDGPU/slp-int-to-fp.ll index 5b73c71c29f13..5a791d3254b02 100644 --- a/llvm/test/CodeGen/AMDGPU/slp-int-to-fp.ll +++ b/llvm/test/CodeGen/AMDGPU/slp-int-to-fp.ll @@ -343,20 +343,20 @@ define void @sitofp_i8_scaled_to_float(ptr addrspace(1) noalias %out, ptr addrsp ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX90A-NEXT: global_load_dwordx2 v[2:3], v[2:3], off ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: v_cvt_f32_i32_sdwa v7, sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 ; GFX90A-NEXT: v_cvt_f32_i32_sdwa v6, sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 -; GFX90A-NEXT: v_cvt_f32_i32_sdwa v9, sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 +; GFX90A-NEXT: v_cvt_f32_i32_sdwa v7, sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 ; GFX90A-NEXT: v_cvt_f32_i32_sdwa v8, sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 -; GFX90A-NEXT: v_cvt_f32_i32_sdwa v11, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 -; GFX90A-NEXT: v_cvt_f32_i32_sdwa v10, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 -; GFX90A-NEXT: v_cvt_f32_i32_sdwa v13, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 -; GFX90A-NEXT: v_cvt_f32_i32_sdwa v12, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 -; GFX90A-NEXT: v_pk_mul_f32 v[6:7], v[6:7], v[4:5] op_sel_hi:[1,0] +; GFX90A-NEXT: v_cvt_f32_i32_sdwa v9, sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 +; GFX90A-NEXT: v_cvt_f32_i32_sdwa v2, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 +; GFX90A-NEXT: v_cvt_f32_i32_sdwa v10, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 +; GFX90A-NEXT: v_cvt_f32_i32_sdwa v11, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 +; GFX90A-NEXT: v_cvt_f32_i32_sdwa v3, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 ; GFX90A-NEXT: v_pk_mul_f32 v[8:9], v[8:9], v[4:5] op_sel_hi:[1,0] -; GFX90A-NEXT: v_pk_mul_f32 v[2:3], v[10:11], v[4:5] op_sel_hi:[1,0] -; GFX90A-NEXT: v_pk_mul_f32 v[4:5], v[12:13], v[4:5] op_sel_hi:[1,0] +; GFX90A-NEXT: v_pk_mul_f32 v[6:7], v[6:7], v[4:5] op_sel_hi:[1,0] +; GFX90A-NEXT: v_pk_mul_f32 v[12:13], v[10:11], v[4:5] op_sel_hi:[1,0] +; GFX90A-NEXT: v_pk_mul_f32 v[10:11], v[2:3], v[4:5] op_sel_hi:[1,0] ; GFX90A-NEXT: global_store_dwordx4 v[0:1], v[6:9], off -; GFX90A-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:16 +; GFX90A-NEXT: global_store_dwordx4 v[0:1], v[10:13], off offset:16 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-coalesced-loads.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-coalesced-loads.ll index 67584277cb976..363e3d13ca96d 100644 --- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-coalesced-loads.ll +++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-coalesced-loads.ll @@ -32,6 +32,9 @@ define float @dequant8_ordered(ptr addrspace(1) %mat, ptr addrspace(1) %vec, hal ; GFX9-NEXT: [[V6:%.*]] = load half, ptr addrspace(1) [[VP6]], align 4 ; GFX9-NEXT: [[VP7:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[VEC]], i64 14 ; GFX9-NEXT: [[V7:%.*]] = load half, ptr addrspace(1) [[VP7]], align 2 +; GFX9-NEXT: [[MP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[MAT]], i64 1 +; GFX9-NEXT: [[MP2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[MAT]], i64 2 +; GFX9-NEXT: [[MP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[MAT]], i64 3 ; GFX9-NEXT: [[MP4:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[MAT]], i64 4 ; GFX9-NEXT: [[Q4:%.*]] = load i8, ptr addrspace(1) [[MP4]], align 2 ; GFX9-NEXT: [[MP5:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[MAT]], i64 5 @@ -44,8 +47,18 @@ define float @dequant8_ordered(ptr addrspace(1) %mat, ptr addrspace(1) %vec, hal ; GFX9-NEXT: [[X1:%.*]] = fpext half [[V1]] to float ; GFX9-NEXT: [[X2:%.*]] = fpext half [[V2]] to float ; GFX9-NEXT: [[X3:%.*]] = fpext half [[V3]] to float -; GFX9-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(1) [[MAT]], align 2 -; GFX9-NEXT: [[TMP1:%.*]] = sitofp <4 x i8> [[TMP0]] to <4 x float> +; GFX9-NEXT: [[Q0:%.*]] = load i8, ptr addrspace(1) [[MAT]], align 2 +; GFX9-NEXT: [[Q1:%.*]] = load i8, ptr addrspace(1) [[MP1]], align 1 +; GFX9-NEXT: [[Q2:%.*]] = load i8, ptr addrspace(1) [[MP2]], align 2 +; GFX9-NEXT: [[Q3:%.*]] = load i8, ptr addrspace(1) [[MP3]], align 1 +; GFX9-NEXT: [[F0:%.*]] = sitofp i8 [[Q0]] to float +; GFX9-NEXT: [[F1:%.*]] = sitofp i8 [[Q1]] to float +; GFX9-NEXT: [[F2:%.*]] = sitofp i8 [[Q2]] to float +; GFX9-NEXT: [[F3:%.*]] = sitofp i8 [[Q3]] to float +; GFX9-NEXT: [[TMP0:%.*]] = insertelement <4 x float> poison, float [[F0]], i64 0 +; GFX9-NEXT: [[TMP9:%.*]] = insertelement <4 x float> [[TMP0]], float [[F1]], i64 1 +; GFX9-NEXT: [[TMP10:%.*]] = insertelement <4 x float> [[TMP9]], float [[F2]], i64 2 +; GFX9-NEXT: [[TMP1:%.*]] = insertelement <4 x float> [[TMP10]], float [[F3]], i64 3 ; GFX9-NEXT: [[TMP2:%.*]] = insertelement <4 x float> poison, float [[ZP]], i64 0 ; GFX9-NEXT: [[TMP3:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <4 x i32> zeroinitializer ; GFX9-NEXT: [[TMP4:%.*]] = fsub contract <4 x float> [[TMP1]], [[TMP3]] @@ -244,12 +257,47 @@ body: ; with one vector store, gfx942 and gfx950 with two, and gfx1030 keeps the ; scalar stores. define void @store4_sitofp(ptr addrspace(1) %in, ptr addrspace(1) %out) { -; GFX9-LABEL: define void @store4_sitofp( -; GFX9-SAME: ptr addrspace(1) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) { -; GFX9-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr addrspace(1) [[IN]], align 4 -; GFX9-NEXT: [[TMP2:%.*]] = sitofp <4 x i8> [[TMP1]] to <4 x float> -; GFX9-NEXT: store <4 x float> [[TMP2]], ptr addrspace(1) [[OUT]], align 16 -; GFX9-NEXT: ret void +; GFX90A-LABEL: define void @store4_sitofp( +; GFX90A-SAME: ptr addrspace(1) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) { +; GFX90A-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i64 1 +; GFX90A-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i64 2 +; GFX90A-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i64 3 +; GFX90A-NEXT: [[Q0:%.*]] = load i8, ptr addrspace(1) [[IN]], align 4 +; GFX90A-NEXT: [[Q1:%.*]] = load i8, ptr addrspace(1) [[P1]], align 1 +; GFX90A-NEXT: [[Q2:%.*]] = load i8, ptr addrspace(1) [[P2]], align 2 +; GFX90A-NEXT: [[Q3:%.*]] = load i8, ptr addrspace(1) [[P3]], align 1 +; GFX90A-NEXT: [[F0:%.*]] = sitofp i8 [[Q0]] to float +; GFX90A-NEXT: [[F1:%.*]] = sitofp i8 [[Q1]] to float +; GFX90A-NEXT: [[F2:%.*]] = sitofp i8 [[Q2]] to float +; GFX90A-NEXT: [[F3:%.*]] = sitofp i8 [[Q3]] to float +; GFX90A-NEXT: [[TMP1:%.*]] = insertelement <4 x float> poison, float [[F0]], i64 0 +; GFX90A-NEXT: [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[F1]], i64 1 +; GFX90A-NEXT: [[TMP3:%.*]] = insertelement <4 x float> [[TMP2]], float [[F2]], i64 2 +; GFX90A-NEXT: [[TMP4:%.*]] = insertelement <4 x float> [[TMP3]], float [[F3]], i64 3 +; GFX90A-NEXT: store <4 x float> [[TMP4]], ptr addrspace(1) [[OUT]], align 16 +; GFX90A-NEXT: ret void +; +; GFX942-950-LABEL: define void @store4_sitofp( +; GFX942-950-SAME: ptr addrspace(1) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) { +; GFX942-950-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i64 1 +; GFX942-950-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i64 2 +; GFX942-950-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i64 3 +; GFX942-950-NEXT: [[O2:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 2 +; GFX942-950-NEXT: [[Q0:%.*]] = load i8, ptr addrspace(1) [[IN]], align 4 +; GFX942-950-NEXT: [[Q1:%.*]] = load i8, ptr addrspace(1) [[P1]], align 1 +; GFX942-950-NEXT: [[F0:%.*]] = sitofp i8 [[Q0]] to float +; GFX942-950-NEXT: [[F1:%.*]] = sitofp i8 [[Q1]] to float +; GFX942-950-NEXT: [[TMP1:%.*]] = insertelement <2 x float> poison, float [[F0]], i64 0 +; GFX942-950-NEXT: [[TMP2:%.*]] = insertelement <2 x float> [[TMP1]], float [[F1]], i64 1 +; GFX942-950-NEXT: [[Q2:%.*]] = load i8, ptr addrspace(1) [[P2]], align 2 +; GFX942-950-NEXT: [[Q3:%.*]] = load i8, ptr addrspace(1) [[P3]], align 1 +; GFX942-950-NEXT: [[F2:%.*]] = sitofp i8 [[Q2]] to float +; GFX942-950-NEXT: [[F3:%.*]] = sitofp i8 [[Q3]] to float +; GFX942-950-NEXT: store <2 x float> [[TMP2]], ptr addrspace(1) [[OUT]], align 16 +; GFX942-950-NEXT: [[TMP3:%.*]] = insertelement <2 x float> poison, float [[F2]], i64 0 +; GFX942-950-NEXT: [[TMP4:%.*]] = insertelement <2 x float> [[TMP3]], float [[F3]], i64 1 +; GFX942-950-NEXT: store <2 x float> [[TMP4]], ptr addrspace(1) [[O2]], align 8 +; GFX942-950-NEXT: ret void ; ; GFX1030-LABEL: define void @store4_sitofp( ; GFX1030-SAME: ptr addrspace(1) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) { @@ -297,12 +345,47 @@ define void @store4_sitofp(ptr addrspace(1) %in, ptr addrspace(1) %out) { ; The same store chain with every byte load aligned to one byte. Global loads ; coalesce at any alignment, so the loads and the conversions stay scalar. define void @store4_sitofp_global_a1(ptr addrspace(1) %in, ptr addrspace(1) %out) { -; GFX9-LABEL: define void @store4_sitofp_global_a1( -; GFX9-SAME: ptr addrspace(1) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) { -; GFX9-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr addrspace(1) [[IN]], align 1 -; GFX9-NEXT: [[TMP2:%.*]] = sitofp <4 x i8> [[TMP1]] to <4 x float> -; GFX9-NEXT: store <4 x float> [[TMP2]], ptr addrspace(1) [[OUT]], align 16 -; GFX9-NEXT: ret void +; GFX90A-LABEL: define void @store4_sitofp_global_a1( +; GFX90A-SAME: ptr addrspace(1) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) { +; GFX90A-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i64 1 +; GFX90A-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i64 2 +; GFX90A-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i64 3 +; GFX90A-NEXT: [[Q0:%.*]] = load i8, ptr addrspace(1) [[IN]], align 1 +; GFX90A-NEXT: [[Q1:%.*]] = load i8, ptr addrspace(1) [[P1]], align 1 +; GFX90A-NEXT: [[Q2:%.*]] = load i8, ptr addrspace(1) [[P2]], align 1 +; GFX90A-NEXT: [[Q3:%.*]] = load i8, ptr addrspace(1) [[P3]], align 1 +; GFX90A-NEXT: [[F0:%.*]] = sitofp i8 [[Q0]] to float +; GFX90A-NEXT: [[F1:%.*]] = sitofp i8 [[Q1]] to float +; GFX90A-NEXT: [[F2:%.*]] = sitofp i8 [[Q2]] to float +; GFX90A-NEXT: [[F3:%.*]] = sitofp i8 [[Q3]] to float +; GFX90A-NEXT: [[TMP1:%.*]] = insertelement <4 x float> poison, float [[F0]], i64 0 +; GFX90A-NEXT: [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[F1]], i64 1 +; GFX90A-NEXT: [[TMP3:%.*]] = insertelement <4 x float> [[TMP2]], float [[F2]], i64 2 +; GFX90A-NEXT: [[TMP4:%.*]] = insertelement <4 x float> [[TMP3]], float [[F3]], i64 3 +; GFX90A-NEXT: store <4 x float> [[TMP4]], ptr addrspace(1) [[OUT]], align 16 +; GFX90A-NEXT: ret void +; +; GFX942-950-LABEL: define void @store4_sitofp_global_a1( +; GFX942-950-SAME: ptr addrspace(1) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) { +; GFX942-950-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i64 1 +; GFX942-950-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i64 2 +; GFX942-950-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i64 3 +; GFX942-950-NEXT: [[O2:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 2 +; GFX942-950-NEXT: [[Q0:%.*]] = load i8, ptr addrspace(1) [[IN]], align 1 +; GFX942-950-NEXT: [[Q1:%.*]] = load i8, ptr addrspace(1) [[P1]], align 1 +; GFX942-950-NEXT: [[F0:%.*]] = sitofp i8 [[Q0]] to float +; GFX942-950-NEXT: [[F1:%.*]] = sitofp i8 [[Q1]] to float +; GFX942-950-NEXT: [[TMP1:%.*]] = insertelement <2 x float> poison, float [[F0]], i64 0 +; GFX942-950-NEXT: [[TMP2:%.*]] = insertelement <2 x float> [[TMP1]], float [[F1]], i64 1 +; GFX942-950-NEXT: [[Q2:%.*]] = load i8, ptr addrspace(1) [[P2]], align 1 +; GFX942-950-NEXT: [[Q3:%.*]] = load i8, ptr addrspace(1) [[P3]], align 1 +; GFX942-950-NEXT: [[F2:%.*]] = sitofp i8 [[Q2]] to float +; GFX942-950-NEXT: [[F3:%.*]] = sitofp i8 [[Q3]] to float +; GFX942-950-NEXT: store <2 x float> [[TMP2]], ptr addrspace(1) [[OUT]], align 16 +; GFX942-950-NEXT: [[TMP3:%.*]] = insertelement <2 x float> poison, float [[F2]], i64 0 +; GFX942-950-NEXT: [[TMP4:%.*]] = insertelement <2 x float> [[TMP3]], float [[F3]], i64 1 +; GFX942-950-NEXT: store <2 x float> [[TMP4]], ptr addrspace(1) [[O2]], align 8 +; GFX942-950-NEXT: ret void ; ; GFX1030-LABEL: define void @store4_sitofp_global_a1( ; GFX1030-SAME: ptr addrspace(1) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) { @@ -351,12 +434,47 @@ define void @store4_sitofp_global_a1(ptr addrspace(1) %in, ptr addrspace(1) %out ; byte loads coalesce into one dword load, so the loads and the conversions ; stay scalar. define void @store4_sitofp_lds_a4(ptr addrspace(3) %in, ptr addrspace(1) %out) { -; GFX9-LABEL: define void @store4_sitofp_lds_a4( -; GFX9-SAME: ptr addrspace(3) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) { -; GFX9-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr addrspace(3) [[IN]], align 4 -; GFX9-NEXT: [[TMP2:%.*]] = sitofp <4 x i8> [[TMP1]] to <4 x float> -; GFX9-NEXT: store <4 x float> [[TMP2]], ptr addrspace(1) [[OUT]], align 16 -; GFX9-NEXT: ret void +; GFX90A-LABEL: define void @store4_sitofp_lds_a4( +; GFX90A-SAME: ptr addrspace(3) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) { +; GFX90A-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[IN]], i32 1 +; GFX90A-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[IN]], i32 2 +; GFX90A-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[IN]], i32 3 +; GFX90A-NEXT: [[Q0:%.*]] = load i8, ptr addrspace(3) [[IN]], align 4 +; GFX90A-NEXT: [[Q1:%.*]] = load i8, ptr addrspace(3) [[P1]], align 1 +; GFX90A-NEXT: [[Q2:%.*]] = load i8, ptr addrspace(3) [[P2]], align 2 +; GFX90A-NEXT: [[Q3:%.*]] = load i8, ptr addrspace(3) [[P3]], align 1 +; GFX90A-NEXT: [[F0:%.*]] = sitofp i8 [[Q0]] to float +; GFX90A-NEXT: [[F1:%.*]] = sitofp i8 [[Q1]] to float +; GFX90A-NEXT: [[F2:%.*]] = sitofp i8 [[Q2]] to float +; GFX90A-NEXT: [[F3:%.*]] = sitofp i8 [[Q3]] to float +; GFX90A-NEXT: [[TMP1:%.*]] = insertelement <4 x float> poison, float [[F0]], i64 0 +; GFX90A-NEXT: [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[F1]], i64 1 +; GFX90A-NEXT: [[TMP3:%.*]] = insertelement <4 x float> [[TMP2]], float [[F2]], i64 2 +; GFX90A-NEXT: [[TMP4:%.*]] = insertelement <4 x float> [[TMP3]], float [[F3]], i64 3 +; GFX90A-NEXT: store <4 x float> [[TMP4]], ptr addrspace(1) [[OUT]], align 16 +; GFX90A-NEXT: ret void +; +; GFX942-950-LABEL: define void @store4_sitofp_lds_a4( +; GFX942-950-SAME: ptr addrspace(3) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) { +; GFX942-950-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[IN]], i32 1 +; GFX942-950-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[IN]], i32 2 +; GFX942-950-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[IN]], i32 3 +; GFX942-950-NEXT: [[O2:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 2 +; GFX942-950-NEXT: [[Q0:%.*]] = load i8, ptr addrspace(3) [[IN]], align 4 +; GFX942-950-NEXT: [[Q1:%.*]] = load i8, ptr addrspace(3) [[P1]], align 1 +; GFX942-950-NEXT: [[F0:%.*]] = sitofp i8 [[Q0]] to float +; GFX942-950-NEXT: [[F1:%.*]] = sitofp i8 [[Q1]] to float +; GFX942-950-NEXT: [[TMP1:%.*]] = insertelement <2 x float> poison, float [[F0]], i64 0 +; GFX942-950-NEXT: [[TMP2:%.*]] = insertelement <2 x float> [[TMP1]], float [[F1]], i64 1 +; GFX942-950-NEXT: store <2 x float> [[TMP2]], ptr addrspace(1) [[OUT]], align 16 +; GFX942-950-NEXT: [[Q2:%.*]] = load i8, ptr addrspace(3) [[P2]], align 2 +; GFX942-950-NEXT: [[Q3:%.*]] = load i8, ptr addrspace(3) [[P3]], align 1 +; GFX942-950-NEXT: [[F2:%.*]] = sitofp i8 [[Q2]] to float +; GFX942-950-NEXT: [[F3:%.*]] = sitofp i8 [[Q3]] to float +; GFX942-950-NEXT: [[TMP3:%.*]] = insertelement <2 x float> poison, float [[F2]], i64 0 +; GFX942-950-NEXT: [[TMP4:%.*]] = insertelement <2 x float> [[TMP3]], float [[F3]], i64 1 +; GFX942-950-NEXT: store <2 x float> [[TMP4]], ptr addrspace(1) [[O2]], align 8 +; GFX942-950-NEXT: ret void ; ; GFX1030-LABEL: define void @store4_sitofp_lds_a4( ; GFX1030-SAME: ptr addrspace(3) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) { @@ -468,8 +586,21 @@ define void @zext4_bv_loop(ptr addrspace(1) %a, ptr addrspace(1) %out) { ; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ] ; CHECK-NEXT: [[OFF:%.*]] = shl i64 [[I]], 2 ; CHECK-NEXT: [[P0:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i64 [[OFF]] -; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(1) [[P0]], align 4 -; CHECK-NEXT: [[V3:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i32> +; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[P0]], i64 1 +; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[P0]], i64 2 +; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[P0]], i64 3 +; CHECK-NEXT: [[L0:%.*]] = load i8, ptr addrspace(1) [[P0]], align 4 +; CHECK-NEXT: [[L1:%.*]] = load i8, ptr addrspace(1) [[P1]], align 1 +; CHECK-NEXT: [[L2:%.*]] = load i8, ptr addrspace(1) [[P2]], align 2 +; CHECK-NEXT: [[L3:%.*]] = load i8, ptr addrspace(1) [[P3]], align 1 +; CHECK-NEXT: [[X0:%.*]] = zext i8 [[L0]] to i32 +; CHECK-NEXT: [[X1:%.*]] = zext i8 [[L1]] to i32 +; CHECK-NEXT: [[X2:%.*]] = zext i8 [[L2]] to i32 +; CHECK-NEXT: [[X3:%.*]] = zext i8 [[L3]] to i32 +; CHECK-NEXT: [[V0:%.*]] = insertelement <4 x i32> poison, i32 [[X0]], i32 0 +; CHECK-NEXT: [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[X1]], i32 1 +; CHECK-NEXT: [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[X2]], i32 2 +; CHECK-NEXT: [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[X3]], i32 3 ; CHECK-NEXT: [[PO:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[OUT]], i64 [[I]] ; CHECK-NEXT: store <4 x i32> [[V3]], ptr addrspace(1) [[PO]], align 16 ; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 @@ -566,7 +697,5 @@ define void @store8_sitofp_half(ptr addrspace(1) %in, ptr addrspace(1) %out) { ret void } ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: -; GFX90A: {{.*}} ; GFX942: {{.*}} -; GFX942-950: {{.*}} ; GFX950: {{.*}} diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/int-reduction-coalesced-loads.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/int-reduction-coalesced-loads.ll index 6a178d20eb717..a043ea67f51ee 100644 --- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/int-reduction-coalesced-loads.ll +++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/int-reduction-coalesced-loads.ll @@ -21,21 +21,68 @@ define i32 @dot8_sext_i8(ptr addrspace(1) %a, ptr addrspace(1) %b) { ; CHECK-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]]) { ; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: [[PA0:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i32 0 +; CHECK-NEXT: [[LA0:%.*]] = load i8, ptr addrspace(1) [[PA0]], align 1 +; CHECK-NEXT: [[XA0:%.*]] = sext i8 [[LA0]] to i32 ; CHECK-NEXT: [[PB0:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[B]], i32 0 -; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(1) [[PA0]], align 1 -; CHECK-NEXT: [[TMP1:%.*]] = sext <4 x i8> [[TMP0]] to <4 x i32> -; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr addrspace(1) [[PB0]], align 1 -; CHECK-NEXT: [[TMP3:%.*]] = sext <4 x i8> [[TMP2]] to <4 x i32> -; CHECK-NEXT: [[TMP4:%.*]] = mul nsw <4 x i32> [[TMP1]], [[TMP3]] +; CHECK-NEXT: [[LB0:%.*]] = load i8, ptr addrspace(1) [[PB0]], align 1 +; CHECK-NEXT: [[XB0:%.*]] = sext i8 [[LB0]] to i32 +; CHECK-NEXT: [[M0:%.*]] = mul nsw i32 [[XA0]], [[XB0]] +; CHECK-NEXT: [[PA1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i32 1 +; CHECK-NEXT: [[LA1:%.*]] = load i8, ptr addrspace(1) [[PA1]], align 1 +; CHECK-NEXT: [[XA1:%.*]] = sext i8 [[LA1]] to i32 +; CHECK-NEXT: [[PB1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[B]], i32 1 +; CHECK-NEXT: [[LB1:%.*]] = load i8, ptr addrspace(1) [[PB1]], align 1 +; CHECK-NEXT: [[XB1:%.*]] = sext i8 [[LB1]] to i32 +; CHECK-NEXT: [[M1:%.*]] = mul nsw i32 [[XA1]], [[XB1]] +; CHECK-NEXT: [[R1:%.*]] = add nsw i32 [[M0]], [[M1]] +; CHECK-NEXT: [[PA2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i32 2 +; CHECK-NEXT: [[LA2:%.*]] = load i8, ptr addrspace(1) [[PA2]], align 1 +; CHECK-NEXT: [[XA2:%.*]] = sext i8 [[LA2]] to i32 +; CHECK-NEXT: [[PB2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[B]], i32 2 +; CHECK-NEXT: [[LB2:%.*]] = load i8, ptr addrspace(1) [[PB2]], align 1 +; CHECK-NEXT: [[XB2:%.*]] = sext i8 [[LB2]] to i32 +; CHECK-NEXT: [[M2:%.*]] = mul nsw i32 [[XA2]], [[XB2]] +; CHECK-NEXT: [[R2:%.*]] = add nsw i32 [[R1]], [[M2]] +; CHECK-NEXT: [[PA3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i32 3 +; CHECK-NEXT: [[LA3:%.*]] = load i8, ptr addrspace(1) [[PA3]], align 1 +; CHECK-NEXT: [[XA3:%.*]] = sext i8 [[LA3]] to i32 +; CHECK-NEXT: [[PB3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[B]], i32 3 +; CHECK-NEXT: [[LB3:%.*]] = load i8, ptr addrspace(1) [[PB3]], align 1 +; CHECK-NEXT: [[XB3:%.*]] = sext i8 [[LB3]] to i32 +; CHECK-NEXT: [[M3:%.*]] = mul nsw i32 [[XA3]], [[XB3]] +; CHECK-NEXT: [[R3:%.*]] = add nsw i32 [[R2]], [[M3]] ; CHECK-NEXT: [[PA4:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i32 4 +; CHECK-NEXT: [[LA4:%.*]] = load i8, ptr addrspace(1) [[PA4]], align 1 +; CHECK-NEXT: [[XA4:%.*]] = sext i8 [[LA4]] to i32 ; CHECK-NEXT: [[PB4:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[B]], i32 4 -; CHECK-NEXT: [[TMP5:%.*]] = load <4 x i8>, ptr addrspace(1) [[PA4]], align 1 -; CHECK-NEXT: [[TMP6:%.*]] = sext <4 x i8> [[TMP5]] to <4 x i32> -; CHECK-NEXT: [[TMP7:%.*]] = load <4 x i8>, ptr addrspace(1) [[PB4]], align 1 -; CHECK-NEXT: [[TMP8:%.*]] = sext <4 x i8> [[TMP7]] to <4 x i32> -; CHECK-NEXT: [[TMP9:%.*]] = mul nsw <4 x i32> [[TMP6]], [[TMP8]] -; CHECK-NEXT: [[RDX_OP:%.*]] = add <4 x i32> [[TMP4]], [[TMP9]] -; CHECK-NEXT: [[R7:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[RDX_OP]]) +; CHECK-NEXT: [[LB4:%.*]] = load i8, ptr addrspace(1) [[PB4]], align 1 +; CHECK-NEXT: [[XB4:%.*]] = sext i8 [[LB4]] to i32 +; CHECK-NEXT: [[M4:%.*]] = mul nsw i32 [[XA4]], [[XB4]] +; CHECK-NEXT: [[R4:%.*]] = add nsw i32 [[R3]], [[M4]] +; CHECK-NEXT: [[PA5:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i32 5 +; CHECK-NEXT: [[LA5:%.*]] = load i8, ptr addrspace(1) [[PA5]], align 1 +; CHECK-NEXT: [[XA5:%.*]] = sext i8 [[LA5]] to i32 +; CHECK-NEXT: [[PB5:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[B]], i32 5 +; CHECK-NEXT: [[LB5:%.*]] = load i8, ptr addrspace(1) [[PB5]], align 1 +; CHECK-NEXT: [[XB5:%.*]] = sext i8 [[LB5]] to i32 +; CHECK-NEXT: [[M5:%.*]] = mul nsw i32 [[XA5]], [[XB5]] +; CHECK-NEXT: [[R5:%.*]] = add nsw i32 [[R4]], [[M5]] +; CHECK-NEXT: [[PA6:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i32 6 +; CHECK-NEXT: [[LA6:%.*]] = load i8, ptr addrspace(1) [[PA6]], align 1 +; CHECK-NEXT: [[XA6:%.*]] = sext i8 [[LA6]] to i32 +; CHECK-NEXT: [[PB6:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[B]], i32 6 +; CHECK-NEXT: [[LB6:%.*]] = load i8, ptr addrspace(1) [[PB6]], align 1 +; CHECK-NEXT: [[XB6:%.*]] = sext i8 [[LB6]] to i32 +; CHECK-NEXT: [[M6:%.*]] = mul nsw i32 [[XA6]], [[XB6]] +; CHECK-NEXT: [[R6:%.*]] = add nsw i32 [[R5]], [[M6]] +; CHECK-NEXT: [[PA7:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i32 7 +; CHECK-NEXT: [[LA7:%.*]] = load i8, ptr addrspace(1) [[PA7]], align 1 +; CHECK-NEXT: [[XA7:%.*]] = sext i8 [[LA7]] to i32 +; CHECK-NEXT: [[PB7:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[B]], i32 7 +; CHECK-NEXT: [[LB7:%.*]] = load i8, ptr addrspace(1) [[PB7]], align 1 +; CHECK-NEXT: [[XB7:%.*]] = sext i8 [[LB7]] to i32 +; CHECK-NEXT: [[M7:%.*]] = mul nsw i32 [[XA7]], [[XB7]] +; CHECK-NEXT: [[R7:%.*]] = add nsw i32 [[R6]], [[M7]] ; CHECK-NEXT: ret i32 [[R7]] ; entry: @@ -110,10 +157,20 @@ define i32 @sum4_zext_i8(ptr addrspace(1) %a) { ; CHECK-SAME: ptr addrspace(1) [[A:%.*]]) { ; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: [[PA0:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i32 0 -; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(1) [[PA0]], align 1 -; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i16> -; CHECK-NEXT: [[TMP2:%.*]] = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> [[TMP1]]) -; CHECK-NEXT: [[R3:%.*]] = zext i16 [[TMP2]] to i32 +; CHECK-NEXT: [[LA0:%.*]] = load i8, ptr addrspace(1) [[PA0]], align 1 +; CHECK-NEXT: [[XA0:%.*]] = zext i8 [[LA0]] to i32 +; CHECK-NEXT: [[PA1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i32 1 +; CHECK-NEXT: [[LA1:%.*]] = load i8, ptr addrspace(1) [[PA1]], align 1 +; CHECK-NEXT: [[XA1:%.*]] = zext i8 [[LA1]] to i32 +; CHECK-NEXT: [[R1:%.*]] = add nsw i32 [[XA0]], [[XA1]] +; CHECK-NEXT: [[PA2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i32 2 +; CHECK-NEXT: [[LA2:%.*]] = load i8, ptr addrspace(1) [[PA2]], align 1 +; CHECK-NEXT: [[XA2:%.*]] = zext i8 [[LA2]] to i32 +; CHECK-NEXT: [[R2:%.*]] = add nsw i32 [[R1]], [[XA2]] +; CHECK-NEXT: [[PA3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i32 3 +; CHECK-NEXT: [[LA3:%.*]] = load i8, ptr addrspace(1) [[PA3]], align 1 +; CHECK-NEXT: [[XA3:%.*]] = zext i8 [[LA3]] to i32 +; CHECK-NEXT: [[R3:%.*]] = add nsw i32 [[R2]], [[XA3]] ; CHECK-NEXT: ret i32 [[R3]] ; entry: @@ -191,10 +248,20 @@ define void @sum4_zext_i8_loop(ptr addrspace(1) %a, ptr addrspace(1) %out) { ; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ] ; CHECK-NEXT: [[OFF:%.*]] = shl i32 [[I]], 2 ; CHECK-NEXT: [[PA0:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i32 [[OFF]] -; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(1) [[PA0]], align 1 -; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i16> -; CHECK-NEXT: [[TMP2:%.*]] = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> [[TMP1]]) -; CHECK-NEXT: [[R3:%.*]] = zext i16 [[TMP2]] to i32 +; CHECK-NEXT: [[LA0:%.*]] = load i8, ptr addrspace(1) [[PA0]], align 1 +; CHECK-NEXT: [[XA0:%.*]] = zext i8 [[LA0]] to i32 +; CHECK-NEXT: [[PA1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[PA0]], i32 1 +; CHECK-NEXT: [[LA1:%.*]] = load i8, ptr addrspace(1) [[PA1]], align 1 +; CHECK-NEXT: [[XA1:%.*]] = zext i8 [[LA1]] to i32 +; CHECK-NEXT: [[R1:%.*]] = add nsw i32 [[XA0]], [[XA1]] +; CHECK-NEXT: [[PA2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[PA0]], i32 2 +; CHECK-NEXT: [[LA2:%.*]] = load i8, ptr addrspace(1) [[PA2]], align 1 +; CHECK-NEXT: [[XA2:%.*]] = zext i8 [[LA2]] to i32 +; CHECK-NEXT: [[R2:%.*]] = add nsw i32 [[R1]], [[XA2]] +; CHECK-NEXT: [[PA3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[PA0]], i32 3 +; CHECK-NEXT: [[LA3:%.*]] = load i8, ptr addrspace(1) [[PA3]], align 1 +; CHECK-NEXT: [[XA3:%.*]] = zext i8 [[LA3]] to i32 +; CHECK-NEXT: [[R3:%.*]] = add nsw i32 [[R2]], [[XA3]] ; CHECK-NEXT: [[PO:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT]], i32 [[I]] ; CHECK-NEXT: store i32 [[R3]], ptr addrspace(1) [[PO]], align 4 ; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i32 [[I]], 1 @@ -239,10 +306,20 @@ define i32 @sum4_zext_i8_lds_a4(ptr addrspace(3) %a) { ; CHECK-SAME: ptr addrspace(3) [[A:%.*]]) { ; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: [[PA0:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[A]], i32 0 -; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(3) [[PA0]], align 4 -; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i16> -; CHECK-NEXT: [[TMP2:%.*]] = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> [[TMP1]]) -; CHECK-NEXT: [[R3:%.*]] = zext i16 [[TMP2]] to i32 +; CHECK-NEXT: [[LA0:%.*]] = load i8, ptr addrspace(3) [[PA0]], align 4 +; CHECK-NEXT: [[XA0:%.*]] = zext i8 [[LA0]] to i32 +; CHECK-NEXT: [[PA1:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[A]], i32 1 +; CHECK-NEXT: [[LA1:%.*]] = load i8, ptr addrspace(3) [[PA1]], align 1 +; CHECK-NEXT: [[XA1:%.*]] = zext i8 [[LA1]] to i32 +; CHECK-NEXT: [[R1:%.*]] = add nsw i32 [[XA0]], [[XA1]] +; CHECK-NEXT: [[PA2:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[A]], i32 2 +; CHECK-NEXT: [[LA2:%.*]] = load i8, ptr addrspace(3) [[PA2]], align 2 +; CHECK-NEXT: [[XA2:%.*]] = zext i8 [[LA2]] to i32 +; CHECK-NEXT: [[R2:%.*]] = add nsw i32 [[R1]], [[XA2]] +; CHECK-NEXT: [[PA3:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[A]], i32 3 +; CHECK-NEXT: [[LA3:%.*]] = load i8, ptr addrspace(3) [[PA3]], align 1 +; CHECK-NEXT: [[XA3:%.*]] = zext i8 [[LA3]] to i32 +; CHECK-NEXT: [[R3:%.*]] = add nsw i32 [[R2]], [[XA3]] ; CHECK-NEXT: ret i32 [[R3]] ; entry: @@ -309,21 +386,68 @@ define i32 @dot8_sext_i8_lds_a16(ptr addrspace(3) %a, ptr addrspace(3) %b) { ; CHECK-SAME: ptr addrspace(3) [[A:%.*]], ptr addrspace(3) [[B:%.*]]) { ; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: [[PA0:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[A]], i32 0 +; CHECK-NEXT: [[LA0:%.*]] = load i8, ptr addrspace(3) [[PA0]], align 16 +; CHECK-NEXT: [[XA0:%.*]] = sext i8 [[LA0]] to i32 ; CHECK-NEXT: [[PB0:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[B]], i32 0 -; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(3) [[PA0]], align 16 -; CHECK-NEXT: [[TMP1:%.*]] = sext <4 x i8> [[TMP0]] to <4 x i32> -; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr addrspace(3) [[PB0]], align 16 -; CHECK-NEXT: [[TMP3:%.*]] = sext <4 x i8> [[TMP2]] to <4 x i32> -; CHECK-NEXT: [[TMP4:%.*]] = mul nsw <4 x i32> [[TMP1]], [[TMP3]] +; CHECK-NEXT: [[LB0:%.*]] = load i8, ptr addrspace(3) [[PB0]], align 16 +; CHECK-NEXT: [[XB0:%.*]] = sext i8 [[LB0]] to i32 +; CHECK-NEXT: [[M0:%.*]] = mul nsw i32 [[XA0]], [[XB0]] +; CHECK-NEXT: [[PA1:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[A]], i32 1 +; CHECK-NEXT: [[LA1:%.*]] = load i8, ptr addrspace(3) [[PA1]], align 1 +; CHECK-NEXT: [[XA1:%.*]] = sext i8 [[LA1]] to i32 +; CHECK-NEXT: [[PB1:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[B]], i32 1 +; CHECK-NEXT: [[LB1:%.*]] = load i8, ptr addrspace(3) [[PB1]], align 1 +; CHECK-NEXT: [[XB1:%.*]] = sext i8 [[LB1]] to i32 +; CHECK-NEXT: [[M1:%.*]] = mul nsw i32 [[XA1]], [[XB1]] +; CHECK-NEXT: [[R1:%.*]] = add nsw i32 [[M0]], [[M1]] +; CHECK-NEXT: [[PA2:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[A]], i32 2 +; CHECK-NEXT: [[LA2:%.*]] = load i8, ptr addrspace(3) [[PA2]], align 2 +; CHECK-NEXT: [[XA2:%.*]] = sext i8 [[LA2]] to i32 +; CHECK-NEXT: [[PB2:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[B]], i32 2 +; CHECK-NEXT: [[LB2:%.*]] = load i8, ptr addrspace(3) [[PB2]], align 2 +; CHECK-NEXT: [[XB2:%.*]] = sext i8 [[LB2]] to i32 +; CHECK-NEXT: [[M2:%.*]] = mul nsw i32 [[XA2]], [[XB2]] +; CHECK-NEXT: [[R2:%.*]] = add nsw i32 [[R1]], [[M2]] +; CHECK-NEXT: [[PA3:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[A]], i32 3 +; CHECK-NEXT: [[LA3:%.*]] = load i8, ptr addrspace(3) [[PA3]], align 1 +; CHECK-NEXT: [[XA3:%.*]] = sext i8 [[LA3]] to i32 +; CHECK-NEXT: [[PB3:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[B]], i32 3 +; CHECK-NEXT: [[LB3:%.*]] = load i8, ptr addrspace(3) [[PB3]], align 1 +; CHECK-NEXT: [[XB3:%.*]] = sext i8 [[LB3]] to i32 +; CHECK-NEXT: [[M3:%.*]] = mul nsw i32 [[XA3]], [[XB3]] +; CHECK-NEXT: [[R3:%.*]] = add nsw i32 [[R2]], [[M3]] ; CHECK-NEXT: [[PA4:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[A]], i32 4 +; CHECK-NEXT: [[LA4:%.*]] = load i8, ptr addrspace(3) [[PA4]], align 4 +; CHECK-NEXT: [[XA4:%.*]] = sext i8 [[LA4]] to i32 ; CHECK-NEXT: [[PB4:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[B]], i32 4 -; CHECK-NEXT: [[TMP5:%.*]] = load <4 x i8>, ptr addrspace(3) [[PA4]], align 4 -; CHECK-NEXT: [[TMP6:%.*]] = sext <4 x i8> [[TMP5]] to <4 x i32> -; CHECK-NEXT: [[TMP7:%.*]] = load <4 x i8>, ptr addrspace(3) [[PB4]], align 4 -; CHECK-NEXT: [[TMP8:%.*]] = sext <4 x i8> [[TMP7]] to <4 x i32> -; CHECK-NEXT: [[TMP9:%.*]] = mul nsw <4 x i32> [[TMP6]], [[TMP8]] -; CHECK-NEXT: [[RDX_OP:%.*]] = add <4 x i32> [[TMP4]], [[TMP9]] -; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[RDX_OP]]) +; CHECK-NEXT: [[LB4:%.*]] = load i8, ptr addrspace(3) [[PB4]], align 4 +; CHECK-NEXT: [[XB4:%.*]] = sext i8 [[LB4]] to i32 +; CHECK-NEXT: [[M4:%.*]] = mul nsw i32 [[XA4]], [[XB4]] +; CHECK-NEXT: [[R4:%.*]] = add nsw i32 [[R3]], [[M4]] +; CHECK-NEXT: [[PA5:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[A]], i32 5 +; CHECK-NEXT: [[LA5:%.*]] = load i8, ptr addrspace(3) [[PA5]], align 1 +; CHECK-NEXT: [[XA5:%.*]] = sext i8 [[LA5]] to i32 +; CHECK-NEXT: [[PB5:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[B]], i32 5 +; CHECK-NEXT: [[LB5:%.*]] = load i8, ptr addrspace(3) [[PB5]], align 1 +; CHECK-NEXT: [[XB5:%.*]] = sext i8 [[LB5]] to i32 +; CHECK-NEXT: [[M5:%.*]] = mul nsw i32 [[XA5]], [[XB5]] +; CHECK-NEXT: [[R5:%.*]] = add nsw i32 [[R4]], [[M5]] +; CHECK-NEXT: [[PA6:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[A]], i32 6 +; CHECK-NEXT: [[LA6:%.*]] = load i8, ptr addrspace(3) [[PA6]], align 2 +; CHECK-NEXT: [[XA6:%.*]] = sext i8 [[LA6]] to i32 +; CHECK-NEXT: [[PB6:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[B]], i32 6 +; CHECK-NEXT: [[LB6:%.*]] = load i8, ptr addrspace(3) [[PB6]], align 2 +; CHECK-NEXT: [[XB6:%.*]] = sext i8 [[LB6]] to i32 +; CHECK-NEXT: [[M6:%.*]] = mul nsw i32 [[XA6]], [[XB6]] +; CHECK-NEXT: [[R6:%.*]] = add nsw i32 [[R5]], [[M6]] +; CHECK-NEXT: [[PA7:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[A]], i32 7 +; CHECK-NEXT: [[LA7:%.*]] = load i8, ptr addrspace(3) [[PA7]], align 1 +; CHECK-NEXT: [[XA7:%.*]] = sext i8 [[LA7]] to i32 +; CHECK-NEXT: [[PB7:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[B]], i32 7 +; CHECK-NEXT: [[LB7:%.*]] = load i8, ptr addrspace(3) [[PB7]], align 1 +; CHECK-NEXT: [[XB7:%.*]] = sext i8 [[LB7]] to i32 +; CHECK-NEXT: [[M7:%.*]] = mul nsw i32 [[XA7]], [[XB7]] +; CHECK-NEXT: [[TMP10:%.*]] = add nsw i32 [[R6]], [[M7]] ; CHECK-NEXT: ret i32 [[TMP10]] ; entry: _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
