https://github.com/MrSidims updated https://github.com/llvm/llvm-project/pull/225342
>From 605c0239e27d5654fe5fedff89695953308dde83 Mon Sep 17 00:00:00 2001 From: Dmitry Sidorov <[email protected]> Date: Thu, 17 Sep 2026 01:40:00 +0200 Subject: [PATCH] [AMDGPU] Price narrow integer to bfloat vector casts A vector lane of 9 to 15 or 17 to 31 bits converted to bfloat got the generic cost, which leaves out the rounding. Such a lane is converted to f32 first like any other narrow lane, so price it as the f32 conversion of the same source plus the rounding. Lanes of 8 and 16 bits keep their cost. --- .../AMDGPU/AMDGPUTargetTransformInfo.cpp | 18 ++-- .../CostModel/AMDGPU/narrow-int-to-bfloat.ll | 90 +++++++++---------- 2 files changed, 56 insertions(+), 52 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp index 089236d299487d..25550ffb5369db 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp @@ -1087,15 +1087,19 @@ InstructionCost GCNTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, if (IsIntToFP) { const unsigned ExtOps = UsesInt64 && SrcBits < 64 ? (IsSigned ? 2 : 1) : 0; if (FPTy->isBFloatTy()) { - if (SrcBits != 8 && SrcBits != 16 && SrcBits != 32 && !UsesInt64) + const bool NarrowLanes = + SrcBits >= 8 && SrcBits < 32 && isa<FixedVectorType>(Src); + if (!NarrowLanes && SrcBits != 8 && SrcBits != 16 && SrcBits != 32 && + !UsesInt64) return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I); // Each integer is converted to f32 first. - unsigned PerElt = UsesInt64 ? ExtOps + (IsSigned ? 12 : 8) : 1; - if (isa<FixedVectorType>(Src) && SrcBits < 32 && - (IsSigned || SrcBits == 16)) { - PerElt = - (ST->hasSDWA() ? 1 : 2) + (SrcBits == 8 && ST->has16BitInsts()); + InstructionCost FloatCost = + Scale(UsesInt64 ? ExtOps + (IsSigned ? 12 : 8) : 1); + if (NarrowLanes) { + auto *FloatTy = + FixedVectorType::get(Type::getFloatTy(Dst->getContext()), NElts); + FloatCost = getCastInstrCost(Opcode, FloatTy, Src, CCH, CostKind); } // Native rounding can convert a pair. With 16 bit instructions the @@ -1108,7 +1112,7 @@ InstructionCost GCNTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, : Scale(!ST->has16BitInsts() ? 1 : ST->hasGFX9Insts() ? 6 : 7); - return Scale(PerElt) + RoundCost; + return FloatCost + RoundCost; } // No instruction converts from a 64 bit integer. diff --git a/llvm/test/Analysis/CostModel/AMDGPU/narrow-int-to-bfloat.ll b/llvm/test/Analysis/CostModel/AMDGPU/narrow-int-to-bfloat.ll index 5a30bc6a82e6e3..0e15b10325bc3c 100644 --- a/llvm/test/Analysis/CostModel/AMDGPU/narrow-int-to-bfloat.ll +++ b/llvm/test/Analysis/CostModel/AMDGPU/narrow-int-to-bfloat.ll @@ -16,14 +16,14 @@ define void @narrow_to_bfloat(<4 x i8> %a, <4 x i12> %b, <4 x i16> %c, <4 x i17> ; GFX6-LABEL: 'narrow_to_bfloat' ; GFX6: Cost Model: Found an estimated cost of 12 for instruction: %as = sitofp <4 x i8> %a to <4 x bfloat> ; GFX6: Cost Model: Found an estimated cost of 8 for instruction: %au = uitofp <4 x i8> %a to <4 x bfloat> -; GFX6: Cost Model: Found an estimated cost of 4 for instruction: %bs = sitofp <4 x i12> %b to <4 x bfloat> -; GFX6: Cost Model: Found an estimated cost of 4 for instruction: %bu = uitofp <4 x i12> %b to <4 x bfloat> +; GFX6: Cost Model: Found an estimated cost of 12 for instruction: %bs = sitofp <4 x i12> %b to <4 x bfloat> +; GFX6: Cost Model: Found an estimated cost of 12 for instruction: %bu = uitofp <4 x i12> %b to <4 x bfloat> ; GFX6: Cost Model: Found an estimated cost of 12 for instruction: %cs = sitofp <4 x i16> %c to <4 x bfloat> ; GFX6: Cost Model: Found an estimated cost of 12 for instruction: %cu = uitofp <4 x i16> %c to <4 x bfloat> -; GFX6: Cost Model: Found an estimated cost of 4 for instruction: %ds = sitofp <4 x i17> %d to <4 x bfloat> -; GFX6: Cost Model: Found an estimated cost of 4 for instruction: %du = uitofp <4 x i17> %d to <4 x bfloat> -; GFX6: Cost Model: Found an estimated cost of 4 for instruction: %es = sitofp <4 x i31> %e to <4 x bfloat> -; GFX6: Cost Model: Found an estimated cost of 4 for instruction: %eu = uitofp <4 x i31> %e to <4 x bfloat> +; GFX6: Cost Model: Found an estimated cost of 12 for instruction: %ds = sitofp <4 x i17> %d to <4 x bfloat> +; GFX6: Cost Model: Found an estimated cost of 12 for instruction: %du = uitofp <4 x i17> %d to <4 x bfloat> +; GFX6: Cost Model: Found an estimated cost of 12 for instruction: %es = sitofp <4 x i31> %e to <4 x bfloat> +; GFX6: Cost Model: Found an estimated cost of 12 for instruction: %eu = uitofp <4 x i31> %e to <4 x bfloat> ; GFX6: Cost Model: Found an estimated cost of 9 for instruction: %fs = sitofp <3 x i12> %f to <3 x bfloat> ; GFX6: Cost Model: Found an estimated cost of 9 for instruction: %fu = uitofp <3 x i12> %f to <3 x bfloat> ; GFX6: Cost Model: Found an estimated cost of 9 for instruction: %gs = sitofp <3 x i17> %g to <3 x bfloat> @@ -36,18 +36,18 @@ define void @narrow_to_bfloat(<4 x i8> %a, <4 x i12> %b, <4 x i16> %c, <4 x i17> ; GFX8-LABEL: 'narrow_to_bfloat' ; GFX8: Cost Model: Found an estimated cost of 36 for instruction: %as = sitofp <4 x i8> %a to <4 x bfloat> ; GFX8: Cost Model: Found an estimated cost of 32 for instruction: %au = uitofp <4 x i8> %a to <4 x bfloat> -; GFX8: Cost Model: Found an estimated cost of 10 for instruction: %bs = sitofp <4 x i12> %b to <4 x bfloat> -; GFX8: Cost Model: Found an estimated cost of 10 for instruction: %bu = uitofp <4 x i12> %b to <4 x bfloat> +; GFX8: Cost Model: Found an estimated cost of 36 for instruction: %bs = sitofp <4 x i12> %b to <4 x bfloat> +; GFX8: Cost Model: Found an estimated cost of 36 for instruction: %bu = uitofp <4 x i12> %b to <4 x bfloat> ; GFX8: Cost Model: Found an estimated cost of 32 for instruction: %cs = sitofp <4 x i16> %c to <4 x bfloat> ; GFX8: Cost Model: Found an estimated cost of 32 for instruction: %cu = uitofp <4 x i16> %c to <4 x bfloat> -; GFX8: Cost Model: Found an estimated cost of 10 for instruction: %ds = sitofp <4 x i17> %d to <4 x bfloat> -; GFX8: Cost Model: Found an estimated cost of 10 for instruction: %du = uitofp <4 x i17> %d to <4 x bfloat> -; GFX8: Cost Model: Found an estimated cost of 10 for instruction: %es = sitofp <4 x i31> %e to <4 x bfloat> -; GFX8: Cost Model: Found an estimated cost of 10 for instruction: %eu = uitofp <4 x i31> %e to <4 x bfloat> -; GFX8: Cost Model: Found an estimated cost of 7 for instruction: %fs = sitofp <3 x i12> %f to <3 x bfloat> -; GFX8: Cost Model: Found an estimated cost of 7 for instruction: %fu = uitofp <3 x i12> %f to <3 x bfloat> -; GFX8: Cost Model: Found an estimated cost of 7 for instruction: %gs = sitofp <3 x i17> %g to <3 x bfloat> -; GFX8: Cost Model: Found an estimated cost of 7 for instruction: %gu = uitofp <3 x i17> %g to <3 x bfloat> +; GFX8: Cost Model: Found an estimated cost of 36 for instruction: %ds = sitofp <4 x i17> %d to <4 x bfloat> +; GFX8: Cost Model: Found an estimated cost of 36 for instruction: %du = uitofp <4 x i17> %d to <4 x bfloat> +; GFX8: Cost Model: Found an estimated cost of 36 for instruction: %es = sitofp <4 x i31> %e to <4 x bfloat> +; GFX8: Cost Model: Found an estimated cost of 36 for instruction: %eu = uitofp <4 x i31> %e to <4 x bfloat> +; GFX8: Cost Model: Found an estimated cost of 27 for instruction: %fs = sitofp <3 x i12> %f to <3 x bfloat> +; GFX8: Cost Model: Found an estimated cost of 27 for instruction: %fu = uitofp <3 x i12> %f to <3 x bfloat> +; GFX8: Cost Model: Found an estimated cost of 27 for instruction: %gs = sitofp <3 x i17> %g to <3 x bfloat> +; GFX8: Cost Model: Found an estimated cost of 27 for instruction: %gu = uitofp <3 x i17> %g to <3 x bfloat> ; GFX8: Cost Model: Found an estimated cost of 16 for instruction: %hs = sitofp <2 x i16> %h to <2 x bfloat> ; GFX8: Cost Model: Found an estimated cost of 16 for instruction: %hu = uitofp <2 x i16> %h to <2 x bfloat> ; GFX8: Cost Model: Found an estimated cost of 24 for instruction: %is = sitofp <3 x i32> %i to <3 x bfloat> @@ -56,18 +56,18 @@ define void @narrow_to_bfloat(<4 x i8> %a, <4 x i12> %b, <4 x i16> %c, <4 x i17> ; GFX9-LABEL: 'narrow_to_bfloat' ; GFX9: Cost Model: Found an estimated cost of 32 for instruction: %as = sitofp <4 x i8> %a to <4 x bfloat> ; GFX9: Cost Model: Found an estimated cost of 28 for instruction: %au = uitofp <4 x i8> %a to <4 x bfloat> -; GFX9: Cost Model: Found an estimated cost of 10 for instruction: %bs = sitofp <4 x i12> %b to <4 x bfloat> -; GFX9: Cost Model: Found an estimated cost of 10 for instruction: %bu = uitofp <4 x i12> %b to <4 x bfloat> +; GFX9: Cost Model: Found an estimated cost of 32 for instruction: %bs = sitofp <4 x i12> %b to <4 x bfloat> +; GFX9: Cost Model: Found an estimated cost of 32 for instruction: %bu = uitofp <4 x i12> %b to <4 x bfloat> ; GFX9: Cost Model: Found an estimated cost of 28 for instruction: %cs = sitofp <4 x i16> %c to <4 x bfloat> ; GFX9: Cost Model: Found an estimated cost of 28 for instruction: %cu = uitofp <4 x i16> %c to <4 x bfloat> -; GFX9: Cost Model: Found an estimated cost of 10 for instruction: %ds = sitofp <4 x i17> %d to <4 x bfloat> -; GFX9: Cost Model: Found an estimated cost of 10 for instruction: %du = uitofp <4 x i17> %d to <4 x bfloat> -; GFX9: Cost Model: Found an estimated cost of 10 for instruction: %es = sitofp <4 x i31> %e to <4 x bfloat> -; GFX9: Cost Model: Found an estimated cost of 10 for instruction: %eu = uitofp <4 x i31> %e to <4 x bfloat> -; GFX9: Cost Model: Found an estimated cost of 7 for instruction: %fs = sitofp <3 x i12> %f to <3 x bfloat> -; GFX9: Cost Model: Found an estimated cost of 7 for instruction: %fu = uitofp <3 x i12> %f to <3 x bfloat> -; GFX9: Cost Model: Found an estimated cost of 7 for instruction: %gs = sitofp <3 x i17> %g to <3 x bfloat> -; GFX9: Cost Model: Found an estimated cost of 7 for instruction: %gu = uitofp <3 x i17> %g to <3 x bfloat> +; GFX9: Cost Model: Found an estimated cost of 32 for instruction: %ds = sitofp <4 x i17> %d to <4 x bfloat> +; GFX9: Cost Model: Found an estimated cost of 32 for instruction: %du = uitofp <4 x i17> %d to <4 x bfloat> +; GFX9: Cost Model: Found an estimated cost of 32 for instruction: %es = sitofp <4 x i31> %e to <4 x bfloat> +; GFX9: Cost Model: Found an estimated cost of 32 for instruction: %eu = uitofp <4 x i31> %e to <4 x bfloat> +; GFX9: Cost Model: Found an estimated cost of 24 for instruction: %fs = sitofp <3 x i12> %f to <3 x bfloat> +; GFX9: Cost Model: Found an estimated cost of 24 for instruction: %fu = uitofp <3 x i12> %f to <3 x bfloat> +; GFX9: Cost Model: Found an estimated cost of 24 for instruction: %gs = sitofp <3 x i17> %g to <3 x bfloat> +; GFX9: Cost Model: Found an estimated cost of 24 for instruction: %gu = uitofp <3 x i17> %g to <3 x bfloat> ; GFX9: Cost Model: Found an estimated cost of 14 for instruction: %hs = sitofp <2 x i16> %h to <2 x bfloat> ; GFX9: Cost Model: Found an estimated cost of 14 for instruction: %hu = uitofp <2 x i16> %h to <2 x bfloat> ; GFX9: Cost Model: Found an estimated cost of 21 for instruction: %is = sitofp <3 x i32> %i to <3 x bfloat> @@ -76,18 +76,18 @@ define void @narrow_to_bfloat(<4 x i8> %a, <4 x i12> %b, <4 x i16> %c, <4 x i17> ; NOSDWA-LABEL: 'narrow_to_bfloat' ; NOSDWA: Cost Model: Found an estimated cost of 36 for instruction: %as = sitofp <4 x i8> %a to <4 x bfloat> ; NOSDWA: Cost Model: Found an estimated cost of 28 for instruction: %au = uitofp <4 x i8> %a to <4 x bfloat> -; NOSDWA: Cost Model: Found an estimated cost of 10 for instruction: %bs = sitofp <4 x i12> %b to <4 x bfloat> -; NOSDWA: Cost Model: Found an estimated cost of 10 for instruction: %bu = uitofp <4 x i12> %b to <4 x bfloat> +; NOSDWA: Cost Model: Found an estimated cost of 36 for instruction: %bs = sitofp <4 x i12> %b to <4 x bfloat> +; NOSDWA: Cost Model: Found an estimated cost of 32 for instruction: %bu = uitofp <4 x i12> %b to <4 x bfloat> ; NOSDWA: Cost Model: Found an estimated cost of 32 for instruction: %cs = sitofp <4 x i16> %c to <4 x bfloat> ; NOSDWA: Cost Model: Found an estimated cost of 32 for instruction: %cu = uitofp <4 x i16> %c to <4 x bfloat> -; NOSDWA: Cost Model: Found an estimated cost of 10 for instruction: %ds = sitofp <4 x i17> %d to <4 x bfloat> -; NOSDWA: Cost Model: Found an estimated cost of 10 for instruction: %du = uitofp <4 x i17> %d to <4 x bfloat> -; NOSDWA: Cost Model: Found an estimated cost of 10 for instruction: %es = sitofp <4 x i31> %e to <4 x bfloat> -; NOSDWA: Cost Model: Found an estimated cost of 10 for instruction: %eu = uitofp <4 x i31> %e to <4 x bfloat> -; NOSDWA: Cost Model: Found an estimated cost of 7 for instruction: %fs = sitofp <3 x i12> %f to <3 x bfloat> -; NOSDWA: Cost Model: Found an estimated cost of 7 for instruction: %fu = uitofp <3 x i12> %f to <3 x bfloat> -; NOSDWA: Cost Model: Found an estimated cost of 7 for instruction: %gs = sitofp <3 x i17> %g to <3 x bfloat> -; NOSDWA: Cost Model: Found an estimated cost of 7 for instruction: %gu = uitofp <3 x i17> %g to <3 x bfloat> +; NOSDWA: Cost Model: Found an estimated cost of 32 for instruction: %ds = sitofp <4 x i17> %d to <4 x bfloat> +; NOSDWA: Cost Model: Found an estimated cost of 32 for instruction: %du = uitofp <4 x i17> %d to <4 x bfloat> +; NOSDWA: Cost Model: Found an estimated cost of 32 for instruction: %es = sitofp <4 x i31> %e to <4 x bfloat> +; NOSDWA: Cost Model: Found an estimated cost of 32 for instruction: %eu = uitofp <4 x i31> %e to <4 x bfloat> +; NOSDWA: Cost Model: Found an estimated cost of 27 for instruction: %fs = sitofp <3 x i12> %f to <3 x bfloat> +; NOSDWA: Cost Model: Found an estimated cost of 24 for instruction: %fu = uitofp <3 x i12> %f to <3 x bfloat> +; NOSDWA: Cost Model: Found an estimated cost of 24 for instruction: %gs = sitofp <3 x i17> %g to <3 x bfloat> +; NOSDWA: Cost Model: Found an estimated cost of 24 for instruction: %gu = uitofp <3 x i17> %g to <3 x bfloat> ; NOSDWA: Cost Model: Found an estimated cost of 16 for instruction: %hs = sitofp <2 x i16> %h to <2 x bfloat> ; NOSDWA: Cost Model: Found an estimated cost of 16 for instruction: %hu = uitofp <2 x i16> %h to <2 x bfloat> ; NOSDWA: Cost Model: Found an estimated cost of 21 for instruction: %is = sitofp <3 x i32> %i to <3 x bfloat> @@ -104,10 +104,10 @@ define void @narrow_to_bfloat(<4 x i8> %a, <4 x i12> %b, <4 x i16> %c, <4 x i17> ; GFX950: Cost Model: Found an estimated cost of 10 for instruction: %du = uitofp <4 x i17> %d to <4 x bfloat> ; GFX950: Cost Model: Found an estimated cost of 10 for instruction: %es = sitofp <4 x i31> %e to <4 x bfloat> ; GFX950: Cost Model: Found an estimated cost of 10 for instruction: %eu = uitofp <4 x i31> %e to <4 x bfloat> -; GFX950: Cost Model: Found an estimated cost of 7 for instruction: %fs = sitofp <3 x i12> %f to <3 x bfloat> -; GFX950: Cost Model: Found an estimated cost of 7 for instruction: %fu = uitofp <3 x i12> %f to <3 x bfloat> -; GFX950: Cost Model: Found an estimated cost of 7 for instruction: %gs = sitofp <3 x i17> %g to <3 x bfloat> -; GFX950: Cost Model: Found an estimated cost of 7 for instruction: %gu = uitofp <3 x i17> %g to <3 x bfloat> +; GFX950: Cost Model: Found an estimated cost of 8 for instruction: %fs = sitofp <3 x i12> %f to <3 x bfloat> +; GFX950: Cost Model: Found an estimated cost of 8 for instruction: %fu = uitofp <3 x i12> %f to <3 x bfloat> +; GFX950: Cost Model: Found an estimated cost of 8 for instruction: %gs = sitofp <3 x i17> %g to <3 x bfloat> +; GFX950: Cost Model: Found an estimated cost of 8 for instruction: %gu = uitofp <3 x i17> %g to <3 x bfloat> ; GFX950: Cost Model: Found an estimated cost of 3 for instruction: %hs = sitofp <2 x i16> %h to <2 x bfloat> ; GFX950: Cost Model: Found an estimated cost of 3 for instruction: %hu = uitofp <2 x i16> %h to <2 x bfloat> ; GFX950: Cost Model: Found an estimated cost of 5 for instruction: %is = sitofp <3 x i32> %i to <3 x bfloat> @@ -116,7 +116,7 @@ define void @narrow_to_bfloat(<4 x i8> %a, <4 x i12> %b, <4 x i16> %c, <4 x i17> ; GFX1250-LABEL: 'narrow_to_bfloat' ; GFX1250: Cost Model: Found an estimated cost of 14 for instruction: %as = sitofp <4 x i8> %a to <4 x bfloat> ; GFX1250: Cost Model: Found an estimated cost of 6 for instruction: %au = uitofp <4 x i8> %a to <4 x bfloat> -; GFX1250: Cost Model: Found an estimated cost of 10 for instruction: %bs = sitofp <4 x i12> %b to <4 x bfloat> +; GFX1250: Cost Model: Found an estimated cost of 14 for instruction: %bs = sitofp <4 x i12> %b to <4 x bfloat> ; GFX1250: Cost Model: Found an estimated cost of 10 for instruction: %bu = uitofp <4 x i12> %b to <4 x bfloat> ; GFX1250: Cost Model: Found an estimated cost of 10 for instruction: %cs = sitofp <4 x i16> %c to <4 x bfloat> ; GFX1250: Cost Model: Found an estimated cost of 10 for instruction: %cu = uitofp <4 x i16> %c to <4 x bfloat> @@ -124,10 +124,10 @@ define void @narrow_to_bfloat(<4 x i8> %a, <4 x i12> %b, <4 x i16> %c, <4 x i17> ; GFX1250: Cost Model: Found an estimated cost of 10 for instruction: %du = uitofp <4 x i17> %d to <4 x bfloat> ; GFX1250: Cost Model: Found an estimated cost of 10 for instruction: %es = sitofp <4 x i31> %e to <4 x bfloat> ; GFX1250: Cost Model: Found an estimated cost of 10 for instruction: %eu = uitofp <4 x i31> %e to <4 x bfloat> -; GFX1250: Cost Model: Found an estimated cost of 7 for instruction: %fs = sitofp <3 x i12> %f to <3 x bfloat> -; GFX1250: Cost Model: Found an estimated cost of 7 for instruction: %fu = uitofp <3 x i12> %f to <3 x bfloat> -; GFX1250: Cost Model: Found an estimated cost of 7 for instruction: %gs = sitofp <3 x i17> %g to <3 x bfloat> -; GFX1250: Cost Model: Found an estimated cost of 7 for instruction: %gu = uitofp <3 x i17> %g to <3 x bfloat> +; GFX1250: Cost Model: Found an estimated cost of 11 for instruction: %fs = sitofp <3 x i12> %f to <3 x bfloat> +; GFX1250: Cost Model: Found an estimated cost of 8 for instruction: %fu = uitofp <3 x i12> %f to <3 x bfloat> +; GFX1250: Cost Model: Found an estimated cost of 8 for instruction: %gs = sitofp <3 x i17> %g to <3 x bfloat> +; GFX1250: Cost Model: Found an estimated cost of 8 for instruction: %gu = uitofp <3 x i17> %g to <3 x bfloat> ; GFX1250: Cost Model: Found an estimated cost of 5 for instruction: %hs = sitofp <2 x i16> %h to <2 x bfloat> ; GFX1250: Cost Model: Found an estimated cost of 5 for instruction: %hu = uitofp <2 x i16> %h to <2 x bfloat> ; GFX1250: Cost Model: Found an estimated cost of 5 for instruction: %is = sitofp <3 x i32> %i to <3 x bfloat> _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
