https://github.com/gbossu created https://github.com/llvm/llvm-project/pull/214471
This is allowing some operations on vscale x 1 types, namely: - load/store - masked load/store - arithmetic instructions like add/sub/mul For those, there is already codegen coverage. See e.g. - llvm/test/CodeGen/AArch64/sve-int-arith.ll - llvm/test/CodeGen/AArch64/sve-load-store-legalisation.ll - llvm/test/CodeGen/AArch64/sve-masked-gather.ll - llvm/test/CodeGen/AArch64/sve-masked-scatter.ll --- <sub>Stack created with <a href="https://github.com/github/gh-stack";>GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback";>Give Feedback 💬</a></sub> From 77e7110974b853e0096429ff718cdbc7577da592 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <[email protected]> Date: Tue, 7 Jul 2026 13:48:07 +0000 Subject: [PATCH] [AArch64][CostModel] Consider some nxv1 operations as legal This is allowing some operations on vscale x 1 types, namely: - load/store - masked load/store - arithmetic instructions like add/sub/mul For those, there is already codegen coverage. See e.g. - llvm/test/CodeGen/AArch64/sve-int-arith.ll - llvm/test/CodeGen/AArch64/sve-load-store-legalisation.ll - llvm/test/CodeGen/AArch64/sve-masked-gather.ll - llvm/test/CodeGen/AArch64/sve-masked-scatter.ll --- .../AArch64/AArch64TargetTransformInfo.cpp | 50 +++++++++------ .../Analysis/CostModel/AArch64/masked_ldst.ll | 63 +++++++------------ .../Analysis/CostModel/AArch64/sve-arith.ll | 6 +- .../Analysis/CostModel/AArch64/sve-ldst.ll | 8 +-- .../force-scalable-vectorization-always.ll | 2 +- .../LoopVectorize/AArch64/invalid-costs.ll | 3 +- .../LoopVectorize/AArch64/scalable-alloca.ll | 1 - .../LoopVectorize/AArch64/scalable-call.ll | 6 -- .../scalable-vectorization-cost-tuning.ll | 6 +- 9 files changed, 65 insertions(+), 80 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index a380cca93d84d..bc2b2a78cad48 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -613,9 +613,12 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting // it. This change will be removed when code-generation for these types is // sufficiently reliable. + // Only allow masked ld/st to pass through to getMemIntrinsicInstrCost(). auto *RetTy = ICA.getReturnType(); if (auto *VTy = dyn_cast<ScalableVectorType>(RetTy)) - if (VTy->getElementCount() == ElementCount::getScalable(1)) + if (VTy->getElementCount() == ElementCount::getScalable(1) && + !is_contained({Intrinsic::masked_load, Intrinsic::masked_store}, + ICA.getID())) return InstructionCost::getInvalid(); switch (ICA.getID()) { @@ -4774,15 +4777,15 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost( // The code-generator is currently not able to handle scalable vectors // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting - // it. This change will be removed when code-generation for these types is - // sufficiently reliable. + // it until all instructions are vetted. + int ISD = TLI->InstructionOpcodeToISD(Opcode); if (auto *VTy = dyn_cast<ScalableVectorType>(Ty)) if (VTy->getElementCount() == ElementCount::getScalable(1)) - return InstructionCost::getInvalid(); + if (!is_contained({ISD::ADD, ISD::SUB, ISD::MUL}, ISD)) + return InstructionCost::getInvalid(); // Legalize the type. std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty); - int ISD = TLI->InstructionOpcodeToISD(Opcode); // TODO: Handle more cost kinds for floating point operations. if (ISD == ISD::FADD || ISD == ISD::FSUB || ISD == ISD::FMUL || @@ -5380,12 +5383,13 @@ AArch64TTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, if (VT->getElementType()->isIntegerTy(1)) return InstructionCost::getInvalid(); - // The code-generator is currently not able to handle scalable vectors - // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting - // it. This change will be removed when code-generation for these types is - // sufficiently reliable. + // <vscale x 1 x eltty> operations require mask adaptation. + // Allow it for normal masked ld/st if (VT->getElementCount() == ElementCount::getScalable(1)) - return InstructionCost::getInvalid(); + return is_contained({Intrinsic::masked_load, Intrinsic::masked_store}, + MICA.getID()) + ? LT.first + 1 + : InstructionCost::getInvalid(); InstructionCost MemOpCost = LT.first; if (MICA.getID() == Intrinsic::masked_expandload) { @@ -5507,17 +5511,23 @@ InstructionCost AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Ty, if (!LT.first.isValid()) return InstructionCost::getInvalid(); - // The code-generator is currently not able to handle scalable vectors - // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting - // it. This change will be removed when code-generation for these types is - // sufficiently reliable. - // We also only support full register predicate loads and stores. - if (auto *VTy = dyn_cast<ScalableVectorType>(Ty)) - if (VTy->getElementCount() == ElementCount::getScalable(1) || - (VTy->getElementType()->isIntegerTy(1) && - !VTy->getElementCount().isKnownMultipleOf( - ElementCount::getScalable(16)))) + if (auto *VTy = dyn_cast<ScalableVectorType>(Ty)) { + // <vscale x 1 x eltty> operations require crafting a new mask. + if (VTy->getElementCount() == ElementCount::getScalable(1)) { + Intrinsic::ID IID = Opcode == Instruction::Load ? Intrinsic::masked_load + : Intrinsic::masked_store; + return getMaskedMemoryOpCost( + MemIntrinsicCostAttributes(IID, Ty, Alignment, AddressSpace), + CostKind) + + 1; + } + + // We only support full register predicate loads and stores. + if (VTy->getElementType()->isIntegerTy(1) && + !VTy->getElementCount().isKnownMultipleOf( + ElementCount::getScalable(16))) return InstructionCost::getInvalid(); + } // TODO: consider latency as well for TCK_SizeAndLatency. if (CostKind == TTI::TCK_CodeSize || CostKind == TTI::TCK_SizeAndLatency) diff --git a/llvm/test/Analysis/CostModel/AArch64/masked_ldst.ll b/llvm/test/Analysis/CostModel/AArch64/masked_ldst.ll index c1ee229ae8e2c..f9d4944d0384b 100644 --- a/llvm/test/Analysis/CostModel/AArch64/masked_ldst.ll +++ b/llvm/test/Analysis/CostModel/AArch64/masked_ldst.ll @@ -72,7 +72,7 @@ define void @scalable() { ; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv2f32 = call <vscale x 2 x float> @llvm.masked.load.nxv2f32.p0(ptr align 8 undef, <vscale x 2 x i1> undef, <vscale x 2 x float> undef) ; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 8 undef, <vscale x 4 x i1> undef, <vscale x 4 x float> undef) ; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 undef, <vscale x 2 x i1> undef, <vscale x 2 x double> undef) -; CHECK-NEXT: Cost Model: Found costs of Invalid for: %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.load.nxv1i64.p0(ptr align 8 undef, <vscale x 1 x i1> undef, <vscale x 1 x i64> undef) +; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.load.nxv1i64.p0(ptr align 8 undef, <vscale x 1 x i1> undef, <vscale x 1 x i64> undef) ; CHECK-NEXT: Cost Model: Found costs of 4 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.load.nxv4i64.p0(ptr align 8 undef, <vscale x 4 x i1> undef, <vscale x 4 x i64> undef) ; CHECK-NEXT: Cost Model: Found costs of 8 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.load.nxv32f16.p0(ptr align 8 undef, <vscale x 32 x i1> undef, <vscale x 32 x half> undef) ; CHECK-NEXT: Cost Model: Found costs of Invalid for: %nxv4i1 = call <vscale x 4 x i1> @llvm.masked.load.nxv4i1.p0(ptr align 16 undef, <vscale x 4 x i1> undef, <vscale x 4 x i1> undef) @@ -250,44 +250,25 @@ define void @scalable_ext_loads() { ret void } +define void @scalable_nxv1() { +; CHECK-LABEL: 'scalable_nxv1' +; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1i8 = call <vscale x 1 x i8> @llvm.masked.load.nxv1i8.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x i8> undef) +; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1i16 = call <vscale x 1 x i16> @llvm.masked.load.nxv1i16.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x i16> undef) +; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1i32 = call <vscale x 1 x i32> @llvm.masked.load.nxv1i32.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x i32> undef) +; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.load.nxv1i64.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x i64> undef) +; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1f16 = call <vscale x 1 x half> @llvm.masked.load.nxv1f16.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x half> undef) +; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1f32 = call <vscale x 1 x float> @llvm.masked.load.nxv1f32.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x float> undef) +; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1f64 = call <vscale x 1 x double> @llvm.masked.load.nxv1f64.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x double> undef) +; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void +; +entry: + %nxv1i8 = call <vscale x 1 x i8> @llvm.masked.load.nxv1i8.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x i8> undef) + %nxv1i16 = call <vscale x 1 x i16> @llvm.masked.load.nxv1i16.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x i16> undef) + %nxv1i32 = call <vscale x 1 x i32> @llvm.masked.load.nxv1i32.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x i32> undef) + %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.load.nxv1i64.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x i64> undef) + %nxv1f16 = call <vscale x 1 x half> @llvm.masked.load.nxv1f16.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x half> undef) + %nxv1f32 = call <vscale x 1 x float> @llvm.masked.load.nxv1f32.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x float> undef) + %nxv1f64 = call <vscale x 1 x double> @llvm.masked.load.nxv1f64.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x double> undef) -declare <2 x i8> @llvm.masked.load.v2i8.p0(ptr, i32, <2 x i1>, <2 x i8>) -declare <4 x i8> @llvm.masked.load.v4i8.p0(ptr, i32, <4 x i1>, <4 x i8>) -declare <8 x i8> @llvm.masked.load.v8i8.p0(ptr, i32, <8 x i1>, <8 x i8>) -declare <16 x i8> @llvm.masked.load.v16i8.p0(ptr, i32, <16 x i1>, <16 x i8>) -declare <2 x i16> @llvm.masked.load.v2i16.p0(ptr, i32, <2 x i1>, <2 x i16>) -declare <4 x i16> @llvm.masked.load.v4i16.p0(ptr, i32, <4 x i1>, <4 x i16>) -declare <8 x i16> @llvm.masked.load.v8i16.p0(ptr, i32, <8 x i1>, <8 x i16>) -declare <2 x i32> @llvm.masked.load.v2i32.p0(ptr, i32, <2 x i1>, <2 x i32>) -declare <4 x i32> @llvm.masked.load.v4i32.p0(ptr, i32, <4 x i1>, <4 x i32>) -declare <2 x i64> @llvm.masked.load.v2i64.p0(ptr, i32, <2 x i1>, <2 x i64>) -declare <4 x i64> @llvm.masked.load.v4i64.p0(ptr, i32, <4 x i1>, <4 x i64>) -declare <2 x half> @llvm.masked.load.v2f16.p0(ptr, i32, <2 x i1>, <2 x half>) -declare <4 x half> @llvm.masked.load.v4f16.p0(ptr, i32, <4 x i1>, <4 x half>) -declare <8 x half> @llvm.masked.load.v8f16.p0(ptr, i32, <8 x i1>, <8 x half>) -declare <32 x half> @llvm.masked.load.v32f16.p0(ptr, i32, <32 x i1>, <32 x half>) -declare <2 x float> @llvm.masked.load.v2f32.p0(ptr, i32, <2 x i1>, <2 x float>) -declare <4 x float> @llvm.masked.load.v4f32.p0(ptr, i32, <4 x i1>, <4 x float>) -declare <2 x double> @llvm.masked.load.v2f64.p0(ptr, i32, <2 x i1>, <2 x double>) -declare <vscale x 4 x i1> @llvm.masked.load.nxv4i1.p0(ptr, i32, <vscale x 4 x i1>, <vscale x 4 x i1>) - - -declare <vscale x 2 x i8> @llvm.masked.load.nxv2i8.p0(ptr, i32, <vscale x 2 x i1>, <vscale x 2 x i8>) -declare <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr, i32, <vscale x 4 x i1>, <vscale x 4 x i8>) -declare <vscale x 8 x i8> @llvm.masked.load.nxv8i8.p0(ptr, i32, <vscale x 8 x i1>, <vscale x 8 x i8>) -declare <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr, i32, <vscale x 16 x i1>, <vscale x 16 x i8>) -declare <vscale x 2 x i16> @llvm.masked.load.nxv2i16.p0(ptr, i32, <vscale x 2 x i1>, <vscale x 2 x i16>) -declare <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr, i32, <vscale x 4 x i1>, <vscale x 4 x i16>) -declare <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr, i32, <vscale x 8 x i1>, <vscale x 8 x i16>) -declare <vscale x 2 x i32> @llvm.masked.load.nxv2i32.p0(ptr, i32, <vscale x 2 x i1>, <vscale x 2 x i32>) -declare <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr, i32, <vscale x 4 x i1>, <vscale x 4 x i32>) -declare <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr, i32, <vscale x 2 x i1>, <vscale x 2 x i64>) -declare <vscale x 4 x i64> @llvm.masked.load.nxv4i64.p0(ptr, i32, <vscale x 4 x i1>, <vscale x 4 x i64>) -declare <vscale x 1 x i64> @llvm.masked.load.nxv1i64.p0(ptr, i32, <vscale x 1 x i1>, <vscale x 1 x i64>) -declare <vscale x 2 x half> @llvm.masked.load.nxv2f16.p0(ptr, i32, <vscale x 2 x i1>, <vscale x 2 x half>) -declare <vscale x 4 x half> @llvm.masked.load.nxv4f16.p0(ptr, i32, <vscale x 4 x i1>, <vscale x 4 x half>) -declare <vscale x 8 x half> @llvm.masked.load.nxv8f16.p0(ptr, i32, <vscale x 8 x i1>, <vscale x 8 x half>) -declare <vscale x 32 x half> @llvm.masked.load.nxv32f16.p0(ptr, i32, <vscale x 32 x i1>, <vscale x 32 x half>) -declare <vscale x 2 x float> @llvm.masked.load.nxv2f32.p0(ptr, i32, <vscale x 2 x i1>, <vscale x 2 x float>) -declare <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr, i32, <vscale x 4 x i1>, <vscale x 4 x float>) -declare <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr, i32, <vscale x 2 x i1>, <vscale x 2 x double>) + ret void +} diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-arith.ll b/llvm/test/Analysis/CostModel/AArch64/sve-arith.ll index 0a9823afdeff5..ecca4fea2e520 100644 --- a/llvm/test/Analysis/CostModel/AArch64/sve-arith.ll +++ b/llvm/test/Analysis/CostModel/AArch64/sve-arith.ll @@ -9,7 +9,7 @@ define void @scalable_add() #0 { ; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv8i16 = add <vscale x 8 x i16> undef, undef ; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv4i32 = add <vscale x 4 x i32> undef, undef ; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv2i64 = add <vscale x 2 x i64> undef, undef -; CHECK-NEXT: Cost Model: Found costs of Invalid for: %nxv1i64 = add <vscale x 1 x i64> undef, undef +; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv1i64 = add <vscale x 1 x i64> undef, undef ; CHECK-NEXT: Cost Model: Found costs of Invalid for: %nxv2i128 = add <vscale x 2 x i128> undef, undef ; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; @@ -30,7 +30,7 @@ define void @scalable_sub() #0 { ; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv8i16 = sub <vscale x 8 x i16> undef, undef ; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv4i32 = sub <vscale x 4 x i32> undef, undef ; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv2i64 = sub <vscale x 2 x i64> undef, undef -; CHECK-NEXT: Cost Model: Found costs of Invalid for: %nxv1i64 = sub <vscale x 1 x i64> undef, undef +; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv1i64 = sub <vscale x 1 x i64> undef, undef ; CHECK-NEXT: Cost Model: Found costs of Invalid for: %nxv2i128 = sub <vscale x 2 x i128> undef, undef ; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; @@ -51,7 +51,7 @@ define void @scalable_mul() #0 { ; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv8i16 = mul <vscale x 8 x i16> undef, undef ; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv4i32 = mul <vscale x 4 x i32> undef, undef ; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv2i64 = mul <vscale x 2 x i64> undef, undef -; CHECK-NEXT: Cost Model: Found costs of Invalid for: %nxv1i64 = mul <vscale x 1 x i64> undef, undef +; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv1i64 = mul <vscale x 1 x i64> undef, undef ; CHECK-NEXT: Cost Model: Found costs of Invalid for: %nxv2i128 = mul <vscale x 2 x i128> undef, undef ; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-ldst.ll b/llvm/test/Analysis/CostModel/AArch64/sve-ldst.ll index b52fddaf8e776..e6e029981156b 100644 --- a/llvm/test/Analysis/CostModel/AArch64/sve-ldst.ll +++ b/llvm/test/Analysis/CostModel/AArch64/sve-ldst.ll @@ -9,7 +9,7 @@ define void @scalable_loads() { ; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %res.nxv8i8 = load <vscale x 8 x i8>, ptr undef, align 8 ; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %res.nxv16i8 = load <vscale x 16 x i8>, ptr undef, align 16 ; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %res.nxv32i8 = load <vscale x 32 x i8>, ptr undef, align 32 -; CHECK-NEXT: Cost Model: Found costs of Invalid for: %res.nxv1i64 = load <vscale x 1 x i64>, ptr undef, align 8 +; CHECK-NEXT: Cost Model: Found costs of 3 for: %res.nxv1i64 = load <vscale x 1 x i64>, ptr undef, align 8 ; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %res.nxv32i1 = load <vscale x 32 x i1>, ptr undef, align 4 ; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %res.nxv16i1 = load <vscale x 16 x i1>, ptr undef, align 2 ; CHECK-NEXT: Cost Model: Found costs of Invalid for: %res.nxv4i1 = load <vscale x 4 x i1>, ptr undef, align 1 @@ -19,7 +19,7 @@ define void @scalable_loads() { ; A510-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %res.nxv8i8 = load <vscale x 8 x i8>, ptr undef, align 8 ; A510-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %res.nxv16i8 = load <vscale x 16 x i8>, ptr undef, align 16 ; A510-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:3 SizeLat:2 for: %res.nxv32i8 = load <vscale x 32 x i8>, ptr undef, align 32 -; A510-NEXT: Cost Model: Found costs of Invalid for: %res.nxv1i64 = load <vscale x 1 x i64>, ptr undef, align 8 +; A510-NEXT: Cost Model: Found costs of 3 for: %res.nxv1i64 = load <vscale x 1 x i64>, ptr undef, align 8 ; A510-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:3 SizeLat:2 for: %res.nxv32i1 = load <vscale x 32 x i1>, ptr undef, align 4 ; A510-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %res.nxv16i1 = load <vscale x 16 x i1>, ptr undef, align 2 ; A510-NEXT: Cost Model: Found costs of Invalid for: %res.nxv4i1 = load <vscale x 4 x i1>, ptr undef, align 1 @@ -40,7 +40,7 @@ define void @scalable_stores() { ; CHECK-NEXT: Cost Model: Found costs of 1 for: store <vscale x 8 x i8> undef, ptr undef, align 8 ; CHECK-NEXT: Cost Model: Found costs of 1 for: store <vscale x 16 x i8> undef, ptr undef, align 16 ; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <vscale x 32 x i8> undef, ptr undef, align 32 -; CHECK-NEXT: Cost Model: Found costs of Invalid for: store <vscale x 1 x i64> undef, ptr undef, align 8 +; CHECK-NEXT: Cost Model: Found costs of 3 for: store <vscale x 1 x i64> undef, ptr undef, align 8 ; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <vscale x 32 x i1> undef, ptr undef, align 4 ; CHECK-NEXT: Cost Model: Found costs of 1 for: store <vscale x 16 x i1> undef, ptr undef, align 2 ; CHECK-NEXT: Cost Model: Found costs of Invalid for: store <vscale x 4 x i1> undef, ptr undef, align 1 @@ -50,7 +50,7 @@ define void @scalable_stores() { ; A510-NEXT: Cost Model: Found costs of 1 for: store <vscale x 8 x i8> undef, ptr undef, align 8 ; A510-NEXT: Cost Model: Found costs of 1 for: store <vscale x 16 x i8> undef, ptr undef, align 16 ; A510-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <vscale x 32 x i8> undef, ptr undef, align 32 -; A510-NEXT: Cost Model: Found costs of Invalid for: store <vscale x 1 x i64> undef, ptr undef, align 8 +; A510-NEXT: Cost Model: Found costs of 3 for: store <vscale x 1 x i64> undef, ptr undef, align 8 ; A510-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <vscale x 32 x i1> undef, ptr undef, align 4 ; A510-NEXT: Cost Model: Found costs of 1 for: store <vscale x 16 x i1> undef, ptr undef, align 2 ; A510-NEXT: Cost Model: Found costs of Invalid for: store <vscale x 4 x i1> undef, ptr undef, align 1 diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/force-scalable-vectorization-always.ll b/llvm/test/Transforms/LoopVectorize/AArch64/force-scalable-vectorization-always.ll index e2ab17e6cfe8f..ad7bf169db96d 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/force-scalable-vectorization-always.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/force-scalable-vectorization-always.ll @@ -11,7 +11,7 @@ define i32 @cost_prefers_fixed_width_vf_but_force_scalable_vf(ptr noalias %dst, ; CHECK: Cost for VF 2: 12 (Estimated cost per lane: 6) ; CHECK: Cost for VF 4: 8 (Estimated cost per lane: 2) ; CHECK: Cost for VF 8: 10 (Estimated cost per lane: 1.25) -; CHECK: Cost for VF vscale x 1: Invalid (Estimated cost per lane: Invalid) +; CHECK: Cost for VF vscale x 1: 11 (Estimated cost per lane: 11) ; CHECK: Cost for VF vscale x 2: 9 (Estimated cost per lane: 4.5) ; CHECK: Cost for VF vscale x 4: 8 (Estimated cost per lane: 2) ; CHECK: LV: Selecting VF: vscale x 4. diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/invalid-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/invalid-costs.ll index 37b6362c12927..ac1f598e1a85c 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/invalid-costs.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/invalid-costs.ll @@ -4,7 +4,8 @@ target triple = "arm64-apple-macosx" -; REMARKS: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): load +; REMARKS: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): ashr +; REMARKS: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): call to llvm.masked.sdiv ; Test case for https://github.com/llvm/llvm-project/issues/160792. define void @replicate_sdiv_conditional(ptr noalias %a, ptr noalias %b, ptr noalias %c) #0 { ; CHECK-LABEL: define void @replicate_sdiv_conditional( diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-alloca.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-alloca.ll index 0ef03c58be97a..57f407cf99be5 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-alloca.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-alloca.ll @@ -3,7 +3,6 @@ ; CHECK-REMARKS: UserVF ignored because of invalid costs. ; CHECK-REMARKS: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): alloca -; CHECK-REMARKS: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): store define void @alloca(ptr %vla, i64 %N) { ; CHECK-LABEL: @alloca( ; CHECK-NOT: <vscale x diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll index 0459b98f2eb1d..f5e32e4646df3 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll @@ -101,9 +101,7 @@ for.end: } ; CHECK-REMARKS: UserVF ignored because of invalid costs. -; CHECK-REMARKS-NEXT: t.c:3:10: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): load ; CHECK-REMARKS-NEXT: t.c:3:20: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin -; CHECK-REMARKS-NEXT: t.c:3:30: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): store define void @vec_sin_no_mapping(ptr noalias nocapture %dst, ptr noalias nocapture readonly %src, i64 %n) { ; CHECK: @vec_sin_no_mapping ; CHECK: call fast <2 x float> @llvm.sin.v2f32 @@ -127,11 +125,9 @@ for.cond.cleanup: } ; CHECK-REMARKS: UserVF ignored because of invalid costs. -; CHECK-REMARKS-NEXT: t.c:3:10: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): load ; CHECK-REMARKS-NEXT: t.c:3:30: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): fadd ; CHECK-REMARKS-NEXT: t.c:3:30: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin ; CHECK-REMARKS-NEXT: t.c:3:20: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin -; CHECK-REMARKS-NEXT: t.c:3:40: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): store define void @vec_sin_no_mapping_ite(ptr noalias nocapture %dst, ptr noalias nocapture readonly %src, i64 %n) { ; CHECK: @vec_sin_no_mapping_ite ; CHECK-NOT: <vscale x @@ -165,9 +161,7 @@ for.cond.cleanup: } ; CHECK-REMARKS: UserVF ignored because of invalid costs. -; CHECK-REMARKS-NEXT: t.c:3:10: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): load ; CHECK-REMARKS-NEXT: t.c:3:20: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin -; CHECK-REMARKS-NEXT: t.c:3:30: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): store define void @vec_sin_fixed_mapping(ptr noalias nocapture %dst, ptr noalias nocapture readonly %src, i64 %n) { ; CHECK: @vec_sin_fixed_mapping ; CHECK: call fast <2 x float> @llvm.sin.v2f32 diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-vectorization-cost-tuning.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-vectorization-cost-tuning.ll index 33b8f22bd048e..8d496944b9d85 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-vectorization-cost-tuning.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-vectorization-cost-tuning.ll @@ -26,7 +26,7 @@ define void @test0(ptr %a, ptr %b, ptr %c) #0 { ; VSCALEFORTUNING1-LABEL: 'test0' -; VSCALEFORTUNING1: Cost for VF vscale x 1: Invalid (Estimated cost per lane: Invalid) +; VSCALEFORTUNING1: Cost for VF vscale x 1: 10 (Estimated cost per lane: 10) ; VSCALEFORTUNING1: Cost for VF vscale x 2: 10 (Estimated cost per lane: 5) ; VSCALEFORTUNING1: Cost for VF vscale x 4: 10 (Estimated cost per lane: 2.5) ; VSCALEFORTUNING1: Cost for VF vscale x 8: 10 (Estimated cost per lane: 1.25) @@ -34,7 +34,7 @@ define void @test0(ptr %a, ptr %b, ptr %c) #0 { ; VSCALEFORTUNING1: LV: Selecting VF: vscale x 16. ; ; VSCALEFORTUNING2-LABEL: 'test0' -; VSCALEFORTUNING2: Cost for VF vscale x 1: Invalid (Estimated cost per lane: Invalid) +; VSCALEFORTUNING2: Cost for VF vscale x 1: 10 (Estimated cost per lane: 5) ; VSCALEFORTUNING2: Cost for VF vscale x 2: 10 (Estimated cost per lane: 2.5) ; VSCALEFORTUNING2: Cost for VF vscale x 4: 10 (Estimated cost per lane: 1.25) ; VSCALEFORTUNING2: Cost for VF vscale x 8: 10 (Estimated cost per lane: 0.625) @@ -42,7 +42,7 @@ define void @test0(ptr %a, ptr %b, ptr %c) #0 { ; VSCALEFORTUNING2: LV: Selecting VF: vscale x 16. ; ; VSCALEFORTUNING1-PREFER-FIXED-LABEL: 'test0' -; VSCALEFORTUNING1-PREFER-FIXED: Cost for VF vscale x 1: Invalid (Estimated cost per lane: Invalid) +; VSCALEFORTUNING1-PREFER-FIXED: Cost for VF vscale x 1: 10 (Estimated cost per lane: 10) ; VSCALEFORTUNING1-PREFER-FIXED: Cost for VF vscale x 2: 10 (Estimated cost per lane: 5) ; VSCALEFORTUNING1-PREFER-FIXED: Cost for VF vscale x 4: 10 (Estimated cost per lane: 2.5) ; VSCALEFORTUNING1-PREFER-FIXED: Cost for VF vscale x 8: 10 (Estimated cost per lane: 1.25) _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
