https://github.com/sdesmalen-arm updated https://github.com/llvm/llvm-project/pull/223393
>From bc3a181f1dfbd3719a2e6864e3d7e7a815c102a9 Mon Sep 17 00:00:00 2001 From: Sander de Smalen <[email protected]> Date: Mon, 14 Sep 2026 08:05:19 +0000 Subject: [PATCH 1/4] [AArch64][SME] Allow more inlining when SME attributes are incompatible. At the moment, 'areInlineCompatible' is very strict as it conservatively disallows inlining any callee if they use intrinsics and have incompatible SME attributes. This PR relaxes those constraints by allowing more intrinsics. It also updates the Clang diagnostic to match the 'new' behaviour that a function is no longer inlined despite 'always_inline' when they are not inline compatible. --- .../clang/Basic/DiagnosticFrontendKinds.td | 2 +- .../AArch64/sme-inline-streaming-attrs.c | 6 +- .../AArch64/AArch64TargetTransformInfo.cpp | 65 ++++++-- .../Inline/AArch64/sme-always-inline.ll | 149 ++++++++++++++++++ 4 files changed, 202 insertions(+), 20 deletions(-) create mode 100644 llvm/test/Transforms/Inline/AArch64/sme-always-inline.ll diff --git a/clang/include/clang/Basic/DiagnosticFrontendKinds.td b/clang/include/clang/Basic/DiagnosticFrontendKinds.td index a10f10502a702..c43758abd2e13 100644 --- a/clang/include/clang/Basic/DiagnosticFrontendKinds.td +++ b/clang/include/clang/Basic/DiagnosticFrontendKinds.td @@ -317,7 +317,7 @@ def err_function_always_inline_attribute_mismatch : Error< "always_inline function %1 and its caller %0 have mismatching %2 attributes">; def warn_function_always_inline_attribute_mismatch : Warning< "always_inline function %1 and its caller %0 have mismatching %2 attributes, " - "inlining may change runtime behaviour">, InGroup<AArch64SMEAttributes>; + "%1 may not be inlined">, InGroup<AArch64SMEAttributes>; def err_function_always_inline_new_za : Error< "always_inline function %0 has new za state">; def err_function_always_inline_new_zt0 diff --git a/clang/test/CodeGen/AArch64/sme-inline-streaming-attrs.c b/clang/test/CodeGen/AArch64/sme-inline-streaming-attrs.c index 68102c9ded40c..ba9f24032d274 100644 --- a/clang/test/CodeGen/AArch64/sme-inline-streaming-attrs.c +++ b/clang/test/CodeGen/AArch64/sme-inline-streaming-attrs.c @@ -26,7 +26,7 @@ void caller(void) { #ifdef TEST_COMPATIBLE void caller_compatible(void) __arm_streaming_compatible { - inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_compatible' have mismatching streaming attributes, inlining may change runtime behaviour}} + inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_compatible' have mismatching streaming attributes, 'inlined_fn' may not be inlined}} inlined_fn_streaming_compatible(); inlined_fn_streaming(); // expected-error {{always_inline function 'inlined_fn_streaming' and its caller 'caller_compatible' have mismatching streaming attributes}} inlined_fn_local(); // expected-error {{always_inline function 'inlined_fn_local' and its caller 'caller_compatible' have mismatching streaming attributes}} @@ -35,7 +35,7 @@ void caller_compatible(void) __arm_streaming_compatible { #ifdef TEST_STREAMING void caller_streaming(void) __arm_streaming { - inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_streaming' have mismatching streaming attributes, inlining may change runtime behaviour}} + inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_streaming' have mismatching streaming attributes, 'inlined_fn' may not be inlined}} inlined_fn_streaming_compatible(); inlined_fn_streaming(); inlined_fn_local(); @@ -45,7 +45,7 @@ void caller_streaming(void) __arm_streaming { #ifdef TEST_LOCALLY __arm_locally_streaming void caller_local(void) { - inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_local' have mismatching streaming attributes, inlining may change runtime behaviour}} + inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_local' have mismatching streaming attributes, 'inlined_fn' may not be inlined}} inlined_fn_streaming_compatible(); inlined_fn_streaming(); inlined_fn_local(); diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index d8e8f58291a8d..b17ee3f4fb35a 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -233,23 +233,56 @@ static bool isSMEABIRoutineCall(const CallInst &CI, SMEAttrs(F->getName(), TLI.getRuntimeLibcallsInfo()).isSMEABIRoutine(); } +static bool isPossiblyIncompatibleIntrinsic(const Instruction *I, + bool ConsiderZA, bool ConsiderSM) { + auto *II = dyn_cast<IntrinsicInst>(I); + if (!II) + return false; + + StringRef Name = II->getCalledFunction()->getName(); + if (ConsiderZA && Name.starts_with("llvm.aarch64.sme.")) + return true; + + if (ConsiderSM) { + if (Name.starts_with("llvm.aarch64.neon") || + Name.starts_with("llvm.aarch64.sve") || + Name.starts_with("llvm.aarch64.sme") || + Name.starts_with("llvm.vscale") || + Name.starts_with("llvm.masked.gather") || + Name.starts_with("llvm.masked.scatter")) { + return true; + } + } + + return false; +} + /// Returns true if the function has explicit operations that can only be /// lowered using incompatible instructions for the selected mode. This also /// returns true if the function F may use or modify ZA state. static bool hasPossibleIncompatibleOps(const Function *F, - const AArch64TargetLowering &TLI) { + const AArch64TargetLowering &TLI, + bool ConsiderZA, bool ConsiderSM) { + assert((ConsiderZA || ConsiderSM) && "No SME state to consider"); for (const BasicBlock &BB : *F) { for (const Instruction &I : BB) { - // Be conservative for now and assume that any call to inline asm or to - // intrinsics could could result in non-streaming ops (e.g. calls to - // @llvm.aarch64.* or @llvm.gather/scatter intrinsics). We can assume that - // all native LLVM instructions can be lowered to compatible instructions. - if (isa<CallInst>(I) && !I.isDebugOrPseudoInst() && - (cast<CallInst>(I).isInlineAsm() || isa<IntrinsicInst>(I) || - isSMEABIRoutineCall(cast<CallInst>(I), TLI))) + // We can assume that all native LLVM instructions can be lowered to + // compatible instructions. + if (!isa<CallInst>(I) || I.isDebugOrPseudoInst()) + continue; + // Be conservative for now and assume that any call to inline asm could + // result in different behaviour. + if (cast<CallInst>(I).isInlineAsm()) + return true; + + if (isSMEABIRoutineCall(cast<CallInst>(I), TLI)) + return true; + + if (isPossiblyIncompatibleIntrinsic(&I, ConsiderZA, ConsiderSM)) return true; } } + return false; } @@ -288,6 +321,9 @@ bool AArch64TTIImpl::areInlineCompatible(const Function *Caller, CallAttrs.callee().hasStreamingInterfaceOrBody()) return false; + if (CallAttrs.callee().isNewZA() || CallAttrs.callee().isNewZT0()) + return false; + // When inlining, we should consider the body of the function, not the // interface. if (CallAttrs.callee().hasStreamingBody()) { @@ -295,16 +331,13 @@ bool AArch64TTIImpl::areInlineCompatible(const Function *Caller, CallAttrs.callee().set(SMEAttrs::SM_Enabled, true); } - if (CallAttrs.callee().isNewZA() || CallAttrs.callee().isNewZT0()) + bool ConsiderZA = + CallAttrs.requiresLazySave() || CallAttrs.requiresPreservingZT0(); + bool ConsiderSM = CallAttrs.requiresSMChange(); + if ((ConsiderZA || ConsiderSM) && + hasPossibleIncompatibleOps(Callee, *getTLI(), ConsiderZA, ConsiderSM)) return false; - if (CallAttrs.requiresLazySave() || CallAttrs.requiresSMChange() || - CallAttrs.requiresPreservingZT0() || - CallAttrs.requiresPreservingAllZAState()) { - if (hasPossibleIncompatibleOps(Callee, *getTLI())) - return false; - } - return BaseT::areInlineCompatible(Caller, Callee); } diff --git a/llvm/test/Transforms/Inline/AArch64/sme-always-inline.ll b/llvm/test/Transforms/Inline/AArch64/sme-always-inline.ll new file mode 100644 index 0000000000000..0cfe565cb1dc3 --- /dev/null +++ b/llvm/test/Transforms/Inline/AArch64/sme-always-inline.ll @@ -0,0 +1,149 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --prefix-filecheck-ir-name bb1 --version 6 +; RUN: opt -passes=always-inline -S < %s | FileCheck %s + +target triple = "aarch64" + +; +; Be cautious about inlining intrinsic calls when their streaming/ZA properties are incompatible. +; + +; llvm.vscale returns a different value in/out of streaming mode, and is not safe to inline. +define i64 @llvm_vscale() alwaysinline { +; CHECK-LABEL: define i64 @llvm_vscale( +; CHECK-SAME: ) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: [[VSCALE:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: ret i64 [[VSCALE]] +; + %vscale = call i64 @llvm.vscale() + ret i64 %vscale +} + +define i64 @compatible_llvm_vscale_caller() { +; CHECK-LABEL: define i64 @compatible_llvm_vscale_caller() { +; CHECK-NEXT: [[VSCALE_I:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: ret i64 [[VSCALE_I]] +; + %vscale = call i64 @llvm_vscale() + ret i64 %vscale +} + +define i64 @incompatible_llvm_vscale_caller() "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define i64 @incompatible_llvm_vscale_caller( +; CHECK-SAME: ) #[[ATTR1:[0-9]+]] { +; CHECK-NEXT: [[VSCALE:%.*]] = call i64 @llvm_vscale() +; CHECK-NEXT: ret i64 [[VSCALE]] +; + %vscale = call i64 @llvm_vscale() + ret i64 %vscale +} + +; NEON intrinsics are not safe in streaming mode +define i32 @neon_intrinsic(<4 x i32> %in) alwaysinline { +; CHECK-LABEL: define i32 @neon_intrinsic( +; CHECK-SAME: <4 x i32> [[IN:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[RES:%.*]] = call i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> [[IN]]) +; CHECK-NEXT: ret i32 [[RES]] +; + %res = call i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> %in) + ret i32 %res +} + +define i32 @compatible_neon_intrinsic_caller(<4 x i32> %in) { +; CHECK-LABEL: define i32 @compatible_neon_intrinsic_caller( +; CHECK-SAME: <4 x i32> [[IN:%.*]]) { +; CHECK-NEXT: [[RES_I:%.*]] = call i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> [[IN]]) +; CHECK-NEXT: ret i32 [[RES_I]] +; + %vscale = call i32 @neon_intrinsic(<4 x i32> %in) + ret i32 %vscale +} + +; a bit of a niche case, but if the caller uses ZA but is not in streaming-mode, a NEON intrinsic is safe. +define i32 @compatible_neon_intrinsic_caller_za(<4 x i32> %in) "aarch64_pstate_za_enabled" { +; CHECK-LABEL: define i32 @compatible_neon_intrinsic_caller_za( +; CHECK-SAME: <4 x i32> [[IN:%.*]]) #[[ATTR2:[0-9]+]] { +; CHECK-NEXT: [[RES_I:%.*]] = call i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> [[IN]]) +; CHECK-NEXT: ret i32 [[RES_I]] +; + %vscale = call i32 @neon_intrinsic(<4 x i32> %in) + ret i32 %vscale +} + +define i32 @incompatible_neon_intrinsic_caller(<4 x i32> %in) "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define i32 @incompatible_neon_intrinsic_caller( +; CHECK-SAME: <4 x i32> [[IN:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: [[VSCALE:%.*]] = call i32 @neon_intrinsic(<4 x i32> [[IN]]) +; CHECK-NEXT: ret i32 [[VSCALE]] +; + %vscale = call i32 @neon_intrinsic(<4 x i32> %in) + ret i32 %vscale +} + +; +; Be cautious about inlining functions that contain calls to SME ABI routines (like __arm_get_current_vg()) +; + +declare i64 @__arm_get_current_vg() + +define i64 @current_vg() alwaysinline { +; CHECK-LABEL: define i64 @current_vg( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: [[VSCALE:%.*]] = call i64 @__arm_get_current_vg() +; CHECK-NEXT: ret i64 [[VSCALE]] +; + %vscale = call i64 @__arm_get_current_vg() + ret i64 %vscale +} + +define i64 @compatible_current_vg_caller() { +; CHECK-LABEL: define i64 @compatible_current_vg_caller() { +; CHECK-NEXT: [[VSCALE_I:%.*]] = call i64 @__arm_get_current_vg() +; CHECK-NEXT: ret i64 [[VSCALE_I]] +; + %vscale = call i64 @current_vg() + ret i64 %vscale +} + +define i64 @incompatible_current_vg_caller() "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define i64 @incompatible_current_vg_caller( +; CHECK-SAME: ) #[[ATTR1]] { +; CHECK-NEXT: [[VSCALE:%.*]] = call i64 @current_vg() +; CHECK-NEXT: ret i64 [[VSCALE]] +; + %vscale = call i64 @current_vg() + ret i64 %vscale +} + +; +; Be cautious about inlining anything with inline asm when streaming/ZA properties are incompatible, +; + +define void @inline_asm() alwaysinline { +; CHECK-LABEL: define void @inline_asm( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: call void asm sideeffect "smstart za +; CHECK-NEXT: ret void +; + call void asm sideeffect "smstart za; svzero za; smstop za", ""() + ret void +} + +define void @compatible_inline_asm() { +; CHECK-LABEL: define void @compatible_inline_asm() { +; CHECK-NEXT: call void asm sideeffect "smstart za +; CHECK-NEXT: ret void +; + call void @inline_asm() + ret void +} + +define void @incompatible_inline_asm() "aarch64_inout_za" { +; CHECK-LABEL: define void @incompatible_inline_asm( +; CHECK-SAME: ) #[[ATTR3:[0-9]+]] { +; CHECK-NEXT: call void @inline_asm() +; CHECK-NEXT: ret void +; + call void @inline_asm() + ret void +} + >From c0c7558cb00531d272582eae9fbdf1fa04535129 Mon Sep 17 00:00:00 2001 From: Sander de Smalen <[email protected]> Date: Mon, 14 Sep 2026 14:27:39 +0000 Subject: [PATCH 2/4] Be more conservative, by disallowing operations on scalable types. Also restore behaviour around inlining calls into agnostic-za functions if they contain e.g. inline asm, which previously caused one of the tests to fail. --- .../AArch64/AArch64TargetTransformInfo.cpp | 13 ++++++-- .../Inline/AArch64/sme-always-inline.ll | 33 +++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index b17ee3f4fb35a..390c2496768d4 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -275,6 +275,14 @@ static bool hasPossibleIncompatibleOps(const Function *F, if (cast<CallInst>(I).isInlineAsm()) return true; + // Be conservative around operations on scalable types, as those may have + // different behaviour in/out of streaming mode. + if (ConsiderSM && (I.getType()->isScalableTy() || + any_of(I.operand_values(), [](const Value *V) { + return V->getType()->isScalableTy(); + }))) + return true; + if (isSMEABIRoutineCall(cast<CallInst>(I), TLI)) return true; @@ -331,8 +339,9 @@ bool AArch64TTIImpl::areInlineCompatible(const Function *Caller, CallAttrs.callee().set(SMEAttrs::SM_Enabled, true); } - bool ConsiderZA = - CallAttrs.requiresLazySave() || CallAttrs.requiresPreservingZT0(); + bool ConsiderZA = CallAttrs.requiresLazySave() || + CallAttrs.requiresPreservingZT0() || + CallAttrs.requiresPreservingAllZAState(); bool ConsiderSM = CallAttrs.requiresSMChange(); if ((ConsiderZA || ConsiderSM) && hasPossibleIncompatibleOps(Callee, *getTLI(), ConsiderZA, ConsiderSM)) diff --git a/llvm/test/Transforms/Inline/AArch64/sme-always-inline.ll b/llvm/test/Transforms/Inline/AArch64/sme-always-inline.ll index 0cfe565cb1dc3..777d6064928d9 100644 --- a/llvm/test/Transforms/Inline/AArch64/sme-always-inline.ll +++ b/llvm/test/Transforms/Inline/AArch64/sme-always-inline.ll @@ -79,6 +79,39 @@ define i32 @incompatible_neon_intrinsic_caller(<4 x i32> %in) "aarch64_pstate_sm ret i32 %vscale } +define i64 @intrinsic_with_scalable_type(ptr %p) alwaysinline { +; CHECK-LABEL: define i64 @intrinsic_with_scalable_type( +; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[LD:%.*]] = load <vscale x 2 x i64>, ptr [[P]], align 16 +; CHECK-NEXT: [[RES:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[LD]]) +; CHECK-NEXT: ret i64 [[RES]] +; + %ld = load <vscale x 2 x i64>, ptr %p + %res = call i64 @llvm.vector.reduce.add(<vscale x 2 x i64> %ld) + ret i64 %res +} + +define i64 @compatible_sve_intrinsic_caller(ptr %p) { +; CHECK-LABEL: define i64 @compatible_sve_intrinsic_caller( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[LD_I:%.*]] = load <vscale x 2 x i64>, ptr [[P]], align 16 +; CHECK-NEXT: [[RES_I:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[LD_I]]) +; CHECK-NEXT: ret i64 [[RES_I]] +; + %res = call i64 @intrinsic_with_scalable_type(ptr %p) + ret i64 %res +} + +define i64 @incompatible_sve_intrinsic_caller(ptr %p) "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define i64 @incompatible_sve_intrinsic_caller( +; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: [[RES:%.*]] = call i64 @intrinsic_with_scalable_type(ptr [[P]]) +; CHECK-NEXT: ret i64 [[RES]] +; + %res = call i64 @intrinsic_with_scalable_type(ptr %p) + ret i64 %res +} + ; ; Be cautious about inlining functions that contain calls to SME ABI routines (like __arm_get_current_vg()) ; >From e51abbbc5eb1553f5625629963f4145950b76f3f Mon Sep 17 00:00:00 2001 From: Sander de Smalen <[email protected]> Date: Thu, 17 Sep 2026 14:51:37 +0000 Subject: [PATCH 3/4] Address comments --- .../AArch64/AArch64TargetTransformInfo.cpp | 143 ++++++++++--- .../Inline/AArch64/sme-always-inline.ll | 200 ++++++++++++++---- .../Inline/AArch64/sme-pstateza-attrs.ll | 10 +- 3 files changed, 275 insertions(+), 78 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index 390c2496768d4..ab89c4e44abef 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -233,30 +233,43 @@ static bool isSMEABIRoutineCall(const CallInst &CI, SMEAttrs(F->getName(), TLI.getRuntimeLibcallsInfo()).isSMEABIRoutine(); } -static bool isPossiblyIncompatibleIntrinsic(const Instruction *I, - bool ConsiderZA, bool ConsiderSM) { - auto *II = dyn_cast<IntrinsicInst>(I); - if (!II) +/// Returns true if \p I is an intrinsic that may not be compatible with a +/// different streaming mode (because it depends on vscale). +static bool isPossiblyIncompatibleIntrinsic(const Instruction *I) { + if (I->isDebugOrPseudoInst()) return false; - StringRef Name = II->getCalledFunction()->getName(); - if (ConsiderZA && Name.starts_with("llvm.aarch64.sme.")) - return true; + if (auto *II = dyn_cast<IntrinsicInst>(I)) { + switch (II->getIntrinsicID()) { + default: + break; + case Intrinsic::vscale: + case Intrinsic::masked_gather: + case Intrinsic::masked_scatter: + return true; + } - if (ConsiderSM) { + StringRef Name = II->getCalledFunction()->getName(); if (Name.starts_with("llvm.aarch64.neon") || Name.starts_with("llvm.aarch64.sve") || - Name.starts_with("llvm.aarch64.sme") || - Name.starts_with("llvm.vscale") || - Name.starts_with("llvm.masked.gather") || - Name.starts_with("llvm.masked.scatter")) { + Name.starts_with("llvm.aarch64.sme")) return true; - } } return false; } +/// Returns true if \p IA has "za" in its clobber list. +static bool hasZAClobber(const InlineAsm *IA) { + for (const InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) { + if (CI.Type != llvm::InlineAsm::ConstraintPrefix::isClobber) + continue; + if (any_of(CI.Codes, [](StringRef S) { return S == "{za}"; })) + return true; + } + return false; +} + /// Returns true if the function has explicit operations that can only be /// lowered using incompatible instructions for the selected mode. This also /// returns true if the function F may use or modify ZA state. @@ -264,30 +277,52 @@ static bool hasPossibleIncompatibleOps(const Function *F, const AArch64TargetLowering &TLI, bool ConsiderZA, bool ConsiderSM) { assert((ConsiderZA || ConsiderSM) && "No SME state to consider"); + + bool HasVLDependentArgsOrRet = + F->getReturnType()->isScalableTy() || + any_of(F->getFunctionType()->params(), + [](const Type *T) { return T->isScalableTy(); }); + for (const BasicBlock &BB : *F) { for (const Instruction &I : BB) { - // We can assume that all native LLVM instructions can be lowered to - // compatible instructions. - if (!isa<CallInst>(I) || I.isDebugOrPseudoInst()) - continue; - // Be conservative for now and assume that any call to inline asm could - // result in different behaviour. - if (cast<CallInst>(I).isInlineAsm()) - return true; - - // Be conservative around operations on scalable types, as those may have - // different behaviour in/out of streaming mode. - if (ConsiderSM && (I.getType()->isScalableTy() || + // Inlining operations on fixed-length vectors when the streaming + // mode does not match, is rejected because performance may be impacted. + // This decision should eventually be moved the cost-model. + if (ConsiderSM && (isa<FixedVectorType>(I.getType()) || any_of(I.operand_values(), [](const Value *V) { - return V->getType()->isScalableTy(); + return isa<FixedVectorType>(V->getType()); }))) return true; - if (isSMEABIRoutineCall(cast<CallInst>(I), TLI)) - return true; + // Inlining operations on scalable vectors is rejected because it is + // a vscale-dependent operation. The only exception is when the interface + // already has vscale-dependent arguments/return value, as the ACLE + // describes that in order for the program to have defined behaviour is + // for vscale to match in both modes. + if (ConsiderSM && !HasVLDependentArgsOrRet) { + if (I.getType()->isScalableTy() || + any_of( + I.operand_values(), + [](const Value *V) { return V->getType()->isScalableTy(); }) || + (isa<GetElementPtrInst>(I) && + cast<GetElementPtrInst>(I).getSourceElementType()->isScalableTy())) + return true; + } - if (isPossiblyIncompatibleIntrinsic(&I, ConsiderZA, ConsiderSM)) - return true; + if (auto *CI = dyn_cast<CallInst>(&I)) { + // Inline asm must be rejected, unless we know that it contains no + // vscale dependent operations and does not use ZA. + if (CI->isInlineAsm() && + (ConsiderSM || (ConsiderZA && hasZAClobber(cast<InlineAsm>( + CI->getCalledOperand()))))) + return true; + + if (isSMEABIRoutineCall(*CI, TLI)) + return true; + + if (ConsiderSM && isPossiblyIncompatibleIntrinsic(&I)) + return true; + } } } @@ -318,6 +353,47 @@ bool AArch64TTIImpl::isMultiversionedFunction(const Function &F) const { return F.hasFnAttribute("fmv-features"); } +/// The compiler must not inline when that may alter the behavior of the +/// program. This is especially relevant around SME which implements different +/// runtime modes and maintains external state through attributes. +/// +/// The compiler must not inline when: +// +/// * The module is compiled with 'strict-fp' and the called function +/// implements a different FP environment than the caller. +/// +/// * The called function has operations that are incompatible in the mode +/// of the caller, e.g.: +/// * inlining non-streaming-only instructions into a streaming function. +/// * inlining streaming-only instructions into a non-streaming function. +/// +/// Inline asm blocks must be entered and exited in the [streaming] mode of +/// the parent function. There is no language-level mechanism to inform the +/// compiler that a particular inline asm block is streaming compatible, so +/// the compiler must reject this as a candidate for inlining. +/// +/// * The called function contains vscale-dependent operations but otherwise +/// does not take/return VL-dependent arguments (see definition in the +/// ACLE (Arm C/C++ Language Extensions)). +/// +/// * The called function sets up new ZA/ZT state into a function that already +/// has ZA or ZT state, as that is not valid as per the ACLE. +/// +/// If the called function has an `alwaysinline` attribute and any of the above +/// conditions is true, then an error should be reported. +/// +/// The compiler should not inline when: +// +/// * The called function has fixed-length vectors and the caller is in +/// streaming mode, as this may cause performance regressions. This should +/// never result in an error to be reported. +/// +/// * The called function sets up new ZA/ZT state into a function that has no +/// ZA and no ZT state, as the compiler currently cannot transfer the +/// attribute to the caller. It may also impact performance, but that should +/// be covered by AArch64TTIImpl::getInlineCallPenalty(). +/// +/// FIXME: Add diagnostics for always_inline. bool AArch64TTIImpl::areInlineCompatible(const Function *Caller, const Function *Callee) const { SMECallAttrs CallAttrs(*Caller, *Callee); @@ -343,6 +419,13 @@ bool AArch64TTIImpl::areInlineCompatible(const Function *Caller, CallAttrs.requiresPreservingZT0() || CallAttrs.requiresPreservingAllZAState(); bool ConsiderSM = CallAttrs.requiresSMChange(); + + // FP environment is interpreted differently between streaming mode and + // non-streaming mode, so are not inline-compatible. + // FIXME: Analyze whether the callee actually has any FP operations. + if (ConsiderSM && (Caller->isStrictFP() || Callee->isStrictFP())) + return false; + if ((ConsiderZA || ConsiderSM) && hasPossibleIncompatibleOps(Callee, *getTLI(), ConsiderZA, ConsiderSM)) return false; diff --git a/llvm/test/Transforms/Inline/AArch64/sme-always-inline.ll b/llvm/test/Transforms/Inline/AArch64/sme-always-inline.ll index 777d6064928d9..b8058f2d60a3c 100644 --- a/llvm/test/Transforms/Inline/AArch64/sme-always-inline.ll +++ b/llvm/test/Transforms/Inline/AArch64/sme-always-inline.ll @@ -8,7 +8,7 @@ target triple = "aarch64" ; ; llvm.vscale returns a different value in/out of streaming mode, and is not safe to inline. -define i64 @llvm_vscale() alwaysinline { +define i64 @llvm_vscale() #0 alwaysinline { ; CHECK-LABEL: define i64 @llvm_vscale( ; CHECK-SAME: ) #[[ATTR0:[0-9]+]] { ; CHECK-NEXT: [[VSCALE:%.*]] = call i64 @llvm.vscale.i64() @@ -18,8 +18,9 @@ define i64 @llvm_vscale() alwaysinline { ret i64 %vscale } -define i64 @compatible_llvm_vscale_caller() { -; CHECK-LABEL: define i64 @compatible_llvm_vscale_caller() { +define i64 @compatible_llvm_vscale_caller() #0 { +; CHECK-LABEL: define i64 @compatible_llvm_vscale_caller( +; CHECK-SAME: ) #[[ATTR1:[0-9]+]] { ; CHECK-NEXT: [[VSCALE_I:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: ret i64 [[VSCALE_I]] ; @@ -27,9 +28,9 @@ define i64 @compatible_llvm_vscale_caller() { ret i64 %vscale } -define i64 @incompatible_llvm_vscale_caller() "aarch64_pstate_sm_enabled" { +define i64 @incompatible_llvm_vscale_caller() #0 "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define i64 @incompatible_llvm_vscale_caller( -; CHECK-SAME: ) #[[ATTR1:[0-9]+]] { +; CHECK-SAME: ) #[[ATTR2:[0-9]+]] { ; CHECK-NEXT: [[VSCALE:%.*]] = call i64 @llvm_vscale() ; CHECK-NEXT: ret i64 [[VSCALE]] ; @@ -37,8 +38,107 @@ define i64 @incompatible_llvm_vscale_caller() "aarch64_pstate_sm_enabled" { ret i64 %vscale } +; It is not safe to inline functions that have vscale-dependent operations in their body +; when the streaming modes don't match up, unless the interface of the callee takes a +; vl-dependent argument. +define ptr @vscale_dependent_op(ptr %p, i64 %k) #0 alwaysinline { +; CHECK-LABEL: define ptr @vscale_dependent_op( +; CHECK-SAME: ptr [[P:%.*]], i64 [[K:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[RES:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P]], i64 [[K]] +; CHECK-NEXT: ret ptr [[RES]] +; + %res = getelementptr <vscale x 4 x i32>, ptr %p, i64 %k + ret ptr %res +} + +define ptr @incompatible_vscale_dependent_operation_sm(ptr %p) #0 "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define ptr @incompatible_vscale_dependent_operation_sm( +; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR2]] { +; CHECK-NEXT: [[RES_I:%.*]] = call ptr @vscale_dependent_op(ptr [[P]], i64 4) +; CHECK-NEXT: ret ptr [[RES_I]] +; + %res = call ptr @vscale_dependent_op(ptr %p, i64 4) + ret ptr %res +} + +define ptr @vscale_dependent_op_vl_dependent_args(ptr %p, i64 %k, <vscale x 4 x i32> %other) #0 alwaysinline { +; CHECK-LABEL: define ptr @vscale_dependent_op_vl_dependent_args( +; CHECK-SAME: ptr [[P:%.*]], i64 [[K:%.*]], <vscale x 4 x i32> [[OTHER:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[RES:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P]], i64 [[K]] +; CHECK-NEXT: store <vscale x 4 x i32> [[OTHER]], ptr [[RES]], align 16 +; CHECK-NEXT: ret ptr [[RES]] +; + %res = getelementptr <vscale x 4 x i32>, ptr %p, i64 %k + store <vscale x 4 x i32> %other, ptr %res + ret ptr %res +} + +define ptr @compatible_vscale_dependent_operation_sm(ptr %p) #0 "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define ptr @compatible_vscale_dependent_operation_sm( +; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR2]] { +; CHECK-NEXT: [[RES:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P]], i64 4 +; CHECK-NEXT: store <vscale x 4 x i32> zeroinitializer, ptr [[RES]], align 16 +; CHECK-NEXT: ret ptr [[RES]] +; + %res = call ptr @vscale_dependent_op_vl_dependent_args(ptr %p, i64 4, <vscale x 4 x i32> zeroinitializer) + ret ptr %res +} + +; functions with fixed-length vectors shouldn't be inlined if the streaming properties don't match +; as performance may be affected. +define void @fixed_length_vector_operation(ptr %p) #0 { +; CHECK-LABEL: define void @fixed_length_vector_operation( +; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[P]], align 16 +; CHECK-NEXT: ret void +; + store <4 x i32> zeroinitializer, ptr %p + ret void +} + +define void @fixed_length_vector_operation_caller_dont_inline(ptr %p) #0 "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define void @fixed_length_vector_operation_caller_dont_inline( +; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR2]] { +; CHECK-NEXT: call void @fixed_length_vector_operation(ptr [[P]]) +; CHECK-NEXT: ret void +; + call void @fixed_length_vector_operation(ptr %p) + ret void +} + +define i32 @strict_fp(i32 %in) #0 strictfp alwaysinline { +; CHECK-LABEL: define i32 @strict_fp( +; CHECK-SAME: i32 [[IN:%.*]]) #[[ATTR3:[0-9]+]] { +; CHECK-NEXT: [[RES:%.*]] = add i32 [[IN]], 42 +; CHECK-NEXT: ret i32 [[RES]] +; + %res = add i32 %in, 42; + ret i32 %res +} + +define i32 @compatible_fp_environment_sm(i32 %in) #0 strictfp { +; CHECK-LABEL: define i32 @compatible_fp_environment_sm( +; CHECK-SAME: i32 [[IN:%.*]]) #[[ATTR4:[0-9]+]] { +; CHECK-NEXT: [[RES_I:%.*]] = add i32 [[IN]], 42 +; CHECK-NEXT: ret i32 [[RES_I]] +; + %res = call i32 @strict_fp(i32 %in) + ret i32 %res +} + +; floating point environment is different in streaming mode, so don't inline. +define i32 @incompatible_fp_environment_sm(i32 %in) #0 "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define i32 @incompatible_fp_environment_sm( +; CHECK-SAME: i32 [[IN:%.*]]) #[[ATTR2]] { +; CHECK-NEXT: [[RES:%.*]] = call i32 @strict_fp(i32 [[IN]]) +; CHECK-NEXT: ret i32 [[RES]] +; + %res = call i32 @strict_fp(i32 %in) + ret i32 %res +} + ; NEON intrinsics are not safe in streaming mode -define i32 @neon_intrinsic(<4 x i32> %in) alwaysinline { +define i32 @neon_intrinsic(<4 x i32> %in) #0 alwaysinline { ; CHECK-LABEL: define i32 @neon_intrinsic( ; CHECK-SAME: <4 x i32> [[IN:%.*]]) #[[ATTR0]] { ; CHECK-NEXT: [[RES:%.*]] = call i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> [[IN]]) @@ -48,20 +148,20 @@ define i32 @neon_intrinsic(<4 x i32> %in) alwaysinline { ret i32 %res } -define i32 @compatible_neon_intrinsic_caller(<4 x i32> %in) { -; CHECK-LABEL: define i32 @compatible_neon_intrinsic_caller( -; CHECK-SAME: <4 x i32> [[IN:%.*]]) { -; CHECK-NEXT: [[RES_I:%.*]] = call i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> [[IN]]) -; CHECK-NEXT: ret i32 [[RES_I]] +define i32 @incompatible_neon_intrinsic_caller(<4 x i32> %in) #0 "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define i32 @incompatible_neon_intrinsic_caller( +; CHECK-SAME: <4 x i32> [[IN:%.*]]) #[[ATTR2]] { +; CHECK-NEXT: [[VSCALE:%.*]] = call i32 @neon_intrinsic(<4 x i32> [[IN]]) +; CHECK-NEXT: ret i32 [[VSCALE]] ; %vscale = call i32 @neon_intrinsic(<4 x i32> %in) ret i32 %vscale } ; a bit of a niche case, but if the caller uses ZA but is not in streaming-mode, a NEON intrinsic is safe. -define i32 @compatible_neon_intrinsic_caller_za(<4 x i32> %in) "aarch64_pstate_za_enabled" { +define i32 @compatible_neon_intrinsic_caller_za(<4 x i32> %in) #0 "aarch64_pstate_za_enabled" { ; CHECK-LABEL: define i32 @compatible_neon_intrinsic_caller_za( -; CHECK-SAME: <4 x i32> [[IN:%.*]]) #[[ATTR2:[0-9]+]] { +; CHECK-SAME: <4 x i32> [[IN:%.*]]) #[[ATTR5:[0-9]+]] { ; CHECK-NEXT: [[RES_I:%.*]] = call i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> [[IN]]) ; CHECK-NEXT: ret i32 [[RES_I]] ; @@ -69,17 +169,7 @@ define i32 @compatible_neon_intrinsic_caller_za(<4 x i32> %in) "aarch64_pstate_z ret i32 %vscale } -define i32 @incompatible_neon_intrinsic_caller(<4 x i32> %in) "aarch64_pstate_sm_enabled" { -; CHECK-LABEL: define i32 @incompatible_neon_intrinsic_caller( -; CHECK-SAME: <4 x i32> [[IN:%.*]]) #[[ATTR1]] { -; CHECK-NEXT: [[VSCALE:%.*]] = call i32 @neon_intrinsic(<4 x i32> [[IN]]) -; CHECK-NEXT: ret i32 [[VSCALE]] -; - %vscale = call i32 @neon_intrinsic(<4 x i32> %in) - ret i32 %vscale -} - -define i64 @intrinsic_with_scalable_type(ptr %p) alwaysinline { +define i64 @intrinsic_with_scalable_type(ptr %p) #0 alwaysinline { ; CHECK-LABEL: define i64 @intrinsic_with_scalable_type( ; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] { ; CHECK-NEXT: [[LD:%.*]] = load <vscale x 2 x i64>, ptr [[P]], align 16 @@ -91,9 +181,9 @@ define i64 @intrinsic_with_scalable_type(ptr %p) alwaysinline { ret i64 %res } -define i64 @compatible_sve_intrinsic_caller(ptr %p) { +define i64 @compatible_sve_intrinsic_caller(ptr %p) #0 { ; CHECK-LABEL: define i64 @compatible_sve_intrinsic_caller( -; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR1]] { ; CHECK-NEXT: [[LD_I:%.*]] = load <vscale x 2 x i64>, ptr [[P]], align 16 ; CHECK-NEXT: [[RES_I:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[LD_I]]) ; CHECK-NEXT: ret i64 [[RES_I]] @@ -102,9 +192,9 @@ define i64 @compatible_sve_intrinsic_caller(ptr %p) { ret i64 %res } -define i64 @incompatible_sve_intrinsic_caller(ptr %p) "aarch64_pstate_sm_enabled" { +define i64 @incompatible_sve_intrinsic_caller(ptr %p) #0 "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define i64 @incompatible_sve_intrinsic_caller( -; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR1]] { +; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR2]] { ; CHECK-NEXT: [[RES:%.*]] = call i64 @intrinsic_with_scalable_type(ptr [[P]]) ; CHECK-NEXT: ret i64 [[RES]] ; @@ -118,7 +208,7 @@ define i64 @incompatible_sve_intrinsic_caller(ptr %p) "aarch64_pstate_sm_enabled declare i64 @__arm_get_current_vg() -define i64 @current_vg() alwaysinline { +define i64 @current_vg() #0 alwaysinline { ; CHECK-LABEL: define i64 @current_vg( ; CHECK-SAME: ) #[[ATTR0]] { ; CHECK-NEXT: [[VSCALE:%.*]] = call i64 @__arm_get_current_vg() @@ -128,8 +218,9 @@ define i64 @current_vg() alwaysinline { ret i64 %vscale } -define i64 @compatible_current_vg_caller() { -; CHECK-LABEL: define i64 @compatible_current_vg_caller() { +define i64 @compatible_current_vg_caller() #0 { +; CHECK-LABEL: define i64 @compatible_current_vg_caller( +; CHECK-SAME: ) #[[ATTR1]] { ; CHECK-NEXT: [[VSCALE_I:%.*]] = call i64 @__arm_get_current_vg() ; CHECK-NEXT: ret i64 [[VSCALE_I]] ; @@ -137,9 +228,9 @@ define i64 @compatible_current_vg_caller() { ret i64 %vscale } -define i64 @incompatible_current_vg_caller() "aarch64_pstate_sm_enabled" { +define i64 @incompatible_current_vg_caller() #0 "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define i64 @incompatible_current_vg_caller( -; CHECK-SAME: ) #[[ATTR1]] { +; CHECK-SAME: ) #[[ATTR2]] { ; CHECK-NEXT: [[VSCALE:%.*]] = call i64 @current_vg() ; CHECK-NEXT: ret i64 [[VSCALE]] ; @@ -151,32 +242,55 @@ define i64 @incompatible_current_vg_caller() "aarch64_pstate_sm_enabled" { ; Be cautious about inlining anything with inline asm when streaming/ZA properties are incompatible, ; -define void @inline_asm() alwaysinline { +define void @inline_asm() #0 alwaysinline { ; CHECK-LABEL: define void @inline_asm( ; CHECK-SAME: ) #[[ATTR0]] { -; CHECK-NEXT: call void asm sideeffect "smstart za +; CHECK-NEXT: call void asm sideeffect "", ""() ; CHECK-NEXT: ret void ; - call void asm sideeffect "smstart za; svzero za; smstop za", ""() + call void asm sideeffect "", ""() ret void } -define void @compatible_inline_asm() { -; CHECK-LABEL: define void @compatible_inline_asm() { -; CHECK-NEXT: call void asm sideeffect "smstart za +define void @incompatible_inline_asm_sm() #0 "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define void @incompatible_inline_asm_sm( +; CHECK-SAME: ) #[[ATTR2]] { +; CHECK-NEXT: call void @inline_asm() ; CHECK-NEXT: ret void ; call void @inline_asm() ret void } -define void @incompatible_inline_asm() "aarch64_inout_za" { -; CHECK-LABEL: define void @incompatible_inline_asm( -; CHECK-SAME: ) #[[ATTR3:[0-9]+]] { -; CHECK-NEXT: call void @inline_asm() +define void @incompatible_inline_asm_za() #0 "aarch64_inout_za" { +; CHECK-LABEL: define void @incompatible_inline_asm_za( +; CHECK-SAME: ) #[[ATTR6:[0-9]+]] { +; CHECK-NEXT: call void @inline_asm_clobbers_za() +; CHECK-NEXT: ret void +; + call void @inline_asm_clobbers_za() + ret void +} + +define void @inline_asm_clobbers_za() #0 alwaysinline { +; CHECK-LABEL: define void @inline_asm_clobbers_za( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: call void asm sideeffect "", "~{za}"() +; CHECK-NEXT: ret void +; + call void asm sideeffect "", "~{za}"() + ret void +} + +; If the inline asm doesn't clobber 'za', we know it doesn't use/touch it. +define void @compatible_inline_asm_za() #0 "aarch64_inout_za" { +; CHECK-LABEL: define void @compatible_inline_asm_za( +; CHECK-SAME: ) #[[ATTR6]] { +; CHECK-NEXT: call void asm sideeffect "", ""() ; CHECK-NEXT: ret void ; call void @inline_asm() ret void } +attributes #0 = { "target-features"="+sve,+sme" } diff --git a/llvm/test/Transforms/Inline/AArch64/sme-pstateza-attrs.ll b/llvm/test/Transforms/Inline/AArch64/sme-pstateza-attrs.ll index 4cd1491611be0..bf0602de53f25 100644 --- a/llvm/test/Transforms/Inline/AArch64/sme-pstateza-attrs.ll +++ b/llvm/test/Transforms/Inline/AArch64/sme-pstateza-attrs.ll @@ -19,7 +19,7 @@ define void @nonza_callee() { ; CHECK-NEXT: ret void ; entry: - call void asm sideeffect "; inlineasm", ""() + call void asm sideeffect "; inlineasm", "~{za}"() call void @inlined_body() ret void } @@ -33,7 +33,7 @@ define void @shared_za_callee() "aarch64_inout_za" { ; CHECK-NEXT: ret void ; entry: - call void asm sideeffect "; inlineasm", ""() + call void asm sideeffect "; inlineasm", "~{za}"() call void @inlined_body() ret void } @@ -45,7 +45,7 @@ define void @new_za_callee() "aarch64_new_za" { ; CHECK-NEXT: call void @inlined_body() ; CHECK-NEXT: ret void ; - call void asm sideeffect "; inlineasm", ""() + call void asm sideeffect "; inlineasm", "~{za}"() call void @inlined_body() ret void } @@ -57,7 +57,7 @@ define void @agnostic_za_callee() "aarch64_za_state_agnostic" { ; CHECK-NEXT: call void @inlined_body() ; CHECK-NEXT: ret void ; - call void asm sideeffect "; inlineasm", ""() + call void asm sideeffect "; inlineasm", "~{za}"() call void @inlined_body() ret void } @@ -386,7 +386,7 @@ define void @nonzt0_callee() { ; CHECK-NEXT: call void @inlined_body() ; CHECK-NEXT: ret void ; - call void asm sideeffect "; inlineasm", ""() + call void asm sideeffect "; inlineasm", "~{za}"() call void @inlined_body() ret void } >From 5038eafe653e897d042be4ae75cb04929eb8a5f9 Mon Sep 17 00:00:00 2001 From: Sander de Smalen <[email protected]> Date: Tue, 22 Sep 2026 09:06:29 +0000 Subject: [PATCH 4/4] Address comments --- .../AArch64/AArch64TargetTransformInfo.cpp | 21 +++++++-------- .../Inline/AArch64/sme-always-inline.ll | 26 ++++++++++++++++++- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index ab89c4e44abef..a87babb37a76a 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -240,20 +240,15 @@ static bool isPossiblyIncompatibleIntrinsic(const Instruction *I) { return false; if (auto *II = dyn_cast<IntrinsicInst>(I)) { - switch (II->getIntrinsicID()) { + unsigned IID = II->getIntrinsicID(); + switch (IID) { default: - break; + return Intrinsic::isTargetIntrinsic(IID); case Intrinsic::vscale: case Intrinsic::masked_gather: case Intrinsic::masked_scatter: return true; } - - StringRef Name = II->getCalledFunction()->getName(); - if (Name.starts_with("llvm.aarch64.neon") || - Name.starts_with("llvm.aarch64.sve") || - Name.starts_with("llvm.aarch64.sme")) - return true; } return false; @@ -278,6 +273,7 @@ static bool hasPossibleIncompatibleOps(const Function *F, bool ConsiderZA, bool ConsiderSM) { assert((ConsiderZA || ConsiderSM) && "No SME state to consider"); + bool IsAlwaysInline = F->hasFnAttribute(Attribute::AlwaysInline); bool HasVLDependentArgsOrRet = F->getReturnType()->isScalableTy() || any_of(F->getFunctionType()->params(), @@ -288,10 +284,11 @@ static bool hasPossibleIncompatibleOps(const Function *F, // Inlining operations on fixed-length vectors when the streaming // mode does not match, is rejected because performance may be impacted. // This decision should eventually be moved the cost-model. - if (ConsiderSM && (isa<FixedVectorType>(I.getType()) || - any_of(I.operand_values(), [](const Value *V) { - return isa<FixedVectorType>(V->getType()); - }))) + if (!IsAlwaysInline && ConsiderSM && + (isa<FixedVectorType>(I.getType()) || + any_of(I.operand_values(), [](const Value *V) { + return isa<FixedVectorType>(V->getType()); + }))) return true; // Inlining operations on scalable vectors is rejected because it is diff --git a/llvm/test/Transforms/Inline/AArch64/sme-always-inline.ll b/llvm/test/Transforms/Inline/AArch64/sme-always-inline.ll index b8058f2d60a3c..a8774f74aea03 100644 --- a/llvm/test/Transforms/Inline/AArch64/sme-always-inline.ll +++ b/llvm/test/Transforms/Inline/AArch64/sme-always-inline.ll @@ -85,7 +85,31 @@ define ptr @compatible_vscale_dependent_operation_sm(ptr %p) #0 "aarch64_pstate_ } ; functions with fixed-length vectors shouldn't be inlined if the streaming properties don't match -; as performance may be affected. +; as performance may be affected. However, when they have the alwaysinline property, they should +; still be inlined. + +define void @fixed_length_vector_operation_alwaysinline(ptr %p) #0 alwaysinline { +; CHECK-LABEL: define void @fixed_length_vector_operation_alwaysinline( +; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[P]], align 16 +; CHECK-NEXT: ret void +; + store <4 x i32> zeroinitializer, ptr %p + ret void +} + +define void @fixed_length_vector_operation_caller_force_inline(ptr %p) #0 "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define void @fixed_length_vector_operation_caller_force_inline( +; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR2]] { +; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[P]], align 16 +; CHECK-NEXT: ret void +; + call void @fixed_length_vector_operation_alwaysinline(ptr %p) + ret void +} + +; functions with fixed-length vectors shouldn't be inlined if the streaming properties don't match +; as performance may be affected. Check they're not inlined without the alwaysinline attribute. define void @fixed_length_vector_operation(ptr %p) #0 { ; CHECK-LABEL: define void @fixed_length_vector_operation( ; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR1]] { _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
