https://github.com/sdesmalen-arm updated https://github.com/llvm/llvm-project/pull/223393
>From bc3a181f1dfbd3719a2e6864e3d7e7a815c102a9 Mon Sep 17 00:00:00 2001 From: Sander de Smalen <[email protected]> Date: Mon, 14 Sep 2026 08:05:19 +0000 Subject: [PATCH 1/2] [AArch64][SME] Allow more inlining when SME attributes are incompatible. At the moment, 'areInlineCompatible' is very strict as it conservatively disallows inlining any callee if they use intrinsics and have incompatible SME attributes. This PR relaxes those constraints by allowing more intrinsics. It also updates the Clang diagnostic to match the 'new' behaviour that a function is no longer inlined despite 'always_inline' when they are not inline compatible. --- .../clang/Basic/DiagnosticFrontendKinds.td | 2 +- .../AArch64/sme-inline-streaming-attrs.c | 6 +- .../AArch64/AArch64TargetTransformInfo.cpp | 65 ++++++-- .../Inline/AArch64/sme-always-inline.ll | 149 ++++++++++++++++++ 4 files changed, 202 insertions(+), 20 deletions(-) create mode 100644 llvm/test/Transforms/Inline/AArch64/sme-always-inline.ll diff --git a/clang/include/clang/Basic/DiagnosticFrontendKinds.td b/clang/include/clang/Basic/DiagnosticFrontendKinds.td index a10f10502a702..c43758abd2e13 100644 --- a/clang/include/clang/Basic/DiagnosticFrontendKinds.td +++ b/clang/include/clang/Basic/DiagnosticFrontendKinds.td @@ -317,7 +317,7 @@ def err_function_always_inline_attribute_mismatch : Error< "always_inline function %1 and its caller %0 have mismatching %2 attributes">; def warn_function_always_inline_attribute_mismatch : Warning< "always_inline function %1 and its caller %0 have mismatching %2 attributes, " - "inlining may change runtime behaviour">, InGroup<AArch64SMEAttributes>; + "%1 may not be inlined">, InGroup<AArch64SMEAttributes>; def err_function_always_inline_new_za : Error< "always_inline function %0 has new za state">; def err_function_always_inline_new_zt0 diff --git a/clang/test/CodeGen/AArch64/sme-inline-streaming-attrs.c b/clang/test/CodeGen/AArch64/sme-inline-streaming-attrs.c index 68102c9ded40c..ba9f24032d274 100644 --- a/clang/test/CodeGen/AArch64/sme-inline-streaming-attrs.c +++ b/clang/test/CodeGen/AArch64/sme-inline-streaming-attrs.c @@ -26,7 +26,7 @@ void caller(void) { #ifdef TEST_COMPATIBLE void caller_compatible(void) __arm_streaming_compatible { - inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_compatible' have mismatching streaming attributes, inlining may change runtime behaviour}} + inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_compatible' have mismatching streaming attributes, 'inlined_fn' may not be inlined}} inlined_fn_streaming_compatible(); inlined_fn_streaming(); // expected-error {{always_inline function 'inlined_fn_streaming' and its caller 'caller_compatible' have mismatching streaming attributes}} inlined_fn_local(); // expected-error {{always_inline function 'inlined_fn_local' and its caller 'caller_compatible' have mismatching streaming attributes}} @@ -35,7 +35,7 @@ void caller_compatible(void) __arm_streaming_compatible { #ifdef TEST_STREAMING void caller_streaming(void) __arm_streaming { - inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_streaming' have mismatching streaming attributes, inlining may change runtime behaviour}} + inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_streaming' have mismatching streaming attributes, 'inlined_fn' may not be inlined}} inlined_fn_streaming_compatible(); inlined_fn_streaming(); inlined_fn_local(); @@ -45,7 +45,7 @@ void caller_streaming(void) __arm_streaming { #ifdef TEST_LOCALLY __arm_locally_streaming void caller_local(void) { - inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_local' have mismatching streaming attributes, inlining may change runtime behaviour}} + inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_local' have mismatching streaming attributes, 'inlined_fn' may not be inlined}} inlined_fn_streaming_compatible(); inlined_fn_streaming(); inlined_fn_local(); diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index d8e8f58291a8d..b17ee3f4fb35a 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -233,23 +233,56 @@ static bool isSMEABIRoutineCall(const CallInst &CI, SMEAttrs(F->getName(), TLI.getRuntimeLibcallsInfo()).isSMEABIRoutine(); } +static bool isPossiblyIncompatibleIntrinsic(const Instruction *I, + bool ConsiderZA, bool ConsiderSM) { + auto *II = dyn_cast<IntrinsicInst>(I); + if (!II) + return false; + + StringRef Name = II->getCalledFunction()->getName(); + if (ConsiderZA && Name.starts_with("llvm.aarch64.sme.")) + return true; + + if (ConsiderSM) { + if (Name.starts_with("llvm.aarch64.neon") || + Name.starts_with("llvm.aarch64.sve") || + Name.starts_with("llvm.aarch64.sme") || + Name.starts_with("llvm.vscale") || + Name.starts_with("llvm.masked.gather") || + Name.starts_with("llvm.masked.scatter")) { + return true; + } + } + + return false; +} + /// Returns true if the function has explicit operations that can only be /// lowered using incompatible instructions for the selected mode. This also /// returns true if the function F may use or modify ZA state. static bool hasPossibleIncompatibleOps(const Function *F, - const AArch64TargetLowering &TLI) { + const AArch64TargetLowering &TLI, + bool ConsiderZA, bool ConsiderSM) { + assert((ConsiderZA || ConsiderSM) && "No SME state to consider"); for (const BasicBlock &BB : *F) { for (const Instruction &I : BB) { - // Be conservative for now and assume that any call to inline asm or to - // intrinsics could could result in non-streaming ops (e.g. calls to - // @llvm.aarch64.* or @llvm.gather/scatter intrinsics). We can assume that - // all native LLVM instructions can be lowered to compatible instructions. - if (isa<CallInst>(I) && !I.isDebugOrPseudoInst() && - (cast<CallInst>(I).isInlineAsm() || isa<IntrinsicInst>(I) || - isSMEABIRoutineCall(cast<CallInst>(I), TLI))) + // We can assume that all native LLVM instructions can be lowered to + // compatible instructions. + if (!isa<CallInst>(I) || I.isDebugOrPseudoInst()) + continue; + // Be conservative for now and assume that any call to inline asm could + // result in different behaviour. + if (cast<CallInst>(I).isInlineAsm()) + return true; + + if (isSMEABIRoutineCall(cast<CallInst>(I), TLI)) + return true; + + if (isPossiblyIncompatibleIntrinsic(&I, ConsiderZA, ConsiderSM)) return true; } } + return false; } @@ -288,6 +321,9 @@ bool AArch64TTIImpl::areInlineCompatible(const Function *Caller, CallAttrs.callee().hasStreamingInterfaceOrBody()) return false; + if (CallAttrs.callee().isNewZA() || CallAttrs.callee().isNewZT0()) + return false; + // When inlining, we should consider the body of the function, not the // interface. if (CallAttrs.callee().hasStreamingBody()) { @@ -295,16 +331,13 @@ bool AArch64TTIImpl::areInlineCompatible(const Function *Caller, CallAttrs.callee().set(SMEAttrs::SM_Enabled, true); } - if (CallAttrs.callee().isNewZA() || CallAttrs.callee().isNewZT0()) + bool ConsiderZA = + CallAttrs.requiresLazySave() || CallAttrs.requiresPreservingZT0(); + bool ConsiderSM = CallAttrs.requiresSMChange(); + if ((ConsiderZA || ConsiderSM) && + hasPossibleIncompatibleOps(Callee, *getTLI(), ConsiderZA, ConsiderSM)) return false; - if (CallAttrs.requiresLazySave() || CallAttrs.requiresSMChange() || - CallAttrs.requiresPreservingZT0() || - CallAttrs.requiresPreservingAllZAState()) { - if (hasPossibleIncompatibleOps(Callee, *getTLI())) - return false; - } - return BaseT::areInlineCompatible(Caller, Callee); } diff --git a/llvm/test/Transforms/Inline/AArch64/sme-always-inline.ll b/llvm/test/Transforms/Inline/AArch64/sme-always-inline.ll new file mode 100644 index 0000000000000..0cfe565cb1dc3 --- /dev/null +++ b/llvm/test/Transforms/Inline/AArch64/sme-always-inline.ll @@ -0,0 +1,149 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --prefix-filecheck-ir-name bb1 --version 6 +; RUN: opt -passes=always-inline -S < %s | FileCheck %s + +target triple = "aarch64" + +; +; Be cautious about inlining intrinsic calls when their streaming/ZA properties are incompatible. +; + +; llvm.vscale returns a different value in/out of streaming mode, and is not safe to inline. +define i64 @llvm_vscale() alwaysinline { +; CHECK-LABEL: define i64 @llvm_vscale( +; CHECK-SAME: ) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: [[VSCALE:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: ret i64 [[VSCALE]] +; + %vscale = call i64 @llvm.vscale() + ret i64 %vscale +} + +define i64 @compatible_llvm_vscale_caller() { +; CHECK-LABEL: define i64 @compatible_llvm_vscale_caller() { +; CHECK-NEXT: [[VSCALE_I:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: ret i64 [[VSCALE_I]] +; + %vscale = call i64 @llvm_vscale() + ret i64 %vscale +} + +define i64 @incompatible_llvm_vscale_caller() "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define i64 @incompatible_llvm_vscale_caller( +; CHECK-SAME: ) #[[ATTR1:[0-9]+]] { +; CHECK-NEXT: [[VSCALE:%.*]] = call i64 @llvm_vscale() +; CHECK-NEXT: ret i64 [[VSCALE]] +; + %vscale = call i64 @llvm_vscale() + ret i64 %vscale +} + +; NEON intrinsics are not safe in streaming mode +define i32 @neon_intrinsic(<4 x i32> %in) alwaysinline { +; CHECK-LABEL: define i32 @neon_intrinsic( +; CHECK-SAME: <4 x i32> [[IN:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[RES:%.*]] = call i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> [[IN]]) +; CHECK-NEXT: ret i32 [[RES]] +; + %res = call i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> %in) + ret i32 %res +} + +define i32 @compatible_neon_intrinsic_caller(<4 x i32> %in) { +; CHECK-LABEL: define i32 @compatible_neon_intrinsic_caller( +; CHECK-SAME: <4 x i32> [[IN:%.*]]) { +; CHECK-NEXT: [[RES_I:%.*]] = call i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> [[IN]]) +; CHECK-NEXT: ret i32 [[RES_I]] +; + %vscale = call i32 @neon_intrinsic(<4 x i32> %in) + ret i32 %vscale +} + +; a bit of a niche case, but if the caller uses ZA but is not in streaming-mode, a NEON intrinsic is safe. +define i32 @compatible_neon_intrinsic_caller_za(<4 x i32> %in) "aarch64_pstate_za_enabled" { +; CHECK-LABEL: define i32 @compatible_neon_intrinsic_caller_za( +; CHECK-SAME: <4 x i32> [[IN:%.*]]) #[[ATTR2:[0-9]+]] { +; CHECK-NEXT: [[RES_I:%.*]] = call i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> [[IN]]) +; CHECK-NEXT: ret i32 [[RES_I]] +; + %vscale = call i32 @neon_intrinsic(<4 x i32> %in) + ret i32 %vscale +} + +define i32 @incompatible_neon_intrinsic_caller(<4 x i32> %in) "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define i32 @incompatible_neon_intrinsic_caller( +; CHECK-SAME: <4 x i32> [[IN:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: [[VSCALE:%.*]] = call i32 @neon_intrinsic(<4 x i32> [[IN]]) +; CHECK-NEXT: ret i32 [[VSCALE]] +; + %vscale = call i32 @neon_intrinsic(<4 x i32> %in) + ret i32 %vscale +} + +; +; Be cautious about inlining functions that contain calls to SME ABI routines (like __arm_get_current_vg()) +; + +declare i64 @__arm_get_current_vg() + +define i64 @current_vg() alwaysinline { +; CHECK-LABEL: define i64 @current_vg( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: [[VSCALE:%.*]] = call i64 @__arm_get_current_vg() +; CHECK-NEXT: ret i64 [[VSCALE]] +; + %vscale = call i64 @__arm_get_current_vg() + ret i64 %vscale +} + +define i64 @compatible_current_vg_caller() { +; CHECK-LABEL: define i64 @compatible_current_vg_caller() { +; CHECK-NEXT: [[VSCALE_I:%.*]] = call i64 @__arm_get_current_vg() +; CHECK-NEXT: ret i64 [[VSCALE_I]] +; + %vscale = call i64 @current_vg() + ret i64 %vscale +} + +define i64 @incompatible_current_vg_caller() "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define i64 @incompatible_current_vg_caller( +; CHECK-SAME: ) #[[ATTR1]] { +; CHECK-NEXT: [[VSCALE:%.*]] = call i64 @current_vg() +; CHECK-NEXT: ret i64 [[VSCALE]] +; + %vscale = call i64 @current_vg() + ret i64 %vscale +} + +; +; Be cautious about inlining anything with inline asm when streaming/ZA properties are incompatible, +; + +define void @inline_asm() alwaysinline { +; CHECK-LABEL: define void @inline_asm( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: call void asm sideeffect "smstart za +; CHECK-NEXT: ret void +; + call void asm sideeffect "smstart za; svzero za; smstop za", ""() + ret void +} + +define void @compatible_inline_asm() { +; CHECK-LABEL: define void @compatible_inline_asm() { +; CHECK-NEXT: call void asm sideeffect "smstart za +; CHECK-NEXT: ret void +; + call void @inline_asm() + ret void +} + +define void @incompatible_inline_asm() "aarch64_inout_za" { +; CHECK-LABEL: define void @incompatible_inline_asm( +; CHECK-SAME: ) #[[ATTR3:[0-9]+]] { +; CHECK-NEXT: call void @inline_asm() +; CHECK-NEXT: ret void +; + call void @inline_asm() + ret void +} + >From d841b7851d47b7a12841709e08c9aaf3c4c7f5f5 Mon Sep 17 00:00:00 2001 From: Sander de Smalen <[email protected]> Date: Mon, 14 Sep 2026 14:27:39 +0000 Subject: [PATCH 2/2] Be more conservative, by disallowing operations on scalable types. Also restore behaviour around inlining calls into agnostic-za functions if they contain e.g. inline asm, which previously caused one of the tests to fail. --- .../AArch64/AArch64TargetTransformInfo.cpp | 11 ++++++- .../Inline/AArch64/sme-always-inline.ll | 33 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index b17ee3f4fb35a..c5319e59bff7d 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -275,6 +275,14 @@ static bool hasPossibleIncompatibleOps(const Function *F, if (cast<CallInst>(I).isInlineAsm()) return true; + // Be conservative around operations on scalable types, as those may have + // different behaviour in/out of streaming mode. + if (ConsiderSM && (I.getType()->isScalableTy() || + any_of(I.operand_values(), [](const Value *V) { + return V->getType()->isScalableTy(); + }))) + return true; + if (isSMEABIRoutineCall(cast<CallInst>(I), TLI)) return true; @@ -332,7 +340,8 @@ bool AArch64TTIImpl::areInlineCompatible(const Function *Caller, } bool ConsiderZA = - CallAttrs.requiresLazySave() || CallAttrs.requiresPreservingZT0(); + CallAttrs.requiresLazySave() || CallAttrs.requiresPreservingZT0() || + CallAttrs.requiresPreservingAllZAState(); bool ConsiderSM = CallAttrs.requiresSMChange(); if ((ConsiderZA || ConsiderSM) && hasPossibleIncompatibleOps(Callee, *getTLI(), ConsiderZA, ConsiderSM)) diff --git a/llvm/test/Transforms/Inline/AArch64/sme-always-inline.ll b/llvm/test/Transforms/Inline/AArch64/sme-always-inline.ll index 0cfe565cb1dc3..777d6064928d9 100644 --- a/llvm/test/Transforms/Inline/AArch64/sme-always-inline.ll +++ b/llvm/test/Transforms/Inline/AArch64/sme-always-inline.ll @@ -79,6 +79,39 @@ define i32 @incompatible_neon_intrinsic_caller(<4 x i32> %in) "aarch64_pstate_sm ret i32 %vscale } +define i64 @intrinsic_with_scalable_type(ptr %p) alwaysinline { +; CHECK-LABEL: define i64 @intrinsic_with_scalable_type( +; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[LD:%.*]] = load <vscale x 2 x i64>, ptr [[P]], align 16 +; CHECK-NEXT: [[RES:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[LD]]) +; CHECK-NEXT: ret i64 [[RES]] +; + %ld = load <vscale x 2 x i64>, ptr %p + %res = call i64 @llvm.vector.reduce.add(<vscale x 2 x i64> %ld) + ret i64 %res +} + +define i64 @compatible_sve_intrinsic_caller(ptr %p) { +; CHECK-LABEL: define i64 @compatible_sve_intrinsic_caller( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[LD_I:%.*]] = load <vscale x 2 x i64>, ptr [[P]], align 16 +; CHECK-NEXT: [[RES_I:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[LD_I]]) +; CHECK-NEXT: ret i64 [[RES_I]] +; + %res = call i64 @intrinsic_with_scalable_type(ptr %p) + ret i64 %res +} + +define i64 @incompatible_sve_intrinsic_caller(ptr %p) "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define i64 @incompatible_sve_intrinsic_caller( +; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: [[RES:%.*]] = call i64 @intrinsic_with_scalable_type(ptr [[P]]) +; CHECK-NEXT: ret i64 [[RES]] +; + %res = call i64 @intrinsic_with_scalable_type(ptr %p) + ret i64 %res +} + ; ; Be cautious about inlining functions that contain calls to SME ABI routines (like __arm_get_current_vg()) ; _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
