https://github.com/sdesmalen-arm updated https://github.com/llvm/llvm-project/pull/223393
>From 455326b678880e25cb2ec74a640ec25f555f732d Mon Sep 17 00:00:00 2001 From: Sander de Smalen <[email protected]> Date: Thu, 1 Oct 2026 08:50:11 +0000 Subject: [PATCH 1/2] Pre-commit tests --- .../Inline/AArch64/sme-pstatesm-attrs.ll | 401 +++++++++++++++--- .../AArch64/sme-pstatesm-inline-transitive.ll | 49 +++ 2 files changed, 387 insertions(+), 63 deletions(-) create mode 100644 llvm/test/Transforms/Inline/AArch64/sme-pstatesm-inline-transitive.ll diff --git a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll index 077a3aa49fb414d..49af107d7e89b41 100644 --- a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll +++ b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 -; RUN: opt < %s -mtriple=aarch64-unknown-linux-gnu -mattr=+sme -S -passes=inline | FileCheck %s +; RUN: opt < %s -mtriple=aarch64-unknown-linux-gnu -mattr=+sve,+sme -S -passes=inline | FileCheck %s declare i32 @llvm.vscale.i32() @@ -7,7 +7,7 @@ declare i32 @llvm.vscale.i32() ; by the other functions below. If we see the call to one of these functions ; being replaced by 'llvm.vscale()', then we know it has been inlined. -define i32 @normal_callee() #0 { +define i32 @normal_callee() { ; CHECK-LABEL: define i32 @normal_callee ; CHECK-SAME: () #[[ATTR1:[0-9]+]] { ; CHECK-NEXT: entry: @@ -19,7 +19,7 @@ entry: ret i32 %res } -define i32 @streaming_callee() #0 "aarch64_pstate_sm_enabled" { +define i32 @streaming_callee() "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define i32 @streaming_callee ; CHECK-SAME: () #[[ATTR2:[0-9]+]] { ; CHECK-NEXT: entry: @@ -31,7 +31,7 @@ entry: ret i32 %res } -define i32 @locally_streaming_callee() #0 "aarch64_pstate_sm_body" { +define i32 @locally_streaming_callee() "aarch64_pstate_sm_body" { ; CHECK-LABEL: define i32 @locally_streaming_callee ; CHECK-SAME: () #[[ATTR3:[0-9]+]] { ; CHECK-NEXT: entry: @@ -43,7 +43,7 @@ entry: ret i32 %res } -define i32 @streaming_compatible_callee() #0 "aarch64_pstate_sm_compatible" { +define i32 @streaming_compatible_callee() "aarch64_pstate_sm_compatible" { ; CHECK-LABEL: define i32 @streaming_compatible_callee ; CHECK-SAME: () #[[ATTR4:[0-9]+]] { ; CHECK-NEXT: entry: @@ -55,7 +55,7 @@ entry: ret i32 %res } -define i32 @streaming_compatible_locally_streaming_callee() #0 "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" { +define i32 @streaming_compatible_locally_streaming_callee() "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" { ; CHECK-LABEL: define i32 @streaming_compatible_locally_streaming_callee ; CHECK-SAME: () #[[ATTR5:[0-9]+]] { ; CHECK-NEXT: entry: @@ -84,7 +84,7 @@ entry: ; [ ] N -> SC ; [ ] N -> N + B ; [ ] N -> SC + B -define i32 @normal_caller_normal_callee_inline() #0 { +define i32 @normal_caller_normal_callee_inline() { ; CHECK-LABEL: define i32 @normal_caller_normal_callee_inline ; CHECK-SAME: () #[[ATTR1]] { ; CHECK-NEXT: entry: @@ -101,7 +101,7 @@ entry: ; [ ] N -> SC ; [ ] N -> N + B ; [ ] N -> SC + B -define i32 @normal_caller_streaming_callee_dont_inline() #0 { +define i32 @normal_caller_streaming_callee_dont_inline() { ; CHECK-LABEL: define i32 @normal_caller_streaming_callee_dont_inline ; CHECK-SAME: () #[[ATTR1]] { ; CHECK-NEXT: entry: @@ -118,7 +118,7 @@ entry: ; [x] N -> SC ; [ ] N -> N + B ; [ ] N -> SC + B -define i32 @normal_caller_streaming_compatible_callee_inline() #0 { +define i32 @normal_caller_streaming_compatible_callee_inline() { ; CHECK-LABEL: define i32 @normal_caller_streaming_compatible_callee_inline ; CHECK-SAME: () #[[ATTR1]] { ; CHECK-NEXT: entry: @@ -135,7 +135,7 @@ entry: ; [ ] N -> SC ; [x] N -> N + B ; [ ] N -> SC + B -define i32 @normal_caller_locally_streaming_callee_dont_inline() #0 { +define i32 @normal_caller_locally_streaming_callee_dont_inline() { ; CHECK-LABEL: define i32 @normal_caller_locally_streaming_callee_dont_inline ; CHECK-SAME: () #[[ATTR1]] { ; CHECK-NEXT: entry: @@ -152,7 +152,7 @@ entry: ; [ ] N -> SC ; [ ] N -> N + B ; [x] N -> SC + B -define i32 @normal_caller_streaming_compatible_locally_streaming_callee_dont_inline() #0 { +define i32 @normal_caller_streaming_compatible_locally_streaming_callee_dont_inline() { ; CHECK-LABEL: define i32 @normal_caller_streaming_compatible_locally_streaming_callee_dont_inline ; CHECK-SAME: () #[[ATTR1]] { ; CHECK-NEXT: entry: @@ -169,7 +169,7 @@ entry: ; [ ] S -> SC ; [ ] S -> N + B ; [ ] S -> SC + B -define i32 @streaming_caller_normal_callee_dont_inline() #0 "aarch64_pstate_sm_enabled" { +define i32 @streaming_caller_normal_callee_dont_inline() "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define i32 @streaming_caller_normal_callee_dont_inline ; CHECK-SAME: () #[[ATTR2]] { ; CHECK-NEXT: entry: @@ -186,7 +186,7 @@ entry: ; [ ] S -> SC ; [ ] S -> N + B ; [ ] S -> SC + B -define i32 @streaming_caller_streaming_callee_inline() #0 "aarch64_pstate_sm_enabled" { +define i32 @streaming_caller_streaming_callee_inline() "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define i32 @streaming_caller_streaming_callee_inline ; CHECK-SAME: () #[[ATTR2]] { ; CHECK-NEXT: entry: @@ -203,7 +203,7 @@ entry: ; [x] S -> SC ; [ ] S -> N + B ; [ ] S -> SC + B -define i32 @streaming_caller_streaming_compatible_callee_inline() #0 "aarch64_pstate_sm_enabled" { +define i32 @streaming_caller_streaming_compatible_callee_inline() "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define i32 @streaming_caller_streaming_compatible_callee_inline ; CHECK-SAME: () #[[ATTR2]] { ; CHECK-NEXT: entry: @@ -220,7 +220,7 @@ entry: ; [ ] S -> SC ; [x] S -> N + B ; [ ] S -> SC + B -define i32 @streaming_caller_locally_streaming_callee_inline() #0 "aarch64_pstate_sm_enabled" { +define i32 @streaming_caller_locally_streaming_callee_inline() "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define i32 @streaming_caller_locally_streaming_callee_inline ; CHECK-SAME: () #[[ATTR2]] { ; CHECK-NEXT: entry: @@ -237,7 +237,7 @@ entry: ; [ ] S -> SC ; [ ] S -> N + B ; [x] S -> SC + B -define i32 @streaming_caller_streaming_compatible_locally_streaming_callee_inline() #0 "aarch64_pstate_sm_enabled" { +define i32 @streaming_caller_streaming_compatible_locally_streaming_callee_inline() "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define i32 @streaming_caller_streaming_compatible_locally_streaming_callee_inline ; CHECK-SAME: () #[[ATTR2]] { ; CHECK-NEXT: entry: @@ -254,7 +254,7 @@ entry: ; [ ] N + B -> SC ; [ ] N + B -> N + B ; [ ] N + B -> SC + B -define i32 @locally_streaming_caller_normal_callee_dont_inline() #0 "aarch64_pstate_sm_body" { +define i32 @locally_streaming_caller_normal_callee_dont_inline() "aarch64_pstate_sm_body" { ; CHECK-LABEL: define i32 @locally_streaming_caller_normal_callee_dont_inline ; CHECK-SAME: () #[[ATTR3]] { ; CHECK-NEXT: entry: @@ -271,7 +271,7 @@ entry: ; [ ] N + B -> SC ; [ ] N + B -> N + B ; [ ] N + B -> SC + B -define i32 @locally_streaming_caller_streaming_callee_inline() #0 "aarch64_pstate_sm_body" { +define i32 @locally_streaming_caller_streaming_callee_inline() "aarch64_pstate_sm_body" { ; CHECK-LABEL: define i32 @locally_streaming_caller_streaming_callee_inline ; CHECK-SAME: () #[[ATTR3]] { ; CHECK-NEXT: entry: @@ -288,7 +288,7 @@ entry: ; [x] N + B -> SC ; [ ] N + B -> N + B ; [ ] N + B -> SC + B -define i32 @locally_streaming_caller_streaming_compatible_callee_inline() #0 "aarch64_pstate_sm_body" { +define i32 @locally_streaming_caller_streaming_compatible_callee_inline() "aarch64_pstate_sm_body" { ; CHECK-LABEL: define i32 @locally_streaming_caller_streaming_compatible_callee_inline ; CHECK-SAME: () #[[ATTR3]] { ; CHECK-NEXT: entry: @@ -305,7 +305,7 @@ entry: ; [ ] N + B -> SC ; [x] N + B -> N + B ; [ ] N + B -> SC + B -define i32 @locally_streaming_caller_locally_streaming_callee_inline() #0 "aarch64_pstate_sm_body" { +define i32 @locally_streaming_caller_locally_streaming_callee_inline() "aarch64_pstate_sm_body" { ; CHECK-LABEL: define i32 @locally_streaming_caller_locally_streaming_callee_inline ; CHECK-SAME: () #[[ATTR3]] { ; CHECK-NEXT: entry: @@ -322,7 +322,7 @@ entry: ; [ ] N + B -> SC ; [ ] N + B -> N + B ; [x] N + B -> SC + B -define i32 @locally_streaming_caller_streaming_compatible_locally_streaming_callee_inline() #0 "aarch64_pstate_sm_body" { +define i32 @locally_streaming_caller_streaming_compatible_locally_streaming_callee_inline() "aarch64_pstate_sm_body" { ; CHECK-LABEL: define i32 @locally_streaming_caller_streaming_compatible_locally_streaming_callee_inline ; CHECK-SAME: () #[[ATTR3]] { ; CHECK-NEXT: entry: @@ -339,7 +339,7 @@ entry: ; [ ] SC -> SC ; [ ] SC -> N + B ; [ ] SC -> SC + B -define i32 @streaming_compatible_caller_normal_callee_dont_inline() #0 "aarch64_pstate_sm_compatible" { +define i32 @streaming_compatible_caller_normal_callee_dont_inline() "aarch64_pstate_sm_compatible" { ; CHECK-LABEL: define i32 @streaming_compatible_caller_normal_callee_dont_inline ; CHECK-SAME: () #[[ATTR4]] { ; CHECK-NEXT: entry: @@ -356,7 +356,7 @@ entry: ; [ ] SC -> SC ; [ ] SC -> N + B ; [ ] SC -> SC + B -define i32 @streaming_compatible_caller_streaming_callee_dont_inline() #0 "aarch64_pstate_sm_compatible" { +define i32 @streaming_compatible_caller_streaming_callee_dont_inline() "aarch64_pstate_sm_compatible" { ; CHECK-LABEL: define i32 @streaming_compatible_caller_streaming_callee_dont_inline ; CHECK-SAME: () #[[ATTR4]] { ; CHECK-NEXT: entry: @@ -373,7 +373,7 @@ entry: ; [x] SC -> SC ; [ ] SC -> N + B ; [ ] SC -> SC + B -define i32 @streaming_compatible_caller_streaming_compatible_callee_inline() #0 "aarch64_pstate_sm_compatible" { +define i32 @streaming_compatible_caller_streaming_compatible_callee_inline() "aarch64_pstate_sm_compatible" { ; CHECK-LABEL: define i32 @streaming_compatible_caller_streaming_compatible_callee_inline ; CHECK-SAME: () #[[ATTR4]] { ; CHECK-NEXT: entry: @@ -390,7 +390,7 @@ entry: ; [ ] SC -> SC ; [x] SC -> N + B ; [ ] SC -> SC + B -define i32 @streaming_compatible_caller_locally_streaming_callee_dont_inline() #0 "aarch64_pstate_sm_compatible" { +define i32 @streaming_compatible_caller_locally_streaming_callee_dont_inline() "aarch64_pstate_sm_compatible" { ; CHECK-LABEL: define i32 @streaming_compatible_caller_locally_streaming_callee_dont_inline ; CHECK-SAME: () #[[ATTR4]] { ; CHECK-NEXT: entry: @@ -407,7 +407,7 @@ entry: ; [ ] SC -> SC ; [ ] SC -> N + B ; [x] SC -> SC + B -define i32 @streaming_compatible_caller_streaming_compatible_locally_streaming_callee_dont_inline() #0 "aarch64_pstate_sm_compatible" { +define i32 @streaming_compatible_caller_streaming_compatible_locally_streaming_callee_dont_inline() "aarch64_pstate_sm_compatible" { ; CHECK-LABEL: define i32 @streaming_compatible_caller_streaming_compatible_locally_streaming_callee_dont_inline ; CHECK-SAME: () #[[ATTR4]] { ; CHECK-NEXT: entry: @@ -423,7 +423,7 @@ entry: ; [ ] SC + B -> SC ; [ ] SC + B -> N + B ; [ ] SC + B -> SC + B -define i32 @streaming_compatible_locally_streaming_caller_normal_callee_dont_inline() #0 "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" { +define i32 @streaming_compatible_locally_streaming_caller_normal_callee_dont_inline() "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" { ; CHECK-LABEL: define i32 @streaming_compatible_locally_streaming_caller_normal_callee_dont_inline ; CHECK-SAME: () #[[ATTR5]] { ; CHECK-NEXT: entry: @@ -440,7 +440,7 @@ entry: ; [ ] SC + B -> SC ; [ ] SC + B -> N + B ; [ ] SC + B -> SC + B -define i32 @streaming_compatible_locally_streaming_caller_streaming_callee_inline() #0 "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" { +define i32 @streaming_compatible_locally_streaming_caller_streaming_callee_inline() "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" { ; CHECK-LABEL: define i32 @streaming_compatible_locally_streaming_caller_streaming_callee_inline ; CHECK-SAME: () #[[ATTR5]] { ; CHECK-NEXT: entry: @@ -457,7 +457,7 @@ entry: ; [x] SC + B -> SC ; [ ] SC + B -> N + B ; [ ] SC + B -> SC + B -define i32 @streaming_compatible_locally_streaming_caller_streaming_compatible_callee_inline() #0 "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" { +define i32 @streaming_compatible_locally_streaming_caller_streaming_compatible_callee_inline() "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" { ; CHECK-LABEL: define i32 @streaming_compatible_locally_streaming_caller_streaming_compatible_callee_inline ; CHECK-SAME: () #[[ATTR5]] { ; CHECK-NEXT: entry: @@ -474,7 +474,7 @@ entry: ; [ ] SC + B -> SC ; [x] SC + B -> N + B ; [ ] SC + B -> SC + B -define i32 @streaming_compatible_locally_streaming_caller_locally_streaming_callee_inline() #0 "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" { +define i32 @streaming_compatible_locally_streaming_caller_locally_streaming_callee_inline() "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" { ; CHECK-LABEL: define i32 @streaming_compatible_locally_streaming_caller_locally_streaming_callee_inline ; CHECK-SAME: () #[[ATTR5]] { ; CHECK-NEXT: entry: @@ -491,7 +491,7 @@ entry: ; [ ] SC + B -> SC ; [ ] SC + B -> N + B ; [x] SC + B -> SC + B -define i32 @streaming_compatible_locally_streaming_caller_and_callee_inline() #0 "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" { +define i32 @streaming_compatible_locally_streaming_caller_and_callee_inline() "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" { ; CHECK-LABEL: define i32 @streaming_compatible_locally_streaming_caller_and_callee_inline ; CHECK-SAME: () #[[ATTR5]] { ; CHECK-NEXT: entry: @@ -503,7 +503,7 @@ entry: ret i32 %res } -define void @normal_callee_with_inlineasm() #0 { +define void @normal_callee_with_inlineasm() { ; CHECK-LABEL: define void @normal_callee_with_inlineasm ; CHECK-SAME: () #[[ATTR1]] { ; CHECK-NEXT: entry: @@ -515,7 +515,7 @@ entry: ret void } -define void @streaming_caller_normal_callee_with_inlineasm_dont_inline() #0 "aarch64_pstate_sm_enabled" { +define void @streaming_caller_normal_callee_with_inlineasm_dont_inline() "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define void @streaming_caller_normal_callee_with_inlineasm_dont_inline ; CHECK-SAME: () #[[ATTR2]] { ; CHECK-NEXT: entry: @@ -527,7 +527,7 @@ entry: ret void } -define i64 @normal_callee_with_intrinsic_call() #0 { +define i64 @normal_callee_with_intrinsic_call() { ; CHECK-LABEL: define i64 @normal_callee_with_intrinsic_call ; CHECK-SAME: () #[[ATTR1]] { ; CHECK-NEXT: entry: @@ -539,7 +539,7 @@ entry: ret i64 %res } -define i64 @streaming_caller_normal_callee_with_intrinsic_call_dont_inline() #0 "aarch64_pstate_sm_enabled" { +define i64 @streaming_caller_normal_callee_with_intrinsic_call_dont_inline() "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define i64 @streaming_caller_normal_callee_with_intrinsic_call_dont_inline ; CHECK-SAME: () #[[ATTR2]] { ; CHECK-NEXT: entry: @@ -553,7 +553,7 @@ entry: declare i64 @llvm.aarch64.sve.cntb(i32) -define i64 @normal_callee_call_sme_state() #0 { +define i64 @normal_callee_call_sme_state() { ; CHECK-LABEL: define i64 @normal_callee_call_sme_state ; CHECK-SAME: () #[[ATTR1]] { ; CHECK-NEXT: entry: @@ -569,7 +569,7 @@ entry: declare {i64, i64} @__arm_sme_state() -define i64 @streaming_caller_normal_callee_call_sme_state_dont_inline() #0 "aarch64_pstate_sm_enabled" { +define i64 @streaming_caller_normal_callee_call_sme_state_dont_inline() "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define i64 @streaming_caller_normal_callee_call_sme_state_dont_inline ; CHECK-SAME: () #[[ATTR2]] { ; CHECK-NEXT: entry: @@ -585,7 +585,7 @@ entry: declare void @nonstreaming_body() -define void @nonstreaming_caller_single_nonstreaming_callee() #0 { +define void @nonstreaming_caller_single_nonstreaming_callee() { ; CHECK-LABEL: define void @nonstreaming_caller_single_nonstreaming_callee ; CHECK-SAME: () #[[ATTR1]] { ; CHECK-NEXT: call void @nonstreaming_body() @@ -595,7 +595,7 @@ define void @nonstreaming_caller_single_nonstreaming_callee() #0 { ret void } -define void @nonstreaming_caller_multiple_nonstreaming_callees() #0 { +define void @nonstreaming_caller_multiple_nonstreaming_callees() { ; CHECK-LABEL: define void @nonstreaming_caller_multiple_nonstreaming_callees ; CHECK-SAME: () #[[ATTR1]] { ; CHECK-NEXT: call void @nonstreaming_body() @@ -608,7 +608,7 @@ define void @nonstreaming_caller_multiple_nonstreaming_callees() #0 { } ; Allow inlining, as inline it would not increase the number of streaming-mode changes. -define void @streaming_caller_to_nonstreaming_callee_with_single_nonstreaming_callee_inline() #0 "aarch64_pstate_sm_enabled" { +define void @streaming_caller_to_nonstreaming_callee_with_single_nonstreaming_callee_inline() "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define void @streaming_caller_to_nonstreaming_callee_with_single_nonstreaming_callee_inline ; CHECK-SAME: () #[[ATTR2]] { ; CHECK-NEXT: call void @nonstreaming_body() @@ -619,7 +619,7 @@ define void @streaming_caller_to_nonstreaming_callee_with_single_nonstreaming_ca } ; Prevent inlining, as inlining it would lead to multiple streaming-mode changes. -define void @streaming_caller_to_nonstreaming_callee_with_multiple_nonstreaming_callees_dont_inline() #0 "aarch64_pstate_sm_enabled" { +define void @streaming_caller_to_nonstreaming_callee_with_multiple_nonstreaming_callees_dont_inline() "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define void @streaming_caller_to_nonstreaming_callee_with_multiple_nonstreaming_callees_dont_inline ; CHECK-SAME: () #[[ATTR2]] { ; CHECK-NEXT: call void @streaming_caller_to_nonstreaming_callee_with_multiple_nonstreaming_callees_dont_inline() @@ -631,7 +631,7 @@ define void @streaming_caller_to_nonstreaming_callee_with_multiple_nonstreaming_ declare void @streaming_compatible_body() "aarch64_pstate_sm_compatible" -define void @nonstreaming_caller_single_streaming_compatible_callee() #0 { +define void @nonstreaming_caller_single_streaming_compatible_callee() { ; CHECK-LABEL: define void @nonstreaming_caller_single_streaming_compatible_callee ; CHECK-SAME: () #[[ATTR1]] { ; CHECK-NEXT: call void @streaming_compatible_body() @@ -641,42 +641,39 @@ define void @nonstreaming_caller_single_streaming_compatible_callee() #0 { ret void } -define void @nonstreaming_caller_multiple_streaming_compatible_callees() #0 { -; CHECK-LABEL: define void @nonstreaming_caller_multiple_streaming_compatible_callees -; CHECK-SAME: () #[[ATTR1]] { -; CHECK-NEXT: call void @streaming_compatible_body() +; Reject inlining; while inlining would remove a streaming-mode change, it may also change runtime behaviour. +define void @streaming_caller_to_nonstreaming_callee_with_single_streamingcompatible_callee_dont_inline() "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define void @streaming_caller_to_nonstreaming_callee_with_single_streamingcompatible_callee_dont_inline +; CHECK-SAME: () #[[ATTR2]] { ; CHECK-NEXT: call void @streaming_compatible_body() ; CHECK-NEXT: ret void ; - call void @streaming_compatible_body() - call void @streaming_compatible_body() + call void @nonstreaming_caller_single_streaming_compatible_callee() ret void } -; Allow inlining, as inline would remove a streaming-mode change. -define void @streaming_caller_to_nonstreaming_callee_with_single_streamingcompatible_callee_inline() #0 "aarch64_pstate_sm_enabled" { -; CHECK-LABEL: define void @streaming_caller_to_nonstreaming_callee_with_single_streamingcompatible_callee_inline -; CHECK-SAME: () #[[ATTR2]] { +define void @nonstreaming_caller_single_streaming_compatible_callee_alwaysinline() alwaysinline { +; CHECK-LABEL: define void @nonstreaming_caller_single_streaming_compatible_callee_alwaysinline +; CHECK-SAME: () #[[ATTR6:[0-9]+]] { ; CHECK-NEXT: call void @streaming_compatible_body() ; CHECK-NEXT: ret void ; - call void @nonstreaming_caller_single_streaming_compatible_callee() + call void @streaming_compatible_body() ret void } -; Allow inlining, as inline would remove several streaming-mode changes. -define void @streaming_caller_to_nonstreaming_callee_with_multiple_streamingcompatible_callees_inline() #0 "aarch64_pstate_sm_enabled" { -; CHECK-LABEL: define void @streaming_caller_to_nonstreaming_callee_with_multiple_streamingcompatible_callees_inline +; Conseratively disallow inlining when forced; it is unclear what the user's intentions were. +define void @streaming_caller_to_nonstreaming_callee_with_single_streamingcompatible_callee_inline() "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define void @streaming_caller_to_nonstreaming_callee_with_single_streamingcompatible_callee_inline ; CHECK-SAME: () #[[ATTR2]] { ; CHECK-NEXT: call void @streaming_compatible_body() -; CHECK-NEXT: call void @streaming_compatible_body() ; CHECK-NEXT: ret void ; - call void @nonstreaming_caller_multiple_streaming_compatible_callees() + call void @nonstreaming_caller_single_streaming_compatible_callee_alwaysinline() ret void } -define void @simple_streaming_function(ptr %ptr) #0 "aarch64_pstate_sm_enabled" { +define void @simple_streaming_function(ptr %ptr) "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define void @simple_streaming_function ; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR2]] { ; CHECK-NEXT: store <vscale x 4 x i32> zeroinitializer, ptr [[PTR]], align 16 @@ -687,7 +684,7 @@ define void @simple_streaming_function(ptr %ptr) #0 "aarch64_pstate_sm_enabled" } ; Don't allow inlining a streaming function into a non-streaming function. -define void @non_streaming_caller_streaming_callee_dont_inline(ptr %ptr) #0 { +define void @non_streaming_caller_streaming_callee_dont_inline(ptr %ptr) { ; CHECK-LABEL: define void @non_streaming_caller_streaming_callee_dont_inline ; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR1]] { ; CHECK-NEXT: call void @simple_streaming_function(ptr [[PTR]]) @@ -697,7 +694,7 @@ define void @non_streaming_caller_streaming_callee_dont_inline(ptr %ptr) #0 { ret void } -define void @simple_locally_streaming_function(ptr %ptr) #0 "aarch64_pstate_sm_body" { +define void @simple_locally_streaming_function(ptr %ptr) "aarch64_pstate_sm_body" { ; CHECK-LABEL: define void @simple_locally_streaming_function ; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR3]] { ; CHECK-NEXT: store <vscale x 4 x i32> zeroinitializer, ptr [[PTR]], align 16 @@ -708,7 +705,7 @@ define void @simple_locally_streaming_function(ptr %ptr) #0 "aarch64_pstate_sm_b } ; Don't allow inlining a locally-streaming function into a non-streaming function. -define void @non_streaming_caller_locally_streaming_callee_dont_inline(ptr %ptr) #0 { +define void @non_streaming_caller_locally_streaming_callee_dont_inline(ptr %ptr) { ; CHECK-LABEL: define void @non_streaming_caller_locally_streaming_callee_dont_inline ; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR1]] { ; CHECK-NEXT: call void @simple_locally_streaming_function(ptr [[PTR]]) @@ -718,4 +715,282 @@ define void @non_streaming_caller_locally_streaming_callee_dont_inline(ptr %ptr) ret void } -attributes #0 = { "target-features"="+sve,+sme" } +; +; Don't inline when there are vscale-dependent operations +; + +; It is not safe to inline functions that have vscale-dependent operations in their body +; when the streaming modes don't match up, unless the interface of the callee takes a +; vl-dependent argument. +define ptr @vscale_dependent_op(ptr %p, i64 %k) { +; CHECK-LABEL: define ptr @vscale_dependent_op +; CHECK-SAME: (ptr [[P:%.*]], i64 [[K:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: [[RES:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P]], i64 [[K]] +; CHECK-NEXT: ret ptr [[RES]] +; + %res = getelementptr <vscale x 4 x i32>, ptr %p, i64 %k + ret ptr %res +} + +define ptr @incompatible_vscale_dependent_operation_sm(ptr %p) "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define ptr @incompatible_vscale_dependent_operation_sm +; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR2]] { +; CHECK-NEXT: [[RES_I:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P]], i64 4 +; CHECK-NEXT: ret ptr [[RES_I]] +; + %res = call ptr @vscale_dependent_op(ptr %p, i64 4) + ret ptr %res +} + +define ptr @vscale_dependent_op_vl_dependent_args(ptr %p, i64 %k, <vscale x 4 x i32> %other) { +; CHECK-LABEL: define ptr @vscale_dependent_op_vl_dependent_args +; CHECK-SAME: (ptr [[P:%.*]], i64 [[K:%.*]], <vscale x 4 x i32> [[OTHER:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: [[RES:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P]], i64 [[K]] +; CHECK-NEXT: store <vscale x 4 x i32> [[OTHER]], ptr [[RES]], align 16 +; CHECK-NEXT: ret ptr [[RES]] +; + %res = getelementptr <vscale x 4 x i32>, ptr %p, i64 %k + store <vscale x 4 x i32> %other, ptr %res + ret ptr %res +} + +define ptr @compatible_vscale_dependent_operation_sm(ptr %p) "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define ptr @compatible_vscale_dependent_operation_sm +; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR2]] { +; CHECK-NEXT: [[RES_I:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P]], i64 4 +; CHECK-NEXT: store <vscale x 4 x i32> zeroinitializer, ptr [[RES_I]], align 16 +; CHECK-NEXT: ret ptr [[RES_I]] +; + %res = call ptr @vscale_dependent_op_vl_dependent_args(ptr %p, i64 4, <vscale x 4 x i32> zeroinitializer) + ret ptr %res +} + +; functions with scalable alloca's shouldn't be inlined if the streaming properties don't match +; as a scalable alloca is a vl-dependent operation. +define void @scalable_alloca_op(ptr %p) { +; CHECK-LABEL: define void @scalable_alloca_op +; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: [[ALLOCA:%.*]] = alloca <vscale x 4 x i32>, align 1 +; CHECK-NEXT: call void [[P]](ptr [[ALLOCA]]) +; CHECK-NEXT: ret void +; + %alloca = alloca <vscale x 4 x i32>, align 1 + call void %p(ptr %alloca) + ret void +} + +define void @incompatible_scalable_alloca_op_caller(ptr %p) "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define void @incompatible_scalable_alloca_op_caller +; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR2]] { +; CHECK-NEXT: [[ALLOCA_I:%.*]] = alloca <vscale x 4 x i32>, align 1 +; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[ALLOCA_I]]) +; CHECK-NEXT: call void [[P]](ptr [[ALLOCA_I]]), !inline_history [[META0:![0-9]+]] +; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[ALLOCA_I]]) +; CHECK-NEXT: ret void +; + call void @scalable_alloca_op(ptr %p) + ret void +} + +; Generic intrinsics that take/return a scalable type, are vscale-dependent operations. +define i64 @intrinsic_with_scalable_type(ptr %p) { +; CHECK-LABEL: define i64 @intrinsic_with_scalable_type +; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: [[LD:%.*]] = load <vscale x 2 x i64>, ptr [[P]], align 16 +; CHECK-NEXT: [[RES:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[LD]]) +; CHECK-NEXT: ret i64 [[RES]] +; + %ld = load <vscale x 2 x i64>, ptr %p + %res = call i64 @llvm.vector.reduce.add(<vscale x 2 x i64> %ld) + ret i64 %res +} + +define i64 @compatible_sve_intrinsic_caller(ptr %p) { +; CHECK-LABEL: define i64 @compatible_sve_intrinsic_caller +; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: [[LD_I:%.*]] = load <vscale x 2 x i64>, ptr [[P]], align 16 +; CHECK-NEXT: [[RES_I:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[LD_I]]) +; CHECK-NEXT: ret i64 [[RES_I]] +; + %res = call i64 @intrinsic_with_scalable_type(ptr %p) + ret i64 %res +} + +define i64 @incompatible_sve_intrinsic_caller(ptr %p) "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define i64 @incompatible_sve_intrinsic_caller +; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR2]] { +; CHECK-NEXT: [[RES:%.*]] = call i64 @intrinsic_with_scalable_type(ptr [[P]]) +; CHECK-NEXT: ret i64 [[RES]] +; + %res = call i64 @intrinsic_with_scalable_type(ptr %p) + ret i64 %res +} + +; +; Don't inline inline-asm when streaming-modes are incompatible, as the asm may +; contain vscale-dependent instructions. +; + +define void @inline_asm() { +; CHECK-LABEL: define void @inline_asm +; CHECK-SAME: () #[[ATTR1]] { +; CHECK-NEXT: call void asm sideeffect "", ""() +; CHECK-NEXT: ret void +; + call void asm sideeffect "", ""() + ret void +} + +define void @incompatible_inline_asm_sm() "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define void @incompatible_inline_asm_sm +; CHECK-SAME: () #[[ATTR2]] { +; CHECK-NEXT: call void @inline_asm() +; CHECK-NEXT: ret void +; + call void @inline_asm() + ret void +} + +; +; Don't inline functions that contain calls to SME ABI routines (like __arm_get_current_vg()) +; + +declare i64 @__arm_get_current_vg() + +define i64 @current_vg() { +; CHECK-LABEL: define i64 @current_vg +; CHECK-SAME: () #[[ATTR1]] { +; CHECK-NEXT: [[VSCALE:%.*]] = call i64 @__arm_get_current_vg() +; CHECK-NEXT: ret i64 [[VSCALE]] +; + %vscale = call i64 @__arm_get_current_vg() + ret i64 %vscale +} + +define i64 @compatible_current_vg_caller() { +; CHECK-LABEL: define i64 @compatible_current_vg_caller +; CHECK-SAME: () #[[ATTR1]] { +; CHECK-NEXT: [[VSCALE_I:%.*]] = call i64 @__arm_get_current_vg() +; CHECK-NEXT: ret i64 [[VSCALE_I]] +; + %vscale = call i64 @current_vg() + ret i64 %vscale +} + +define i64 @incompatible_current_vg_caller() "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define i64 @incompatible_current_vg_caller +; CHECK-SAME: () #[[ATTR2]] { +; CHECK-NEXT: [[VSCALE:%.*]] = call i64 @current_vg() +; CHECK-NEXT: ret i64 [[VSCALE]] +; + %vscale = call i64 @current_vg() + ret i64 %vscale +} + +; +; Don't inline NEON intrinsics into streaming mode. +; + +define i32 @neon_intrinsic(<4 x i32> %in) { +; CHECK-LABEL: define i32 @neon_intrinsic +; CHECK-SAME: (<4 x i32> [[IN:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: [[RES:%.*]] = call i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> [[IN]]) +; CHECK-NEXT: ret i32 [[RES]] +; + %res = call i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> %in) + ret i32 %res +} + +define i32 @incompatible_neon_intrinsic_caller(<4 x i32> %in) "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define i32 @incompatible_neon_intrinsic_caller +; CHECK-SAME: (<4 x i32> [[IN:%.*]]) #[[ATTR2]] { +; CHECK-NEXT: [[VSCALE:%.*]] = call i32 @neon_intrinsic(<4 x i32> [[IN]]) +; CHECK-NEXT: ret i32 [[VSCALE]] +; + %vscale = call i32 @neon_intrinsic(<4 x i32> %in) + ret i32 %vscale +} + +; a bit of a niche case, but if the caller uses ZA but is not in streaming-mode, a NEON intrinsic is safe. +define i32 @compatible_neon_intrinsic_caller_za(<4 x i32> %in) "aarch64_pstate_za_enabled" { +; CHECK-LABEL: define i32 @compatible_neon_intrinsic_caller_za +; CHECK-SAME: (<4 x i32> [[IN:%.*]]) #[[ATTR7:[0-9]+]] { +; CHECK-NEXT: [[RES_I:%.*]] = call i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> [[IN]]) +; CHECK-NEXT: ret i32 [[RES_I]] +; + %vscale = call i32 @neon_intrinsic(<4 x i32> %in) + ret i32 %vscale +} + +; +; Don't inline fixed-length vector code, as performance may be affected. +; + +; Functions with fixed-length vectors shouldn't be inlined if the streaming properties don't match +; as performance may be affected. However, when they have the alwaysinline property, they should +; still be inlined. + +define void @fixed_length_vector_operation_alwaysinline(ptr %p) alwaysinline { +; CHECK-LABEL: define void @fixed_length_vector_operation_alwaysinline +; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR6]] { +; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[P]], align 16 +; CHECK-NEXT: ret void +; + store <4 x i32> zeroinitializer, ptr %p + ret void +} + +define void @fixed_length_vector_operation_caller_force_inline(ptr %p) "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define void @fixed_length_vector_operation_caller_force_inline +; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR2]] { +; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[P]], align 16 +; CHECK-NEXT: ret void +; + call void @fixed_length_vector_operation_alwaysinline(ptr %p) + ret void +} + +define void @fixed_length_vector_operation(ptr %p) { +; CHECK-LABEL: define void @fixed_length_vector_operation +; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[P]], align 16 +; CHECK-NEXT: ret void +; + store <4 x i32> zeroinitializer, ptr %p + ret void +} + +define void @fixed_length_vector_operation_caller_dont_inline(ptr %p) "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define void @fixed_length_vector_operation_caller_dont_inline +; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR2]] { +; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[P]], align 16 +; CHECK-NEXT: ret void +; + call void @fixed_length_vector_operation(ptr %p) + ret void +} + +; +; strictfp and streaming-mode +; + +define float @strict_fp(float %in) strictfp { +; CHECK-LABEL: define float @strict_fp +; CHECK-SAME: (float [[IN:%.*]]) #[[ATTR8:[0-9]+]] { +; CHECK-NEXT: [[RES:%.*]] = fadd float [[IN]], 4.200000e+01 +; CHECK-NEXT: ret float [[RES]] +; + %res = fadd float %in, 42.0; + ret float %res +} + +; floating point environment is different in streaming mode, so don't inline. +define float @incompatible_fp_environment_sm(float %in) strictfp "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define float @incompatible_fp_environment_sm +; CHECK-SAME: (float [[IN:%.*]]) #[[ATTR9:[0-9]+]] { +; CHECK-NEXT: [[RES_I:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[IN]], float 4.200000e+01, metadata !"round.tonearest", metadata !"fpexcept.ignore") #[[ATTR14:[0-9]+]] +; CHECK-NEXT: ret float [[RES_I]] +; + %res = call float @strict_fp(float %in) + ret float %res +} diff --git a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-inline-transitive.ll b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-inline-transitive.ll new file mode 100644 index 000000000000000..21242b304591de7 --- /dev/null +++ b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-inline-transitive.ll @@ -0,0 +1,49 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 +; RUN: opt -S -passes=inline -inlinedefault-threshold=1 < %s | FileCheck %s --check-prefixes=CHECK,TH1 +; RUN: opt -S -passes=inline -inlinedefault-threshold=25 < %s | FileCheck %s --check-prefixes=CHECK,TH25 + +target triple = "aarch64" + +declare void @streaming_f() "aarch64_pstate_sm_enabled" + +define void @c() #0 "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define void @c +; CHECK-SAME: () #[[ATTR1:[0-9]+]] { +; CHECK-NEXT: call void @streaming_f() +; CHECK-NEXT: call void @streaming_f() +; CHECK-NEXT: ret void +; + call void @streaming_f() + call void @streaming_f() + ret void +} + +; Don't inline the call to @c, as inlining a streaming function into a non-streaming function is considered incompatible. +define void @b() #0 { +; CHECK-LABEL: define void @b +; CHECK-SAME: () #[[ATTR2:[0-9]+]] { +; CHECK-NEXT: call void @c() +; CHECK-NEXT: ret void +; + call void @c() + ret void +} + +; Inline the call to @c in @a, as that avoids streaming mode changes. +define void @a() #0 "aarch64_pstate_sm_enabled" { +; TH1-LABEL: define void @a +; TH1-SAME: () #[[ATTR1]] { +; TH1-NEXT: call void @c() +; TH1-NEXT: ret void +; +; TH25-LABEL: define void @a +; TH25-SAME: () #[[ATTR1]] { +; TH25-NEXT: call void @streaming_f() +; TH25-NEXT: call void @streaming_f() +; TH25-NEXT: ret void +; + call void @b() + ret void +} + +attributes #0 = { "target-features"="+sve,+sme" } >From 69d50be077d2ce512edd3a8e72b98ef012e73d3f Mon Sep 17 00:00:00 2001 From: Sander de Smalen <[email protected]> Date: Mon, 14 Sep 2026 08:05:19 +0000 Subject: [PATCH 2/2] [AArch64][SME] Allow more inlining when SME attributes are incompatible. At the moment, 'areInlineCompatible' is very strict as it conservatively disallows inlining any callee if they use intrinsics and have incompatible SME attributes. This PR relaxes those constraints by allowing more intrinsics. It also updates the Clang diagnostic to match the 'new' behaviour that a function is no longer inlined despite 'always_inline' when they are not inline compatible. --- .../clang/Basic/DiagnosticFrontendKinds.td | 2 +- .../AArch64/sme-inline-streaming-attrs.c | 6 +- .../AArch64/AArch64TargetTransformInfo.cpp | 161 ++++++++++++++++-- .../sme-pstatesm-attrs-low-threshold.ll | 47 ----- .../Inline/AArch64/sme-pstatesm-attrs.ll | 19 +-- 5 files changed, 158 insertions(+), 77 deletions(-) delete mode 100644 llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs-low-threshold.ll diff --git a/clang/include/clang/Basic/DiagnosticFrontendKinds.td b/clang/include/clang/Basic/DiagnosticFrontendKinds.td index 031858610ede2fd..e88031a13c7fcd0 100644 --- a/clang/include/clang/Basic/DiagnosticFrontendKinds.td +++ b/clang/include/clang/Basic/DiagnosticFrontendKinds.td @@ -317,7 +317,7 @@ def err_function_always_inline_attribute_mismatch : Error< "always_inline function %1 and its caller %0 have mismatching %2 attributes">; def warn_function_always_inline_attribute_mismatch : Warning< "always_inline function %1 and its caller %0 have mismatching %2 attributes, " - "inlining may change runtime behaviour">, InGroup<AArch64SMEAttributes>; + "%1 may not be inlined">, InGroup<AArch64SMEAttributes>; def err_function_always_inline_new_za : Error< "always_inline function %0 has new za state">; def err_function_always_inline_new_zt0 diff --git a/clang/test/CodeGen/AArch64/sme-inline-streaming-attrs.c b/clang/test/CodeGen/AArch64/sme-inline-streaming-attrs.c index 68102c9ded40c4a..ba9f24032d2745a 100644 --- a/clang/test/CodeGen/AArch64/sme-inline-streaming-attrs.c +++ b/clang/test/CodeGen/AArch64/sme-inline-streaming-attrs.c @@ -26,7 +26,7 @@ void caller(void) { #ifdef TEST_COMPATIBLE void caller_compatible(void) __arm_streaming_compatible { - inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_compatible' have mismatching streaming attributes, inlining may change runtime behaviour}} + inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_compatible' have mismatching streaming attributes, 'inlined_fn' may not be inlined}} inlined_fn_streaming_compatible(); inlined_fn_streaming(); // expected-error {{always_inline function 'inlined_fn_streaming' and its caller 'caller_compatible' have mismatching streaming attributes}} inlined_fn_local(); // expected-error {{always_inline function 'inlined_fn_local' and its caller 'caller_compatible' have mismatching streaming attributes}} @@ -35,7 +35,7 @@ void caller_compatible(void) __arm_streaming_compatible { #ifdef TEST_STREAMING void caller_streaming(void) __arm_streaming { - inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_streaming' have mismatching streaming attributes, inlining may change runtime behaviour}} + inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_streaming' have mismatching streaming attributes, 'inlined_fn' may not be inlined}} inlined_fn_streaming_compatible(); inlined_fn_streaming(); inlined_fn_local(); @@ -45,7 +45,7 @@ void caller_streaming(void) __arm_streaming { #ifdef TEST_LOCALLY __arm_locally_streaming void caller_local(void) { - inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_local' have mismatching streaming attributes, inlining may change runtime behaviour}} + inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_local' have mismatching streaming attributes, 'inlined_fn' may not be inlined}} inlined_fn_streaming_compatible(); inlined_fn_streaming(); inlined_fn_local(); diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index ddea78fa4e80839..19a9b07389c6aec 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -237,23 +237,100 @@ static bool isSMEABIRoutineCall(const CallInst &CI, SMEAttrs(F->getName(), TLI.getRuntimeLibcallsInfo()).isSMEABIRoutine(); } +/// Returns true if \p I is an intrinsic that may not be compatible with a +/// different streaming mode (because it depends on vscale). +static bool isPossiblyIncompatibleIntrinsic(const Instruction *I) { + if (I->isDebugOrPseudoInst()) + return false; + + if (auto *II = dyn_cast<IntrinsicInst>(I)) { + unsigned IID = II->getIntrinsicID(); + switch (IID) { + default: + return Intrinsic::isTargetIntrinsic(IID); + case Intrinsic::vscale: + case Intrinsic::masked_gather: + case Intrinsic::masked_scatter: + return true; + } + } + + return false; +} + /// Returns true if the function has explicit operations that can only be /// lowered using incompatible instructions for the selected mode. This also /// returns true if the function F may use or modify ZA state. static bool hasPossibleIncompatibleOps(const Function *F, - const AArch64TargetLowering &TLI) { + const AArch64TargetLowering &TLI, + bool ConsiderZA, bool ConsiderZT, + bool ConsiderSM) { + assert((ConsiderZA || ConsiderZT || ConsiderSM) && + "No SME state to consider"); + + bool IsAlwaysInline = F->hasFnAttribute(Attribute::AlwaysInline); + bool HasVLDependentArgsOrRet = + F->getReturnType()->isScalableTy() || + any_of(F->getFunctionType()->params(), + [](const Type *T) { return T->isScalableTy(); }); + for (const BasicBlock &BB : *F) { for (const Instruction &I : BB) { - // Be conservative for now and assume that any call to inline asm or to - // intrinsics could could result in non-streaming ops (e.g. calls to - // @llvm.aarch64.* or @llvm.gather/scatter intrinsics). We can assume that - // all native LLVM instructions can be lowered to compatible instructions. - if (isa<CallInst>(I) && !I.isDebugOrPseudoInst() && - (cast<CallInst>(I).isInlineAsm() || isa<IntrinsicInst>(I) || - isSMEABIRoutineCall(cast<CallInst>(I), TLI))) + // Inlining operations on fixed-length vectors when the streaming + // mode does not match, is rejected because performance may be impacted. + // This decision should eventually be moved the cost-model. + if (!IsAlwaysInline && ConsiderSM && + (isa<FixedVectorType>(I.getType()) || + any_of(I.operand_values(), [](const Value *V) { + return isa<FixedVectorType>(V->getType()); + }))) + return true; + + // Inlining operations on scalable vectors is rejected because it is + // a vscale-dependent operation. The only exception is when the interface + // already has vscale-dependent arguments/return value, as the ACLE + // describes that in order for the program to have defined behaviour is + // for vscale to match in both modes. + if (ConsiderSM && !HasVLDependentArgsOrRet) { + if (I.getType()->isScalableTy() || + any_of( + I.operand_values(), + [](const Value *V) { return V->getType()->isScalableTy(); }) || + (isa<GetElementPtrInst>(I) && cast<GetElementPtrInst>(I) + .getSourceElementType() + ->isScalableTy()) || + (isa<AllocaInst>(I) && cast<AllocaInst>(I).isScalable())) + return true; + } + + auto *CB = dyn_cast<CallBase>(&I); + if (!CB) + continue; + + // Inline asm must be rejected as it could use SME state. + if (CB->isInlineAsm()) return true; + + if (auto *CI = dyn_cast<CallInst>(&I)) { + // If the callee has calls to streaming compatible functions, then those + // may have vl-dependent statements. Be cautious about inlining such + // calls, as the streaming-compatible calls would otherwise be executed + // in a different streaming mode. + if (ConsiderSM && CI->getCalledFunction()) { + SMEAttrs CalleeAttrs(*CI->getCalledFunction()); + if (CalleeAttrs.hasStreamingCompatibleInterface()) + return true; + } + + if (isSMEABIRoutineCall(*CI, TLI)) + return true; + + if (ConsiderSM && isPossiblyIncompatibleIntrinsic(&I)) + return true; + } } } + return false; } @@ -281,6 +358,51 @@ bool AArch64TTIImpl::isMultiversionedFunction(const Function &F) const { return F.hasFnAttribute("fmv-features"); } +/// The compiler must not inline when that may alter the behavior of the +/// program. This is especially relevant around SME which implements different +/// runtime modes and maintains external state through attributes. +/// +/// The compiler must not inline when: +// +/// * The module is compiled with 'strict-fp' and the called function +/// implements a different FP environment than the caller. +/// +/// * The called function has operations that are incompatible in the mode +/// of the caller, e.g.: +/// * inlining non-streaming-only instructions into a streaming function. +/// * inlining streaming-only instructions into a non-streaming function. +/// +/// Inline asm blocks must be entered and exited in the [streaming] mode of +/// the parent function. There is no language-level mechanism to inform the +/// compiler that a particular inline asm block is streaming compatible, so +/// the compiler must reject this as a candidate for inlining. +/// +/// * The called function contains vscale-dependent operations but otherwise +/// does not take/return VL-dependent arguments (see definition in the +/// ACLE (Arm C/C++ Language Extensions)). +/// +/// * The called function sets up new ZA/ZT state into a function that already +/// has ZA or ZT state, as that is not valid as per the ACLE. +/// +/// If the called function has an `alwaysinline` attribute and any of the above +/// conditions is true, then an error should be reported. +/// +/// The compiler should not inline when: +// +/// * The called function has fixed-length vectors and the caller is in +/// streaming mode, as this may cause performance regressions. This should +/// never result in an error to be reported. +/// +/// * The called function sets up new ZA/ZT state into a function that has no +/// ZA and no ZT state, as the compiler currently cannot transfer the +/// attribute to the caller. It may also impact performance, but that should +/// be covered by AArch64TTIImpl::getInlineCallPenalty(). +/// +/// If a callee is incompatible with its caller, then the inliner should prevent +/// inlining. LLVM's `alwaysinline` attribute has no impact on whether a callee +/// is inline compatible. This is especially relevant with streaming-mode +/// attributes, where `alwaysinline`ing a function into a function with a +/// different streaming mode may otherwise impact runtime behaviour. bool AArch64TTIImpl::areInlineCompatible(const Function *Caller, const Function *Callee) const { SMECallAttrs CallAttrs(*Caller, *Callee); @@ -292,6 +414,9 @@ bool AArch64TTIImpl::areInlineCompatible(const Function *Caller, CallAttrs.callee().hasStreamingInterfaceOrBody()) return false; + if (CallAttrs.callee().isNewZA() || CallAttrs.callee().isNewZT0()) + return false; + // When inlining, we should consider the body of the function, not the // interface. if (CallAttrs.callee().hasStreamingBody()) { @@ -299,15 +424,21 @@ bool AArch64TTIImpl::areInlineCompatible(const Function *Caller, CallAttrs.callee().set(SMEAttrs::SM_Enabled, true); } - if (CallAttrs.callee().isNewZA() || CallAttrs.callee().isNewZT0()) + bool ConsiderZA = CallAttrs.requiresZASave(); + bool ConsiderZT = CallAttrs.requiresPreservingZT0() || + CallAttrs.requiresPreservingAllZAState(); + bool ConsiderSM = CallAttrs.requiresSMChange(); + + // FP environment is interpreted differently between streaming mode and + // non-streaming mode, so are not inline-compatible. + // FIXME: Analyze whether the callee actually has any FP operations. + if (ConsiderSM && (Caller->isStrictFP() || Callee->isStrictFP())) return false; - if (CallAttrs.requiresLazySave() || CallAttrs.requiresSMChange() || - CallAttrs.requiresPreservingZT0() || - CallAttrs.requiresPreservingAllZAState()) { - if (hasPossibleIncompatibleOps(Callee, *getTLI())) - return false; - } + if ((ConsiderZA || ConsiderZT || ConsiderSM) && + hasPossibleIncompatibleOps(Callee, *getTLI(), ConsiderZA, ConsiderZT, + ConsiderSM)) + return false; return BaseT::areInlineCompatible(Caller, Callee); } diff --git a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs-low-threshold.ll b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs-low-threshold.ll deleted file mode 100644 index 8a608a1b8e156cf..000000000000000 --- a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs-low-threshold.ll +++ /dev/null @@ -1,47 +0,0 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 -; RUN: opt < %s -mtriple=aarch64-unknown-linux-gnu -mattr=+sme -S -passes=inline -inlinedefault-threshold=1 | FileCheck %s - -; This test sets the inline-threshold to 1 such that by default the call to @streaming_callee is not inlined. -; However, if the call to @streaming_callee requires a streaming-mode change, it should always inline the call because the streaming-mode change is more expensive. -target triple = "aarch64" - -declare void @streaming_compatible_f() #0 "aarch64_pstate_sm_compatible" - -; Function @non_streaming_callee doesn't contain any operations that may use ZA -; state and therefore can be legally inlined into a normal function. -define void @non_streaming_callee() #0 { -; CHECK-LABEL: define void @non_streaming_callee -; CHECK-SAME: () #[[ATTR1:[0-9]+]] { -; CHECK-NEXT: call void @streaming_compatible_f() -; CHECK-NEXT: call void @streaming_compatible_f() -; CHECK-NEXT: ret void -; - call void @streaming_compatible_f() - call void @streaming_compatible_f() - ret void -} - -; Inline call to @non_streaming_callee to remove a streaming mode change. -define void @streaming_caller_inline() #0 "aarch64_pstate_sm_enabled" { -; CHECK-LABEL: define void @streaming_caller_inline -; CHECK-SAME: () #[[ATTR2:[0-9]+]] { -; CHECK-NEXT: call void @streaming_compatible_f() -; CHECK-NEXT: call void @streaming_compatible_f() -; CHECK-NEXT: ret void -; - call void @non_streaming_callee() - ret void -} - -; Don't inline call to @non_streaming_callee when the inline-threshold is set to 1, because it does not eliminate a streaming-mode change. -define void @non_streaming_caller_dont_inline() #0 { -; CHECK-LABEL: define void @non_streaming_caller_dont_inline -; CHECK-SAME: () #[[ATTR1]] { -; CHECK-NEXT: call void @non_streaming_callee() -; CHECK-NEXT: ret void -; - call void @non_streaming_callee() - ret void -} - -attributes #0 = { "target-features"="+sve,+sme" } diff --git a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll index 49af107d7e89b41..a55d52b970e1785 100644 --- a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll +++ b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll @@ -645,7 +645,7 @@ define void @nonstreaming_caller_single_streaming_compatible_callee() { define void @streaming_caller_to_nonstreaming_callee_with_single_streamingcompatible_callee_dont_inline() "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define void @streaming_caller_to_nonstreaming_callee_with_single_streamingcompatible_callee_dont_inline ; CHECK-SAME: () #[[ATTR2]] { -; CHECK-NEXT: call void @streaming_compatible_body() +; CHECK-NEXT: call void @nonstreaming_caller_single_streaming_compatible_callee() ; CHECK-NEXT: ret void ; call void @nonstreaming_caller_single_streaming_compatible_callee() @@ -666,7 +666,7 @@ define void @nonstreaming_caller_single_streaming_compatible_callee_alwaysinline define void @streaming_caller_to_nonstreaming_callee_with_single_streamingcompatible_callee_inline() "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define void @streaming_caller_to_nonstreaming_callee_with_single_streamingcompatible_callee_inline ; CHECK-SAME: () #[[ATTR2]] { -; CHECK-NEXT: call void @streaming_compatible_body() +; CHECK-NEXT: call void @nonstreaming_caller_single_streaming_compatible_callee_alwaysinline() ; CHECK-NEXT: ret void ; call void @nonstreaming_caller_single_streaming_compatible_callee_alwaysinline() @@ -735,8 +735,8 @@ define ptr @vscale_dependent_op(ptr %p, i64 %k) { define ptr @incompatible_vscale_dependent_operation_sm(ptr %p) "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define ptr @incompatible_vscale_dependent_operation_sm ; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR2]] { -; CHECK-NEXT: [[RES_I:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P]], i64 4 -; CHECK-NEXT: ret ptr [[RES_I]] +; CHECK-NEXT: [[RES:%.*]] = call ptr @vscale_dependent_op(ptr [[P]], i64 4) +; CHECK-NEXT: ret ptr [[RES]] ; %res = call ptr @vscale_dependent_op(ptr %p, i64 4) ret ptr %res @@ -782,10 +782,7 @@ define void @scalable_alloca_op(ptr %p) { define void @incompatible_scalable_alloca_op_caller(ptr %p) "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define void @incompatible_scalable_alloca_op_caller ; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR2]] { -; CHECK-NEXT: [[ALLOCA_I:%.*]] = alloca <vscale x 4 x i32>, align 1 -; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[ALLOCA_I]]) -; CHECK-NEXT: call void [[P]](ptr [[ALLOCA_I]]), !inline_history [[META0:![0-9]+]] -; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[ALLOCA_I]]) +; CHECK-NEXT: call void @scalable_alloca_op(ptr [[P]]) ; CHECK-NEXT: ret void ; call void @scalable_alloca_op(ptr %p) @@ -963,7 +960,7 @@ define void @fixed_length_vector_operation(ptr %p) { define void @fixed_length_vector_operation_caller_dont_inline(ptr %p) "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define void @fixed_length_vector_operation_caller_dont_inline ; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR2]] { -; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[P]], align 16 +; CHECK-NEXT: call void @fixed_length_vector_operation(ptr [[P]]) ; CHECK-NEXT: ret void ; call void @fixed_length_vector_operation(ptr %p) @@ -988,8 +985,8 @@ define float @strict_fp(float %in) strictfp { define float @incompatible_fp_environment_sm(float %in) strictfp "aarch64_pstate_sm_enabled" { ; CHECK-LABEL: define float @incompatible_fp_environment_sm ; CHECK-SAME: (float [[IN:%.*]]) #[[ATTR9:[0-9]+]] { -; CHECK-NEXT: [[RES_I:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[IN]], float 4.200000e+01, metadata !"round.tonearest", metadata !"fpexcept.ignore") #[[ATTR14:[0-9]+]] -; CHECK-NEXT: ret float [[RES_I]] +; CHECK-NEXT: [[RES:%.*]] = call float @strict_fp(float [[IN]]) +; CHECK-NEXT: ret float [[RES]] ; %res = call float @strict_fp(float %in) ret float %res _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
