https://github.com/daniel-donenfeld updated https://github.com/llvm/llvm-project/pull/205639
>From 51d3d250b5db6983f099ce33cd9a77c286e82e02 Mon Sep 17 00:00:00 2001 From: Daniel Donenfeld <[email protected]> Date: Tue, 2 Jun 2026 20:15:39 +0000 Subject: [PATCH 1/5] Bug fix for device function pointer having same name as a generated callprototype --- llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp | 2 +- llvm/lib/Target/NVPTX/NVPTXInstrInfo.td | 2 +- llvm/test/CodeGen/NVPTX/call_bitcast_byval.ll | 2 +- llvm/test/CodeGen/NVPTX/callchain.ll | 2 +- .../NVPTX/callprototype-local-label.ll | 17 ++++++++++ .../CodeGen/NVPTX/convert-call-to-indirect.ll | 16 +++++----- llvm/test/CodeGen/NVPTX/indirect_byval.ll | 8 ++--- .../CodeGen/NVPTX/lower-args-gridconstant.ll | 32 +++++++++---------- llvm/test/CodeGen/NVPTX/noreturn.ll | 4 +-- llvm/test/CodeGen/NVPTX/param-align.ll | 10 +++--- 10 files changed, 56 insertions(+), 39 deletions(-) create mode 100644 llvm/test/CodeGen/NVPTX/callprototype-local-label.ll diff --git a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp index 2486818493204..b91e3da92bf63 100644 --- a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp @@ -735,7 +735,7 @@ void NVPTXAsmPrinter::emitCallPrototype(const CallBase &CB, const auto PtrVT = TLI->getPointerTy(DL); Type *RetTy = CB.getFunctionType()->getReturnType(); - O << "prototype_" << UniqueCallSite << " : .callprototype "; + O << "$L__prototype_" << UniqueCallSite << " : .callprototype "; if (RetTy->isVoidTy() || RetTy->isEmptyTy()) { O << "()"; diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td index 4c3c7792f277b..99d9bb6af0628 100644 --- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td +++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td @@ -1940,7 +1940,7 @@ foreach is_convergent = [0, 1] in { NVPTXInst<(outs), (ins ADDR_base:$addr, CallOperand:$rets, CallOperand:$params, i32imm:$proto), - "call${rets:RetList} $addr, (${params:ParamList}), prototype_$proto;">; + "call${rets:RetList} $addr, (${params:ParamList}), $$L__prototype_$proto;">; def CALL_UNI # convergent_suffix : NVPTXInst<(outs), diff --git a/llvm/test/CodeGen/NVPTX/call_bitcast_byval.ll b/llvm/test/CodeGen/NVPTX/call_bitcast_byval.ll index 2a36c4162ba02..3cd64105b65c3 100644 --- a/llvm/test/CodeGen/NVPTX/call_bitcast_byval.ll +++ b/llvm/test/CodeGen/NVPTX/call_bitcast_byval.ll @@ -13,7 +13,7 @@ target triple = "nvptx64-nvidia-cuda" %"class.sycl::_V1::detail::half_impl::half" = type { half } %complex_half = type { half, half } -; CHECK: prototype_0 : .callprototype (.param .align 2 .b8 _[4]) _ (.param .b32 _, .param .b32 _, .param .align 2 .b8 _[4]); +; CHECK: $L__prototype_0 : .callprototype (.param .align 2 .b8 _[4]) _ (.param .b32 _, .param .b32 _, .param .align 2 .b8 _[4]); ; CHECK: .param .align 2 .b8 param2[4]; ; CHECK: .param .align 2 .b8 retval0[4]; ; CHECK-DAG: st.param.b16 [param2], %rs{{[0-9]+}}; diff --git a/llvm/test/CodeGen/NVPTX/callchain.ll b/llvm/test/CodeGen/NVPTX/callchain.ll index 847d8b80ad38a..be61a9fbe3291 100644 --- a/llvm/test/CodeGen/NVPTX/callchain.ll +++ b/llvm/test/CodeGen/NVPTX/callchain.ll @@ -4,7 +4,7 @@ target triple = "nvptx" define void @foo(ptr %ptr) { -; CHECK: prototype_0 : .callprototype ()_ () +; CHECK: $L__prototype_0 : .callprototype ()_ () tail call void %ptr() ret void } diff --git a/llvm/test/CodeGen/NVPTX/callprototype-local-label.ll b/llvm/test/CodeGen/NVPTX/callprototype-local-label.ll new file mode 100644 index 0000000000000..371282a28745c --- /dev/null +++ b/llvm/test/CodeGen/NVPTX/callprototype-local-label.ll @@ -0,0 +1,17 @@ +; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_75 | FileCheck %s +; RUN: %if ptxas-sm_75 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_75 | %ptxas-verify --compile-only -arch=sm_75 %} + +target triple = "nvptx64-nvidia-cuda" + +@prototype_0 = addrspace(1) global ptr null, align 8 + +define i32 @call_via_prototype_0(i32 %a, i32 %b, i32 %c, i32 %d) { +; CHECK-LABEL: call_via_prototype_0( +; CHECK: ld.global.{{u|b}}64 {{%rd[0-9]+}}, [prototype_0]; +; CHECK: $L__prototype_0 : .callprototype (.param .b32 _) _ (.param .b32 _, .param .b32 _, .param .b32 _, .param .b32 _); +; CHECK-NEXT: call (retval0), %rd{{[0-9]+}}, (param0, param1, param2, param3), $L__prototype_0; +; CHECK-NOT: prototype_0 : .callprototype + %fp = load ptr, ptr addrspace(1) @prototype_0, align 8 + %ret = call i32 %fp(i32 %a, i32 %b, i32 %c, i32 %d) + ret i32 %ret +} diff --git a/llvm/test/CodeGen/NVPTX/convert-call-to-indirect.ll b/llvm/test/CodeGen/NVPTX/convert-call-to-indirect.ll index 79be7ed2f18f1..e04c7fdef81df 100644 --- a/llvm/test/CodeGen/NVPTX/convert-call-to-indirect.ll +++ b/llvm/test/CodeGen/NVPTX/convert-call-to-indirect.ll @@ -10,7 +10,7 @@ define %struct.64 @test_return_type_mismatch(ptr %p) { ; CHECK-LABEL: test_return_type_mismatch( ; CHECK: { ; CHECK-NEXT: .reg .b64 %rd<32>; -; CHECK-NEXT: prototype_0 : .callprototype (.param .align 1 .b8 _[8]) _ (.param .b64 _); +; CHECK-NEXT: $L__prototype_0 : .callprototype (.param .align 1 .b8 _[8]) _ (.param .b64 _); ; CHECK-NEXT: // %bb.0: ; CHECK-NEXT: ld.param.b64 %rd1, [test_return_type_mismatch_param_0]; ; CHECK-NEXT: { // callseq 0, 0 @@ -18,7 +18,7 @@ define %struct.64 @test_return_type_mismatch(ptr %p) { ; CHECK-NEXT: .param .align 1 .b8 retval0[8]; ; CHECK-NEXT: st.param.b64 [param0], %rd1; ; CHECK-NEXT: mov.b64 %rd2, callee; -; CHECK-NEXT: call (retval0), %rd2, (param0), prototype_0; +; CHECK-NEXT: call (retval0), %rd2, (param0), $L__prototype_0; ; CHECK-NEXT: ld.param.b8 %rd3, [retval0+7]; ; CHECK-NEXT: ld.param.b8 %rd4, [retval0+6]; ; CHECK-NEXT: ld.param.b8 %rd5, [retval0+5]; @@ -66,14 +66,14 @@ define i64 @test_param_type_mismatch(ptr %p) { ; CHECK-LABEL: test_param_type_mismatch( ; CHECK: { ; CHECK-NEXT: .reg .b64 %rd<3>; -; CHECK-NEXT: prototype_1 : .callprototype (.param .b64 _) _ (.param .b64 _); +; CHECK-NEXT: $L__prototype_1 : .callprototype (.param .b64 _) _ (.param .b64 _); ; CHECK-NEXT: // %bb.0: ; CHECK-NEXT: { // callseq 1, 0 ; CHECK-NEXT: .param .b64 param0; ; CHECK-NEXT: .param .b64 retval0; ; CHECK-NEXT: st.param.b64 [param0], 7; ; CHECK-NEXT: mov.b64 %rd1, callee; -; CHECK-NEXT: call (retval0), %rd1, (param0), prototype_1; +; CHECK-NEXT: call (retval0), %rd1, (param0), $L__prototype_1; ; CHECK-NEXT: ld.param.b64 %rd2, [retval0]; ; CHECK-NEXT: } // callseq 1 ; CHECK-NEXT: st.param.b64 [func_retval0], %rd2; @@ -86,7 +86,7 @@ define i64 @test_param_count_mismatch(ptr %p) { ; CHECK-LABEL: test_param_count_mismatch( ; CHECK: { ; CHECK-NEXT: .reg .b64 %rd<4>; -; CHECK-NEXT: prototype_2 : .callprototype (.param .b64 _) _ (.param .b64 _, .param .b64 _); +; CHECK-NEXT: $L__prototype_2 : .callprototype (.param .b64 _) _ (.param .b64 _, .param .b64 _); ; CHECK-NEXT: // %bb.0: ; CHECK-NEXT: ld.param.b64 %rd1, [test_param_count_mismatch_param_0]; ; CHECK-NEXT: { // callseq 2, 0 @@ -96,7 +96,7 @@ define i64 @test_param_count_mismatch(ptr %p) { ; CHECK-NEXT: st.param.b64 [param0], %rd1; ; CHECK-NEXT: st.param.b64 [param1], 7; ; CHECK-NEXT: mov.b64 %rd2, callee; -; CHECK-NEXT: call (retval0), %rd2, (param0, param1), prototype_2; +; CHECK-NEXT: call (retval0), %rd2, (param0, param1), $L__prototype_2; ; CHECK-NEXT: ld.param.b64 %rd3, [retval0]; ; CHECK-NEXT: } // callseq 2 ; CHECK-NEXT: st.param.b64 [func_retval0], %rd3; @@ -112,7 +112,7 @@ define %struct.64 @test_return_type_mismatch_variadic(ptr %p) { ; CHECK-NEXT: .reg .b64 %SP; ; CHECK-NEXT: .reg .b64 %SPL; ; CHECK-NEXT: .reg .b64 %rd<33>; -; CHECK-NEXT: prototype_3 : .callprototype (.param .align 1 .b8 _[8]) _ (.param .b64 _, .param .b64 _); +; CHECK-NEXT: $L__prototype_3 : .callprototype (.param .align 1 .b8 _[8]) _ (.param .b64 _, .param .b64 _); ; CHECK-NEXT: // %bb.0: ; CHECK-NEXT: mov.b64 %SPL, __local_depot3; ; CHECK-NEXT: ld.param.b64 %rd1, [test_return_type_mismatch_variadic_param_0]; @@ -124,7 +124,7 @@ define %struct.64 @test_return_type_mismatch_variadic(ptr %p) { ; CHECK-NEXT: st.param.b64 [param1], %rd2; ; CHECK-NEXT: st.param.b64 [param0], %rd1; ; CHECK-NEXT: mov.b64 %rd3, callee_variadic; -; CHECK-NEXT: call (retval0), %rd3, (param0, param1), prototype_3; +; CHECK-NEXT: call (retval0), %rd3, (param0, param1), $L__prototype_3; ; CHECK-NEXT: ld.param.b8 %rd4, [retval0+7]; ; CHECK-NEXT: ld.param.b8 %rd5, [retval0+6]; ; CHECK-NEXT: ld.param.b8 %rd6, [retval0+5]; diff --git a/llvm/test/CodeGen/NVPTX/indirect_byval.ll b/llvm/test/CodeGen/NVPTX/indirect_byval.ll index 7a13bcd1f6674..a5a394386dc76 100644 --- a/llvm/test/CodeGen/NVPTX/indirect_byval.ll +++ b/llvm/test/CodeGen/NVPTX/indirect_byval.ll @@ -18,7 +18,7 @@ define internal i32 @foo() { ; CHECK-NEXT: .reg .b16 %rs<2>; ; CHECK-NEXT: .reg .b32 %r<2>; ; CHECK-NEXT: .reg .b64 %rd<4>; -; CHECK-NEXT: prototype_0 : .callprototype (.param .b32 _) _ (.param .align 1 .b8 _[1], .param .b64 _); +; CHECK-NEXT: $L__prototype_0 : .callprototype (.param .b32 _) _ (.param .align 1 .b8 _[1], .param .b64 _); ; CHECK-NEXT: // %bb.0: // %entry ; CHECK-NEXT: mov.b64 %SPL, __local_depot0; ; CHECK-NEXT: add.u64 %rd1, %SPL, 0; @@ -31,7 +31,7 @@ define internal i32 @foo() { ; CHECK-NEXT: st.param.b64 [param1], %rd2; ; CHECK-NEXT: ld.local.b8 %rs1, [%SPL+1]; ; CHECK-NEXT: st.param.b8 [param0], %rs1; -; CHECK-NEXT: call (retval0), %rd3, (param0, param1), prototype_0; +; CHECK-NEXT: call (retval0), %rd3, (param0, param1), $L__prototype_0; ; CHECK-NEXT: ld.param.b32 %r1, [retval0]; ; CHECK-NEXT: } // callseq 0 ; CHECK-NEXT: st.param.b32 [func_retval0], %r1; @@ -53,7 +53,7 @@ define internal i32 @bar() { ; CHECK-NEXT: .reg .b64 %SPL; ; CHECK-NEXT: .reg .b32 %r<2>; ; CHECK-NEXT: .reg .b64 %rd<5>; -; CHECK-NEXT: prototype_1 : .callprototype (.param .b32 _) _ (.param .align 8 .b8 _[8], .param .b64 _); +; CHECK-NEXT: $L__prototype_1 : .callprototype (.param .b32 _) _ (.param .align 8 .b8 _[8], .param .b64 _); ; CHECK-NEXT: // %bb.0: // %entry ; CHECK-NEXT: mov.b64 %SPL, __local_depot1; ; CHECK-NEXT: add.u64 %rd1, %SPL, 0; @@ -66,7 +66,7 @@ define internal i32 @bar() { ; CHECK-NEXT: st.param.b64 [param1], %rd2; ; CHECK-NEXT: ld.local.b64 %rd4, [%SPL+8]; ; CHECK-NEXT: st.param.b64 [param0], %rd4; -; CHECK-NEXT: call (retval0), %rd3, (param0, param1), prototype_1; +; CHECK-NEXT: call (retval0), %rd3, (param0, param1), $L__prototype_1; ; CHECK-NEXT: ld.param.b32 %r1, [retval0]; ; CHECK-NEXT: } // callseq 1 ; CHECK-NEXT: st.param.b32 [func_retval0], %r1; diff --git a/llvm/test/CodeGen/NVPTX/lower-args-gridconstant.ll b/llvm/test/CodeGen/NVPTX/lower-args-gridconstant.ll index 5b23f009fea6c..90e1e29410027 100644 --- a/llvm/test/CodeGen/NVPTX/lower-args-gridconstant.ll +++ b/llvm/test/CodeGen/NVPTX/lower-args-gridconstant.ll @@ -154,7 +154,7 @@ define ptx_kernel void @grid_const_escape(ptr byval(%struct.s) align 4 "nvvm.gri ; PTX-DEFAULT-LABEL: grid_const_escape( ; PTX-DEFAULT: { ; PTX-DEFAULT-NEXT: .reg .b64 %rd<4>; -; PTX-DEFAULT-NEXT: prototype_0 : .callprototype (.param .b32 _) _ (.param .b64 _); +; PTX-DEFAULT-NEXT: $L__prototype_0 : .callprototype (.param .b32 _) _ (.param .b64 _); ; PTX-DEFAULT-NEXT: // %bb.0: ; PTX-DEFAULT-NEXT: mov.b64 %rd1, grid_const_escape_param_0; ; PTX-DEFAULT-NEXT: cvta.param.u64 %rd2, %rd1; @@ -163,7 +163,7 @@ define ptx_kernel void @grid_const_escape(ptr byval(%struct.s) align 4 "nvvm.gri ; PTX-DEFAULT-NEXT: .param .b32 retval0; ; PTX-DEFAULT-NEXT: st.param.b64 [param0], %rd2; ; PTX-DEFAULT-NEXT: mov.b64 %rd3, escape; -; PTX-DEFAULT-NEXT: call (retval0), %rd3, (param0), prototype_0; +; PTX-DEFAULT-NEXT: call (retval0), %rd3, (param0), $L__prototype_0; ; PTX-DEFAULT-NEXT: } // callseq 0 ; PTX-DEFAULT-NEXT: ret; ; @@ -171,7 +171,7 @@ define ptx_kernel void @grid_const_escape(ptr byval(%struct.s) align 4 "nvvm.gri ; PTX-SHORT-PTR: { ; PTX-SHORT-PTR-NEXT: .reg .b32 %r<2>; ; PTX-SHORT-PTR-NEXT: .reg .b64 %rd<4>; -; PTX-SHORT-PTR-NEXT: prototype_0 : .callprototype (.param .b32 _) _ (.param .b64 _); +; PTX-SHORT-PTR-NEXT: $L__prototype_0 : .callprototype (.param .b32 _) _ (.param .b64 _); ; PTX-SHORT-PTR-NEXT: // %bb.0: ; PTX-SHORT-PTR-NEXT: mov.b32 %r1, grid_const_escape_param_0; ; PTX-SHORT-PTR-NEXT: cvt.u64.u32 %rd1, %r1; @@ -181,7 +181,7 @@ define ptx_kernel void @grid_const_escape(ptr byval(%struct.s) align 4 "nvvm.gri ; PTX-SHORT-PTR-NEXT: .param .b32 retval0; ; PTX-SHORT-PTR-NEXT: st.param.b64 [param0], %rd2; ; PTX-SHORT-PTR-NEXT: mov.b64 %rd3, escape; -; PTX-SHORT-PTR-NEXT: call (retval0), %rd3, (param0), prototype_0; +; PTX-SHORT-PTR-NEXT: call (retval0), %rd3, (param0), $L__prototype_0; ; PTX-SHORT-PTR-NEXT: } // callseq 0 ; PTX-SHORT-PTR-NEXT: ret; ; OPT-LABEL: define ptx_kernel void @grid_const_escape( @@ -201,7 +201,7 @@ define ptx_kernel void @multiple_grid_const_escape(ptr byval(%struct.s) align 4 ; PTX-DEFAULT-NEXT: .reg .b64 %SPL; ; PTX-DEFAULT-NEXT: .reg .b32 %r<2>; ; PTX-DEFAULT-NEXT: .reg .b64 %rd<8>; -; PTX-DEFAULT-NEXT: prototype_1 : .callprototype (.param .b32 _) _ (.param .b64 _, .param .b64 _, .param .b64 _); +; PTX-DEFAULT-NEXT: $L__prototype_1 : .callprototype (.param .b32 _) _ (.param .b64 _, .param .b64 _, .param .b64 _); ; PTX-DEFAULT-NEXT: // %bb.0: ; PTX-DEFAULT-NEXT: mov.b64 %SPL, __local_depot4; ; PTX-DEFAULT-NEXT: mov.b64 %rd1, multiple_grid_const_escape_param_0; @@ -221,7 +221,7 @@ define ptx_kernel void @multiple_grid_const_escape(ptr byval(%struct.s) align 4 ; PTX-DEFAULT-NEXT: st.param.b64 [param1], %rd6; ; PTX-DEFAULT-NEXT: st.param.b64 [param0], %rd4; ; PTX-DEFAULT-NEXT: mov.b64 %rd7, escape3; -; PTX-DEFAULT-NEXT: call (retval0), %rd7, (param0, param1, param2), prototype_1; +; PTX-DEFAULT-NEXT: call (retval0), %rd7, (param0, param1, param2), $L__prototype_1; ; PTX-DEFAULT-NEXT: } // callseq 1 ; PTX-DEFAULT-NEXT: ret; ; @@ -232,7 +232,7 @@ define ptx_kernel void @multiple_grid_const_escape(ptr byval(%struct.s) align 4 ; PTX-SHORT-PTR-NEXT: .reg .b32 %SPL; ; PTX-SHORT-PTR-NEXT: .reg .b32 %r<5>; ; PTX-SHORT-PTR-NEXT: .reg .b64 %rd<8>; -; PTX-SHORT-PTR-NEXT: prototype_1 : .callprototype (.param .b32 _) _ (.param .b64 _, .param .b64 _, .param .b64 _); +; PTX-SHORT-PTR-NEXT: $L__prototype_1 : .callprototype (.param .b32 _) _ (.param .b64 _, .param .b64 _, .param .b64 _); ; PTX-SHORT-PTR-NEXT: // %bb.0: ; PTX-SHORT-PTR-NEXT: mov.b32 %SPL, __local_depot4; ; PTX-SHORT-PTR-NEXT: mov.b32 %r1, multiple_grid_const_escape_param_0; @@ -255,7 +255,7 @@ define ptx_kernel void @multiple_grid_const_escape(ptr byval(%struct.s) align 4 ; PTX-SHORT-PTR-NEXT: st.param.b64 [param1], %rd6; ; PTX-SHORT-PTR-NEXT: st.param.b64 [param0], %rd4; ; PTX-SHORT-PTR-NEXT: mov.b64 %rd7, escape3; -; PTX-SHORT-PTR-NEXT: call (retval0), %rd7, (param0, param1, param2), prototype_1; +; PTX-SHORT-PTR-NEXT: call (retval0), %rd7, (param0, param1, param2), $L__prototype_1; ; PTX-SHORT-PTR-NEXT: } // callseq 1 ; PTX-SHORT-PTR-NEXT: ret; ; OPT-LABEL: define ptx_kernel void @multiple_grid_const_escape( @@ -362,7 +362,7 @@ define ptx_kernel void @grid_const_partial_escape(ptr byval(i32) align 4 "nvvm.g ; PTX-DEFAULT: { ; PTX-DEFAULT-NEXT: .reg .b32 %r<3>; ; PTX-DEFAULT-NEXT: .reg .b64 %rd<6>; -; PTX-DEFAULT-NEXT: prototype_2 : .callprototype (.param .b32 _) _ (.param .b64 _); +; PTX-DEFAULT-NEXT: $L__prototype_2 : .callprototype (.param .b32 _) _ (.param .b64 _); ; PTX-DEFAULT-NEXT: // %bb.0: ; PTX-DEFAULT-NEXT: mov.b64 %rd1, grid_const_partial_escape_param_0; ; PTX-DEFAULT-NEXT: ld.param.b64 %rd2, [grid_const_partial_escape_param_1]; @@ -376,7 +376,7 @@ define ptx_kernel void @grid_const_partial_escape(ptr byval(i32) align 4 "nvvm.g ; PTX-DEFAULT-NEXT: .param .b32 retval0; ; PTX-DEFAULT-NEXT: st.param.b64 [param0], %rd4; ; PTX-DEFAULT-NEXT: mov.b64 %rd5, escape; -; PTX-DEFAULT-NEXT: call (retval0), %rd5, (param0), prototype_2; +; PTX-DEFAULT-NEXT: call (retval0), %rd5, (param0), $L__prototype_2; ; PTX-DEFAULT-NEXT: } // callseq 2 ; PTX-DEFAULT-NEXT: ret; ; @@ -384,7 +384,7 @@ define ptx_kernel void @grid_const_partial_escape(ptr byval(i32) align 4 "nvvm.g ; PTX-SHORT-PTR: { ; PTX-SHORT-PTR-NEXT: .reg .b32 %r<4>; ; PTX-SHORT-PTR-NEXT: .reg .b64 %rd<6>; -; PTX-SHORT-PTR-NEXT: prototype_2 : .callprototype (.param .b32 _) _ (.param .b64 _); +; PTX-SHORT-PTR-NEXT: $L__prototype_2 : .callprototype (.param .b32 _) _ (.param .b64 _); ; PTX-SHORT-PTR-NEXT: // %bb.0: ; PTX-SHORT-PTR-NEXT: mov.b32 %r1, grid_const_partial_escape_param_0; ; PTX-SHORT-PTR-NEXT: ld.param.b64 %rd1, [grid_const_partial_escape_param_1]; @@ -399,7 +399,7 @@ define ptx_kernel void @grid_const_partial_escape(ptr byval(i32) align 4 "nvvm.g ; PTX-SHORT-PTR-NEXT: .param .b32 retval0; ; PTX-SHORT-PTR-NEXT: st.param.b64 [param0], %rd4; ; PTX-SHORT-PTR-NEXT: mov.b64 %rd5, escape; -; PTX-SHORT-PTR-NEXT: call (retval0), %rd5, (param0), prototype_2; +; PTX-SHORT-PTR-NEXT: call (retval0), %rd5, (param0), $L__prototype_2; ; PTX-SHORT-PTR-NEXT: } // callseq 2 ; PTX-SHORT-PTR-NEXT: ret; ; OPT-LABEL: define ptx_kernel void @grid_const_partial_escape( @@ -422,7 +422,7 @@ define ptx_kernel i32 @grid_const_partial_escapemem(ptr byval(%struct.s) align 4 ; PTX-DEFAULT: { ; PTX-DEFAULT-NEXT: .reg .b32 %r<4>; ; PTX-DEFAULT-NEXT: .reg .b64 %rd<6>; -; PTX-DEFAULT-NEXT: prototype_3 : .callprototype (.param .b32 _) _ (.param .b64 _); +; PTX-DEFAULT-NEXT: $L__prototype_3 : .callprototype (.param .b32 _) _ (.param .b64 _); ; PTX-DEFAULT-NEXT: // %bb.0: ; PTX-DEFAULT-NEXT: mov.b64 %rd1, grid_const_partial_escapemem_param_0; ; PTX-DEFAULT-NEXT: ld.param.b64 %rd2, [grid_const_partial_escapemem_param_1]; @@ -437,7 +437,7 @@ define ptx_kernel i32 @grid_const_partial_escapemem(ptr byval(%struct.s) align 4 ; PTX-DEFAULT-NEXT: .param .b32 retval0; ; PTX-DEFAULT-NEXT: st.param.b64 [param0], %rd4; ; PTX-DEFAULT-NEXT: mov.b64 %rd5, escape; -; PTX-DEFAULT-NEXT: call (retval0), %rd5, (param0), prototype_3; +; PTX-DEFAULT-NEXT: call (retval0), %rd5, (param0), $L__prototype_3; ; PTX-DEFAULT-NEXT: } // callseq 3 ; PTX-DEFAULT-NEXT: st.param.b32 [func_retval0], %r3; ; PTX-DEFAULT-NEXT: ret; @@ -446,7 +446,7 @@ define ptx_kernel i32 @grid_const_partial_escapemem(ptr byval(%struct.s) align 4 ; PTX-SHORT-PTR: { ; PTX-SHORT-PTR-NEXT: .reg .b32 %r<5>; ; PTX-SHORT-PTR-NEXT: .reg .b64 %rd<6>; -; PTX-SHORT-PTR-NEXT: prototype_3 : .callprototype (.param .b32 _) _ (.param .b64 _); +; PTX-SHORT-PTR-NEXT: $L__prototype_3 : .callprototype (.param .b32 _) _ (.param .b64 _); ; PTX-SHORT-PTR-NEXT: // %bb.0: ; PTX-SHORT-PTR-NEXT: mov.b32 %r1, grid_const_partial_escapemem_param_0; ; PTX-SHORT-PTR-NEXT: ld.param.b64 %rd1, [grid_const_partial_escapemem_param_1]; @@ -462,7 +462,7 @@ define ptx_kernel i32 @grid_const_partial_escapemem(ptr byval(%struct.s) align 4 ; PTX-SHORT-PTR-NEXT: .param .b32 retval0; ; PTX-SHORT-PTR-NEXT: st.param.b64 [param0], %rd4; ; PTX-SHORT-PTR-NEXT: mov.b64 %rd5, escape; -; PTX-SHORT-PTR-NEXT: call (retval0), %rd5, (param0), prototype_3; +; PTX-SHORT-PTR-NEXT: call (retval0), %rd5, (param0), $L__prototype_3; ; PTX-SHORT-PTR-NEXT: } // callseq 3 ; PTX-SHORT-PTR-NEXT: st.param.b32 [func_retval0], %r4; ; PTX-SHORT-PTR-NEXT: ret; diff --git a/llvm/test/CodeGen/NVPTX/noreturn.ll b/llvm/test/CodeGen/NVPTX/noreturn.ll index 0062e62756d36..416418c7ffde7 100644 --- a/llvm/test/CodeGen/NVPTX/noreturn.ll +++ b/llvm/test/CodeGen/NVPTX/noreturn.ll @@ -32,8 +32,8 @@ define ptx_kernel void @ignore_kernel_noreturn() #0 { } ; CHECK-LABEL: .entry callprototype_noreturn( -; CHECK: prototype_{{[0-9]+}} : .callprototype ()_ (.param .b32 _) .noreturn; -; CHECK: prototype_{{[0-9]+}} : .callprototype (.param .b32 _) _ (.param .b32 _); +; CHECK: {{[$]}}L__prototype_{{[0-9]+}} : .callprototype ()_ (.param .b32 _) .noreturn; +; CHECK: {{[$]}}L__prototype_{{[0-9]+}} : .callprototype (.param .b32 _) _ (.param .b32 _); define ptx_kernel void @callprototype_noreturn(i32) { %fn = load ptr, ptr addrspace(1) @function_pointer diff --git a/llvm/test/CodeGen/NVPTX/param-align.ll b/llvm/test/CodeGen/NVPTX/param-align.ll index c85080fdf295a..7f0a1646962ae 100644 --- a/llvm/test/CodeGen/NVPTX/param-align.ll +++ b/llvm/test/CodeGen/NVPTX/param-align.ll @@ -55,17 +55,17 @@ declare ptr @getfp(i32 %n) define ptx_device void @t6() { ; CHECK: .func t6 %fp = call ptr @getfp(i32 0) -; CHECK: prototype_2 : .callprototype ()_ (.param .align 8 .b8 _[8]); +; CHECK: $L__prototype_2 : .callprototype ()_ (.param .align 8 .b8 _[8]); call void %fp(ptr byval(double) null); %fp2 = call ptr @getfp(i32 1) -; NOALIGN4: prototype_4 : .callprototype ()_ (.param .align 2 .b8 _[4]); -; ALIGN4: prototype_4 : .callprototype ()_ (.param .align 4 .b8 _[4]); +; NOALIGN4: $L__prototype_4 : .callprototype ()_ (.param .align 2 .b8 _[4]); +; ALIGN4: $L__prototype_4 : .callprototype ()_ (.param .align 4 .b8 _[4]); call void %fp(ptr byval(%struct.half2) null); %fp3 = call ptr @getfp(i32 2) -; NOALIGN4: prototype_6 : .callprototype ()_ (.param .align 1 .b8 _[1]); -; ALIGN4: prototype_6 : .callprototype ()_ (.param .align 4 .b8 _[1]); +; NOALIGN4: $L__prototype_6 : .callprototype ()_ (.param .align 1 .b8 _[1]); +; ALIGN4: $L__prototype_6 : .callprototype ()_ (.param .align 4 .b8 _[1]); call void %fp(ptr byval(i8) null); ret void } >From 5e607fa797d669f53ceafb13566a8899c10e027f Mon Sep 17 00:00:00 2001 From: Daniel Donenfeld <[email protected]> Date: Wed, 24 Jun 2026 21:34:42 +0000 Subject: [PATCH 2/5] Fix missed test --- llvm/test/CodeGen/NVPTX/vaargs.ll | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/llvm/test/CodeGen/NVPTX/vaargs.ll b/llvm/test/CodeGen/NVPTX/vaargs.ll index 8ff429a16b74c..34ff8c18e2ca0 100644 --- a/llvm/test/CodeGen/NVPTX/vaargs.ll +++ b/llvm/test/CodeGen/NVPTX/vaargs.ll @@ -216,7 +216,7 @@ define i32 @test_foo(i32 %i, i64 %l, double %d, ptr %p) { ; CHECK32-NEXT: .reg .b32 %SPL; ; CHECK32-NEXT: .reg .b32 %r<9>; ; CHECK32-NEXT: .reg .b64 %rd<3>; -; CHECK32-NEXT: prototype_1 : .callprototype (.param .b32 _) _ (.param .b32 _, .param .b32 _); +; CHECK32-NEXT: $L__prototype_1 : .callprototype (.param .b32 _) _ (.param .b32 _, .param .b32 _); ; CHECK32-NEXT: // %bb.0: // %entry ; CHECK32-NEXT: mov.b32 %SPL, __local_depot1; ; CHECK32-NEXT: ld.param.b32 %r2, [test_foo_param_3]; @@ -238,7 +238,7 @@ define i32 @test_foo(i32 %i, i64 %l, double %d, ptr %p) { ; CHECK32-NEXT: .param .b32 retval0; ; CHECK32-NEXT: st.param.b32 [param1], %r3; ; CHECK32-NEXT: st.param.b32 [param0], 4; -; CHECK32-NEXT: call (retval0), %r7, (param0, param1), prototype_1; +; CHECK32-NEXT: call (retval0), %r7, (param0, param1), $L__prototype_1; ; CHECK32-NEXT: ld.param.b32 %r8, [retval0]; ; CHECK32-NEXT: } // callseq 1 ; CHECK32-NEXT: st.param.b32 [func_retval0], %r8; @@ -251,7 +251,7 @@ define i32 @test_foo(i32 %i, i64 %l, double %d, ptr %p) { ; CHECK64-NEXT: .reg .b64 %SPL; ; CHECK64-NEXT: .reg .b32 %r<3>; ; CHECK64-NEXT: .reg .b64 %rd<9>; -; CHECK64-NEXT: prototype_1 : .callprototype (.param .b32 _) _ (.param .b32 _, .param .b64 _); +; CHECK64-NEXT: $L__prototype_1 : .callprototype (.param .b32 _) _ (.param .b32 _, .param .b64 _); ; CHECK64-NEXT: // %bb.0: // %entry ; CHECK64-NEXT: mov.b64 %SPL, __local_depot1; ; CHECK64-NEXT: ld.param.b64 %rd3, [test_foo_param_3]; @@ -273,7 +273,7 @@ define i32 @test_foo(i32 %i, i64 %l, double %d, ptr %p) { ; CHECK64-NEXT: .param .b32 retval0; ; CHECK64-NEXT: st.param.b64 [param1], %rd4; ; CHECK64-NEXT: st.param.b32 [param0], 4; -; CHECK64-NEXT: call (retval0), %rd8, (param0, param1), prototype_1; +; CHECK64-NEXT: call (retval0), %rd8, (param0, param1), $L__prototype_1; ; CHECK64-NEXT: ld.param.b32 %r2, [retval0]; ; CHECK64-NEXT: } // callseq 1 ; CHECK64-NEXT: st.param.b32 [func_retval0], %r2; >From 96d4e985639a339b898b643456bc6faf6d4a9072 Mon Sep 17 00:00:00 2001 From: Daniel Donenfeld <[email protected]> Date: Mon, 29 Jun 2026 19:54:26 +0000 Subject: [PATCH 3/5] Add cuda C++ lit test for prototype collision --- .../CodeGenCUDA/callprototype-local-label.cu | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 clang/test/CodeGenCUDA/callprototype-local-label.cu diff --git a/clang/test/CodeGenCUDA/callprototype-local-label.cu b/clang/test/CodeGenCUDA/callprototype-local-label.cu new file mode 100644 index 0000000000000..6f4bb2de0b02a --- /dev/null +++ b/clang/test/CodeGenCUDA/callprototype-local-label.cu @@ -0,0 +1,29 @@ +// REQUIRES: nvptx-registered-target +// RUN: %clang_cc1 -triple nvptx64-nvidia-cuda -target-cpu sm_75 \ +// RUN: -fcuda-is-device -S -o - -x cuda %s \ +// RUN: | FileCheck %s + +// Test that a global named 'prototype_0' does not cause a callprototype label +// collision: the label must be '$L__prototype_0', not 'prototype_0'. +// extern "C" is used to keep the PTX global name unmangled. + +#define __device__ __attribute__((device)) + +extern "C" { + +__device__ int simple_func() { return 42; } + +__device__ int (*prototype_0)(int, int, int, int) = nullptr; +__device__ int call_via_prototype_0(int a, int b, int c, int d) { + if (prototype_0 != nullptr) + return prototype_0(a, b, c, d); + return a + b + c + d; +} + +} // extern "C" + +// CHECK: .visible .global .align 8 .u64 prototype_0; +// CHECK-LABEL: .visible .func (.param .b32 func_retval0) call_via_prototype_0( +// CHECK: $L__prototype_0 : .callprototype (.param .b32 _) _ (.param .b32 _, .param .b32 _, .param .b32 _, .param .b32 _); +// CHECK-NEXT: call (retval0), %rd{{[0-9]+}}, (param0, param1, param2, param3), $L__prototype_0; +// CHECK-NOT: prototype_0 : .callprototype >From 7da8e0fb92301ddcbbe963159ebc01a5d3828f01 Mon Sep 17 00:00:00 2001 From: Daniel Donenfeld <[email protected]> Date: Mon, 29 Jun 2026 20:31:48 +0000 Subject: [PATCH 4/5] Fix tests after rebase --- clang/test/CodeGenCUDA/callprototype-local-label.cu | 2 +- llvm/test/CodeGen/NVPTX/callprototype-local-label.ll | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/clang/test/CodeGenCUDA/callprototype-local-label.cu b/clang/test/CodeGenCUDA/callprototype-local-label.cu index 6f4bb2de0b02a..9e5f3d868a6d5 100644 --- a/clang/test/CodeGenCUDA/callprototype-local-label.cu +++ b/clang/test/CodeGenCUDA/callprototype-local-label.cu @@ -25,5 +25,5 @@ __device__ int call_via_prototype_0(int a, int b, int c, int d) { // CHECK: .visible .global .align 8 .u64 prototype_0; // CHECK-LABEL: .visible .func (.param .b32 func_retval0) call_via_prototype_0( // CHECK: $L__prototype_0 : .callprototype (.param .b32 _) _ (.param .b32 _, .param .b32 _, .param .b32 _, .param .b32 _); -// CHECK-NEXT: call (retval0), %rd{{[0-9]+}}, (param0, param1, param2, param3), $L__prototype_0; +// CHECK: call (retval0), %rd{{[0-9]+}}, (param0, param1, param2, param3), $L__prototype_0; // CHECK-NOT: prototype_0 : .callprototype diff --git a/llvm/test/CodeGen/NVPTX/callprototype-local-label.ll b/llvm/test/CodeGen/NVPTX/callprototype-local-label.ll index 371282a28745c..894a5dc97d11b 100644 --- a/llvm/test/CodeGen/NVPTX/callprototype-local-label.ll +++ b/llvm/test/CodeGen/NVPTX/callprototype-local-label.ll @@ -7,9 +7,9 @@ target triple = "nvptx64-nvidia-cuda" define i32 @call_via_prototype_0(i32 %a, i32 %b, i32 %c, i32 %d) { ; CHECK-LABEL: call_via_prototype_0( -; CHECK: ld.global.{{u|b}}64 {{%rd[0-9]+}}, [prototype_0]; ; CHECK: $L__prototype_0 : .callprototype (.param .b32 _) _ (.param .b32 _, .param .b32 _, .param .b32 _, .param .b32 _); -; CHECK-NEXT: call (retval0), %rd{{[0-9]+}}, (param0, param1, param2, param3), $L__prototype_0; +; CHECK: ld.global.{{u|b}}64 {{%rd[0-9]+}}, [prototype_0]; +; CHECK: call (retval0), %rd{{[0-9]+}}, (param0, param1, param2, param3), $L__prototype_0; ; CHECK-NOT: prototype_0 : .callprototype %fp = load ptr, ptr addrspace(1) @prototype_0, align 8 %ret = call i32 %fp(i32 %a, i32 %b, i32 %c, i32 %d) >From fa6960ba45333a167a8e7537fbe41c5e95c5be83 Mon Sep 17 00:00:00 2001 From: Daniel Donenfeld <[email protected]> Date: Tue, 18 Aug 2026 17:29:07 +0000 Subject: [PATCH 5/5] Add initial way of using MCSymbol to unique the call prototype name --- llvm/lib/Target/NVPTX/CMakeLists.txt | 1 + llvm/lib/Target/NVPTX/NVPTX.h | 3 + llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp | 15 ++-- llvm/lib/Target/NVPTX/NVPTXInstrInfo.td | 6 +- .../Target/NVPTX/NVPTXMachineFunctionInfo.h | 26 +++++- .../NVPTX/NVPTXPreEmitSymbolLowering.cpp | 90 +++++++++++++++++++ llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp | 5 ++ .../NVPTX/callprototype-dollar-local-label.ll | 19 ++++ 8 files changed, 154 insertions(+), 11 deletions(-) create mode 100644 llvm/lib/Target/NVPTX/NVPTXPreEmitSymbolLowering.cpp create mode 100644 llvm/test/CodeGen/NVPTX/callprototype-dollar-local-label.ll diff --git a/llvm/lib/Target/NVPTX/CMakeLists.txt b/llvm/lib/Target/NVPTX/CMakeLists.txt index 98d285f763478..3fd38eab69341 100644 --- a/llvm/lib/Target/NVPTX/CMakeLists.txt +++ b/llvm/lib/Target/NVPTX/CMakeLists.txt @@ -36,6 +36,7 @@ set(NVPTXCodeGen_sources NVPTXMCExpr.cpp NVPTXMarkKernelPtrsGlobal.cpp NVPTXPeephole.cpp + NVPTXPreEmitSymbolLowering.cpp NVPTXPrologEpilogPass.cpp NVPTXPromoteParamAlign.cpp NVPTXProxyRegErasure.cpp diff --git a/llvm/lib/Target/NVPTX/NVPTX.h b/llvm/lib/Target/NVPTX/NVPTX.h index f7899e72abb20..6feb4b9b87846 100644 --- a/llvm/lib/Target/NVPTX/NVPTX.h +++ b/llvm/lib/Target/NVPTX/NVPTX.h @@ -67,6 +67,8 @@ MachineFunctionPass *createNVPTXPeepholeLegacyPass(); MachineFunctionPass *createNVPTXProxyRegErasureLegacyPass(); MachineFunctionPass *createNVPTXForwardParamsLegacyPass(); MachineFunctionPass *createNVPTXAddressFolderLegacyPass(); +MachineFunctionPass * +createNVPTXPreEmitSymbolLoweringLegacyPass(const TargetMachine &TM); void initializeNVVMReflectLegacyPassPass(PassRegistry &); void initializeGenericToNVVMLegacyPassPass(PassRegistry &); @@ -74,6 +76,7 @@ void initializeNVPTXAllocaHoistingLegacyPassPass(PassRegistry &); void initializeNVPTXAsmPrinterPass(PassRegistry &); void initializeNVPTXAssignValidGlobalNamesLegacyPassPass(PassRegistry &); void initializeNVPTXAtomicLowerLegacyPassPass(PassRegistry &); +void initializeNVPTXPreEmitSymbolLoweringLegacyPass(PassRegistry &); void initializeNVPTXCtorDtorLoweringLegacyPass(PassRegistry &); void initializeNVPTXLowerAggrCopiesLegacyPassPass(PassRegistry &); void initializeNVPTXLowerAllocaLegacyPassPass(PassRegistry &); diff --git a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp index b91e3da92bf63..7f16f8ad8a5f4 100644 --- a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp @@ -253,7 +253,7 @@ class NVPTXAsmPrinter : public AsmPrinter { void encodeDebugInfoRegisterNumbers(const MachineFunction &MF); void printReturnValStr(const Function *, raw_ostream &O); void printReturnValStr(const MachineFunction &MF, raw_ostream &O); - void emitCallPrototype(const CallBase &CB, unsigned UniqueCallSite, + void emitCallPrototype(const CallBase &CB, MCSymbol *PrototypeSymbol, raw_ostream &O) const; void emitJumpTable(const MachineJumpTableEntry &MJT, unsigned MJTI) const; @@ -605,6 +605,8 @@ MCOperand NVPTXAsmPrinter::lowerOperand(const MachineOperand &MO) { case MachineOperand::MO_MachineBasicBlock: return MCOperand::createExpr( MCSymbolRefExpr::create(MO.getMBB()->getSymbol(), OutContext)); + case MachineOperand::MO_MCSymbol: + return GetSymbolRef(MO.getMCSymbol()); case MachineOperand::MO_ExternalSymbol: return GetSymbolRef(GetExternalSymbolSymbol(MO.getSymbolName())); case MachineOperand::MO_JumpTableIndex: @@ -727,7 +729,7 @@ void NVPTXAsmPrinter::printReturnValStr(const MachineFunction &MF, } void NVPTXAsmPrinter::emitCallPrototype(const CallBase &CB, - unsigned UniqueCallSite, + MCSymbol *PrototypeSymbol, raw_ostream &O) const { const DataLayout &DL = getDataLayout(); const NVPTXSubtarget &STI = MF->getSubtarget<NVPTXSubtarget>(); @@ -735,7 +737,8 @@ void NVPTXAsmPrinter::emitCallPrototype(const CallBase &CB, const auto PtrVT = TLI->getPointerTy(DL); Type *RetTy = CB.getFunctionType()->getReturnType(); - O << "$L__prototype_" << UniqueCallSite << " : .callprototype "; + PrototypeSymbol->print(O, MAI); + O << " : .callprototype "; if (RetTy->isVoidTy() || RetTy->isEmptyTy()) { O << "()"; @@ -937,8 +940,10 @@ void NVPTXAsmPrinter::emitFunctionBodyStart() { emitDemotedVars(&MF->getFunction(), O); const auto *MFI = MF->getInfo<NVPTXMachineFunctionInfo>(); - for (const auto &[Id, CB] : MFI->getCallPrototypes()) - emitCallPrototype(*CB, Id, O); + for (const auto &Entry : MFI->getCallPrototypes()) { + const auto &Prototype = Entry.second; + emitCallPrototype(*Prototype.CB, Prototype.Symbol, O); + } OutStreamer->emitRawText(O.str()); diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td index 99d9bb6af0628..9118dee885177 100644 --- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td +++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td @@ -1938,9 +1938,9 @@ foreach is_convergent = [0, 1] in { let isCall = 1, isConvergent = is_convergent in { def CALL # convergent_suffix : NVPTXInst<(outs), - (ins ADDR_base:$addr, CallOperand:$rets, CallOperand:$params, - i32imm:$proto), - "call${rets:RetList} $addr, (${params:ParamList}), $$L__prototype_$proto;">; + (ins ADDR_base:$addr, CallOperand:$rets, CallOperand:$params, + ADDR_base:$proto), + "call${rets:RetList} $addr, (${params:ParamList}), $proto;">; def CALL_UNI # convergent_suffix : NVPTXInst<(outs), diff --git a/llvm/lib/Target/NVPTX/NVPTXMachineFunctionInfo.h b/llvm/lib/Target/NVPTX/NVPTXMachineFunctionInfo.h index 866dba0942c01..e32ba5a83ff7b 100644 --- a/llvm/lib/Target/NVPTX/NVPTXMachineFunctionInfo.h +++ b/llvm/lib/Target/NVPTX/NVPTXMachineFunctionInfo.h @@ -20,8 +20,15 @@ namespace llvm { class CallBase; +class MCSymbol; class NVPTXMachineFunctionInfo : public MachineFunctionInfo { +public: + struct CallPrototype { + const CallBase *CB; + MCSymbol *Symbol = nullptr; + }; + private: /// Stores a mapping from index to symbol name for image handles that are /// replaced with image references @@ -29,7 +36,7 @@ class NVPTXMachineFunctionInfo : public MachineFunctionInfo { /// Stores a mapping from a unique call-site id to the call instruction that /// needs an indirect-call prototype emitted. - std::map<unsigned, const CallBase *> CallPrototypes; + std::map<unsigned, CallPrototype> CallPrototypes; public: NVPTXMachineFunctionInfo(const Function &F, const TargetSubtargetInfo *STI) {} @@ -61,10 +68,23 @@ class NVPTXMachineFunctionInfo : public MachineFunctionInfo { } void addCallPrototype(unsigned Id, const CallBase *CB) { - CallPrototypes.try_emplace(Id, CB); + CallPrototypes.try_emplace(Id, CallPrototype{CB}); + } + + void setCallPrototypeSymbol(unsigned Id, MCSymbol *Symbol) { + auto It = CallPrototypes.find(Id); + assert(It != CallPrototypes.end() && "unknown call prototype"); + It->second.Symbol = Symbol; + } + + MCSymbol *getCallPrototypeSymbol(unsigned Id) const { + auto It = CallPrototypes.find(Id); + assert(It != CallPrototypes.end() && "unknown call prototype"); + assert(It->second.Symbol && "call prototype symbol not set"); + return It->second.Symbol; } - const std::map<unsigned, const CallBase *> &getCallPrototypes() const { + const std::map<unsigned, CallPrototype> &getCallPrototypes() const { return CallPrototypes; } }; diff --git a/llvm/lib/Target/NVPTX/NVPTXPreEmitSymbolLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXPreEmitSymbolLowering.cpp new file mode 100644 index 0000000000000..722279bbe06e7 --- /dev/null +++ b/llvm/lib/Target/NVPTX/NVPTXPreEmitSymbolLowering.cpp @@ -0,0 +1,90 @@ +//===-- NVPTXPreEmitSymbolLowering.cpp - Lower symbols before emission -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "NVPTX.h" +#include "NVPTXMachineFunctionInfo.h" +#include "NVPTXRegisterInfo.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/MachineOperand.h" +#include "llvm/IR/GlobalValue.h" +#include "llvm/IR/Module.h" +#include "llvm/MC/MCSymbol.h" +#include "llvm/Target/TargetMachine.h" + +using namespace llvm; + +namespace { + +class NVPTXPreEmitSymbolLoweringLegacy : public MachineFunctionPass { + const TargetMachine *TM = nullptr; + +public: + static char ID; + + NVPTXPreEmitSymbolLoweringLegacy() : MachineFunctionPass(ID) {} + NVPTXPreEmitSymbolLoweringLegacy(const TargetMachine &TM) + : MachineFunctionPass(ID), TM(&TM) {} + + bool doInitialization(Module &M) override { + assert(TM && "TargetMachine must be set"); + for (const GlobalValue &GV : M.global_values()) + TM->getSymbol(&GV); + return false; + } + + bool runOnMachineFunction(MachineFunction &MF) override { + auto *MFI = MF.getInfo<NVPTXMachineFunctionInfo>(); + if (MFI->getCallPrototypes().empty()) + return false; + + for (const auto &[Id, Prototype] : MFI->getCallPrototypes()) { + if (!Prototype.Symbol) { + MCSymbol *Symbol = MF.getContext().createTempSymbol( + "prototype_" + Twine(Id), /*AlwaysAddSuffix=*/false); + MFI->setCallPrototypeSymbol(Id, Symbol); + } + } + + bool Changed = false; + for (MachineBasicBlock &MBB : MF) { + for (MachineInstr &MI : MBB) { + if (MI.getOpcode() != NVPTX::CALL && MI.getOpcode() != NVPTX::CALL_conv) + continue; + + MachineOperand &Proto = MI.getOperand(3); + if (Proto.isImm()) { + Proto.ChangeToMCSymbol(MFI->getCallPrototypeSymbol(Proto.getImm())); + Changed = true; + } + assert(Proto.isMCSymbol() && + "call prototype operand must be rewritten to a symbol"); + } + } + + return Changed; + } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesCFG(); + MachineFunctionPass::getAnalysisUsage(AU); + } +}; + +} // namespace + +char NVPTXPreEmitSymbolLoweringLegacy::ID = 0; + +INITIALIZE_PASS(NVPTXPreEmitSymbolLoweringLegacy, "nvptx-pre-emit-symbol-lowering", + "NVPTX Pre-Emit Symbol Lowering", false, false) + +MachineFunctionPass * +llvm::createNVPTXPreEmitSymbolLoweringLegacyPass(const TargetMachine &TM) { + return new NVPTXPreEmitSymbolLoweringLegacy(TM); +} diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp index 037672e06c18e..2fb48c540a7e8 100644 --- a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp @@ -136,6 +136,7 @@ class NVPTXPassConfig : public TargetPassConfig { bool addInstSelector() override; void addPreRegAlloc() override; void addPostRegAlloc() override; + void addPreEmitPass2() override; FunctionPass *createTargetRegisterAllocator(bool) override; void addFastRegAlloc() override; @@ -356,6 +357,10 @@ void NVPTXPassConfig::addPostRegAlloc() { } } +void NVPTXPassConfig::addPreEmitPass2() { + addPass(createNVPTXPreEmitSymbolLoweringLegacyPass(getNVPTXTargetMachine())); +} + FunctionPass *NVPTXPassConfig::createTargetRegisterAllocator(bool) { return nullptr; // No reg alloc } diff --git a/llvm/test/CodeGen/NVPTX/callprototype-dollar-local-label.ll b/llvm/test/CodeGen/NVPTX/callprototype-dollar-local-label.ll new file mode 100644 index 0000000000000..f0effa497d3ef --- /dev/null +++ b/llvm/test/CodeGen/NVPTX/callprototype-dollar-local-label.ll @@ -0,0 +1,19 @@ +; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_75 | FileCheck %s +; RUN: %if ptxas-sm_75 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_75 | %ptxas-verify --compile-only -arch=sm_75 %} + +target triple = "nvptx64-nvidia-cuda" + +@"$L__prototype_0" = addrspace(1) global ptr null, align 8 +@"$L__prototype_00" = addrspace(1) global i32 7, align 4 + +define i32 @call_via_dollar_prototype_0(i32 %a, i32 %b, i32 %c, i32 %d) { +; CHECK-DAG: .visible .global .align 8 .u64 $L__prototype_0; +; CHECK-DAG: .visible .global .align 4 .u32 $L__prototype_00 = 7; +; CHECK-LABEL: call_via_dollar_prototype_0( +; CHECK: $L__prototype_01 : .callprototype (.param .b32 _) _ (.param .b32 _, .param .b32 _, .param .b32 _, .param .b32 _); +; CHECK: ld.global.{{u|b}}64 {{%rd[0-9]+}}, [$L__prototype_0]; +; CHECK: call (retval0), %rd{{[0-9]+}}, (param0, param1, param2, param3), $L__prototype_01; + %fp = load ptr, ptr addrspace(1) @"$L__prototype_0", align 8 + %ret = call i32 %fp(i32 %a, i32 %b, i32 %c, i32 %d) + ret i32 %ret +} _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
