llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-clang-codegen Author: Weibo He (NewSigma) <details> <summary>Changes</summary> As mentioned in #<!-- -->151067, current design of `llvm.coro.end` mixes two functionalities: querying where we are and lowering to some code. This patch separate these functionalities into independent intrinsics by introducing a new intrinsic `llvm.coro.where`. --- Patch is 149.25 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/153404.diff 147 Files Affected: - (modified) clang/lib/CodeGen/CGCoroutine.cpp (+8-6) - (modified) clang/test/CodeGenCoroutines/coro-builtins.c (+1-1) - (modified) clang/test/CodeGenCoroutines/coro-eh-cleanup.cpp (+2-2) - (modified) clang/test/CodeGenCoroutines/coro-lambda.cpp (+1-1) - (modified) clang/test/CodeGenCoroutines/coro-params.cpp (+2-2) - (modified) llvm/docs/Coroutines.rst (+38-13) - (modified) llvm/include/llvm/IR/Intrinsics.td (+3-2) - (modified) llvm/include/llvm/Transforms/Coroutines/CoroInstr.h (+12) - (modified) llvm/include/llvm/Transforms/Coroutines/CoroShape.h (+2) - (modified) llvm/lib/Transforms/Coroutines/CoroCleanup.cpp (+1-1) - (modified) llvm/lib/Transforms/Coroutines/CoroCloner.h (+1) - (modified) llvm/lib/Transforms/Coroutines/CoroSplit.cpp (+21-8) - (modified) llvm/lib/Transforms/Coroutines/Coroutines.cpp (+4) - (modified) llvm/test/Analysis/GlobalsModRef/nonescaping-noalias.ll (+2-2) - (modified) llvm/test/Assembler/auto_upgrade_intrinsics.ll (+3-3) - (modified) llvm/test/Transforms/Coroutines/ArgAddr.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-align16.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-align32.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-align64-02.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-align64.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-align8-02.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-align8.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-alloc-with-param-O0.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-alloc-with-param-O2.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-alloca-01.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-alloca-02.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-alloca-03.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-alloca-04.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-alloca-05.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-alloca-06.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-alloca-07.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-alloca-08.ll (+3-3) - (modified) llvm/test/Transforms/Coroutines/coro-alloca-loop-carried-address.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-alloca-outside-frame.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-alloca-with-addrspace.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-async-addr-lifetime-infinite-loop-bug.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-async-addr-lifetime-start-bug.ll (+3-3) - (modified) llvm/test/Transforms/Coroutines/coro-async-coro-id-async-bug.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-async-declaration.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-async-dyn-align.ll (+3-3) - (modified) llvm/test/Transforms/Coroutines/coro-async-end-bug.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-async-no-cse-swift-async-context-addr.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-async-nomerge.ll (+3-3) - (modified) llvm/test/Transforms/Coroutines/coro-async-phi.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-async.ll (+12-12) - (modified) llvm/test/Transforms/Coroutines/coro-await-suspend-handle-in-ramp.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-await-suspend-lower-invoke.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-await-suspend-lower.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-byval-param.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-catchswitch-cleanuppad.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-catchswitch.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-debug-O2.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-debug-coro-frame.ll (+3-3) - (modified) llvm/test/Transforms/Coroutines/coro-debug-dbg.values-not_used_in_frame.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-debug-dbg.values.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-debug-frame-variable.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-debug-spill-dbg.declare.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-debug.ll (+4-4) - (modified) llvm/test/Transforms/Coroutines/coro-early-twice.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-eh-aware-edge-split-00.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-eh-aware-edge-split-01.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-eh-aware-edge-split-02.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-frame-arrayalloca.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-00.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-01.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-02.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-03.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-04.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-05.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-frame-unreachable.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-frame.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-lifetime-end.ll (+4-4) - (modified) llvm/test/Transforms/Coroutines/coro-materialize.ll (+6-6) - (modified) llvm/test/Transforms/Coroutines/coro-noalias-param.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-only-destroy-when-complete.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-padding.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-param-copy.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-readnone-02.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-readnone.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-retcon-alloca-opaque-ptr.ll (+3-3) - (modified) llvm/test/Transforms/Coroutines/coro-retcon-alloca.ll (+6-6) - (modified) llvm/test/Transforms/Coroutines/coro-retcon-frame.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-retcon-once-private.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-retcon-once-value.ll (+3-3) - (modified) llvm/test/Transforms/Coroutines/coro-retcon-once-value2.ll (+4-4) - (modified) llvm/test/Transforms/Coroutines/coro-retcon-opaque-ptr.ll (+3-3) - (modified) llvm/test/Transforms/Coroutines/coro-retcon-remat.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-retcon-resume-values.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-retcon-resume-values2.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-retcon-unreachable.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-retcon-value.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-retcon.ll (+4-4) - (modified) llvm/test/Transforms/Coroutines/coro-spill-after-phi.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-spill-corobegin.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-spill-defs-before-corobegin.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-spill-promise-02.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-spill-promise.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-spill-suspend.ll (+1-1) - (modified) llvm/test/Transforms/Coroutines/coro-split-00.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-split-01.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-split-02.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-split-alloc.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-split-dbg-labels.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-split-dbg.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-split-eh-00.ll (+4-3) - (modified) llvm/test/Transforms/Coroutines/coro-split-eh-01.ll (+3-3) - (modified) llvm/test/Transforms/Coroutines/coro-split-final-suspend.ll (+5-4) - (modified) llvm/test/Transforms/Coroutines/coro-split-hidden.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-split-musttail-chain-pgo-counter-promo.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-split-musttail-ppc64le.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-split-musttail.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-split-musttail1.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-split-musttail10.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-split-musttail12.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-split-musttail13.ll (+3-3) - (modified) llvm/test/Transforms/Coroutines/coro-split-musttail2.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-split-musttail3.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-split-musttail4.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-split-musttail5.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-split-musttail6.ll (+3-3) - (modified) llvm/test/Transforms/Coroutines/coro-split-musttail7.ll (+3-3) - (modified) llvm/test/Transforms/Coroutines/coro-split-musttail8.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-split-musttail9.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-split-no-lifetime.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-split-noinline.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-split-sink-lifetime-01.ll (+3-3) - (modified) llvm/test/Transforms/Coroutines/coro-split-sink-lifetime-02.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-split-sink-lifetime-03.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-split-sink-lifetime-04.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/coro-swifterror.ll (+3-3) - (modified) llvm/test/Transforms/Coroutines/coro-zero-alloca.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/ex0.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/ex1.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/ex2.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/ex3.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/ex4.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/ex5.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/no-suspend.ll (+9-9) - (modified) llvm/test/Transforms/Coroutines/phi-coro-end.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/remarks.ll (+2-2) - (modified) llvm/test/Transforms/Coroutines/swift-async-dbg.ll (+3-3) - (modified) llvm/test/Transforms/FunctionAttrs/noreturn.ll (+2-2) - (modified) llvm/test/Transforms/LICM/licm-coroutine.ll (+2-2) - (modified) llvm/test/Transforms/LICM/sink-with-coroutine.ll (+4-5) - (modified) llvm/unittests/Transforms/Coroutines/ExtraRematTest.cpp (+4-4) - (modified) mlir/test/Target/LLVMIR/Import/intrinsic.ll (+2-2) - (modified) mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir (+3-3) ``````````diff diff --git a/clang/lib/CodeGen/CGCoroutine.cpp b/clang/lib/CodeGen/CGCoroutine.cpp index 827385f9c1a1f..cbc38516a0815 100644 --- a/clang/lib/CodeGen/CGCoroutine.cpp +++ b/clang/lib/CodeGen/CGCoroutine.cpp @@ -575,17 +575,19 @@ struct CallCoroEnd final : public EHScopeStack::Cleanup { llvm::Function *CoroEndFn = CGM.getIntrinsic(llvm::Intrinsic::coro_end); // See if we have a funclet bundle to associate coro.end with. (WinEH) auto Bundles = getBundlesForCoroEnd(CGF); - auto *CoroEnd = - CGF.Builder.CreateCall(CoroEndFn, - {NullPtr, CGF.Builder.getTrue(), - llvm::ConstantTokenNone::get(CoroEndFn->getContext())}, - Bundles); + CGF.Builder.CreateCall( + CoroEndFn, + {NullPtr, CGF.Builder.getTrue(), + llvm::ConstantTokenNone::get(CoroEndFn->getContext())}, + Bundles); if (Bundles.empty()) { // Otherwise, (landingpad model), create a conditional branch that leads // either to a cleanup block or a block with EH resume instruction. auto *ResumeBB = CGF.getEHResumeBlock(/*isCleanup=*/true); auto *CleanupContBB = CGF.createBasicBlock("cleanup.cont"); - CGF.Builder.CreateCondBr(CoroEnd, ResumeBB, CleanupContBB); + auto *CoroWhereFn = CGM.getIntrinsic(llvm::Intrinsic::coro_where); + auto *CoroWhere = CGF.Builder.CreateCall(CoroWhereFn); + CGF.Builder.CreateCondBr(CoroWhere, ResumeBB, CleanupContBB); CGF.EmitBlock(CleanupContBB); } } diff --git a/clang/test/CodeGenCoroutines/coro-builtins.c b/clang/test/CodeGenCoroutines/coro-builtins.c index 79f119b2b60ff..0c2553274f09f 100644 --- a/clang/test/CodeGenCoroutines/coro-builtins.c +++ b/clang/test/CodeGenCoroutines/coro-builtins.c @@ -37,7 +37,7 @@ void f(int n) { // CHECK-NEXT: call ptr @llvm.coro.free(token %[[COROID]], ptr %[[FRAME]]) __builtin_coro_free(__builtin_coro_frame()); - // CHECK-NEXT: call i1 @llvm.coro.end(ptr %[[FRAME]], i1 false, token none) + // CHECK-NEXT: call void @llvm.coro.end(ptr %[[FRAME]], i1 false, token none) __builtin_coro_end(__builtin_coro_frame(), 0); // CHECK-NEXT: call i8 @llvm.coro.suspend(token none, i1 true) diff --git a/clang/test/CodeGenCoroutines/coro-eh-cleanup.cpp b/clang/test/CodeGenCoroutines/coro-eh-cleanup.cpp index 725cf8faa6b4c..d849ccef17681 100644 --- a/clang/test/CodeGenCoroutines/coro-eh-cleanup.cpp +++ b/clang/test/CodeGenCoroutines/coro-eh-cleanup.cpp @@ -60,7 +60,7 @@ coro_t f() { // CHECK: [[COROENDBB]]: // CHECK-NEXT: %[[CLPAD:.+]] = cleanuppad within none -// CHECK-NEXT: call i1 @llvm.coro.end(ptr null, i1 true, token none) [ "funclet"(token %[[CLPAD]]) ] +// CHECK-NEXT: call void @llvm.coro.end(ptr null, i1 true, token none) [ "funclet"(token %[[CLPAD]]) ] // CHECK-NEXT: cleanupret from %[[CLPAD]] unwind label // CHECK-LPAD: @_Z1fv( @@ -76,7 +76,7 @@ coro_t f() { // CHECK-LPAD: to label %{{.+}} unwind label %[[UNWINDBB:.+]] // CHECK-LPAD: [[UNWINDBB]]: -// CHECK-LPAD: %[[I1RESUME:.+]] = call i1 @llvm.coro.end(ptr null, i1 true, token none) +// CHECK-LPAD: %[[I1RESUME:.+]] = call i1 @llvm.coro.where() // CHECK-LPAD: br i1 %[[I1RESUME]], label %[[EHRESUME:.+]], label // CHECK-LPAD: [[EHRESUME]]: // CHECK-LPAD-NEXT: %[[exn:.+]] = load ptr, ptr %exn.slot, align 8 diff --git a/clang/test/CodeGenCoroutines/coro-lambda.cpp b/clang/test/CodeGenCoroutines/coro-lambda.cpp index 26c51070f9e2d..b24a190ab41fb 100644 --- a/clang/test/CodeGenCoroutines/coro-lambda.cpp +++ b/clang/test/CodeGenCoroutines/coro-lambda.cpp @@ -55,4 +55,4 @@ void f() { // CHECK: alloca %"struct.Task::promise_type" // CHECK: call token @llvm.coro.id( // CHECK: call i8 @llvm.coro.suspend( -// CHECK: call i1 @llvm.coro.end( +// CHECK: call void @llvm.coro.end( diff --git a/clang/test/CodeGenCoroutines/coro-params.cpp b/clang/test/CodeGenCoroutines/coro-params.cpp index 719726cca29c5..79e77a21017fa 100644 --- a/clang/test/CodeGenCoroutines/coro-params.cpp +++ b/clang/test/CodeGenCoroutines/coro-params.cpp @@ -117,7 +117,7 @@ void f(int val, MoveOnly moParam, MoveAndCopy mcParam, TrivialABI trivialParam) // CHECK-NEXT: call ptr @llvm.coro.free( // The original trivial_abi parameter is destroyed when returning from the ramp. - // CHECK: call i1 @llvm.coro.end + // CHECK: call void @llvm.coro.end // CHECK: call void @_ZN10TrivialABID1Ev(ptr {{[^,]*}} %[[TrivialAlloca]]) } @@ -242,6 +242,6 @@ void msabi(MSParm p) { co_return; // The local alloca is used for the destructor call at the end of the ramp. - // MSABI: call i1 @llvm.coro.end + // MSABI: call void @llvm.coro.end // MSABI: call void @"??1MSParm@@QEAA@XZ"(ptr{{.*}} %[[ParamAlloca]]) } diff --git a/llvm/docs/Coroutines.rst b/llvm/docs/Coroutines.rst index dde73c9c3cc23..d9114eb1d1aa6 100644 --- a/llvm/docs/Coroutines.rst +++ b/llvm/docs/Coroutines.rst @@ -303,7 +303,7 @@ The LLVM IR for this coroutine looks like this: call void @free(ptr %mem) br label %suspend suspend: - %unused = call i1 @llvm.coro.end(ptr %hdl, i1 false, token none) + call void @llvm.coro.end(ptr %hdl, i1 false, token none) ret ptr %hdl } @@ -637,7 +637,7 @@ store the current value produced by a coroutine. call void @free(ptr %mem) br label %suspend suspend: - %unused = call i1 @llvm.coro.end(ptr %hdl, i1 false, token none) + call void @llvm.coro.end(ptr %hdl, i1 false, token none) ret ptr %hdl } @@ -806,7 +806,7 @@ The LLVM IR for a coroutine using a Coroutine with a custom ABI looks like: call void @free(ptr %mem) br label %suspend suspend: - %unused = call i1 @llvm.coro.end(ptr %hdl, i1 false, token none) + call void @llvm.coro.end(ptr %hdl, i1 false, token none) ret ptr %hdl } @@ -1444,7 +1444,7 @@ A frontend should emit function attribute `presplitcoroutine` for the coroutine. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ :: - declare i1 @llvm.coro.end(ptr <handle>, i1 <unwind>, token <result.token>) + declare void @llvm.coro.end(ptr <handle>, i1 <unwind>, token <result.token>) Overview: """"""""" @@ -1502,7 +1502,8 @@ For landingpad based exception model, it is expected that frontend uses the .. code-block:: llvm ehcleanup: - %InResumePart = call i1 @llvm.coro.end(ptr null, i1 true, token none) + call void @llvm.coro.end(ptr null, i1 true, token none) + %InResumePart = call i1 @llvm.coro.where() br i1 %InResumePart, label %eh.resume, label %cleanup.cont cleanup.cont: @@ -1515,7 +1516,7 @@ For landingpad based exception model, it is expected that frontend uses the %lpad.val29 = insertvalue { ptr, i32 } %lpad.val, i32 %sel, 1 resume { ptr, i32 } %lpad.val29 -The `CoroSpit` pass replaces `coro.end` with ``True`` in the resume functions, +The `CoroSpit` pass replaces `coro.where` with ``True`` in the resume functions, thus leading to immediate unwind to the caller, whereas in start function it is replaced with ``False``, thus allowing to proceed to the rest of the cleanup code that is only needed during initial invocation of the coroutine. @@ -1527,7 +1528,7 @@ referring to an enclosing cleanuppad as follows: ehcleanup: %tok = cleanuppad within none [] - %unused = call i1 @llvm.coro.end(ptr null, i1 true, token none) [ "funclet"(token %tok) ] + call void @llvm.coro.end(ptr null, i1 true, token none) [ "funclet"(token %tok) ] cleanupret from %tok unwind label %RestOfTheCleanup The `CoroSplit` pass, if the funclet bundle is present, will insert @@ -1592,7 +1593,7 @@ The number of arguments must match the return type of the continuation function: cleanup: %tok = call token (...) @llvm.coro.end.results(i8 %val) - call i1 @llvm.coro.end(ptr %hdl, i1 0, token %tok) + call void @llvm.coro.end(ptr %hdl, i1 0, token %tok) unreachable ... @@ -1604,7 +1605,7 @@ The number of arguments must match the return type of the continuation function: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ :: - declare i1 @llvm.coro.end.async(ptr <handle>, i1 <unwind>, ...) + declare void @llvm.coro.end.async(ptr <handle>, i1 <unwind>, ...) Overview: """"""""" @@ -1635,10 +1636,10 @@ the function call. .. code-block:: llvm - call i1 (ptr, i1, ...) @llvm.coro.end.async( - ptr %hdl, i1 0, - ptr @must_tail_call_return, - ptr %ctxt, ptr %task, ptr %actor) + call void (ptr, i1, ...) @llvm.coro.end.async( + ptr %hdl, i1 0, + ptr @must_tail_call_return, + ptr %ctxt, ptr %task, ptr %actor) unreachable .. _coro.suspend: @@ -2117,6 +2118,30 @@ Example: %hdl.result = ... ; get address of returned coroutine handle ret ptr %hdl.result +'llvm.coro.where' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: + + declare i1 @llvm.coro.where() + +Overview: +""""""""" + +The '``llvm.coro.where``' intrinsic returns a bool value that marks coroutine resume +part and start part. + +Arguments: +"""""""""" + +None + +Semantics: +"""""""""" + +The `CoroSpit` pass replaces `coro.where` with ``True`` in the resume functions, +whereas in start function it is replaced with ``False``, thus allowing the frontend +separate resume part and start part. + Coroutine Transformation Passes =============================== CoroEarly diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td index e0ee12391b31d..0d113e0ccb854 100644 --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -1775,12 +1775,13 @@ def int_coro_free : Intrinsic<[llvm_ptr_ty], [llvm_token_ty, llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly, ReadOnly<ArgIndex<1>>, NoCapture<ArgIndex<1>>]>; -def int_coro_end : Intrinsic<[llvm_i1_ty], [llvm_ptr_ty, llvm_i1_ty, llvm_token_ty], []>; +def int_coro_end : Intrinsic<[], [llvm_ptr_ty, llvm_i1_ty, llvm_token_ty], []>; def int_coro_end_results : Intrinsic<[llvm_token_ty], [llvm_vararg_ty]>; def int_coro_end_async - : Intrinsic<[llvm_i1_ty], [llvm_ptr_ty, llvm_i1_ty, llvm_vararg_ty], []>; + : Intrinsic<[], [llvm_ptr_ty, llvm_i1_ty, llvm_vararg_ty], []>; def int_coro_frame : Intrinsic<[llvm_ptr_ty], [], [IntrNoMem]>; +def int_coro_where : Intrinsic<[llvm_i1_ty], [], [IntrNoMem]>; def int_coro_noop : Intrinsic<[llvm_ptr_ty], [], [IntrNoMem]>; def int_coro_size : Intrinsic<[llvm_anyint_ty], [], [IntrNoMem]>; def int_coro_align : Intrinsic<[llvm_anyint_ty], [], [IntrNoMem]>; diff --git a/llvm/include/llvm/Transforms/Coroutines/CoroInstr.h b/llvm/include/llvm/Transforms/Coroutines/CoroInstr.h index 0688068167ae6..36ec76c554d50 100644 --- a/llvm/include/llvm/Transforms/Coroutines/CoroInstr.h +++ b/llvm/include/llvm/Transforms/Coroutines/CoroInstr.h @@ -428,6 +428,18 @@ class CoroFrameInst : public IntrinsicInst { } }; +/// This represents the llvm.coro.where instruction. +class CoroWhereInst : public IntrinsicInst { +public: + // Methods to support type inquiry through isa, cast, and dyn_cast: + static bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::coro_where; + } + static bool classof(const Value *V) { + return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); + } +}; + /// This represents the llvm.coro.free instruction. class CoroFreeInst : public IntrinsicInst { enum { IdArg, FrameArg }; diff --git a/llvm/include/llvm/Transforms/Coroutines/CoroShape.h b/llvm/include/llvm/Transforms/Coroutines/CoroShape.h index c54081de2d9da..83697200e061d 100644 --- a/llvm/include/llvm/Transforms/Coroutines/CoroShape.h +++ b/llvm/include/llvm/Transforms/Coroutines/CoroShape.h @@ -53,6 +53,7 @@ enum class ABI { struct Shape { CoroBeginInst *CoroBegin = nullptr; SmallVector<AnyCoroEndInst *, 4> CoroEnds; + SmallVector<CoroWhereInst *, 2> CoroWheres; SmallVector<CoroSizeInst *, 2> CoroSizes; SmallVector<CoroAlignInst *, 2> CoroAligns; SmallVector<AnyCoroSuspendInst *, 4> CoroSuspends; @@ -65,6 +66,7 @@ struct Shape { void clear() { CoroBegin = nullptr; CoroEnds.clear(); + CoroWheres.clear(); CoroSizes.clear(); CoroAligns.clear(); CoroSuspends.clear(); diff --git a/llvm/lib/Transforms/Coroutines/CoroCleanup.cpp b/llvm/lib/Transforms/Coroutines/CoroCleanup.cpp index c00e9c7bbee06..65b5a6fc18c0f 100644 --- a/llvm/lib/Transforms/Coroutines/CoroCleanup.cpp +++ b/llvm/lib/Transforms/Coroutines/CoroCleanup.cpp @@ -75,8 +75,8 @@ bool Lowerer::lower(Function &F) { case Intrinsic::coro_subfn_addr: lowerSubFn(Builder, cast<CoroSubFnInst>(II)); break; - case Intrinsic::coro_end: case Intrinsic::coro_suspend_retcon: + case Intrinsic::coro_where: if (IsPrivateAndUnprocessed) { II->replaceAllUsesWith(PoisonValue::get(II->getType())); } else diff --git a/llvm/lib/Transforms/Coroutines/CoroCloner.h b/llvm/lib/Transforms/Coroutines/CoroCloner.h index d1887980fb3bc..78de08879c99e 100644 --- a/llvm/lib/Transforms/Coroutines/CoroCloner.h +++ b/llvm/lib/Transforms/Coroutines/CoroCloner.h @@ -120,6 +120,7 @@ class BaseCloner { void replaceRetconOrAsyncSuspendUses(); void replaceCoroSuspends(); void replaceCoroEnds(); + void replaceCoroWhere(); void replaceSwiftErrorOps(); void salvageDebugInfo(); void handleFinalSuspend(); diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp index 180ac9c61e7df..5961cbf726f1a 100644 --- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp +++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp @@ -388,10 +388,6 @@ static void replaceCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, replaceUnwindCoroEnd(End, Shape, FramePtr, InResume, CG); else replaceFallthroughCoroEnd(End, Shape, FramePtr, InResume, CG); - - auto &Context = End->getContext(); - End->replaceAllUsesWith(InResume ? ConstantInt::getTrue(Context) - : ConstantInt::getFalse(Context)); End->eraseFromParent(); } @@ -562,6 +558,15 @@ void coro::BaseCloner::replaceCoroEnds() { } } +void coro::BaseCloner::replaceCoroWhere() { + auto &Ctx = OrigF.getContext(); + for (auto *CW : Shape.CoroWheres) { + auto *NewCW = cast<CoroWhereInst>(VMap[CW]); + NewCW->replaceAllUsesWith(ConstantInt::getTrue(Ctx)); + NewCW->eraseFromParent(); + } +} + static void replaceSwiftErrorOps(Function &F, coro::Shape &Shape, ValueToValueMapTy *VMap) { if (Shape.ABI == coro::ABI::Async && Shape.CoroSuspends.empty()) @@ -1077,6 +1082,8 @@ void coro::BaseCloner::create() { // Remove coro.end intrinsics. replaceCoroEnds(); + replaceCoroWhere(); + // Salvage debug info that points into the coroutine frame. salvageDebugInfo(); } @@ -1951,11 +1958,16 @@ static void removeCoroEndsFromRampFunction(const coro::Shape &Shape) { replaceCoroEnd(End, Shape, Shape.FramePtr, /*in resume*/ false, nullptr); } } else { - for (llvm::AnyCoroEndInst *End : Shape.CoroEnds) { - auto &Context = End->getContext(); - End->replaceAllUsesWith(ConstantInt::getFalse(Context)); + for (llvm::AnyCoroEndInst *End : Shape.CoroEnds) End->eraseFromParent(); - } + } +} + +static void removeCoroWhereFromRampFunction(const coro::Shape &Shape) { + for (auto *CW : Shape.CoroWheres) { + auto &Ctx = CW->getContext(); + CW->replaceAllUsesWith(ConstantInt::getFalse(Ctx)); + CW->eraseFromParent(); } } @@ -2020,6 +2032,7 @@ static void doSplitCoroutine(Function &F, SmallVectorImpl<Function *> &Clones, coro::salvageDebugInfo(ArgToAllocaMap, *DVR, false /*UseEntryValue*/); removeCoroEndsFromRampFunction(Shape); + removeCoroWhereFromRampFunction(Shape); if (shouldCreateNoAllocVariant) SwitchCoroutineSplitter::createNoAllocVariant(F, Shape, Clones); diff --git a/llvm/lib/Transforms/Coroutines/Coroutines.cpp b/llvm/lib/Transforms/Coroutines/Coroutines.cpp index ac93f748ce65c..7c3cbe62ab7b1 100644 --- a/llvm/lib/Transforms/Coroutines/Coroutines.cpp +++ b/llvm/lib/Transforms/Coroutines/Coroutines.cpp @@ -93,6 +93,7 @@ static Intrinsic::ID NonOverloadedCoroIntrinsics[] = { Intrinsic::coro_save, Intrinsic::coro_subfn_addr, Intrinsic::coro_suspend, + Intrinsic::coro_where, }; bool coro::isSuspendBlock(BasicBlock *BB) { @@ -275,6 +276,9 @@ void coro::Shape::analyze(Function &F, } } break; + case Intrinsic::coro_where: + CoroWheres.push_back(cast<CoroWhereInst>(II)); + break; case Intrinsic::coro_promise: assert(CoroPromise == nullptr && "CoroEarly must ensure coro.promise unique"); diff --git a/llvm/test/Analysis/GlobalsModRef/nonescaping-noalias.ll b/llvm/test/Analysis/GlobalsModRef/nonescaping-noalias.ll index eed93cf0df8ef..e2eb4f6e7b9e9 100644 --- a/llvm/test/Analysis/GlobalsModRef/nonescaping-noalias.ll +++ b/llvm/test/Analysis/GlobalsModRef/nonescaping-noalias.ll @@ -62,7 +62,7 @@ define ptr @test1_tls_noopt(ptr %coro, ptr %param) presplitcoroutine { ; CHECK-NEXT: store i32 [[V]], ptr [[PARAM]], align 4 ; CHECK-NEXT: ret ptr [[CORO]] ; CHECK: suspend: -; CHECK-NEXT: [[TMP1:%.*]] = call i1 @llvm.coro.end(ptr [[CORO]], i1 false, token none) +; CHECK-NEXT: call void @llvm.coro.end(ptr [[CORO]], i1 false, token none) ; CHECK-NEXT: ret ptr [[CORO]] ; entry: @@ -79,7 +79,7 @@ resume: ret ptr %coro suspend: - call i1 @llvm.coro.end(ptr %coro, i1 0, token none) + call void @llvm.coro.end(ptr %coro, i1 0, token none) ret ptr %coro } diff --git a/llvm/test/Assembler/auto_upgrade_intrinsics.ll b/llvm/test/Assembler/auto_upgrade_intrinsics.ll index 37cb49650f6bd..64d4a3ba7c802 100644 --- a/llvm/test/Assembler/auto_upgrade_intrinsics.ll +++ b/llvm/test/Assembler/auto_upgrade_intrinsics.ll @@ -47,11 +47,11 @@ entry: ret void } -declare i1 @llvm.coro.end(ptr, i1) +declare void @llvm.coro.end(ptr, i1) define void @test.coro.end(ptr %ptr) { ; CHECK-LABEL: @test.coro.end( -; CHECK: call i1 @llvm.coro.end(ptr %ptr, i1 false, token none) - call i1 @llvm.coro.end(ptr %ptr, i1 false) +; CHECK: call void @llvm.coro.end(ptr %ptr, i1 false, token none) + call void @llvm.coro.end(ptr %ptr, i1 false) ret void } diff --git a/llvm/test/Transforms/Coroutines/ArgAddr.ll b/llvm/test/Transforms/Coroutines/ArgAddr.ll index ab70836508101..9328c67459077 100644 --- a/llvm/test/Transforms/Coroutines/ArgAddr.ll +++ b/llvm/test/Transforms/Coroutines/ArgAddr.ll @@ -45,7 +45,7 @@ coro_Cleanup: br label %coro_Suspend coro_Suspend: - call i1 @llvm.coro.end(ptr null, i1 false, token none) + call void @llvm.coro.end(ptr null, i1 false, token none) ret ptr %1 } @@ -69,7 +69,7 @@ declare i32 @llvm.coro.size.i32() declare ptr @llvm.coro.begin(token, ptr) declare i8 @llvm.coro.suspend(token, i1) declare ptr @llvm.coro.free(token, ptr) -declare i1 @llvm.coro.end(ptr, i1, token) +declare void @llvm.coro.end(ptr, i1, token) declare void @llvm.coro.resume(ptr) declare void @llvm.coro.destroy(ptr) diff --git a/llvm/test/Transforms/Coroutines/coro-align16.ll b/llvm/test/Transforms/Coroutines/coro-align16.ll index 39902be9149e8..afdca77e8af3e 100644 --- a/llvm/test/Transforms/Coroutines/coro-align16.ll +++ b/llvm/test/Transforms/Coroutines/coro-align16.ll @@ -24,7 +24,7 @@ cleanup: br label %suspend suspend: - call i1 @llvm.coro.end(ptr %hdl, i1 0, token none) + call void @llvm.coro.end(ptr %hdl, i1 0, token none) ret ptr %hdl } @@ -44,7 +44,7 @@ declare void @llvm.coro.destroy(ptr) declare token @llvm.coro.id(i32, ptr, ptr, ptr) declare i1 @llvm.coro.alloc(token) declare ptr @llvm.coro.begin(token, ptr) -declare i1 @llvm.coro.end(ptr, i1, token) +declare void @llvm.coro.end(ptr, i1, token) declare void @capture_call(ptr) declare void @nocapture_call(ptr nocapture) diff --git a/llvm/test/Transforms/Coroutines/coro-align32.ll b/llvm/test/Transforms/Coroutines/coro-align32.ll index 3d910e951259b..9e82ec83011f5 100644 --- a/llvm/test/Transforms/Coroutines/coro-align32.ll +++ b/llvm/test/Transforms/Coroutines/coro-align32.ll @@ -28,7 +28,7 @@ cleanup... [truncated] `````````` </details> https://github.com/llvm/llvm-project/pull/153404 _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
