llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-coroutines @llvm/pr-subscribers-clang-codegen Author: Weibo He (NewSigma) <details> <summary>Changes</summary> `CoroElide` requires that we see the `llvm.coro.destroy` intrinsic. However, we never emit the intrinsic if a coroutine flows off the final suspending point, and HALO does not work, even for trivial cases. See https://godbolt.org/z/3KqfzMY7e ``` C++ #include <coroutine> struct coro { using promise_type = coro; auto get_return_object() noexcept { return coro{std::coroutine_handle<coro>::from_promise(*this)}; } std::suspend_never initial_suspend() noexcept { return {}; } std::suspend_never final_suspend() noexcept { return {}; } void return_void() noexcept {} void unhandled_exception() {} std::coroutine_handle<> handle; }; coro fn() { co_await std::suspend_always{}; } int main() { fn().handle.resume(); } ``` This patch proposes a optimization hint `llvm.coro.dead` for cases where `llvm.coro.destroy` is not available so that HALO works. It contains 4 commits: - [CoroSplit] Keep coro.free for resume/destroy Keep these coro.free so that we can elide them - [Coroutines][NFC] Elide coro.free based on frame instead of coro.id coro.id is unavailable in resumers - [CoroElide][IR] Add llvm.coro.dead ME part - [clang][CodeGen] Emit coro.dead for coroutines Final FE part: see if we break anything --- Patch is 35.50 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/185336.diff 27 Files Affected: - (modified) clang/lib/CodeGen/CGCoroutine.cpp (+3) - (modified) clang/test/CodeGenCoroutines/coro-elide.cpp (+26-1) - (modified) llvm/docs/Coroutines.rst (+43) - (modified) llvm/include/llvm/IR/Intrinsics.td (+1) - (modified) llvm/lib/Transforms/Coroutines/CoroCleanup.cpp (+8-6) - (modified) llvm/lib/Transforms/Coroutines/CoroElide.cpp (+31-28) - (modified) llvm/lib/Transforms/Coroutines/CoroInternal.h (+1-1) - (modified) llvm/lib/Transforms/Coroutines/CoroSplit.cpp (+8-11) - (modified) llvm/lib/Transforms/Coroutines/Coroutines.cpp (+4-8) - (modified) llvm/test/Transforms/Coroutines/coro-alloc-with-param-O0.ll (+4-2) - (modified) llvm/test/Transforms/Coroutines/coro-alloc-with-param-O2.ll (+4-2) - (modified) llvm/test/Transforms/Coroutines/coro-alloca-07.ll (+4-2) - (modified) llvm/test/Transforms/Coroutines/coro-await-suspend-lower-invoke.ll (+4-2) - (modified) llvm/test/Transforms/Coroutines/coro-await-suspend-lower.ll (+4-2) - (modified) llvm/test/Transforms/Coroutines/coro-eh-aware-edge-split-00.ll (+1-1) - (modified) llvm/test/Transforms/Coroutines/coro-eh-aware-edge-split-01.ll (+2-1) - (modified) llvm/test/Transforms/Coroutines/coro-eh-aware-edge-split-02.ll (+2-1) - (modified) llvm/test/Transforms/Coroutines/coro-frame-arrayalloca.ll (+4-2) - (modified) llvm/test/Transforms/Coroutines/coro-frame.ll (+4-2) - (modified) llvm/test/Transforms/Coroutines/coro-only-destroy-when-complete.ll (+10-2) - (modified) llvm/test/Transforms/Coroutines/coro-padding.ll (+4-2) - (modified) llvm/test/Transforms/Coroutines/coro-spill-corobegin.ll (+4-2) - (modified) llvm/test/Transforms/Coroutines/coro-spill-promise-02.ll (+4-2) - (modified) llvm/test/Transforms/Coroutines/coro-spill-promise.ll (+4-2) - (modified) llvm/test/Transforms/Coroutines/coro-spill-suspend.ll (+4-2) - (modified) llvm/test/Transforms/Coroutines/coro-split-tbaa-md.ll (+4-2) - (modified) llvm/test/Transforms/Coroutines/coro-zero-alloca.ll (+4-2) ``````````diff diff --git a/clang/lib/CodeGen/CGCoroutine.cpp b/clang/lib/CodeGen/CGCoroutine.cpp index 7282c42420657..9d9d2450c3d68 100644 --- a/clang/lib/CodeGen/CGCoroutine.cpp +++ b/clang/lib/CodeGen/CGCoroutine.cpp @@ -643,6 +643,9 @@ struct CallCoroDelete final : public EHScopeStack::Cleanup { // No longer need old terminator. InsertPt->eraseFromParent(); CGF.Builder.SetInsertPoint(AfterFreeBB); + + auto *CoroDeadFn = CGF.CGM.getIntrinsic(llvm::Intrinsic::coro_dead); + CGF.Builder.CreateCall(CoroDeadFn, {CGF.CurCoro.Data->CoroBegin}); } explicit CallCoroDelete(Stmt *DeallocStmt) : Deallocate(DeallocStmt) {} }; diff --git a/clang/test/CodeGenCoroutines/coro-elide.cpp b/clang/test/CodeGenCoroutines/coro-elide.cpp index d7569c3b4d087..1902dd64b5e5a 100644 --- a/clang/test/CodeGenCoroutines/coro-elide.cpp +++ b/clang/test/CodeGenCoroutines/coro-elide.cpp @@ -1,8 +1,32 @@ -// This tests that the coroutine elide optimization could happen succesfully. // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -std=c++20 -O2 -emit-llvm %s -o - | FileCheck %s #include "Inputs/coroutine.h" +namespace { +struct coro { + using promise_type = coro; + + auto get_return_object() noexcept { return coro{std::coroutine_handle<coro>::from_promise(*this)}; } + std::suspend_never initial_suspend() noexcept { return {}; } + std::suspend_never final_suspend() noexcept { return {}; } + void return_void() noexcept {} + void unhandled_exception() {} + + std::coroutine_handle<> handle; +}; +} + +void flowoff() { + []() -> coro { + co_await std::suspend_always{}; + }().handle.resume(); +} + +// Tests that the coroutine elide optimization could happen if control flows off the end of the coroutine +// CHECK-LABEL: define{{.*}} void @_Z7flowoffv +// CHECK-NEXT: entry: +// CHECK-NEXT: ret void + struct Task { struct promise_type { struct FinalAwaiter { @@ -58,5 +82,6 @@ Task task1() { co_return co_await task0(); } +// Tests that the coroutine elide optimization could happen if handle.destroy() is invoked // CHECK-LABEL: define{{.*}} void @_Z5task1v.resume // CHECK-NOT: call{{.*}}_Znwm diff --git a/llvm/docs/Coroutines.rst b/llvm/docs/Coroutines.rst index 0e6b49c84acee..34de250ce5e4f 100644 --- a/llvm/docs/Coroutines.rst +++ b/llvm/docs/Coroutines.rst @@ -873,6 +873,8 @@ the coroutine destroy function. Otherwise it is replaced with an indirect call based on the function pointer for the destroy function stored in the coroutine frame. Destroying a coroutine that is not suspended leads to undefined behavior. +This intrinsic implies `coro.dead`. + .. _coro.resume: 'llvm.coro.resume' Intrinsic @@ -1169,6 +1171,47 @@ Example (standard deallocation functions): call void @free(ptr %mem) ret void +.. _coro.dead: + +'llvm.coro.dead' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: + + declare void @llvm.coro.dead(ptr <frame>) + +Overview: +""""""""" + +The '``llvm.coro.dead``' intrinsic is an optimization hint to help Heap Allocation eLision Optimization(HALO). + +Arguments: +"""""""""" + +The argument is a pointer to the coroutine frame. This should be the same +pointer that was returned by prior `coro.begin` call. + +Semantics: +"""""""""" + +A frontend can delegate this intrinsic to indicate that the coroutine frame is dead, allowing +coroutines that are not explicitly destroyed via `coro.destroy` to be elided. + +Example: +""""""""""""""""""""""""""""""""""""""" + +.. code-block:: llvm + + cleanup: + %mem = call ptr @llvm.coro.free(token %id, ptr %frame) + %mem_not_null = icmp ne ptr %mem, null + br i1 %mem_not_null, label %if.then, label %if.end + if.then: + call void @CustomFree(ptr %mem) + br label %if.end + if.end: + call void @llvm.coro.dead(ptr %frame) + ret void + .. _coro.alloc: 'llvm.coro.alloc' Intrinsic diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td index 5b5fffaa48951..d3ca2529dc1bd 100644 --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -1849,6 +1849,7 @@ def int_coro_free : Intrinsic<[llvm_ptr_ty], [llvm_token_ty, llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly, ReadOnly<ArgIndex<1>>, NoCapture<ArgIndex<1>>]>; +def int_coro_dead : Intrinsic<[], [llvm_ptr_ty], [IntrNoMem]>; def int_coro_end : Intrinsic<[], [llvm_ptr_ty, llvm_i1_ty, llvm_token_ty], []>; def int_coro_end_results : Intrinsic<[llvm_token_ty], [llvm_vararg_ty]>; def int_coro_end_async diff --git a/llvm/lib/Transforms/Coroutines/CoroCleanup.cpp b/llvm/lib/Transforms/Coroutines/CoroCleanup.cpp index c44eaddd7ee55..2ba4c4d953f7e 100644 --- a/llvm/lib/Transforms/Coroutines/CoroCleanup.cpp +++ b/llvm/lib/Transforms/Coroutines/CoroCleanup.cpp @@ -108,6 +108,8 @@ bool Lowerer::lower(Function &F) { case Intrinsic::coro_free: II->replaceAllUsesWith(II->getArgOperand(1)); break; + case Intrinsic::coro_dead: + break; case Intrinsic::coro_alloc: II->replaceAllUsesWith(ConstantInt::getTrue(Context)); break; @@ -256,12 +258,12 @@ void NoopCoroElider::eraseFromWorklist(Instruction *I) { static bool declaresCoroCleanupIntrinsics(const Module &M) { return coro::declaresIntrinsics( - M, - {Intrinsic::coro_alloc, Intrinsic::coro_begin, Intrinsic::coro_subfn_addr, - Intrinsic::coro_free, Intrinsic::coro_id, Intrinsic::coro_id_retcon, - Intrinsic::coro_id_async, Intrinsic::coro_id_retcon_once, - Intrinsic::coro_noop, Intrinsic::coro_async_size_replace, - Intrinsic::coro_async_resume, Intrinsic::coro_begin_custom_abi}); + M, {Intrinsic::coro_alloc, Intrinsic::coro_begin, + Intrinsic::coro_subfn_addr, Intrinsic::coro_free, + Intrinsic::coro_dead, Intrinsic::coro_id, Intrinsic::coro_id_retcon, + Intrinsic::coro_id_async, Intrinsic::coro_id_retcon_once, + Intrinsic::coro_noop, Intrinsic::coro_async_size_replace, + Intrinsic::coro_async_resume, Intrinsic::coro_begin_custom_abi}); } PreservedAnalyses CoroCleanupPass::run(Module &M, diff --git a/llvm/lib/Transforms/Coroutines/CoroElide.cpp b/llvm/lib/Transforms/Coroutines/CoroElide.cpp index 1c8d4a8592d60..adff67a472fb4 100644 --- a/llvm/lib/Transforms/Coroutines/CoroElide.cpp +++ b/llvm/lib/Transforms/Coroutines/CoroElide.cpp @@ -73,7 +73,8 @@ class CoroIdElider { SmallVector<CoroBeginInst *, 1> CoroBegins; SmallVector<CoroAllocInst *, 1> CoroAllocs; SmallVector<CoroSubFnInst *, 4> ResumeAddr; - DenseMap<CoroBeginInst *, SmallVector<CoroSubFnInst *, 4>> DestroyAddr; + SmallVector<CoroSubFnInst *, 4> DestroyAddr; + DenseMap<CoroBeginInst *, SmallVector<IntrinsicInst *, 4>> BeginDeadMap; }; } // end anonymous namespace @@ -177,23 +178,31 @@ CoroIdElider::CoroIdElider(CoroIdInst *CoroId, FunctionElideInfo &FEI, CoroAllocs.push_back(CA); } - // Collect all coro.subfn.addrs associated with coro.begin. - // Note, we only devirtualize the calls if their coro.subfn.addr refers to - // coro.begin directly. If we run into cases where this check is too - // conservative, we can consider relaxing the check. for (CoroBeginInst *CB : CoroBegins) { - for (User *U : CB->users()) - if (auto *II = dyn_cast<CoroSubFnInst>(U)) + for (User *U : CB->users()) { + auto &LifetimeHints = BeginDeadMap[CB]; + // Collect all coro.subfn.addrs associated with coro.begin. + // Note, we only devirtualize the calls if their coro.subfn.addr refers to + // coro.begin directly. If we run into cases where this check is too + // conservative, we can consider relaxing the check. + if (auto *II = dyn_cast<CoroSubFnInst>(U)) { switch (II->getIndex()) { case CoroSubFnInst::ResumeIndex: ResumeAddr.push_back(II); break; case CoroSubFnInst::DestroyIndex: - DestroyAddr[CB].push_back(II); + LifetimeHints.push_back(II); + DestroyAddr.push_back(II); break; default: llvm_unreachable("unexpected coro.subfn.addr constant"); } + } + + auto *II = dyn_cast<IntrinsicInst>(U); + if (II && II->getIntrinsicID() == Intrinsic::coro_dead) + LifetimeHints.push_back(II); + } } } @@ -228,6 +237,7 @@ void CoroIdElider::elideHeapAllocations(uint64_t FrameSize, Align FrameAlign) { new BitCastInst(Frame, PointerType::getUnqual(C), "vFrame", InsertPt); for (auto *CB : CoroBegins) { + coro::elideCoroFree(CB); CB->replaceAllUsesWith(FrameVoidPtr); CB->eraseFromParent(); } @@ -239,8 +249,8 @@ void CoroIdElider::elideHeapAllocations(uint64_t FrameSize, Align FrameAlign) { bool CoroIdElider::canCoroBeginEscape( const CoroBeginInst *CB, const SmallPtrSetImpl<BasicBlock *> &TIs) const { - const auto &It = DestroyAddr.find(CB); - assert(It != DestroyAddr.end()); + const auto &It = BeginDeadMap.find(CB); + assert(It != BeginDeadMap.end()); // Limit the number of blocks we visit. unsigned Limit = 32 * (1 + It->second.size()); @@ -249,8 +259,8 @@ bool CoroIdElider::canCoroBeginEscape( Worklist.push_back(CB->getParent()); SmallPtrSet<const BasicBlock *, 32> Visited; - // Consider basicblock of coro.destroy as visited one, so that we - // skip the path pass through coro.destroy. + // Consider basicblock of coro.dead as visited one, so that we + // skip the path pass through coro.dead. for (auto *DA : It->second) Visited.insert(DA->getParent()); @@ -326,11 +336,11 @@ bool CoroIdElider::lifetimeEligibleForElide() const { if (CoroAllocs.empty()) return false; - // Check that for every coro.begin there is at least one coro.destroy directly + // Check that for every coro.begin there is at least one coro.dead directly // referencing the SSA value of that coro.begin along each // non-exceptional path. // - // If the value escaped, then coro.destroy would have been referencing a + // If the value escaped, then coro.dead would have been referencing a // memory location storing that value and not the virtual register. SmallPtrSet<BasicBlock *, 8> Terminators; @@ -346,21 +356,16 @@ bool CoroIdElider::lifetimeEligibleForElide() const { Terminators.insert(&B); } - // Filter out the coro.destroy that lie along exceptional paths. + // Filter out the coro.dead that lie along exceptional paths. for (const auto *CB : CoroBegins) { - auto It = DestroyAddr.find(CB); - - // FIXME: If we have not found any destroys for this coro.begin, we - // disqualify this elide. - if (It == DestroyAddr.end()) + auto It = BeginDeadMap.find(CB); + if (It == BeginDeadMap.end()) return false; - const auto &CorrespondingDestroyAddrs = It->second; - - // If every terminators is dominated by coro.destroy, we could know the + // If every terminators is dominated by coro.dead, we could know the // corresponding coro.begin wouldn't escape. auto DominatesTerminator = [&](auto *TI) { - return llvm::any_of(CorrespondingDestroyAddrs, [&](auto *Destroy) { + return llvm::any_of(It->second, [&](auto *Destroy) { return DT.dominates(Destroy, TI->getTerminator()); }); }; @@ -370,7 +375,7 @@ bool CoroIdElider::lifetimeEligibleForElide() const { // Otherwise canCoroBeginEscape would decide whether there is any paths from // coro.begin to Terminators which not pass through any of the - // coro.destroys. This is a slower analysis. + // coro.dead. This is a slower analysis. // // canCoroBeginEscape is relatively slow, so we avoid to run it as much as // possible. @@ -400,8 +405,7 @@ bool CoroIdElider::attemptElide() { EligibleForElide ? CoroSubFnInst::CleanupIndex : CoroSubFnInst::DestroyIndex); - for (auto &It : DestroyAddr) - replaceWithConstant(DestroyAddrConstant, It.second); + replaceWithConstant(DestroyAddrConstant, DestroyAddr); auto FrameSizeAndAlign = getFrameLayout(cast<Function>(ResumeAddrConstant)); @@ -410,7 +414,6 @@ bool CoroIdElider::attemptElide() { if (EligibleForElide && FrameSizeAndAlign) { elideHeapAllocations(FrameSizeAndAlign->first, FrameSizeAndAlign->second); - coro::replaceCoroFree(CoroId, /*Elide=*/true); NumOfCoroElided++; #ifndef NDEBUG diff --git a/llvm/lib/Transforms/Coroutines/CoroInternal.h b/llvm/lib/Transforms/Coroutines/CoroInternal.h index cc47a557ee5c0..319e600870091 100644 --- a/llvm/lib/Transforms/Coroutines/CoroInternal.h +++ b/llvm/lib/Transforms/Coroutines/CoroInternal.h @@ -21,7 +21,7 @@ namespace llvm::coro { bool isSuspendBlock(BasicBlock *BB); bool declaresAnyIntrinsic(const Module &M); bool declaresIntrinsics(const Module &M, ArrayRef<Intrinsic::ID> List); -void replaceCoroFree(CoroIdInst *CoroId, bool Elide); +void elideCoroFree(Value *FramePtr); /// Replaces all @llvm.coro.alloc intrinsics calls associated with a given /// call @llvm.coro.id instruction with boolean value false. diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp index 587f581ded8d5..d43519ab0d3e9 100644 --- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp +++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp @@ -1100,10 +1100,9 @@ void coro::SwitchCloner::create() { // Clone the function coro::BaseCloner::create(); - // Eliminate coro.free from the clones, replacing it with 'null' in cleanup, - // to suppress deallocation code. - coro::replaceCoroFree(cast<CoroIdInst>(VMap[Shape.CoroBegin->getId()]), - /*Elide=*/FKind == coro::CloneKind::SwitchCleanup); + // Replacing coro.free with 'null' in cleanup to suppress deallocation code. + if (FKind == coro::CloneKind::SwitchCleanup) + elideCoroFree(NewFramePtr); } static void updateAsyncFuncPointerContextSize(coro::Shape &Shape) { @@ -1163,10 +1162,9 @@ static void handleNoSuspendCoroutine(coro::Shape &Shape) { auto *CoroBegin = Shape.CoroBegin; switch (Shape.ABI) { case coro::ABI::Switch: { - auto SwitchId = Shape.getSwitchCoroId(); - auto *AllocInst = SwitchId->getCoroAlloc(); - coro::replaceCoroFree(SwitchId, /*Elide=*/AllocInst != nullptr); - if (AllocInst) { + if (auto *AllocInst = Shape.getSwitchCoroId()->getCoroAlloc()) { + coro::elideCoroFree(CoroBegin); + IRBuilder<> Builder(AllocInst); // Create an alloca for a byte array of the frame size auto *FrameTy = ArrayType::get(Type::getInt8Ty(Builder.getContext()), @@ -1441,9 +1439,8 @@ struct SwitchCoroutineSplitter { if (Shape.CoroBegin) { auto *NewCoroBegin = cast_if_present<CoroBeginInst>(VMap[Shape.CoroBegin]); - auto *NewCoroId = cast<CoroIdInst>(NewCoroBegin->getId()); - coro::replaceCoroFree(NewCoroId, /*Elide=*/true); - coro::suppressCoroAllocs(NewCoroId); + coro::elideCoroFree(NewCoroBegin); + coro::suppressCoroAllocs(cast<CoroIdInst>(NewCoroBegin->getId())); NewCoroBegin->replaceAllUsesWith(NoAllocF->getArg(FrameIdx)); NewCoroBegin->eraseFromParent(); } diff --git a/llvm/lib/Transforms/Coroutines/Coroutines.cpp b/llvm/lib/Transforms/Coroutines/Coroutines.cpp index 2922e39a85e81..a68a5bf1623b5 100644 --- a/llvm/lib/Transforms/Coroutines/Coroutines.cpp +++ b/llvm/lib/Transforms/Coroutines/Coroutines.cpp @@ -118,11 +118,10 @@ bool coro::declaresIntrinsics(const Module &M, ArrayRef<Intrinsic::ID> List) { return false; } -// Replace all coro.frees associated with the provided CoroId either with 'null' -// if Elide is true and with its frame parameter otherwise. -void coro::replaceCoroFree(CoroIdInst *CoroId, bool Elide) { +// Replace all coro.frees associated with the provided frame with 'null' +void coro::elideCoroFree(Value *FramePtr) { SmallVector<CoroFreeInst *, 4> CoroFrees; - for (User *U : CoroId->users()) + for (User *U : FramePtr->users()) if (auto CF = dyn_cast<CoroFreeInst>(U)) CoroFrees.push_back(CF); @@ -130,10 +129,7 @@ void coro::replaceCoroFree(CoroIdInst *CoroId, bool Elide) { return; Value *Replacement = - Elide - ? ConstantPointerNull::get(PointerType::get(CoroId->getContext(), 0)) - : CoroFrees.front()->getFrame(); - + ConstantPointerNull::get(PointerType::get(FramePtr->getContext(), 0)); for (CoroFreeInst *CF : CoroFrees) { CF->replaceAllUsesWith(Replacement); CF->eraseFromParent(); diff --git a/llvm/test/Transforms/Coroutines/coro-alloc-with-param-O0.ll b/llvm/test/Transforms/Coroutines/coro-alloc-with-param-O0.ll index 2db096f13136d..77f383a4ff387 100644 --- a/llvm/test/Transforms/Coroutines/coro-alloc-with-param-O0.ll +++ b/llvm/test/Transforms/Coroutines/coro-alloc-with-param-O0.ll @@ -69,14 +69,16 @@ declare void @free(ptr) ; CHECK-NEXT: [[THIS_RELOAD_ADDR:%.*]] = getelementptr inbounds i8, ptr [[HDL]], i64 16 ; CHECK-NEXT: [[THIS_RELOAD:%.*]] = load i64, ptr [[THIS_RELOAD_ADDR]], align 4 ; CHECK-NEXT: call void @print2(i64 [[THIS_RELOAD]]) -; CHECK-NEXT: call void @free(ptr [[HDL]]) +; CHECK-NEXT: [[MEM:%.*]] = call ptr @llvm.coro.free(token poison, ptr [[HDL]]) +; CHECK-NEXT: call void @free(ptr [[MEM]]) ; CHECK-NEXT: ret void ; ; ; CHECK-LABEL: define internal fastcc void @f_copy.destroy( ; CHECK-SAME: ptr noundef nonnull align 8 dereferenceable(32) [[HDL:%.*]]) { ; CHECK-NEXT: [[ENTRY_DESTROY:.*:]] -; CHECK-NEXT: call void @free(ptr [[HDL]]) +; CHECK-NEXT: [[MEM:%.*]] = call ptr @llvm.coro.free(token poison, ptr [[HDL]]) +; CHECK-NEXT: call void @free(ptr [[MEM]]) ; CHECK-NEXT: ret void ; ; diff --git a/llvm/test/Transforms/Coroutines/coro-alloc-with-param-O2.ll b/llvm/test/Transforms/Coroutines/coro-alloc-with-param-O2.ll index e079d8114cc23..953f937b62e17 100644 --- a/llvm/test/Transforms/Coroutines/coro-alloc-with-param-O2.ll +++ b/llvm/test/Transforms/Coroutines/coro-alloc-with-param-O2.ll @@ -64,14 +64,16 @@ declare void @free(ptr) ; CHECK-NEXT: [[THIS_RELOAD_ADDR:%.*]] = getelementptr inbounds i8, ptr [[HDL]], i64 16 ; CHECK-NEXT: [[THIS_RELOAD:%.*]] = load i64, ptr [[THIS_RELOAD_ADDR]], align 4 ; CHECK-NEXT: call void @print2(i64 [[THIS_RELOAD]]) -; CHECK-NEXT: call void @free(ptr [[HDL]]) +; CHECK-NEXT: [[MEM:%.*]] = call ptr @llvm.coro.free(token poison, ptr [[HDL]]) +; CHECK-NEXT: call void @free(ptr [[MEM]]) ; CHECK-NEXT: ret void ; ; ; CHECK-LABEL: define internal fastcc void @f_direct.destroy( ; CHECK-SAME: ptr noundef nonnull align 8 dereferenceable(32) [[HDL:%.*]]) { ; CHECK-NEXT: [[ENTRY_DESTROY:.*:]] -; CHECK-NEXT: call void @free(ptr [[HDL]]) +; CHECK-NEXT: [[MEM:%.*]] = call ptr @llvm.coro.free(token poison, ptr [[HDL]]) +; CHECK-NEXT: call void @free(ptr [[MEM]]) ; CHECK-NEXT: ret void ; ; diff --git a/llvm/test/Transforms/Coroutines/coro-alloca-07.ll b/llvm/test/Transforms/Coroutines/coro-alloca-07.ll index 76aca567f785b..4c86abb1e8dc2 100644 --- a/llvm/test/Transforms/Coroutines/coro-alloca-07.ll +++ b/llvm/test/Transforms/Coroutines/coro-alloca-07.ll @@ -90,14 +90,16 @@ declare void @free(ptr) ; CHECK-NEXT: [[ALIAS_PHI_RELOAD_ADDR:%.*]] = getelementptr inbounds i8, ptr [[HDL]], i64 32 ; CHECK-NEXT: [[ALIAS_PHI_RELOAD:%.*]] = load ptr, ptr [[ALIAS_PHI_RELOAD_ADDR]], align 8 ; CHECK-NEXT: call void @print(ptr [[ALIAS_PHI_RELOAD]]) -; CHECK-NEXT: call void @free(ptr [[HDL]]) +; CHECK-NEXT: [[MEM:%.*]] = call ptr @llvm.coro.free(token poison, ptr [[HDL]]) +; CHECK-NEXT: call void @free(ptr [[MEM]]) ; CHECK-NEXT: ret void ; ; ; CHECK-LABEL: define internal fastcc void @f.destroy( ; CHECK-SAME: ptr noundef nonnull align 8 dereferenceable(48) [[HDL:%.*]]) { ; CHECK-NEXT: [[ENTRY_DESTROY:.*:]] -; CHECK-NEXT: call void @free(ptr [[HDL]]) +; CHECK-NEXT: [[MEM:%.*]] = call ptr @llvm.coro.free(token poison, ptr [[HDL]]) +; CHECK-NEXT: call void @free(ptr [[MEM]]) ; CHECK-NEXT: ret void ; ; diff --git a/llvm/test/Transforms/Coroutines/coro-await-suspend-lower-invoke.ll b/llvm/test/Transforms/Coroutines/coro-await-suspend-lower-invoke.ll index 72bb8fcf5b610..78540d9363cc6 100644 --- a/llvm/test/Transforms/Coroutines/coro-await-suspend-lower-invoke.ll +++ b/llvm/test/Transforms/Coroutines/coro-await-suspend-lower-invoke.ll @@ -167,7 +167,8 @@ declare... [truncated] `````````` </details> https://github.com/llvm/llvm-project/pull/185336 _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
