https://github.com/etiennep-chromium updated https://github.com/llvm/llvm-project/pull/212331
>From 5413ef689946031f8abb8b56fd8fe05f47052efb Mon Sep 17 00:00:00 2001 From: Etienne Pierre-doray <[email protected]> Date: Mon, 27 Jul 2026 19:14:00 +0000 Subject: [PATCH 1/8] [Win32][Coroutine] Fix inalloca arguments lifetime in coroutines On Win32 x86, arguments passed by value with non-trivial constructors/destructors require `inalloca`. If a coroutine suspends during the evaluation of these arguments, the stack frame is popped, destroying the already constructed arguments. This fix detects when a call inside a coroutine requires `inalloca` and contains suspend points in its arguments. In this case, we: 1. Force all arguments of the call to be evaluated into temporaries in the coroutine frame (which survive suspension). 2. Delay the `llvm.stacksave` to the end of argument evaluation (after resume). 3. Allocate the `inalloca` struct on the stack after resume and copy the temporaries into it. 4. Emit lifetime markers (`llvm.lifetime.start`/`llvm.lifetime.end`) for the `inalloca` alloca to prevent it from being moved to the coroutine frame by CoroSplit. 5. Safely clean up the temporaries in the frame. If an argument is non-copyable/non-movable, we now emit a compilation error. BUG=LLVM#59382 TAG=agy CONV=f4ad8337-ec22-4db0-a3a2-b3c51fb6619b --- clang/lib/CodeGen/CGCall.cpp | 221 ++++++++++++++++++++++++++--------- clang/lib/CodeGen/CGCall.h | 5 + 2 files changed, 171 insertions(+), 55 deletions(-) diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp index c0e2456891e9d..b86e509dde0d2 100644 --- a/clang/lib/CodeGen/CGCall.cpp +++ b/clang/lib/CodeGen/CGCall.cpp @@ -27,7 +27,9 @@ #include "clang/AST/Decl.h" #include "clang/AST/DeclCXX.h" #include "clang/AST/DeclObjC.h" +#include "clang/AST/ExprCXX.h" #include "clang/AST/RecordLayout.h" +#include "clang/AST/RecursiveASTVisitor.h" #include "clang/Basic/CodeGenOptions.h" #include "clang/Basic/TargetInfo.h" #include "clang/CodeGen/CGFunctionInfo.h" @@ -4788,6 +4790,27 @@ static bool isObjCMethodWithTypeParams(const ObjCMethodDecl *method) { } #endif +namespace { +class CoroSuspendFinder : public RecursiveASTVisitor<CoroSuspendFinder> { +public: + bool FoundSuspend = false; + bool VisitCoawaitExpr(CoawaitExpr *) { + FoundSuspend = true; + return false; // Stop traversal + } + bool VisitCoyieldExpr(CoyieldExpr *) { + FoundSuspend = true; + return false; // Stop traversal + } +}; +} // namespace + +static bool containsCoroSuspend(const Expr *E) { + CoroSuspendFinder Finder; + Finder.TraverseStmt(const_cast<Expr *>(E)); + return Finder.FoundSuspend; +} + /// EmitCallArgs - Emit call arguments for a function. void CodeGenFunction::EmitCallArgs( CallArgList &Args, PrototypeWrapper Prototype, @@ -4886,11 +4909,25 @@ void CodeGenFunction::EmitCallArgs( std::swap(Args.back(), *(&Args.back() - 1)); }; - // Insert a stack save if we're going to need any inalloca args. if (hasInAllocaArgs(CGM, ExplicitCC, ArgTypes)) { assert(getTarget().getTriple().getArch() == llvm::Triple::x86 && "inalloca only supported on x86"); - Args.allocateArgumentMemory(*this); + if (isCoroutine()) { + bool CallHasSuspend = false; + for (const Expr *A : ArgRange) { + bool hasSuspend = containsCoroSuspend(A); + if (hasSuspend) { + CallHasSuspend = true; + break; + } + } + if (CallHasSuspend) { + Args.setForceWriteback(true); + } + } + if (!Args.shouldForceWriteback()) { + Args.allocateArgumentMemory(*this); + } } // Evaluate each argument in the appropriate order. @@ -4926,6 +4963,10 @@ void CodeGenFunction::EmitCallArgs( } } + if (Args.shouldForceWriteback()) { + Args.allocateArgumentMemory(*this); + } + if (!LeftToRight) { // Un-reverse the arguments we just evaluated so they match up with the LLVM // IR function. @@ -5023,39 +5064,42 @@ void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E, // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee. // However, we still have to push an EH-only cleanup in case we unwind before // we make it to the call. - if (type->isRecordType() && - type->castAsRecordDecl()->isParamDestroyedInCallee()) { - // If we're using inalloca, use the argument memory. Otherwise, use a - // temporary. - AggValueSlot Slot = args.isUsingInAlloca() - ? createPlaceholderSlot(*this, type) - : CreateAggTemp(type, "agg.tmp"); - - bool DestroyedInCallee = true, NeedsCleanup = true; - if (const auto *RD = type->getAsCXXRecordDecl()) - DestroyedInCallee = RD->hasNonTrivialDestructor(); - else - NeedsCleanup = type.isDestructedType(); - - if (DestroyedInCallee) - Slot.setExternallyDestructed(); - - EmitAggExpr(E, Slot); - RValue RV = Slot.asRValue(); - args.add(RV, type); - - if (DestroyedInCallee && NeedsCleanup) { - // Create a no-op GEP between the placeholder and the cleanup so we can - // RAUW it successfully. It also serves as a marker of the first - // instruction where the cleanup is active. - pushFullExprCleanup<DestroyUnpassedArg>(NormalAndEHCleanup, - Slot.getAddress(), type); - // This unreachable is a temporary marker which will be removed later. - llvm::Instruction *IsActive = - Builder.CreateFlagLoad(llvm::Constant::getNullValue(Int8PtrTy)); - args.addArgCleanupDeactivation(EHStack.stable_begin(), IsActive); + if (type->isRecordType()) { + bool paramDestroyed = type->castAsRecordDecl()->isParamDestroyedInCallee(); + if (paramDestroyed) { + // If we're using inalloca, use the argument memory. Otherwise, use a + // temporary. + bool usePlaceholder = + args.isUsingInAlloca() && !args.shouldForceWriteback(); + AggValueSlot Slot = usePlaceholder ? createPlaceholderSlot(*this, type) + : CreateAggTemp(type, "agg.tmp"); + + bool DestroyedInCallee = true, NeedsCleanup = true; + if (const auto *RD = type->getAsCXXRecordDecl()) + DestroyedInCallee = RD->hasNonTrivialDestructor(); + else + NeedsCleanup = type.isDestructedType(); + + if (DestroyedInCallee && !args.shouldForceWriteback()) + Slot.setExternallyDestructed(); + + EmitAggExpr(E, Slot); + RValue RV = Slot.asRValue(); + args.add(RV, type); + + if (DestroyedInCallee && NeedsCleanup && !args.shouldForceWriteback()) { + // Create a no-op GEP between the placeholder and the cleanup so we can + // RAUW it successfully. It also serves as a marker of the first + // instruction where the cleanup is active. + pushFullExprCleanup<DestroyUnpassedArg>(NormalAndEHCleanup, + Slot.getAddress(), type); + // This unreachable is a temporary marker which will be removed later. + llvm::Instruction *IsActive = + Builder.CreateFlagLoad(llvm::Constant::getNullValue(Int8PtrTy)); + args.addArgCleanupDeactivation(EHStack.stable_begin(), IsActive); + } + return; } - return; } if (HasAggregateEvalKind && isa<ImplicitCastExpr>(E) && @@ -5404,9 +5448,13 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, llvm::Instruction *IP = CallArgs.getStackBase(); llvm::AllocaInst *AI; if (IP) { - IP = IP->getNextNode(); - AI = new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(), "argmem", - IP->getIterator()); + if (llvm::Instruction *Next = IP->getNextNode()) { + AI = new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(), "argmem", + Next->getIterator()); + } else { + AI = new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(), "argmem", + IP->getParent()); + } } else { AI = CreateTempAlloca(ArgStruct, "argmem"); } @@ -5415,6 +5463,9 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, AI->setUsedWithInAlloca(true); assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca()); ArgMemory = RawAddress(AI, ArgStruct, Align); + if (isCoroutine()) { + EmitLifetimeStart(AI); + } } ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo); @@ -5498,26 +5549,83 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, RawAddress Addr = I->hasLValue() ? I->getKnownLValue().getAddress() : I->getKnownRValue().getAggregateAddress(); - llvm::Instruction *Placeholder = - cast<llvm::Instruction>(Addr.getPointer()); - - if (!ArgInfo.getInAllocaIndirect()) { - // Replace the placeholder with the appropriate argument slot GEP. - CGBuilderTy::InsertPoint IP = Builder.saveIP(); - Builder.SetInsertPoint(Placeholder); - Addr = Builder.CreateStructGEP(ArgMemory, - ArgInfo.getInAllocaFieldIndex()); - Builder.restoreIP(IP); + if (CallArgs.shouldForceWriteback()) { + RawAddress Dest = RawAddress::invalid(); + if (!ArgInfo.getInAllocaIndirect()) { + Dest = Builder.CreateStructGEP(ArgMemory, + ArgInfo.getInAllocaFieldIndex()); + } else { + Dest = CreateMemTemp(info_it->type, "inalloca.indirect.tmp"); + Address ArgSlot = Builder.CreateStructGEP( + ArgMemory, ArgInfo.getInAllocaFieldIndex()); + Builder.CreateStore(Dest.getPointer(), ArgSlot); + } + const CXXRecordDecl *RD = info_it->type->getAsCXXRecordDecl(); + if (RD && !RD->isTriviallyCopyable()) { + const CXXConstructorDecl *CopyCtor = nullptr; + const CXXConstructorDecl *MoveCtor = nullptr; + for (const CXXConstructorDecl *C : RD->ctors()) { + if (C->isDeleted()) + continue; + if (C->isMoveConstructor()) { + MoveCtor = C; + } else if (C->isCopyConstructor()) { + if (!CopyCtor || C->getParamDecl(0) + ->getType() + ->getPointeeType() + .isConstQualified()) + CopyCtor = C; + } + } + const CXXConstructorDecl *Ctor = MoveCtor ? MoveCtor : CopyCtor; + if (!Ctor || Ctor->isDeleted()) { + CGM.Error(Loc, "coroutine argument must be copyable or movable " + "on this target"); + EmitAggregateCopy(MakeAddrLValue(Dest, info_it->type), + MakeAddrLValue(Addr, info_it->type), + info_it->type, AggValueSlot::DoesNotOverlap); + } else { + CallArgList CtorArgs; + llvm::Value *ThisPtr = getAsNaturalPointerTo( + Dest, Ctor->getThisType()->getPointeeType()); + CtorArgs.add(RValue::get(ThisPtr), Ctor->getThisType()); + QualType ParamTy = Ctor->getParamDecl(0)->getType(); + llvm::Value *SrcPtr = + getAsNaturalPointerTo(Addr, ParamTy->getPointeeType()); + CtorArgs.add(RValue::get(SrcPtr), ParamTy); + EmitCXXConstructorCall(Ctor, Ctor_Complete, + /*ForVirtualBase*/ false, + /*Delegating*/ false, Dest, CtorArgs, + AggValueSlot::DoesNotOverlap, Loc, + /*NewPointerIsChecked*/ false); + } + } else { + EmitAggregateCopy(MakeAddrLValue(Dest, info_it->type), + MakeAddrLValue(Addr, info_it->type), + info_it->type, AggValueSlot::DoesNotOverlap); + } } else { - // For indirect things such as overaligned structs, replace the - // placeholder with a regular aggregate temporary alloca. Store the - // address of this alloca into the struct. - Addr = CreateMemTemp(info_it->type, "inalloca.indirect.tmp"); - Address ArgSlot = Builder.CreateStructGEP( - ArgMemory, ArgInfo.getInAllocaFieldIndex()); - Builder.CreateStore(Addr.getPointer(), ArgSlot); + llvm::Instruction *Placeholder = + cast<llvm::Instruction>(Addr.getPointer()); + + if (!ArgInfo.getInAllocaIndirect()) { + // Replace the placeholder with the appropriate argument slot GEP. + CGBuilderTy::InsertPoint IP = Builder.saveIP(); + Builder.SetInsertPoint(Placeholder); + Addr = Builder.CreateStructGEP(ArgMemory, + ArgInfo.getInAllocaFieldIndex()); + Builder.restoreIP(IP); + } else { + // For indirect things such as overaligned structs, replace the + // placeholder with a regular aggregate temporary alloca. Store the + // address of this alloca into the struct. + Addr = CreateMemTemp(info_it->type, "inalloca.indirect.tmp"); + Address ArgSlot = Builder.CreateStructGEP( + ArgMemory, ArgInfo.getInAllocaFieldIndex()); + Builder.CreateStore(Addr.getPointer(), ArgSlot); + } + deferPlaceholderReplacement(Placeholder, Addr.getPointer()); } - deferPlaceholderReplacement(Placeholder, Addr.getPointer()); } else if (ArgInfo.getInAllocaIndirect()) { // Make a temporary alloca and store the address of it into the argument // struct. @@ -6289,6 +6397,9 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // The stack cleanup for inalloca arguments has to run out of the normal // lexical order, so deactivate it and run it manually here. CallArgs.freeArgumentMemory(*this); + if (isCoroutine() && ArgMemory.isValid()) { + EmitLifetimeEnd(ArgMemory.getPointer()); + } // Extract the return value. RValue Ret; diff --git a/clang/lib/CodeGen/CGCall.h b/clang/lib/CodeGen/CGCall.h index 145992652934f..488296439eb0c 100644 --- a/clang/lib/CodeGen/CGCall.h +++ b/clang/lib/CodeGen/CGCall.h @@ -357,6 +357,9 @@ class CallArgList : public SmallVector<CallArg, 8> { std::reverse(Writebacks.begin(), Writebacks.end()); } + bool shouldForceWriteback() const { return ForceWriteback; } + void setForceWriteback(bool V) { ForceWriteback = V; } + private: SmallVector<Writeback, 1> Writebacks; @@ -367,6 +370,8 @@ class CallArgList : public SmallVector<CallArg, 8> { /// The stacksave call. It dominates all of the argument evaluation. llvm::CallInst *StackBase = nullptr; + + bool ForceWriteback = false; }; /// FunctionArgList - Type for representing both the decl and type >From 37df429465f43951864ccc41667d02d329d1af6c Mon Sep 17 00:00:00 2001 From: Etienne Pierre-doray <[email protected]> Date: Mon, 27 Jul 2026 20:32:56 +0000 Subject: [PATCH 2/8] CR-efriedma-quic --- clang/lib/CodeGen/CGCall.cpp | 257 ++++++++++++++-------------- clang/lib/CodeGen/CGCall.h | 5 - clang/lib/CodeGen/CGExpr.cpp | 5 + clang/lib/CodeGen/CGExprAgg.cpp | 50 ++++++ clang/lib/CodeGen/CodeGenFunction.h | 12 ++ clang/lib/Sema/SemaExpr.cpp | 65 +++++++ 6 files changed, 260 insertions(+), 134 deletions(-) diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp index b86e509dde0d2..0312b456df6cd 100644 --- a/clang/lib/CodeGen/CGCall.cpp +++ b/clang/lib/CodeGen/CGCall.cpp @@ -4811,6 +4811,46 @@ static bool containsCoroSuspend(const Expr *E) { return Finder.FoundSuspend; } +static const MaterializeTemporaryExpr *getMTEToPreEvaluate(const Expr *E) { + while (true) { + E = E->IgnoreParens(); + if (auto *Cleanups = dyn_cast<ExprWithCleanups>(E)) { + E = Cleanups->getSubExpr(); + continue; + } + if (auto *Bind = dyn_cast<CXXBindTemporaryExpr>(E)) { + E = Bind->getSubExpr(); + continue; + } + break; + } + + if (auto *CE = dyn_cast<CXXConstructExpr>(E)) { + if (CE->getNumArgs() > 0) { + const Expr *Arg = CE->getArg(0); + while (true) { + Arg = Arg->IgnoreParens(); + if (auto *MTE = dyn_cast<MaterializeTemporaryExpr>(Arg)) + return MTE; + if (auto *Cleanups = dyn_cast<ExprWithCleanups>(Arg)) { + Arg = Cleanups->getSubExpr(); + continue; + } + if (auto *Bind = dyn_cast<CXXBindTemporaryExpr>(Arg)) { + Arg = Bind->getSubExpr(); + continue; + } + if (auto *ICE = dyn_cast<ImplicitCastExpr>(Arg)) { + Arg = ICE->getSubExpr(); + continue; + } + break; + } + } + } + return nullptr; +} + /// EmitCallArgs - Emit call arguments for a function. void CodeGenFunction::EmitCallArgs( CallArgList &Args, PrototypeWrapper Prototype, @@ -4909,27 +4949,46 @@ void CodeGenFunction::EmitCallArgs( std::swap(Args.back(), *(&Args.back() - 1)); }; - if (hasInAllocaArgs(CGM, ExplicitCC, ArgTypes)) { - assert(getTarget().getTriple().getArch() == llvm::Triple::x86 && - "inalloca only supported on x86"); - if (isCoroutine()) { - bool CallHasSuspend = false; - for (const Expr *A : ArgRange) { - bool hasSuspend = containsCoroSuspend(A); - if (hasSuspend) { - CallHasSuspend = true; - break; - } + bool CallHasSuspend = false; + if (isCoroutine()) { + for (const Expr *A : ArgRange) { + if (containsCoroSuspend(A)) { + CallHasSuspend = true; + break; } - if (CallHasSuspend) { - Args.setForceWriteback(true); + } + } + + SmallVector<const MaterializeTemporaryExpr *, 4> MTEsToPreEvaluate; + if (CallHasSuspend && hasInAllocaArgs(CGM, ExplicitCC, ArgTypes)) { + for (const Expr *A : ArgRange) { + if (auto *MTE = getMTEToPreEvaluate(A)) { + MTEsToPreEvaluate.push_back(MTE); } } - if (!Args.shouldForceWriteback()) { - Args.allocateArgumentMemory(*this); + } + + if (!MTEsToPreEvaluate.empty()) { + if (LeftToRight) { + for (const MaterializeTemporaryExpr *MTE : MTEsToPreEvaluate) { + LValue LV = EmitMaterializeTemporaryExpr(MTE); + PreEvaluatedMaterializedTemporaries[MTE] = LV; + } + } else { + for (int i = MTEsToPreEvaluate.size() - 1; i >= 0; --i) { + const MaterializeTemporaryExpr *MTE = MTEsToPreEvaluate[i]; + LValue LV = EmitMaterializeTemporaryExpr(MTE); + PreEvaluatedMaterializedTemporaries[MTE] = LV; + } } } + if (hasInAllocaArgs(CGM, ExplicitCC, ArgTypes)) { + assert(getTarget().getTriple().getArch() == llvm::Triple::x86 && + "inalloca only supported on x86"); + Args.allocateArgumentMemory(*this); + } + // Evaluate each argument in the appropriate order. size_t CallArgsStart = Args.size(); for (unsigned I = 0, E = ArgTypes.size(); I != E; ++I) { @@ -4963,10 +5022,6 @@ void CodeGenFunction::EmitCallArgs( } } - if (Args.shouldForceWriteback()) { - Args.allocateArgumentMemory(*this); - } - if (!LeftToRight) { // Un-reverse the arguments we just evaluated so they match up with the LLVM // IR function. @@ -4975,6 +5030,10 @@ void CodeGenFunction::EmitCallArgs( // Reverse the writebacks to match the MSVC ABI. Args.reverseWritebacks(); } + + for (const auto *MTE : MTEsToPreEvaluate) { + PreEvaluatedMaterializedTemporaries.erase(MTE); + } } namespace { @@ -5064,42 +5123,39 @@ void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E, // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee. // However, we still have to push an EH-only cleanup in case we unwind before // we make it to the call. - if (type->isRecordType()) { - bool paramDestroyed = type->castAsRecordDecl()->isParamDestroyedInCallee(); - if (paramDestroyed) { - // If we're using inalloca, use the argument memory. Otherwise, use a - // temporary. - bool usePlaceholder = - args.isUsingInAlloca() && !args.shouldForceWriteback(); - AggValueSlot Slot = usePlaceholder ? createPlaceholderSlot(*this, type) - : CreateAggTemp(type, "agg.tmp"); - - bool DestroyedInCallee = true, NeedsCleanup = true; - if (const auto *RD = type->getAsCXXRecordDecl()) - DestroyedInCallee = RD->hasNonTrivialDestructor(); - else - NeedsCleanup = type.isDestructedType(); - - if (DestroyedInCallee && !args.shouldForceWriteback()) - Slot.setExternallyDestructed(); - - EmitAggExpr(E, Slot); - RValue RV = Slot.asRValue(); - args.add(RV, type); - - if (DestroyedInCallee && NeedsCleanup && !args.shouldForceWriteback()) { - // Create a no-op GEP between the placeholder and the cleanup so we can - // RAUW it successfully. It also serves as a marker of the first - // instruction where the cleanup is active. - pushFullExprCleanup<DestroyUnpassedArg>(NormalAndEHCleanup, - Slot.getAddress(), type); - // This unreachable is a temporary marker which will be removed later. - llvm::Instruction *IsActive = - Builder.CreateFlagLoad(llvm::Constant::getNullValue(Int8PtrTy)); - args.addArgCleanupDeactivation(EHStack.stable_begin(), IsActive); - } - return; + if (type->isRecordType() && + type->castAsRecordDecl()->isParamDestroyedInCallee()) { + // If we're using inalloca, use the argument memory. Otherwise, use a + // temporary. + AggValueSlot Slot = args.isUsingInAlloca() + ? createPlaceholderSlot(*this, type) + : CreateAggTemp(type, "agg.tmp"); + + bool DestroyedInCallee = true, NeedsCleanup = true; + if (const auto *RD = type->getAsCXXRecordDecl()) + DestroyedInCallee = RD->hasNonTrivialDestructor(); + else + NeedsCleanup = type.isDestructedType(); + + if (DestroyedInCallee) + Slot.setExternallyDestructed(); + + EmitAggExpr(E, Slot); + RValue RV = Slot.asRValue(); + args.add(RV, type); + + if (DestroyedInCallee && NeedsCleanup) { + // Create a no-op GEP between the placeholder and the cleanup so we can + // RAUW it successfully. It also serves as a marker of the first + // instruction where the cleanup is active. + pushFullExprCleanup<DestroyUnpassedArg>(NormalAndEHCleanup, + Slot.getAddress(), type); + // This unreachable is a temporary marker which will be removed later. + llvm::Instruction *IsActive = + Builder.CreateFlagLoad(llvm::Constant::getNullValue(Int8PtrTy)); + args.addArgCleanupDeactivation(EHStack.stable_begin(), IsActive); } + return; } if (HasAggregateEvalKind && isa<ImplicitCastExpr>(E) && @@ -5549,83 +5605,26 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, RawAddress Addr = I->hasLValue() ? I->getKnownLValue().getAddress() : I->getKnownRValue().getAggregateAddress(); - if (CallArgs.shouldForceWriteback()) { - RawAddress Dest = RawAddress::invalid(); - if (!ArgInfo.getInAllocaIndirect()) { - Dest = Builder.CreateStructGEP(ArgMemory, - ArgInfo.getInAllocaFieldIndex()); - } else { - Dest = CreateMemTemp(info_it->type, "inalloca.indirect.tmp"); - Address ArgSlot = Builder.CreateStructGEP( - ArgMemory, ArgInfo.getInAllocaFieldIndex()); - Builder.CreateStore(Dest.getPointer(), ArgSlot); - } - const CXXRecordDecl *RD = info_it->type->getAsCXXRecordDecl(); - if (RD && !RD->isTriviallyCopyable()) { - const CXXConstructorDecl *CopyCtor = nullptr; - const CXXConstructorDecl *MoveCtor = nullptr; - for (const CXXConstructorDecl *C : RD->ctors()) { - if (C->isDeleted()) - continue; - if (C->isMoveConstructor()) { - MoveCtor = C; - } else if (C->isCopyConstructor()) { - if (!CopyCtor || C->getParamDecl(0) - ->getType() - ->getPointeeType() - .isConstQualified()) - CopyCtor = C; - } - } - const CXXConstructorDecl *Ctor = MoveCtor ? MoveCtor : CopyCtor; - if (!Ctor || Ctor->isDeleted()) { - CGM.Error(Loc, "coroutine argument must be copyable or movable " - "on this target"); - EmitAggregateCopy(MakeAddrLValue(Dest, info_it->type), - MakeAddrLValue(Addr, info_it->type), - info_it->type, AggValueSlot::DoesNotOverlap); - } else { - CallArgList CtorArgs; - llvm::Value *ThisPtr = getAsNaturalPointerTo( - Dest, Ctor->getThisType()->getPointeeType()); - CtorArgs.add(RValue::get(ThisPtr), Ctor->getThisType()); - QualType ParamTy = Ctor->getParamDecl(0)->getType(); - llvm::Value *SrcPtr = - getAsNaturalPointerTo(Addr, ParamTy->getPointeeType()); - CtorArgs.add(RValue::get(SrcPtr), ParamTy); - EmitCXXConstructorCall(Ctor, Ctor_Complete, - /*ForVirtualBase*/ false, - /*Delegating*/ false, Dest, CtorArgs, - AggValueSlot::DoesNotOverlap, Loc, - /*NewPointerIsChecked*/ false); - } - } else { - EmitAggregateCopy(MakeAddrLValue(Dest, info_it->type), - MakeAddrLValue(Addr, info_it->type), - info_it->type, AggValueSlot::DoesNotOverlap); - } + llvm::Instruction *Placeholder = + cast<llvm::Instruction>(Addr.getPointer()); + + if (!ArgInfo.getInAllocaIndirect()) { + // Replace the placeholder with the appropriate argument slot GEP. + CGBuilderTy::InsertPoint IP = Builder.saveIP(); + Builder.SetInsertPoint(Placeholder); + Addr = Builder.CreateStructGEP(ArgMemory, + ArgInfo.getInAllocaFieldIndex()); + Builder.restoreIP(IP); } else { - llvm::Instruction *Placeholder = - cast<llvm::Instruction>(Addr.getPointer()); - - if (!ArgInfo.getInAllocaIndirect()) { - // Replace the placeholder with the appropriate argument slot GEP. - CGBuilderTy::InsertPoint IP = Builder.saveIP(); - Builder.SetInsertPoint(Placeholder); - Addr = Builder.CreateStructGEP(ArgMemory, - ArgInfo.getInAllocaFieldIndex()); - Builder.restoreIP(IP); - } else { - // For indirect things such as overaligned structs, replace the - // placeholder with a regular aggregate temporary alloca. Store the - // address of this alloca into the struct. - Addr = CreateMemTemp(info_it->type, "inalloca.indirect.tmp"); - Address ArgSlot = Builder.CreateStructGEP( - ArgMemory, ArgInfo.getInAllocaFieldIndex()); - Builder.CreateStore(Addr.getPointer(), ArgSlot); - } - deferPlaceholderReplacement(Placeholder, Addr.getPointer()); + // For indirect things such as overaligned structs, replace the + // placeholder with a regular aggregate temporary alloca. Store the + // address of this alloca into the struct. + Addr = CreateMemTemp(info_it->type, "inalloca.indirect.tmp"); + Address ArgSlot = Builder.CreateStructGEP( + ArgMemory, ArgInfo.getInAllocaFieldIndex()); + Builder.CreateStore(Addr.getPointer(), ArgSlot); } + deferPlaceholderReplacement(Placeholder, Addr.getPointer()); } else if (ArgInfo.getInAllocaIndirect()) { // Make a temporary alloca and store the address of it into the argument // struct. diff --git a/clang/lib/CodeGen/CGCall.h b/clang/lib/CodeGen/CGCall.h index 488296439eb0c..145992652934f 100644 --- a/clang/lib/CodeGen/CGCall.h +++ b/clang/lib/CodeGen/CGCall.h @@ -357,9 +357,6 @@ class CallArgList : public SmallVector<CallArg, 8> { std::reverse(Writebacks.begin(), Writebacks.end()); } - bool shouldForceWriteback() const { return ForceWriteback; } - void setForceWriteback(bool V) { ForceWriteback = V; } - private: SmallVector<Writeback, 1> Writebacks; @@ -370,8 +367,6 @@ class CallArgList : public SmallVector<CallArg, 8> { /// The stacksave call. It dominates all of the argument evaluation. llvm::CallInst *StackBase = nullptr; - - bool ForceWriteback = false; }; /// FunctionArgList - Type for representing both the decl and type diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp index 23802cdeb4811..bd08e056798e3 100644 --- a/clang/lib/CodeGen/CGExpr.cpp +++ b/clang/lib/CodeGen/CGExpr.cpp @@ -513,6 +513,11 @@ static bool isAAPCS(const TargetInfo &TargetInfo) { LValue CodeGenFunction:: EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { + auto It = PreEvaluatedMaterializedTemporaries.find(M); + if (It != PreEvaluatedMaterializedTemporaries.end()) { + return It->second; + } + const Expr *E = M->getSubExpr(); assert((!M->getExtendingDecl() || !isa<VarDecl>(M->getExtendingDecl()) || diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp index 3a4291719da74..9d58c3e6aa36d 100644 --- a/clang/lib/CodeGen/CGExprAgg.cpp +++ b/clang/lib/CodeGen/CGExprAgg.cpp @@ -754,8 +754,58 @@ void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType, // Visitor Methods //===----------------------------------------------------------------------===// +static void emitNonTrivialCopyOrMove(CodeGenFunction &CGF, AggValueSlot Dest, + LValue SrcLV, QualType Type) { + const CXXRecordDecl *RD = Type->getAsCXXRecordDecl(); + assert(RD && "Type must be a CXXRecordDecl"); + + const CXXConstructorDecl *CopyCtor = nullptr; + const CXXConstructorDecl *MoveCtor = nullptr; + for (const CXXConstructorDecl *C : RD->ctors()) { + if (C->isDeleted()) + continue; + if (C->isMoveConstructor()) { + MoveCtor = C; + } else if (C->isCopyConstructor()) { + if (!CopyCtor || + C->getParamDecl(0)->getType()->getPointeeType().isConstQualified()) + CopyCtor = C; + } + } + + const CXXConstructorDecl *Ctor = MoveCtor ? MoveCtor : CopyCtor; + assert(Ctor && "No copy or move constructor found"); + + CallArgList CtorArgs; + llvm::Value *ThisPtr = CGF.getAsNaturalPointerTo( + Dest.getAddress(), Ctor->getThisType()->getPointeeType()); + CtorArgs.add(RValue::get(ThisPtr), Ctor->getThisType()); + + QualType ParamTy = Ctor->getParamDecl(0)->getType(); + llvm::Value *SrcPtr = + CGF.getAsNaturalPointerTo(SrcLV.getAddress(), ParamTy->getPointeeType()); + CtorArgs.add(RValue::get(SrcPtr), ParamTy); + + CGF.EmitCXXConstructorCall(Ctor, Ctor_Complete, + /*ForVirtualBase*/ false, + /*Delegating*/ false, Dest.getAddress(), CtorArgs, + AggValueSlot::DoesNotOverlap, SourceLocation(), + /*NewPointerIsChecked*/ false); +} + void AggExprEmitter::VisitMaterializeTemporaryExpr( MaterializeTemporaryExpr *E) { + if (CGF.hasPreEvaluatedTemporary(E)) { + LValue SrcLV = CGF.getPreEvaluatedTemporary(E); + QualType Type = E->getType(); + const CXXRecordDecl *RD = Type->getAsCXXRecordDecl(); + if (RD && !RD->isTriviallyCopyable()) { + emitNonTrivialCopyOrMove(CGF, Dest, SrcLV, Type); + } else { + EmitFinalDestCopy(Type, SrcLV); + } + return; + } Visit(E->getSubExpr()); } diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h index fd474c09044ef..9ed5e9b532b62 100644 --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -1772,6 +1772,8 @@ class CodeGenFunction : public CodeGenTypeCache { /// expressions. llvm::DenseMap<const OpaqueValueExpr *, LValue> OpaqueLValues; llvm::DenseMap<const OpaqueValueExpr *, RValue> OpaqueRValues; + llvm::DenseMap<const MaterializeTemporaryExpr *, LValue> + PreEvaluatedMaterializedTemporaries; // VLASizeMap - This keeps track of the associated size for each VLA type. // We track this by the size expression rather than the type itself because @@ -3098,6 +3100,16 @@ class CodeGenFunction : public CodeGenTypeCache { /// already been emitted. bool isOpaqueValueEmitted(const OpaqueValueExpr *E); + bool hasPreEvaluatedTemporary(const MaterializeTemporaryExpr *M) const { + return PreEvaluatedMaterializedTemporaries.count(M); + } + LValue getPreEvaluatedTemporary(const MaterializeTemporaryExpr *M) { + auto It = PreEvaluatedMaterializedTemporaries.find(M); + assert(It != PreEvaluatedMaterializedTemporaries.end() && + "MTE is not pre-evaluated"); + return It->second; + } + /// Get the index of the current ArrayInitLoopExpr, if any. llvm::Value *getArrayInitIndex() { return ArrayInitIndex; } diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp index 9fd8c6a0a5451..632049ca354c7 100644 --- a/clang/lib/Sema/SemaExpr.cpp +++ b/clang/lib/Sema/SemaExpr.cpp @@ -6209,6 +6209,55 @@ Sema::ConvertArgumentsForCall(CallExpr *Call, Expr *Fn, return false; } +namespace { +struct CoroSuspendFinder : DynamicRecursiveASTVisitor { + bool FoundSuspend = false; + + CoroSuspendFinder() { ShouldVisitImplicitCode = true; } + + bool VisitCoawaitExpr(CoawaitExpr *E) override { + FoundSuspend = true; + return false; // Stop traversal + } + bool VisitCoyieldExpr(CoyieldExpr *E) override { + FoundSuspend = true; + return false; // Stop traversal + } +}; +} // namespace + +static bool containsCoroSuspend(const Expr *E) { + if (!E) + return false; + CoroSuspendFinder Finder; + Finder.TraverseStmt(const_cast<Expr *>(E)); + return Finder.FoundSuspend; +} + +static bool isWin32InAllocaRecord(ASTContext &Context, QualType Ty) { + const RecordType *RT = Ty->getAs<RecordType>(); + if (!RT) + return false; + const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getDecl()); + if (!RD) + return false; + + const llvm::Triple &Triple = Context.getTargetInfo().getTriple(); + if (Triple.getArch() != llvm::Triple::x86 || !Triple.isOSWindows()) + return false; + if (!Context.getTargetInfo().getCXXABI().isMicrosoft()) + return false; + + if (RD->canPassInRegisters()) + return false; + + TypeInfo Info = Context.getTypeInfo(Context.getCanonicalTagType(RD)); + if (Info.isAlignRequired() && Info.Align > 4) + return false; // passed indirectly + + return true; +} + bool Sema::GatherArgumentsForCall(SourceLocation CallLoc, FunctionDecl *FDecl, const FunctionProtoType *Proto, unsigned FirstParam, ArrayRef<Expr *> Args, @@ -6216,6 +6265,15 @@ bool Sema::GatherArgumentsForCall(SourceLocation CallLoc, FunctionDecl *FDecl, VariadicCallType CallType, bool AllowExplicit, bool IsListInitialization) { unsigned NumParams = Proto->getNumParams(); + bool CallHasSuspend = false; + if (getCurFunction() && getCurFunction()->isCoroutine()) { + for (const Expr *A : Args) { + if (containsCoroSuspend(A)) { + CallHasSuspend = true; + break; + } + } + } bool Invalid = false; size_t ArgIx = 0; // Continue to check argument types (even if we have too few/many args). @@ -6275,6 +6333,13 @@ bool Sema::GatherArgumentsForCall(SourceLocation CallLoc, FunctionDecl *FDecl, : diag::warn_obt_discarded_at_function_boundary) << Arg->getType() << ProtoArgType; } + if (CallHasSuspend && Arg->isPRValue() && + isWin32InAllocaRecord(Context, ProtoArgType)) { + ExprResult Materialized = + CreateMaterializeTemporaryExpr(Arg->getType(), Arg, false); + if (!Materialized.isInvalid()) + Arg = Materialized.get(); + } ExprResult ArgE = PerformCopyInitialization( Entity, SourceLocation(), Arg, IsListInitialization, AllowExplicit); >From d134ed5ed5be8f70248b3be2682cef0becfdeb55 Mon Sep 17 00:00:00 2001 From: Etienne Pierre-doray <[email protected]> Date: Wed, 29 Jul 2026 10:49:40 +0000 Subject: [PATCH 3/8] Self review --- clang/include/clang/AST/Expr.h | 4 +++ clang/lib/AST/Expr.cpp | 16 +++++++++++ clang/lib/CodeGen/CGCall.cpp | 27 ++++-------------- clang/lib/CodeGen/CGExprAgg.cpp | 49 +++------------------------------ clang/lib/CodeGen/CGExprCXX.cpp | 11 +++++++- clang/lib/Sema/SemaExpr.cpp | 27 +----------------- 6 files changed, 40 insertions(+), 94 deletions(-) diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h index a0ab599fa82d2..7ba25ea1295f8 100644 --- a/clang/include/clang/AST/Expr.h +++ b/clang/include/clang/AST/Expr.h @@ -247,6 +247,10 @@ class Expr : public ValueStmt { return static_cast<bool>(getDependence() & ExprDependence::Error); } + /// Whether this expression contains a coroutine suspend point + /// (co_await or co_yield). + bool containsCoroutineSuspendPoints() const; + /// getExprLoc - Return the preferred location for the arrow when diagnosing /// a problem with a generic expression. SourceLocation getExprLoc() const LLVM_READONLY; diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp index 64d61dbc3d128..91d7fe0ffec70 100644 --- a/clang/lib/AST/Expr.cpp +++ b/clang/lib/AST/Expr.cpp @@ -3986,6 +3986,22 @@ bool Expr::HasSideEffects(const ASTContext &Ctx, return false; } +static bool containsCoroutineSuspendPoints(const Stmt *S) { + if (!S) + return false; + if (isa<CoawaitExpr>(S) || isa<CoyieldExpr>(S) || + isa<DependentCoawaitExpr>(S)) + return true; + for (const Stmt *Child : S->children()) + if (Child && containsCoroutineSuspendPoints(Child)) + return true; + return false; +} + +bool Expr::containsCoroutineSuspendPoints() const { + return ::containsCoroutineSuspendPoints(this); +} + FPOptions Expr::getFPFeaturesInEffect(const LangOptions &LO) const { if (auto Call = dyn_cast<CallExpr>(this)) return Call->getFPFeaturesInEffect(LO); diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp index 0312b456df6cd..4cebb5b735dca 100644 --- a/clang/lib/CodeGen/CGCall.cpp +++ b/clang/lib/CodeGen/CGCall.cpp @@ -4790,27 +4790,6 @@ static bool isObjCMethodWithTypeParams(const ObjCMethodDecl *method) { } #endif -namespace { -class CoroSuspendFinder : public RecursiveASTVisitor<CoroSuspendFinder> { -public: - bool FoundSuspend = false; - bool VisitCoawaitExpr(CoawaitExpr *) { - FoundSuspend = true; - return false; // Stop traversal - } - bool VisitCoyieldExpr(CoyieldExpr *) { - FoundSuspend = true; - return false; // Stop traversal - } -}; -} // namespace - -static bool containsCoroSuspend(const Expr *E) { - CoroSuspendFinder Finder; - Finder.TraverseStmt(const_cast<Expr *>(E)); - return Finder.FoundSuspend; -} - static const MaterializeTemporaryExpr *getMTEToPreEvaluate(const Expr *E) { while (true) { E = E->IgnoreParens(); @@ -4952,7 +4931,7 @@ void CodeGenFunction::EmitCallArgs( bool CallHasSuspend = false; if (isCoroutine()) { for (const Expr *A : ArgRange) { - if (containsCoroSuspend(A)) { + if (A && A->containsCoroutineSuspendPoints()) { CallHasSuspend = true; break; } @@ -5517,6 +5496,10 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, auto Align = CallInfo.getArgStructAlignment(); AI->setAlignment(Align.getAsAlign()); AI->setUsedWithInAlloca(true); + if (isCoroutine()) { + AI->setMetadata(llvm::LLVMContext::MD_coro_outside_frame, + llvm::MDNode::get(CGM.getLLVMContext(), {})); + } assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca()); ArgMemory = RawAddress(AI, ArgStruct, Align); if (isCoroutine()) { diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp index 9d58c3e6aa36d..fe4f15c2ad882 100644 --- a/clang/lib/CodeGen/CGExprAgg.cpp +++ b/clang/lib/CodeGen/CGExprAgg.cpp @@ -754,56 +754,15 @@ void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType, // Visitor Methods //===----------------------------------------------------------------------===// -static void emitNonTrivialCopyOrMove(CodeGenFunction &CGF, AggValueSlot Dest, - LValue SrcLV, QualType Type) { - const CXXRecordDecl *RD = Type->getAsCXXRecordDecl(); - assert(RD && "Type must be a CXXRecordDecl"); - - const CXXConstructorDecl *CopyCtor = nullptr; - const CXXConstructorDecl *MoveCtor = nullptr; - for (const CXXConstructorDecl *C : RD->ctors()) { - if (C->isDeleted()) - continue; - if (C->isMoveConstructor()) { - MoveCtor = C; - } else if (C->isCopyConstructor()) { - if (!CopyCtor || - C->getParamDecl(0)->getType()->getPointeeType().isConstQualified()) - CopyCtor = C; - } - } - - const CXXConstructorDecl *Ctor = MoveCtor ? MoveCtor : CopyCtor; - assert(Ctor && "No copy or move constructor found"); - - CallArgList CtorArgs; - llvm::Value *ThisPtr = CGF.getAsNaturalPointerTo( - Dest.getAddress(), Ctor->getThisType()->getPointeeType()); - CtorArgs.add(RValue::get(ThisPtr), Ctor->getThisType()); - - QualType ParamTy = Ctor->getParamDecl(0)->getType(); - llvm::Value *SrcPtr = - CGF.getAsNaturalPointerTo(SrcLV.getAddress(), ParamTy->getPointeeType()); - CtorArgs.add(RValue::get(SrcPtr), ParamTy); - - CGF.EmitCXXConstructorCall(Ctor, Ctor_Complete, - /*ForVirtualBase*/ false, - /*Delegating*/ false, Dest.getAddress(), CtorArgs, - AggValueSlot::DoesNotOverlap, SourceLocation(), - /*NewPointerIsChecked*/ false); -} - void AggExprEmitter::VisitMaterializeTemporaryExpr( MaterializeTemporaryExpr *E) { if (CGF.hasPreEvaluatedTemporary(E)) { LValue SrcLV = CGF.getPreEvaluatedTemporary(E); QualType Type = E->getType(); - const CXXRecordDecl *RD = Type->getAsCXXRecordDecl(); - if (RD && !RD->isTriviallyCopyable()) { - emitNonTrivialCopyOrMove(CGF, Dest, SrcLV, Type); - } else { - EmitFinalDestCopy(Type, SrcLV); - } + assert((!Type->getAsCXXRecordDecl() || + Type->getAsCXXRecordDecl()->isTriviallyCopyable()) && + "Non-trivially copyable MTE should not be visited as aggregate when pre-evaluated"); + EmitFinalDestCopy(Type, SrcLV); return; } Visit(E->getSubExpr()); diff --git a/clang/lib/CodeGen/CGExprCXX.cpp b/clang/lib/CodeGen/CGExprCXX.cpp index 82300c3ede183..03d551dda62db 100644 --- a/clang/lib/CodeGen/CGExprCXX.cpp +++ b/clang/lib/CodeGen/CGExprCXX.cpp @@ -624,7 +624,16 @@ void CodeGenFunction::EmitCXXConstructExpr(const CXXConstructExpr *E, return; // Elide the constructor if we're constructing from a temporary. - if (getLangOpts().ElideConstructors && E->isElidable()) { + bool DisableElision = false; + if (E->getNumArgs() > 0) { + if (const auto *MTE = dyn_cast<MaterializeTemporaryExpr>( + E->getArg(0)->IgnoreImpCasts())) { + if (hasPreEvaluatedTemporary(MTE)) + DisableElision = true; + } + } + + if (getLangOpts().ElideConstructors && E->isElidable() && !DisableElision) { // FIXME: This only handles the simplest case, where the source object // is passed directly as the first argument to the constructor. // This should also handle stepping though implicit casts and diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp index 632049ca354c7..5777efbf958f1 100644 --- a/clang/lib/Sema/SemaExpr.cpp +++ b/clang/lib/Sema/SemaExpr.cpp @@ -6209,31 +6209,6 @@ Sema::ConvertArgumentsForCall(CallExpr *Call, Expr *Fn, return false; } -namespace { -struct CoroSuspendFinder : DynamicRecursiveASTVisitor { - bool FoundSuspend = false; - - CoroSuspendFinder() { ShouldVisitImplicitCode = true; } - - bool VisitCoawaitExpr(CoawaitExpr *E) override { - FoundSuspend = true; - return false; // Stop traversal - } - bool VisitCoyieldExpr(CoyieldExpr *E) override { - FoundSuspend = true; - return false; // Stop traversal - } -}; -} // namespace - -static bool containsCoroSuspend(const Expr *E) { - if (!E) - return false; - CoroSuspendFinder Finder; - Finder.TraverseStmt(const_cast<Expr *>(E)); - return Finder.FoundSuspend; -} - static bool isWin32InAllocaRecord(ASTContext &Context, QualType Ty) { const RecordType *RT = Ty->getAs<RecordType>(); if (!RT) @@ -6268,7 +6243,7 @@ bool Sema::GatherArgumentsForCall(SourceLocation CallLoc, FunctionDecl *FDecl, bool CallHasSuspend = false; if (getCurFunction() && getCurFunction()->isCoroutine()) { for (const Expr *A : Args) { - if (containsCoroSuspend(A)) { + if (A && A->containsCoroutineSuspendPoints()) { CallHasSuspend = true; break; } >From a28bee92fcf5fbf3121cb5ec58aee0bd4766d1a8 Mon Sep 17 00:00:00 2001 From: Etienne Pierre-doray <[email protected]> Date: Wed, 29 Jul 2026 11:05:18 +0000 Subject: [PATCH 4/8] Self review --- clang/lib/CodeGen/CGCall.cpp | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp index 4cebb5b735dca..e87f332f34b76 100644 --- a/clang/lib/CodeGen/CGCall.cpp +++ b/clang/lib/CodeGen/CGCall.cpp @@ -4928,18 +4928,8 @@ void CodeGenFunction::EmitCallArgs( std::swap(Args.back(), *(&Args.back() - 1)); }; - bool CallHasSuspend = false; - if (isCoroutine()) { - for (const Expr *A : ArgRange) { - if (A && A->containsCoroutineSuspendPoints()) { - CallHasSuspend = true; - break; - } - } - } - SmallVector<const MaterializeTemporaryExpr *, 4> MTEsToPreEvaluate; - if (CallHasSuspend && hasInAllocaArgs(CGM, ExplicitCC, ArgTypes)) { + if (isCoroutine() && hasInAllocaArgs(CGM, ExplicitCC, ArgTypes)) { for (const Expr *A : ArgRange) { if (auto *MTE = getMTEToPreEvaluate(A)) { MTEsToPreEvaluate.push_back(MTE); >From 3cfb63fea8960d31f15aefc5612964a2309caf64 Mon Sep 17 00:00:00 2001 From: Etienne Pierre-doray <[email protected]> Date: Wed, 29 Jul 2026 11:15:23 +0000 Subject: [PATCH 5/8] Add test --- .../CodeGenCoroutines/coro-win32-inalloca.cpp | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 clang/test/CodeGenCoroutines/coro-win32-inalloca.cpp diff --git a/clang/test/CodeGenCoroutines/coro-win32-inalloca.cpp b/clang/test/CodeGenCoroutines/coro-win32-inalloca.cpp new file mode 100644 index 0000000000000..34d1d0d81d943 --- /dev/null +++ b/clang/test/CodeGenCoroutines/coro-win32-inalloca.cpp @@ -0,0 +1,93 @@ +// RUN: %clang_cc1 -std=c++20 -triple=i686-pc-windows-msvc -emit-llvm -o - %s -disable-llvm-passes | FileCheck %s + +namespace std { +template <typename R, typename... Args> +struct coroutine_traits { + using promise_type = typename R::promise_type; +}; + +template <class Promise = void> struct coroutine_handle { + coroutine_handle() = default; + static coroutine_handle from_address(void *) noexcept; +}; +template <> struct coroutine_handle<void> { + static coroutine_handle from_address(void *) noexcept; + coroutine_handle() = default; + template <class PromiseType> + coroutine_handle(coroutine_handle<PromiseType>) noexcept; +}; +} // namespace std + +struct suspend_always { + bool await_ready() noexcept { return false; } + void await_suspend(std::coroutine_handle<>) noexcept {} + void await_resume() noexcept {} +}; + +struct suspend_never { + bool await_ready() noexcept { return true; } + void await_suspend(std::coroutine_handle<>) noexcept {} + void await_resume() noexcept {} +}; + +struct task { + struct promise_type { + task get_return_object() { return {}; } + suspend_never initial_suspend() { return {}; } + suspend_never final_suspend() noexcept { return {}; } + void return_void() {} + void unhandled_exception() {} + }; +}; + +struct Noisy { + int val; + Noisy(int v); + Noisy(const Noisy&) = delete; + Noisy(Noisy&& o) noexcept; + ~Noisy(); +}; + +struct Awaiter { + bool await_ready() noexcept { return false; } + void await_suspend(std::coroutine_handle<>) noexcept {} + Noisy await_resume() noexcept; +}; + +void consume_two(Noisy x, Noisy y); + +// CHECK-LABEL: define dso_local void @"?my_task@@YA?AUtask@@XZ"( +task my_task() { + // CHECK: %[[MTE_Y:.+]] = alloca %struct.Noisy, + // CHECK: %[[MTE_X:.+]] = alloca %struct.Noisy, + + // Evaluate Noisy(42) before suspend: + // CHECK: call x86_thiscallcc noundef ptr @"??0Noisy@@QAE@H@Z"(ptr {{[^,]*}} %[[MTE_Y]], i32 noundef 42) + + // Suspend for co_await Awaiter{}: + // CHECK: call void @llvm.coro.await.suspend.void( + // CHECK: call i8 @llvm.coro.suspend( + + // After resume: + // CHECK: call x86_thiscallcc void @"?await_resume@Awaiter@@QAE?AUNoisy@@XZ"(ptr {{[^,]*}} %{{.*}}, ptr dead_on_unwind writable sret(%struct.Noisy) align 4 %[[MTE_X]]) + + // Allocate inalloca: + // CHECK: %[[STACKSAVE:.+]] = call ptr @llvm.stacksave.p0() + // CHECK: %[[ARGMEM:.+]] = alloca inalloca <{ %struct.Noisy, %struct.Noisy }>, align 4, !coro.outside.frame ![[METADATA_NUM:[0-9]+]] + + // Move y (pre-evaluated Noisy(42)) to inalloca: + // CHECK: %[[GEP_Y:.+]] = getelementptr inbounds nuw <{ %struct.Noisy, %struct.Noisy }>, ptr %[[ARGMEM]], i32 0, i32 1 + // CHECK: call x86_thiscallcc noundef ptr @"??0Noisy@@QAE@$$QAU0@@Z"(ptr {{[^,]*}} %[[GEP_Y]], ptr noundef nonnull align 4 dereferenceable(4) %[[MTE_Y]]) + + // Move x (co_await Awaiter{} result) to inalloca: + // CHECK: %[[GEP_X:.+]] = getelementptr inbounds nuw <{ %struct.Noisy, %struct.Noisy }>, ptr %[[ARGMEM]], i32 0, i32 0 + // CHECK: call x86_thiscallcc noundef ptr @"??0Noisy@@QAE@$$QAU0@@Z"(ptr {{[^,]*}} %[[GEP_X]], ptr noundef nonnull align 4 dereferenceable(4) %[[MTE_X]]) + + // Lifetime start and call: + // CHECK: call void @llvm.lifetime.start.p0(ptr %[[ARGMEM]]) + // CHECK: call void @"?consume_two@@YAXUNoisy@@0@Z"(ptr inalloca(<{ %struct.Noisy, %struct.Noisy }>) %[[ARGMEM]]) + + consume_two(co_await Awaiter{}, Noisy(42)); +} + +// CHECK: ![[METADATA_NUM]] = !{} >From aa554816facaeb02bb60783fa180c1a20529fc6b Mon Sep 17 00:00:00 2001 From: Etienne Pierre-doray <[email protected]> Date: Wed, 29 Jul 2026 11:22:21 +0000 Subject: [PATCH 6/8] Format --- clang/lib/CodeGen/CGExprAgg.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp index 1eb0b65e1f491..0c0dde5aa5178 100644 --- a/clang/lib/CodeGen/CGExprAgg.cpp +++ b/clang/lib/CodeGen/CGExprAgg.cpp @@ -782,7 +782,8 @@ void AggExprEmitter::VisitMaterializeTemporaryExpr( QualType Type = E->getType(); assert((!Type->getAsCXXRecordDecl() || Type->getAsCXXRecordDecl()->isTriviallyCopyable()) && - "Non-trivially copyable MTE should not be visited as aggregate when pre-evaluated"); + "Non-trivially copyable MTE should not be visited as aggregate when " + "pre-evaluated"); EmitFinalDestCopy(Type, SrcLV); return; } >From 2dff9384a3d53b87cd49a819326f6346531fb745 Mon Sep 17 00:00:00 2001 From: Etienne Pierre-doray <[email protected]> Date: Wed, 29 Jul 2026 18:32:12 +0000 Subject: [PATCH 7/8] Fix lambda expressions --- clang/lib/AST/Expr.cpp | 9 +++++++++ clang/lib/Sema/SemaExpr.cpp | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp index 22a697fe09327..0bcd38ae97450 100644 --- a/clang/lib/AST/Expr.cpp +++ b/clang/lib/AST/Expr.cpp @@ -4004,6 +4004,15 @@ static bool containsCoroutineSuspendPoints(const Stmt *S) { if (isa<CoawaitExpr>(S) || isa<CoyieldExpr>(S) || isa<DependentCoawaitExpr>(S)) return true; + + if (auto *LE = dyn_cast<LambdaExpr>(S)) { + for (const Expr *Init : LE->capture_inits()) { + if (Init && containsCoroutineSuspendPoints(Init)) + return true; + } + return false; + } + for (const Stmt *Child : S->children()) if (Child && containsCoroutineSuspendPoints(Child)) return true; diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp index 5f71404fb8c5f..15475c2ec8773 100644 --- a/clang/lib/Sema/SemaExpr.cpp +++ b/clang/lib/Sema/SemaExpr.cpp @@ -6350,7 +6350,8 @@ bool Sema::GatherArgumentsForCall(SourceLocation CallLoc, FunctionDecl *FDecl, << Arg->getType() << ProtoArgType; } if (CallHasSuspend && Arg->isPRValue() && - isWin32InAllocaRecord(Context, ProtoArgType)) { + (isWin32InAllocaRecord(Context, ProtoArgType) || + Arg->containsCoroutineSuspendPoints())) { ExprResult Materialized = CreateMaterializeTemporaryExpr(Arg->getType(), Arg, false); if (!Materialized.isInvalid()) >From b5c45875f35e3c67fc72c591d66444d8874d8778 Mon Sep 17 00:00:00 2001 From: Etienne Pierre-doray <[email protected]> Date: Wed, 29 Jul 2026 18:58:54 +0000 Subject: [PATCH 8/8] Add AST node --- clang/include/clang/AST/ComputeDependence.h | 2 + clang/include/clang/AST/ExprCXX.h | 49 +++++++++++++++++++ clang/include/clang/AST/RecursiveASTVisitor.h | 7 +++ clang/include/clang/Basic/StmtNodes.td | 1 + .../include/clang/Serialization/ASTBitCodes.h | 1 + clang/lib/AST/ComputeDependence.cpp | 5 ++ clang/lib/AST/ExprCXX.cpp | 6 +++ clang/lib/AST/ExprConstant.cpp | 1 + clang/lib/AST/StmtPrinter.cpp | 5 ++ clang/lib/AST/StmtProfile.cpp | 5 ++ clang/lib/CodeGen/CGCall.cpp | 47 ++---------------- clang/lib/CodeGen/CGExprAgg.cpp | 7 +++ clang/lib/CodeGen/CGExprComplex.cpp | 5 ++ clang/lib/CodeGen/CGExprScalar.cpp | 5 ++ clang/lib/Sema/SemaExpr.cpp | 20 +++++--- clang/lib/Sema/TreeTransform.h | 6 +++ clang/lib/Serialization/ASTReaderStmt.cpp | 11 +++++ clang/lib/Serialization/ASTWriterStmt.cpp | 8 +++ 18 files changed, 143 insertions(+), 48 deletions(-) diff --git a/clang/include/clang/AST/ComputeDependence.h b/clang/include/clang/AST/ComputeDependence.h index 3a3c86842501a..0da1e99bf523c 100644 --- a/clang/include/clang/AST/ComputeDependence.h +++ b/clang/include/clang/AST/ComputeDependence.h @@ -69,6 +69,7 @@ class PackIndexingExpr; class SubstNonTypeTemplateParmExpr; class CoroutineSuspendExpr; class DependentCoawaitExpr; +class CoroutineSuspendParameterBypassExpr; class CXXNewExpr; class CXXPseudoDestructorExpr; class OverloadExpr; @@ -161,6 +162,7 @@ ExprDependence computeDependence(PackIndexingExpr *E); ExprDependence computeDependence(SubstNonTypeTemplateParmExpr *E); ExprDependence computeDependence(CoroutineSuspendExpr *E); ExprDependence computeDependence(DependentCoawaitExpr *E); +ExprDependence computeDependence(CoroutineSuspendParameterBypassExpr *E); ExprDependence computeDependence(CXXNewExpr *E); ExprDependence computeDependence(CXXPseudoDestructorExpr *E); ExprDependence computeDependence(OverloadExpr *E, bool KnownDependent, diff --git a/clang/include/clang/AST/ExprCXX.h b/clang/include/clang/AST/ExprCXX.h index d3d3b9c6d6326..65af878212ff7 100644 --- a/clang/include/clang/AST/ExprCXX.h +++ b/clang/include/clang/AST/ExprCXX.h @@ -5554,6 +5554,55 @@ class CXXReflectExpr : public Expr { /// Helper that selects an expression from an InitListExpr depending on the /// current expansion index. See 'CXXExpansionStmtPattern' for how this is used. +/// Represents a temporary that needs to be pre-evaluated in a coroutine +/// because it is passed to a call that contains suspend points, and we +/// want to delay inalloca allocation until after all suspends. +class CoroutineSuspendParameterBypassExpr : public Expr { + friend class ASTStmtReader; + + Stmt *SubExpr = nullptr; + Stmt *MoveExpr = nullptr; + + CoroutineSuspendParameterBypassExpr(Expr *SubExpr, Expr *MoveExpr) + : Expr(CoroutineSuspendParameterBypassExprClass, MoveExpr->getType(), + MoveExpr->getValueKind(), MoveExpr->getObjectKind()), + SubExpr(SubExpr), MoveExpr(MoveExpr) { + setDependence(computeDependence(this)); + } + +public: + CoroutineSuspendParameterBypassExpr(EmptyShell Empty) + : Expr(CoroutineSuspendParameterBypassExprClass, Empty) {} + + static CoroutineSuspendParameterBypassExpr * + Create(const ASTContext &C, Expr *SubExpr, Expr *MoveExpr); + + Expr *getSubExpr() { return cast<Expr>(SubExpr); } + const Expr *getSubExpr() const { return cast<Expr>(SubExpr); } + void setSubExpr(Expr *E) { SubExpr = E; } + + Expr *getMoveExpr() { return cast<Expr>(MoveExpr); } + const Expr *getMoveExpr() const { return cast<Expr>(MoveExpr); } + void setMoveExpr(Expr *E) { MoveExpr = E; } + + SourceLocation getBeginLoc() const LLVM_READONLY { + return SubExpr->getBeginLoc(); + } + SourceLocation getEndLoc() const LLVM_READONLY { + return MoveExpr->getEndLoc(); + } + + static bool classof(const Stmt *T) { + return T->getStmtClass() == CoroutineSuspendParameterBypassExprClass; + } + + child_range children() { return child_range(&MoveExpr, &MoveExpr + 1); } + + const_child_range children() const { + return const_child_range(&MoveExpr, &MoveExpr + 1); + } +}; + class CXXExpansionSelectExpr : public Expr { friend class ASTStmtReader; diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h index cdf8a71d54cc9..2fae8e87ee917 100644 --- a/clang/include/clang/AST/RecursiveASTVisitor.h +++ b/clang/include/clang/AST/RecursiveASTVisitor.h @@ -3163,6 +3163,13 @@ DEF_TRAVERSE_STMT(CoyieldExpr, { } }) +DEF_TRAVERSE_STMT(CoroutineSuspendParameterBypassExpr, { + if (!getDerived().shouldVisitImplicitCode()) { + TRY_TO_TRAVERSE_OR_ENQUEUE_STMT(S->getSubExpr()); + ShouldVisitChildren = false; + } +}) + DEF_TRAVERSE_STMT(ConceptSpecializationExpr, { TRY_TO(TraverseConceptReference(S->getConceptReference())); }) diff --git a/clang/include/clang/Basic/StmtNodes.td b/clang/include/clang/Basic/StmtNodes.td index f5fa397c92ef3..4c1dd6ba01661 100644 --- a/clang/include/clang/Basic/StmtNodes.td +++ b/clang/include/clang/Basic/StmtNodes.td @@ -181,6 +181,7 @@ def CoroutineSuspendExpr : StmtNode<Expr, 1>; def CoawaitExpr : StmtNode<CoroutineSuspendExpr>; def DependentCoawaitExpr : StmtNode<Expr>; def CoyieldExpr : StmtNode<CoroutineSuspendExpr>; +def CoroutineSuspendParameterBypassExpr : StmtNode<Expr>; // C++20 Concepts expressions def ConceptSpecializationExpr : StmtNode<Expr>; diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h index 671341488278e..b57472e56f832 100644 --- a/clang/include/clang/Serialization/ASTBitCodes.h +++ b/clang/include/clang/Serialization/ASTBitCodes.h @@ -2063,6 +2063,7 @@ enum StmtCode { EXPR_COAWAIT, EXPR_COYIELD, EXPR_DEPENDENT_COAWAIT, + EXPR_COROUTINE_SUSPEND_PARAMETER_BYPASS, // FixedPointLiteral EXPR_FIXEDPOINT_LITERAL, diff --git a/clang/lib/AST/ComputeDependence.cpp b/clang/lib/AST/ComputeDependence.cpp index a819bb6dec599..1cc40fb158cfb 100644 --- a/clang/lib/AST/ComputeDependence.cpp +++ b/clang/lib/AST/ComputeDependence.cpp @@ -344,6 +344,11 @@ ExprDependence clang::computeDependence(CXXBindTemporaryExpr *E) { return E->getSubExpr()->getDependence(); } +ExprDependence +clang::computeDependence(CoroutineSuspendParameterBypassExpr *E) { + return E->getMoveExpr()->getDependence(); +} + ExprDependence clang::computeDependence(CXXScalarValueInitExpr *E) { auto D = toExprDependenceForImpliedType(E->getType()->getDependence()); if (auto *TSI = E->getTypeSourceInfo()) diff --git a/clang/lib/AST/ExprCXX.cpp b/clang/lib/AST/ExprCXX.cpp index 6c1cde6540d85..6ccad263760f8 100644 --- a/clang/lib/AST/ExprCXX.cpp +++ b/clang/lib/AST/ExprCXX.cpp @@ -1132,6 +1132,12 @@ CXXBindTemporaryExpr *CXXBindTemporaryExpr::Create(const ASTContext &C, return new (C) CXXBindTemporaryExpr(Temp, SubExpr); } +CoroutineSuspendParameterBypassExpr * +CoroutineSuspendParameterBypassExpr::Create(const ASTContext &C, Expr *SubExpr, + Expr *MoveExpr) { + return new (C) CoroutineSuspendParameterBypassExpr(SubExpr, MoveExpr); +} + CXXTemporaryObjectExpr::CXXTemporaryObjectExpr( CXXConstructorDecl *Cons, QualType Ty, TypeSourceInfo *TSI, ArrayRef<Expr *> Args, SourceRange ParenOrBraceRange, diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp index 9d69de2a7c6fd..5a0ed07d21e97 100644 --- a/clang/lib/AST/ExprConstant.cpp +++ b/clang/lib/AST/ExprConstant.cpp @@ -22378,6 +22378,7 @@ static ICEDiag CheckICE(const Expr* E, const ASTContext &Ctx) { case Expr::CXXParenListInitExprClass: case Expr::HLSLOutArgExprClass: case Expr::CXXExpansionSelectExprClass: + case Expr::CoroutineSuspendParameterBypassExprClass: return ICEDiag(IK_NotICE, E->getBeginLoc()); case Expr::MemberExprClass: { diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp index 877191d456b35..2f29325bab682 100644 --- a/clang/lib/AST/StmtPrinter.cpp +++ b/clang/lib/AST/StmtPrinter.cpp @@ -2386,6 +2386,11 @@ void StmtPrinter::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *Node) { PrintExpr(Node->getSubExpr()); } +void StmtPrinter::VisitCoroutineSuspendParameterBypassExpr( + CoroutineSuspendParameterBypassExpr *Node) { + PrintExpr(Node->getSubExpr()); +} + void StmtPrinter::VisitCXXTemporaryObjectExpr(CXXTemporaryObjectExpr *Node) { Node->getType().print(OS, Policy); if (Node->isStdInitListInitialization()) diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp index 00c132f1ed9e0..dd9b7c06cd78c 100644 --- a/clang/lib/AST/StmtProfile.cpp +++ b/clang/lib/AST/StmtProfile.cpp @@ -2175,6 +2175,11 @@ void StmtProfiler::VisitCXXBindTemporaryExpr(const CXXBindTemporaryExpr *S) { const_cast<CXXDestructorDecl *>(S->getTemporary()->getDestructor())); } +void StmtProfiler::VisitCoroutineSuspendParameterBypassExpr( + const CoroutineSuspendParameterBypassExpr *S) { + VisitExpr(S); +} + void StmtProfiler::VisitCXXConstructExpr(const CXXConstructExpr *S) { VisitExpr(S); VisitDecl(S->getConstructor()); diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp index 17f9c8abd7497..6cb6d7e249ac7 100644 --- a/clang/lib/CodeGen/CGCall.cpp +++ b/clang/lib/CodeGen/CGCall.cpp @@ -5031,46 +5031,6 @@ static bool isObjCMethodWithTypeParams(const ObjCMethodDecl *method) { } #endif -static const MaterializeTemporaryExpr *getMTEToPreEvaluate(const Expr *E) { - while (true) { - E = E->IgnoreParens(); - if (auto *Cleanups = dyn_cast<ExprWithCleanups>(E)) { - E = Cleanups->getSubExpr(); - continue; - } - if (auto *Bind = dyn_cast<CXXBindTemporaryExpr>(E)) { - E = Bind->getSubExpr(); - continue; - } - break; - } - - if (auto *CE = dyn_cast<CXXConstructExpr>(E)) { - if (CE->getNumArgs() > 0) { - const Expr *Arg = CE->getArg(0); - while (true) { - Arg = Arg->IgnoreParens(); - if (auto *MTE = dyn_cast<MaterializeTemporaryExpr>(Arg)) - return MTE; - if (auto *Cleanups = dyn_cast<ExprWithCleanups>(Arg)) { - Arg = Cleanups->getSubExpr(); - continue; - } - if (auto *Bind = dyn_cast<CXXBindTemporaryExpr>(Arg)) { - Arg = Bind->getSubExpr(); - continue; - } - if (auto *ICE = dyn_cast<ImplicitCastExpr>(Arg)) { - Arg = ICE->getSubExpr(); - continue; - } - break; - } - } - } - return nullptr; -} - /// EmitCallArgs - Emit call arguments for a function. void CodeGenFunction::EmitCallArgs( CallArgList &Args, PrototypeWrapper Prototype, @@ -5172,8 +5132,11 @@ void CodeGenFunction::EmitCallArgs( SmallVector<const MaterializeTemporaryExpr *, 4> MTEsToPreEvaluate; if (isCoroutine() && hasInAllocaArgs(CGM, ExplicitCC, ArgTypes)) { for (const Expr *A : ArgRange) { - if (auto *MTE = getMTEToPreEvaluate(A)) { - MTEsToPreEvaluate.push_back(MTE); + if (auto *Bypass = dyn_cast<CoroutineSuspendParameterBypassExpr>(A)) { + if (auto *MTE = + dyn_cast<MaterializeTemporaryExpr>(Bypass->getSubExpr())) { + MTEsToPreEvaluate.push_back(MTE); + } } } } diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp index 0c0dde5aa5178..87673afc287aa 100644 --- a/clang/lib/CodeGen/CGExprAgg.cpp +++ b/clang/lib/CodeGen/CGExprAgg.cpp @@ -200,6 +200,8 @@ class AggExprEmitter : public StmtVisitor<AggExprEmitter> { void VisitCXXTypeidExpr(CXXTypeidExpr *E) { EmitAggLoadOfLValue(E); } void VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *E); void VisitOpaqueValueExpr(OpaqueValueExpr *E); + void VisitCoroutineSuspendParameterBypassExpr( + CoroutineSuspendParameterBypassExpr *E); void VisitPseudoObjectExpr(PseudoObjectExpr *E) { if (E->isGLValue()) { @@ -798,6 +800,11 @@ void AggExprEmitter::VisitOpaqueValueExpr(OpaqueValueExpr *e) { EmitFinalDestCopy(e->getType(), CGF.getOrCreateOpaqueLValueMapping(e)); } +void AggExprEmitter::VisitCoroutineSuspendParameterBypassExpr( + CoroutineSuspendParameterBypassExpr *E) { + Visit(E->getMoveExpr()); +} + void AggExprEmitter::VisitCompoundLiteralExpr(CompoundLiteralExpr *E) { if (Dest.isPotentiallyAliased()) { // Just emit a load of the lvalue + a copy, because our compound literal diff --git a/clang/lib/CodeGen/CGExprComplex.cpp b/clang/lib/CodeGen/CGExprComplex.cpp index 350cbd18c7ed7..7c08d9eaff87d 100644 --- a/clang/lib/CodeGen/CGExprComplex.cpp +++ b/clang/lib/CodeGen/CGExprComplex.cpp @@ -169,6 +169,11 @@ class ComplexExprEmitter return CGF.getOrCreateOpaqueRValueMapping(E).getComplexVal(); } + ComplexPairTy VisitCoroutineSuspendParameterBypassExpr( + CoroutineSuspendParameterBypassExpr *E) { + return Visit(E->getMoveExpr()); + } + ComplexPairTy VisitPseudoObjectExpr(PseudoObjectExpr *E) { return CGF.EmitPseudoObjectRValue(E).getComplexVal(); } diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp index 8783b43846434..d3064d2498a03 100644 --- a/clang/lib/CodeGen/CGExprScalar.cpp +++ b/clang/lib/CodeGen/CGExprScalar.cpp @@ -600,6 +600,11 @@ class ScalarExprEmitter return CGF.getOrCreateOpaqueRValueMapping(E).getScalarVal(); } + Value *VisitCoroutineSuspendParameterBypassExpr( + CoroutineSuspendParameterBypassExpr *E) { + return Visit(E->getMoveExpr()); + } + Value *VisitOpenACCAsteriskSizeExpr(OpenACCAsteriskSizeExpr *E) { llvm_unreachable("Codegen for this isn't defined/implemented"); } diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp index 15475c2ec8773..be204239c72ba 100644 --- a/clang/lib/Sema/SemaExpr.cpp +++ b/clang/lib/Sema/SemaExpr.cpp @@ -6349,21 +6349,29 @@ bool Sema::GatherArgumentsForCall(SourceLocation CallLoc, FunctionDecl *FDecl, : diag::warn_obt_discarded_at_function_boundary) << Arg->getType() << ProtoArgType; } - if (CallHasSuspend && Arg->isPRValue() && - (isWin32InAllocaRecord(Context, ProtoArgType) || - Arg->containsCoroutineSuspendPoints())) { + bool NeedsBypass = CallHasSuspend && Arg->isPRValue() && + (isWin32InAllocaRecord(Context, ProtoArgType) || + Arg->containsCoroutineSuspendPoints()); + Expr *PreBypassArg = Arg; + if (NeedsBypass) { ExprResult Materialized = CreateMaterializeTemporaryExpr(Arg->getType(), Arg, false); if (!Materialized.isInvalid()) - Arg = Materialized.get(); + PreBypassArg = Materialized.get(); } - ExprResult ArgE = PerformCopyInitialization( - Entity, SourceLocation(), Arg, IsListInitialization, AllowExplicit); + ExprResult ArgE = + PerformCopyInitialization(Entity, SourceLocation(), PreBypassArg, + IsListInitialization, AllowExplicit); if (ArgE.isInvalid()) return true; Arg = ArgE.getAs<Expr>(); + + if (NeedsBypass) { + Arg = CoroutineSuspendParameterBypassExpr::Create(Context, PreBypassArg, + Arg); + } } else { assert(Param && "can't use default arguments without a known callee"); diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h index 0725664a4e050..b8563c29df616 100644 --- a/clang/lib/Sema/TreeTransform.h +++ b/clang/lib/Sema/TreeTransform.h @@ -15964,6 +15964,12 @@ TreeTransform<Derived>::TransformCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) { return getDerived().TransformExpr(E->getSubExpr()); } +template <typename Derived> +ExprResult TreeTransform<Derived>::TransformCoroutineSuspendParameterBypassExpr( + CoroutineSuspendParameterBypassExpr *E) { + return getDerived().TransformExpr(E->getSubExpr()); +} + /// Transform a C++ expression that contains cleanups that should /// be run after the expression is evaluated. /// diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp index 6cde6c1816dc7..313719d16d293 100644 --- a/clang/lib/Serialization/ASTReaderStmt.cpp +++ b/clang/lib/Serialization/ASTReaderStmt.cpp @@ -512,6 +512,13 @@ void ASTStmtReader::VisitDependentCoawaitExpr(DependentCoawaitExpr *E) { SubExpr = Record.readSubStmt(); } +void ASTStmtReader::VisitCoroutineSuspendParameterBypassExpr( + CoroutineSuspendParameterBypassExpr *E) { + VisitExpr(E); + E->setSubExpr(cast_or_null<Expr>(Record.readSubExpr())); + E->setMoveExpr(cast_or_null<Expr>(Record.readSubExpr())); +} + void ASTStmtReader::VisitCapturedStmt(CapturedStmt *S) { VisitStmt(S); Record.skipInts(1); @@ -4555,6 +4562,10 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { S = new (Context) DependentCoawaitExpr(Empty); break; + case EXPR_COROUTINE_SUSPEND_PARAMETER_BYPASS: + S = new (Context) CoroutineSuspendParameterBypassExpr(Empty); + break; + case EXPR_CONCEPT_SPECIALIZATION: { S = new (Context) ConceptSpecializationExpr(Empty); break; diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp index 10443d42df8c0..a4b993740186c 100644 --- a/clang/lib/Serialization/ASTWriterStmt.cpp +++ b/clang/lib/Serialization/ASTWriterStmt.cpp @@ -487,6 +487,14 @@ void ASTStmtWriter::VisitDependentCoawaitExpr(DependentCoawaitExpr *E) { Code = serialization::EXPR_DEPENDENT_COAWAIT; } +void ASTStmtWriter::VisitCoroutineSuspendParameterBypassExpr( + CoroutineSuspendParameterBypassExpr *E) { + VisitExpr(E); + Record.AddStmt(E->getSubExpr()); + Record.AddStmt(E->getMoveExpr()); + Code = serialization::EXPR_COROUTINE_SUSPEND_PARAMETER_BYPASS; +} + static void addConstraintSatisfaction(ASTRecordWriter &Record, const ASTConstraintSatisfaction &Satisfaction) { _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
