https://github.com/kevinsala updated https://github.com/llvm/llvm-project/pull/152830
>From f66e5faa93cf2f40bd2a6bd7a95abddf78fb6076 Mon Sep 17 00:00:00 2001 From: Kevin Sala <[email protected]> Date: Fri, 8 Aug 2025 11:04:06 -0700 Subject: [PATCH 1/3] [OpenMP] Add codegen support for dyn_groupprivate clause --- clang/lib/CodeGen/CGOpenMPRuntime.cpp | 40 ++++++++++++------- .../llvm/Frontend/OpenMP/OMPIRBuilder.h | 7 +++- llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 10 ++++- 3 files changed, 39 insertions(+), 18 deletions(-) diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp index a5f2f0efa2c3b..d9121827a813a 100644 --- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp +++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp @@ -9489,18 +9489,30 @@ static llvm::Value *emitDeviceID( return DeviceID; } -static llvm::Value *emitDynCGGroupMem(const OMPExecutableDirective &D, - CodeGenFunction &CGF) { - llvm::Value *DynCGroupMem = CGF.Builder.getInt32(0); - - if (auto *DynMemClause = D.getSingleClause<OMPXDynCGroupMemClause>()) { - CodeGenFunction::RunCleanupsScope DynCGroupMemScope(CGF); - llvm::Value *DynCGroupMemVal = CGF.EmitScalarExpr( - DynMemClause->getSize(), /*IgnoreResultAssign=*/true); - DynCGroupMem = CGF.Builder.CreateIntCast(DynCGroupMemVal, CGF.Int32Ty, - /*isSigned=*/false); - } - return DynCGroupMem; +static std::pair<llvm::Value *, bool> +emitDynCGroupMem(const OMPExecutableDirective &D, CodeGenFunction &CGF) { + llvm::Value *DynGP = CGF.Builder.getInt32(0); + bool DynGPFallback = false; + + if (auto *DynGPClause = D.getSingleClause<OMPDynGroupprivateClause>()) { + CodeGenFunction::RunCleanupsScope DynGPScope(CGF); + llvm::Value *DynGPVal = + CGF.EmitScalarExpr(DynGPClause->getSize(), /*IgnoreResultAssign=*/true); + DynGP = CGF.Builder.CreateIntCast(DynGPVal, CGF.Int32Ty, + /*isSigned=*/false); + DynGPFallback = (DynGPClause->getFirstDynGroupprivateModifier() != + OMPC_DYN_GROUPPRIVATE_strict && + DynGPClause->getSecondDynGroupprivateModifier() != + OMPC_DYN_GROUPPRIVATE_strict); + } else if (auto *OMPXDynCGClause = + D.getSingleClause<OMPXDynCGroupMemClause>()) { + CodeGenFunction::RunCleanupsScope DynCGMemScope(CGF); + llvm::Value *DynCGMemVal = CGF.EmitScalarExpr(OMPXDynCGClause->getSize(), + /*IgnoreResultAssign=*/true); + DynGP = CGF.Builder.CreateIntCast(DynCGMemVal, CGF.Int32Ty, + /*isSigned=*/false); + } + return {DynGP, DynGPFallback}; } static void genMapInfoForCaptures( MappableExprsHandler &MEHandler, CodeGenFunction &CGF, @@ -9710,7 +9722,7 @@ static void emitTargetCallKernelLaunch( llvm::Value *RTLoc = OMPRuntime->emitUpdateLocation(CGF, D.getBeginLoc()); llvm::Value *NumIterations = OMPRuntime->emitTargetNumIterationsCall(CGF, D, SizeEmitter); - llvm::Value *DynCGGroupMem = emitDynCGGroupMem(D, CGF); + auto [DynCGroupMem, DynCGroupMemFallback] = emitDynCGroupMem(D, CGF); llvm::OpenMPIRBuilder::InsertPointTy AllocaIP( CGF.AllocaInsertPt->getParent(), CGF.AllocaInsertPt->getIterator()); @@ -9720,7 +9732,7 @@ static void emitTargetCallKernelLaunch( llvm::OpenMPIRBuilder::TargetKernelArgs Args( NumTargetItems, RTArgs, NumIterations, NumTeams, NumThreads, - DynCGGroupMem, HasNoWait); + DynCGroupMem, HasNoWait, DynCGroupMemFallback); llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(OMPRuntime->getOMPBuilder().emitKernelLaunch( diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h index 19a4058b64382..ebc50eecb551e 100644 --- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h +++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h @@ -2341,17 +2341,20 @@ class OpenMPIRBuilder { Value *DynCGGroupMem = nullptr; /// True if the kernel has 'no wait' clause. bool HasNoWait = false; + /// True if the dynamic shared memory may fallback. + bool MayFallbackDynCGroupMem = false; // Constructors for TargetKernelArgs. TargetKernelArgs() {} TargetKernelArgs(unsigned NumTargetItems, TargetDataRTArgs RTArgs, Value *NumIterations, ArrayRef<Value *> NumTeams, ArrayRef<Value *> NumThreads, Value *DynCGGroupMem, - bool HasNoWait) + bool HasNoWait, bool MayFallbackDynCGroupMem) : NumTargetItems(NumTargetItems), RTArgs(RTArgs), NumIterations(NumIterations), NumTeams(NumTeams), NumThreads(NumThreads), DynCGGroupMem(DynCGGroupMem), - HasNoWait(HasNoWait) {} + HasNoWait(HasNoWait), + MayFallbackDynCGroupMem(MayFallbackDynCGroupMem) {} }; /// Create the kernel args vector used by emitTargetKernel. This function diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp index 170224616ac64..e600508d347cb 100644 --- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp +++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp @@ -506,7 +506,13 @@ void OpenMPIRBuilder::getKernelArgsVector(TargetKernelArgs &KernelArgs, auto Int32Ty = Type::getInt32Ty(Builder.getContext()); constexpr const size_t MaxDim = 3; Value *ZeroArray = Constant::getNullValue(ArrayType::get(Int32Ty, MaxDim)); - Value *Flags = Builder.getInt64(KernelArgs.HasNoWait); + + Value *HasNoWaitFlag = Builder.getInt64(KernelArgs.HasNoWait); + Value *MayFallbackDynCGroupMemFlag = + Builder.getInt64(KernelArgs.MayFallbackDynCGroupMem); + MayFallbackDynCGroupMemFlag = + Builder.CreateShl(MayFallbackDynCGroupMemFlag, 2); + Value *Flags = Builder.CreateOr(HasNoWaitFlag, MayFallbackDynCGroupMemFlag); assert(!KernelArgs.NumTeams.empty() && !KernelArgs.NumThreads.empty()); @@ -7891,7 +7897,7 @@ static void emitTargetCall( KArgs = OpenMPIRBuilder::TargetKernelArgs(NumTargetItems, RTArgs, TripCount, NumTeamsC, NumThreadsC, - DynCGGroupMem, HasNoWait); + DynCGGroupMem, HasNoWait, false); // Assume no error was returned because TaskBodyCB and // EmitTargetCallFallbackCB don't produce any. >From 3a2fe705a6c11e484ed232a166fd332634fd8a70 Mon Sep 17 00:00:00 2001 From: Kevin Sala <[email protected]> Date: Fri, 24 Oct 2025 18:57:14 -0700 Subject: [PATCH 2/3] Update for fallback complex modifier --- clang/lib/CodeGen/CGOpenMPRuntime.cpp | 26 ++++++++++++++----- .../llvm/Frontend/OpenMP/OMPConstants.h | 10 +++++++ .../llvm/Frontend/OpenMP/OMPIRBuilder.h | 11 ++++---- llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 16 ++++++------ 4 files changed, 44 insertions(+), 19 deletions(-) diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp index 48168193778b8..2d08e1a04ff03 100644 --- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp +++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp @@ -10000,10 +10000,10 @@ static llvm::Value *emitDeviceID( return DeviceID; } -static std::pair<llvm::Value *, bool> +static std::pair<llvm::Value *, OMPDynGroupprivateFallbackType> emitDynCGroupMem(const OMPExecutableDirective &D, CodeGenFunction &CGF) { llvm::Value *DynGP = CGF.Builder.getInt32(0); - bool DynGPFallback = false; + OMPDynGroupprivateFallbackType DynGPFallback; if (auto *DynGPClause = D.getSingleClause<OMPDynGroupprivateClause>()) { CodeGenFunction::RunCleanupsScope DynGPScope(CGF); @@ -10011,10 +10011,22 @@ emitDynCGroupMem(const OMPExecutableDirective &D, CodeGenFunction &CGF) { CGF.EmitScalarExpr(DynGPClause->getSize(), /*IgnoreResultAssign=*/true); DynGP = CGF.Builder.CreateIntCast(DynGPVal, CGF.Int32Ty, /*isSigned=*/false); - DynGPFallback = (DynGPClause->getFirstDynGroupprivateModifier() != - OMPC_DYN_GROUPPRIVATE_strict && - DynGPClause->getSecondDynGroupprivateModifier() != - OMPC_DYN_GROUPPRIVATE_strict); + auto FallbackModifier = DynGPClause->getDynGroupprivateFallbackModifier(); + switch (FallbackModifier) { + case OMPC_DYN_GROUPPRIVATE_FALLBACK_abort: + DynGPFallback = OMPDynGroupprivateFallbackType::Abort; + break; + case OMPC_DYN_GROUPPRIVATE_FALLBACK_null: + DynGPFallback = OMPDynGroupprivateFallbackType::Null; + break; + case OMPC_DYN_GROUPPRIVATE_FALLBACK_default_mem: + case OMPC_DYN_GROUPPRIVATE_FALLBACK_unknown: + // This is the default for dyn_groupprivate. + DynGPFallback = OMPDynGroupprivateFallbackType::DefaultMem; + break; + default: + llvm_unreachable("Unknown fallback modifier for OpenMP dyn_groupprivate"); + } } else if (auto *OMPXDynCGClause = D.getSingleClause<OMPXDynCGroupMemClause>()) { CodeGenFunction::RunCleanupsScope DynCGMemScope(CGF); @@ -10022,9 +10034,11 @@ emitDynCGroupMem(const OMPExecutableDirective &D, CodeGenFunction &CGF) { /*IgnoreResultAssign=*/true); DynGP = CGF.Builder.CreateIntCast(DynCGMemVal, CGF.Int32Ty, /*isSigned=*/false); + DynGPFallback = OMPDynGroupprivateFallbackType::Abort; } return {DynGP, DynGPFallback}; } + static void genMapInfoForCaptures( MappableExprsHandler &MEHandler, CodeGenFunction &CGF, const CapturedStmt &CS, llvm::SmallVectorImpl<llvm::Value *> &CapturedVars, diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h b/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h index 7bec7e0c6736d..1ac9ac040468c 100644 --- a/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h +++ b/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h @@ -190,6 +190,16 @@ enum class OMPScheduleType { LLVM_MARK_AS_BITMASK_ENUM(/* LargestValue */ ModifierMask) }; +/// The fallback types for the dyn_groupprivate clause. +enum class OMPDynGroupprivateFallbackType : uint64_t { + /// Abort the execution. + Abort = 0, + /// Return null pointer. + Null = 1, + /// Allocate from a implementation defined memory space. + DefaultMem = 2 +}; + // Default OpenMP mapper name suffix. inline constexpr const char *OmpDefaultMapperName = ".omp.default.mapper"; diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h index f20b58409302f..aa370606c6539 100644 --- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h +++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h @@ -2449,20 +2449,21 @@ class OpenMPIRBuilder { Value *DynCGGroupMem = nullptr; /// True if the kernel has 'no wait' clause. bool HasNoWait = false; - /// True if the dynamic shared memory may fallback. - bool MayFallbackDynCGroupMem = false; + /// The fallback mechanism for the shared memory. + omp::OMPDynGroupprivateFallbackType DynCGroupMemFallback = + omp::OMPDynGroupprivateFallbackType::Abort; // Constructors for TargetKernelArgs. TargetKernelArgs() {} TargetKernelArgs(unsigned NumTargetItems, TargetDataRTArgs RTArgs, Value *NumIterations, ArrayRef<Value *> NumTeams, ArrayRef<Value *> NumThreads, Value *DynCGGroupMem, - bool HasNoWait, bool MayFallbackDynCGroupMem) + bool HasNoWait, + omp::OMPDynGroupprivateFallbackType DynCGroupMemFallback) : NumTargetItems(NumTargetItems), RTArgs(RTArgs), NumIterations(NumIterations), NumTeams(NumTeams), NumThreads(NumThreads), DynCGGroupMem(DynCGGroupMem), - HasNoWait(HasNoWait), - MayFallbackDynCGroupMem(MayFallbackDynCGroupMem) {} + HasNoWait(HasNoWait), DynCGroupMemFallback(DynCGroupMemFallback) {} }; /// Create the kernel args vector used by emitTargetKernel. This function diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp index 4cfe038c36eee..9f80e3eb61bfc 100644 --- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp +++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp @@ -532,11 +532,11 @@ void OpenMPIRBuilder::getKernelArgsVector(TargetKernelArgs &KernelArgs, Value *ZeroArray = Constant::getNullValue(ArrayType::get(Int32Ty, MaxDim)); Value *HasNoWaitFlag = Builder.getInt64(KernelArgs.HasNoWait); - Value *MayFallbackDynCGroupMemFlag = - Builder.getInt64(KernelArgs.MayFallbackDynCGroupMem); - MayFallbackDynCGroupMemFlag = - Builder.CreateShl(MayFallbackDynCGroupMemFlag, 2); - Value *Flags = Builder.CreateOr(HasNoWaitFlag, MayFallbackDynCGroupMemFlag); + + Value *DynCGroupMemFallbackFlag = + Builder.getInt64(static_cast<uint64_t>(KernelArgs.DynCGroupMemFallback)); + DynCGroupMemFallbackFlag = Builder.CreateShl(DynCGroupMemFallbackFlag, 2); + Value *Flags = Builder.CreateOr(HasNoWaitFlag, DynCGroupMemFallbackFlag); assert(!KernelArgs.NumTeams.empty() && !KernelArgs.NumThreads.empty()); @@ -8368,9 +8368,9 @@ static void emitTargetCall( // TODO: Use correct DynCGGroupMem Value *DynCGGroupMem = Builder.getInt32(0); - KArgs = OpenMPIRBuilder::TargetKernelArgs(NumTargetItems, RTArgs, TripCount, - NumTeamsC, NumThreadsC, - DynCGGroupMem, HasNoWait, false); + KArgs = OpenMPIRBuilder::TargetKernelArgs( + NumTargetItems, RTArgs, TripCount, NumTeamsC, NumThreadsC, + DynCGGroupMem, HasNoWait, OMPDynGroupprivateFallbackType::Abort); // Assume no error was returned because TaskBodyCB and // EmitTargetCallFallbackCB don't produce any. >From c4905e056722d56d2b096c6e7e30c9580b2a4461 Mon Sep 17 00:00:00 2001 From: Kevin Sala <[email protected]> Date: Thu, 30 Oct 2025 14:30:00 -0700 Subject: [PATCH 3/3] Fix initialization of a fallback variable --- clang/lib/CodeGen/CGOpenMPRuntime.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp index 2d08e1a04ff03..74076eb5a9733 100644 --- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp +++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp @@ -10003,7 +10003,7 @@ static llvm::Value *emitDeviceID( static std::pair<llvm::Value *, OMPDynGroupprivateFallbackType> emitDynCGroupMem(const OMPExecutableDirective &D, CodeGenFunction &CGF) { llvm::Value *DynGP = CGF.Builder.getInt32(0); - OMPDynGroupprivateFallbackType DynGPFallback; + auto DynGPFallback = OMPDynGroupprivateFallbackType::Abort; if (auto *DynGPClause = D.getSingleClause<OMPDynGroupprivateClause>()) { CodeGenFunction::RunCleanupsScope DynGPScope(CGF); @@ -10034,7 +10034,6 @@ emitDynCGroupMem(const OMPExecutableDirective &D, CodeGenFunction &CGF) { /*IgnoreResultAssign=*/true); DynGP = CGF.Builder.CreateIntCast(DynCGMemVal, CGF.Int32Ty, /*isSigned=*/false); - DynGPFallback = OMPDynGroupprivateFallbackType::Abort; } return {DynGP, DynGPFallback}; } _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
