[PATCH] D157738: [OpenMP] Emit offloading entries for indirect target variables

2023-08-24 Thread Joseph Huber via Phabricator via cfe-commits
This revision was automatically updated to reflect the committed changes.
Closed by commit rG9da61aed751e: [OpenMP] Emit offloading entries for indirect 
target variables (authored by jhuber6).

Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D157738/new/

https://reviews.llvm.org/D157738

Files:
  clang/lib/CodeGen/CGOpenMPRuntime.cpp
  clang/lib/CodeGen/CGOpenMPRuntime.h
  clang/lib/CodeGen/CodeGenModule.cpp
  clang/test/OpenMP/target_indirect_codegen.cpp
  llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
  llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp

Index: llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
===
--- llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5705,9 +5705,10 @@
 
 void OpenMPIRBuilder::createOffloadEntry(Constant *ID, Constant *Addr,
  uint64_t Size, int32_t Flags,
- GlobalValue::LinkageTypes) {
+ GlobalValue::LinkageTypes,
+ StringRef Name) {
   if (!Config.isGPU()) {
-emitOffloadingEntry(ID, Addr->getName(), Size, Flags);
+emitOffloadingEntry(ID, Name.empty() ? Addr->getName() : Name, Size, Flags);
 return;
   }
   // TODO: Add support for global variables on the device after declare target
@@ -5867,13 +5868,20 @@
 
   // Hidden or internal symbols on the device are not externally visible.
   // We should not attempt to register them by creating an offloading
-  // entry.
+  // entry. Indirect variables are handled separately on the device.
   if (auto *GV = dyn_cast(CE->getAddress()))
-if (GV->hasLocalLinkage() || GV->hasHiddenVisibility())
+if ((GV->hasLocalLinkage() || GV->hasHiddenVisibility()) &&
+Flags != OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect)
   continue;
 
-  createOffloadEntry(CE->getAddress(), CE->getAddress(), CE->getVarSize(),
- Flags, CE->getLinkage());
+  // Indirect globals need to use a special name that doesn't match the name
+  // of the associated host global.
+  if (Flags == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect)
+createOffloadEntry(CE->getAddress(), CE->getAddress(), CE->getVarSize(),
+   Flags, CE->getLinkage(), CE->getVarName());
+  else
+createOffloadEntry(CE->getAddress(), CE->getAddress(), CE->getVarSize(),
+   Flags, CE->getLinkage());
 
 } else {
   llvm_unreachable("Unsupported entry kind.");
@@ -6218,8 +6226,13 @@
   }
   return;
 }
-OffloadEntriesDeviceGlobalVar.try_emplace(VarName, OffloadingEntriesNum,
-  Addr, VarSize, Flags, Linkage);
+if (Flags == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect)
+  OffloadEntriesDeviceGlobalVar.try_emplace(VarName, OffloadingEntriesNum,
+Addr, VarSize, Flags, Linkage,
+VarName.str());
+else
+  OffloadEntriesDeviceGlobalVar.try_emplace(
+  VarName, OffloadingEntriesNum, Addr, VarSize, Flags, Linkage, "");
 ++OffloadingEntriesNum;
   }
 }
Index: llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
===
--- llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -326,6 +326,8 @@
 OMPTargetGlobalVarEntryEnter = 0x2,
 /// Mark the entry as having no declare target entry kind.
 OMPTargetGlobalVarEntryNone = 0x3,
+/// Mark the entry as a declare target indirect global.
+OMPTargetGlobalVarEntryIndirect = 0x4,
   };
 
   /// Kind of device clause for declare target variables
@@ -349,6 +351,7 @@
 /// Type of the global variable.
 int64_t VarSize;
 GlobalValue::LinkageTypes Linkage;
+const std::string VarName;
 
   public:
 OffloadEntryInfoDeviceGlobalVar()
@@ -359,13 +362,15 @@
 explicit OffloadEntryInfoDeviceGlobalVar(unsigned Order, Constant *Addr,
  int64_t VarSize,
  OMPTargetGlobalVarEntryKind Flags,
- GlobalValue::LinkageTypes Linkage)
+ GlobalValue::LinkageTypes Linkage,
+ const std::string &VarName)
 : OffloadEntryInfo(OffloadingEntryInfoDeviceGlobalVar, Order, Flags),
-  VarSize(VarSize), Linkage(Linkage) {
+  VarSize(VarSize), Linkage(Linkage), VarName(VarName) {
   setAddress(Addr);
 }
 
 int64_t getVarSize() const { return VarSize; }
+StringRef getVarName() const { return VarName; }
 void setVar

[PATCH] D157738: [OpenMP] Emit offloading entries for indirect target variables

2023-08-14 Thread Joseph Huber via Phabricator via cfe-commits
jhuber6 updated this revision to Diff 550140.
jhuber6 added a comment.

Address comments


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D157738/new/

https://reviews.llvm.org/D157738

Files:
  clang/lib/CodeGen/CGOpenMPRuntime.cpp
  clang/lib/CodeGen/CGOpenMPRuntime.h
  clang/lib/CodeGen/CodeGenModule.cpp
  clang/test/OpenMP/target_indirect_codegen.cpp
  llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
  llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp

Index: llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
===
--- llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5525,9 +5525,10 @@
 
 void OpenMPIRBuilder::createOffloadEntry(Constant *ID, Constant *Addr,
  uint64_t Size, int32_t Flags,
- GlobalValue::LinkageTypes) {
+ GlobalValue::LinkageTypes,
+ StringRef Name) {
   if (!Config.isGPU()) {
-emitOffloadingEntry(ID, Addr->getName(), Size, Flags);
+emitOffloadingEntry(ID, Name.empty() ? Addr->getName() : Name, Size, Flags);
 return;
   }
   // TODO: Add support for global variables on the device after declare target
@@ -5687,13 +5688,20 @@
 
   // Hidden or internal symbols on the device are not externally visible.
   // We should not attempt to register them by creating an offloading
-  // entry.
+  // entry. Indirect variables are handled separately on the device.
   if (auto *GV = dyn_cast(CE->getAddress()))
-if (GV->hasLocalLinkage() || GV->hasHiddenVisibility())
+if ((GV->hasLocalLinkage() || GV->hasHiddenVisibility()) &&
+Flags != OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect)
   continue;
 
-  createOffloadEntry(CE->getAddress(), CE->getAddress(), CE->getVarSize(),
- Flags, CE->getLinkage());
+  // Indirect globals need to use a special name that doesn't match the name
+  // of the associated host global.
+  if (Flags == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect)
+createOffloadEntry(CE->getAddress(), CE->getAddress(), CE->getVarSize(),
+   Flags, CE->getLinkage(), CE->getVarName());
+  else
+createOffloadEntry(CE->getAddress(), CE->getAddress(), CE->getVarSize(),
+   Flags, CE->getLinkage());
 
 } else {
   llvm_unreachable("Unsupported entry kind.");
@@ -6038,8 +6046,13 @@
   }
   return;
 }
-OffloadEntriesDeviceGlobalVar.try_emplace(VarName, OffloadingEntriesNum,
-  Addr, VarSize, Flags, Linkage);
+if (Flags == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect)
+  OffloadEntriesDeviceGlobalVar.try_emplace(VarName, OffloadingEntriesNum,
+Addr, VarSize, Flags, Linkage,
+VarName.str());
+else
+  OffloadEntriesDeviceGlobalVar.try_emplace(
+  VarName, OffloadingEntriesNum, Addr, VarSize, Flags, Linkage, "");
 ++OffloadingEntriesNum;
   }
 }
Index: llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
===
--- llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -326,6 +326,8 @@
 OMPTargetGlobalVarEntryEnter = 0x2,
 /// Mark the entry as having no declare target entry kind.
 OMPTargetGlobalVarEntryNone = 0x3,
+/// Mark the entry as a declare target indirect global.
+OMPTargetGlobalVarEntryIndirect = 0x4,
   };
 
   /// Kind of device clause for declare target variables
@@ -349,6 +351,7 @@
 /// Type of the global variable.
 int64_t VarSize;
 GlobalValue::LinkageTypes Linkage;
+const std::string VarName;
 
   public:
 OffloadEntryInfoDeviceGlobalVar()
@@ -359,13 +362,15 @@
 explicit OffloadEntryInfoDeviceGlobalVar(unsigned Order, Constant *Addr,
  int64_t VarSize,
  OMPTargetGlobalVarEntryKind Flags,
- GlobalValue::LinkageTypes Linkage)
+ GlobalValue::LinkageTypes Linkage,
+ const std::string &VarName)
 : OffloadEntryInfo(OffloadingEntryInfoDeviceGlobalVar, Order, Flags),
-  VarSize(VarSize), Linkage(Linkage) {
+  VarSize(VarSize), Linkage(Linkage), VarName(VarName) {
   setAddress(Addr);
 }
 
 int64_t getVarSize() const { return VarSize; }
+StringRef getVarName() const { return VarName; }
 void setVarSize(int64_t Size) { VarSize = Size; }
 GlobalValue::LinkageTypes getLinkage() const { return Linkag

[PATCH] D157738: [OpenMP] Emit offloading entries for indirect target variables

2023-08-14 Thread Joseph Huber via Phabricator via cfe-commits
jhuber6 marked an inline comment as done.
jhuber6 added inline comments.



Comment at: clang/lib/CodeGen/CGOpenMPRuntime.cpp:1996-1997
+llvm::GlobalValue *GV) {
+  std::optional ActiveAttr =
+  OMPDeclareTargetDeclAttr::getActiveAttr(FD);
+

arsenm wrote:
> not a huge fan of std::optional
This is pretty far entrenched in the Clang handling for this attribute so I 
don't intend to change it here.



Comment at: clang/lib/CodeGen/CGOpenMPRuntime.cpp:2022
+  OMPBuilder.OffloadInfoManager.registerDeviceGlobalVarEntryInfo(
+  Name, Addr, CGM.getContext().getTypeSize(CGM.getContext().VoidPtrTy) / 8,
+  llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect,

arsenm wrote:
> isn't there a store size?
Yeah I can use that instead.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D157738/new/

https://reviews.llvm.org/D157738

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits


[PATCH] D157738: [OpenMP] Emit offloading entries for indirect target variables

2023-08-14 Thread Jon Chesterfield via Phabricator via cfe-commits
JonChesterfield added a comment.

Ok, I'm really sure this needs to be reflected in the type system. &foo for 
some target foo gets to be larger than 8 bytes and we use a different calling 
convention for it. Otherwise however we carve this the type erasure is going to 
make unrelated calls acquire the dynamic search overhead.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D157738/new/

https://reviews.llvm.org/D157738

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits


[PATCH] D157738: [OpenMP] Emit offloading entries for indirect target variables

2023-08-14 Thread Matt Arsenault via Phabricator via cfe-commits
arsenm added inline comments.



Comment at: clang/lib/CodeGen/CGOpenMPRuntime.cpp:1996-1997
+llvm::GlobalValue *GV) {
+  std::optional ActiveAttr =
+  OMPDeclareTargetDeclAttr::getActiveAttr(FD);
+

not a huge fan of std::optional



Comment at: clang/lib/CodeGen/CGOpenMPRuntime.cpp:2017
+/*isConstant=*/true,
+llvm::GlobalValue::ExternalLinkage, GV, 
Name);
+Addr->setVisibility(llvm::GlobalValue::ProtectedVisibility);

target global address space



Comment at: clang/lib/CodeGen/CGOpenMPRuntime.cpp:2022
+  OMPBuilder.OffloadInfoManager.registerDeviceGlobalVarEntryInfo(
+  Name, Addr, CGM.getContext().getTypeSize(CGM.getContext().VoidPtrTy) / 8,
+  llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect,

isn't there a store size?


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D157738/new/

https://reviews.llvm.org/D157738

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits


[PATCH] D157738: [OpenMP] Emit offloading entries for indirect target variables

2023-08-14 Thread Jon Chesterfield via Phabricator via cfe-commits
JonChesterfield added a comment.

If calling an indirect function pointer on the GPU requires a table lookup 
(keyed by host function addresses, which I didn't think we knew at GPU compile 
time), and we cannot distinguish indirect function pointers from function 
pointers, then this feature must send _every_ indirect call on the GPU through 
the table search in case it hits in the table and then branch on the value if 
it doesn't.

So if we have a few indirect openmp functions and then we call a function which 
makes indirect calls of a function pointer which happened to be unrelated to 
this feature, it's going to search that table anyway. Say qsort on a 
non-inlined comparison function.

This feature, as I understand it so far, therefore induces a global slowdown on 
every call to an unknown function. Slow function calls are a bad thing.

What am I missing?


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D157738/new/

https://reviews.llvm.org/D157738

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits


[PATCH] D157738: [OpenMP] Emit offloading entries for indirect target variables

2023-08-14 Thread Ravi Narayanaswamy via Phabricator via cfe-commits
RaviNarayanaswamy added a comment.

In D157738#4586465 , @JonChesterfield 
wrote:

>> calling device functions via their associated host pointer
>
> What does this mean? Defining a function foo such that the host and each 
> individual target each have their own machine code for it, such that &foo on 
> the host can be copied over to the target and then invoked to mean call the 
> function on the local target with the same name?
>
> If so, calling through the pointer &foo on the GPU doing a logarithmic search 
> through a table to choose a function address to branch to sounds like 
> something that will codegen into very slow code. Does it do that search on 
> every call?
>
> Is there an ambition to have &foo on the host and &foo on the (each) target 
> return the same value, in the pointer equality sense?
>
> Searching the linked spec for indirect finds the following
>
>> If the indirect clause is present and invoked-by-fptr evaluates to true, any 
>> procedures that appear in a to clause on the directive may be called with an 
>> indirect device invocation. If the indirect clause is present and 
>> invoked-by-fptr does not evaluate to true, any procedures that appear in a 
>> to clause on the directive may not be called with an indirect device 
>> invocation. Unless otherwise specified by an indirect clause, procedures may 
>> not be called with an indirect device invocation.
>
> Which tells me that the indirect clause means procedures can be called with 
> an indirect device invocation. Searching for the expression "indirect device 
> invocation" finds that paragraph and nothing else. So... where does the spec 
> say what this thing is?

Only indirect calls on the device will do a search the table.  The spec does 
not say how it should be implemented.  One could do the  translation at the 
target region when it is mapped on the host but this  will not handle all the  
cases.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D157738/new/

https://reviews.llvm.org/D157738

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits


[PATCH] D157738: [OpenMP] Emit offloading entries for indirect target variables

2023-08-14 Thread Joseph Huber via Phabricator via cfe-commits
jhuber6 added a comment.

In D157738#4586465 , @JonChesterfield 
wrote:

>> calling device functions via their associated host pointer
>
> What does this mean? Defining a function foo such that the host and each 
> individual target each have their own machine code for it, such that &foo on 
> the host can be copied over to the target and then invoked to mean call the 
> function on the local target with the same name?
>
> If so, calling through the pointer &foo on the GPU doing a logarithmic search 
> through a table to choose a function address to branch to sounds like 
> something that will codegen into very slow code. Does it do that search on 
> every call?
>
> Is there an ambition to have &foo on the host and &foo on the target return 
> the same value, in the pointer equality sense?

That's exactly what it means, the mapping is only done for targets with 
`indirect` declared on them. The indirect calls themselves, I think @jdoerfert 
is implementing some specialization? He just asked me to implement this since 
it's related to copying of virtual classes to the device.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D157738/new/

https://reviews.llvm.org/D157738

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits


[PATCH] D157738: [OpenMP] Emit offloading entries for indirect target variables

2023-08-14 Thread Jon Chesterfield via Phabricator via cfe-commits
JonChesterfield added a comment.

> calling device functions via their associated host pointer

What does this mean? Defining a function foo such that the host and each 
individual target each have their own machine code for it, such that &foo can 
be copied over to the target and then invoked to mean call the function on the 
local target with the same name?

If so, calling through the pointer &foo on the GPU doing a logarithmic search 
through a table to choose a function address to branch to sounds like something 
that will codegen into very slow code. Does it do that search on every call?


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D157738/new/

https://reviews.llvm.org/D157738

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits


[PATCH] D157738: [OpenMP] Emit offloading entries for indirect target variables

2023-08-11 Thread Joseph Huber via Phabricator via cfe-commits
jhuber6 created this revision.
jhuber6 added reviewers: jdoerfert, tianshilei1992, ye-luo, ABataev, 
RaviNarayanaswamy.
Herald added subscribers: guansong, hiraditya, yaxunl.
Herald added a project: All.
jhuber6 requested review of this revision.
Herald added subscribers: llvm-commits, cfe-commits, jplehr, sstefan1.
Herald added projects: clang, LLVM.

OpenMP 5.1 allows emission of the `indirect` clause on declare target
functions, see 
https://www.openmp.org/spec-html/5.1/openmpsu70.html#x98-1080002.14.7.
The intended use of this is to permit calling device functions via their
associated host pointer. In order to do this the first step will be
building a map associating these variables. Doing this will require the
same offloading entry handling we use for other kernels and globals.

We intentionally emit a new global on the device side. Although it's
possible to look up the device function's address directly, this would
require changing the visibility and would prevent us from making static
functions indirect. Also, the CUDA toolchain will optimize out unused
functions and using a global prevents that. The downside is that the
runtime will need to read the global and copy its value, but there
shouldn't be any other costs.

Note that this patch just performs the codegen, currently this new
offloading entry type is unused and will be ignored by the runtime.


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D157738

Files:
  clang/lib/CodeGen/CGOpenMPRuntime.cpp
  clang/lib/CodeGen/CGOpenMPRuntime.h
  clang/lib/CodeGen/CodeGenModule.cpp
  clang/test/OpenMP/target_indirect_codegen.cpp
  llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
  llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp

Index: llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
===
--- llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5525,9 +5525,10 @@
 
 void OpenMPIRBuilder::createOffloadEntry(Constant *ID, Constant *Addr,
  uint64_t Size, int32_t Flags,
- GlobalValue::LinkageTypes) {
+ GlobalValue::LinkageTypes,
+ StringRef Name) {
   if (!Config.isGPU()) {
-emitOffloadingEntry(ID, Addr->getName(), Size, Flags);
+emitOffloadingEntry(ID, Name.empty() ? Addr->getName() : Name, Size, Flags);
 return;
   }
   // TODO: Add support for global variables on the device after declare target
@@ -5687,13 +5688,20 @@
 
   // Hidden or internal symbols on the device are not externally visible.
   // We should not attempt to register them by creating an offloading
-  // entry.
+  // entry. Indirect variables are handled separately on the device.
   if (auto *GV = dyn_cast(CE->getAddress()))
-if (GV->hasLocalLinkage() || GV->hasHiddenVisibility())
+if ((GV->hasLocalLinkage() || GV->hasHiddenVisibility()) &&
+Flags != OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect)
   continue;
 
-  createOffloadEntry(CE->getAddress(), CE->getAddress(), CE->getVarSize(),
- Flags, CE->getLinkage());
+  // Indirect globals need to use a special name that doesn't match the name
+  // of the associated host global.
+  if (Flags == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect)
+createOffloadEntry(CE->getAddress(), CE->getAddress(), CE->getVarSize(),
+   Flags, CE->getLinkage(), CE->getVarName());
+  else
+createOffloadEntry(CE->getAddress(), CE->getAddress(), CE->getVarSize(),
+   Flags, CE->getLinkage());
 
 } else {
   llvm_unreachable("Unsupported entry kind.");
@@ -6038,8 +6046,13 @@
   }
   return;
 }
-OffloadEntriesDeviceGlobalVar.try_emplace(VarName, OffloadingEntriesNum,
-  Addr, VarSize, Flags, Linkage);
+if (Flags == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect)
+  OffloadEntriesDeviceGlobalVar.try_emplace(VarName, OffloadingEntriesNum,
+Addr, VarSize, Flags, Linkage,
+VarName.str());
+else
+  OffloadEntriesDeviceGlobalVar.try_emplace(
+  VarName, OffloadingEntriesNum, Addr, VarSize, Flags, Linkage, "");
 ++OffloadingEntriesNum;
   }
 }
Index: llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
===
--- llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -326,6 +326,8 @@
 OMPTargetGlobalVarEntryEnter = 0x2,
 /// Mark the entry as having no declare target entry kind.
 OMPTargetGlobalVarEntryNone = 0x3,
+/// Mark the entry as a declare target indirect global.
+OMPTargetGlobalVarE