Author: Arseniy Obolenskiy Date: 2026-10-14T14:20:42+02:00 New Revision: faf60e71655fb39133a24d89c12c55b75b2021d7
URL: https://github.com/llvm/llvm-project/commit/faf60e71655fb39133a24d89c12c55b75b2021d7 DIFF: https://github.com/llvm/llvm-project/commit/faf60e71655fb39133a24d89c12c55b75b2021d7.diff LOG: [CIR][HIP] Support managed variables (#230910) Lower HIP __managed__ variables in CIR as classic codegen does - Rename variable x to `x.managed` and add a pointer x that the runtime fills in - Access the variable through that pointer - Register it with __hipRegisterManagedVar Added: clang/test/CIR/CodeGenHIP/managed-var.hip Modified: clang/lib/CIR/CodeGen/CIRGenCUDANV.cpp clang/lib/CIR/CodeGen/CIRGenModule.cpp clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp clang/test/CIR/CodeGenCUDA/template-class-static-member.cu Removed: ################################################################################ diff --git a/clang/lib/CIR/CodeGen/CIRGenCUDANV.cpp b/clang/lib/CIR/CodeGen/CIRGenCUDANV.cpp index 79153ca788756..8f7d65b827507 100644 --- a/clang/lib/CIR/CodeGen/CIRGenCUDANV.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenCUDANV.cpp @@ -62,6 +62,7 @@ class CIRGenNVCUDARuntime : public CIRGenCUDARuntime { void emitDeviceStubBodyNew(CIRGenFunction &cgf, cir::FuncOp fn, FunctionArgList &args); void recordDeviceBinary(); + void transformManagedVars(); mlir::Value prepareKernelArgs(CIRGenFunction &cgf, mlir::Location loc, FunctionArgList &args); mlir::Operation *getKernelHandle(cir::FuncOp fn, GlobalDecl gd) override; @@ -581,7 +582,76 @@ void CIRGenNVCUDARuntime::recordDeviceBinary() { mlir::StringAttr::get(bytes, fatbinTy)); } +// Uses of a managed variable load its address from a pointer the runtime fills. +void CIRGenNVCUDARuntime::transformManagedVars() { + CIRGenBuilderTy &builder = cgm.getBuilder(); + mlir::ModuleOp module = cgm.getModule(); + for (auto &&info : deviceVars) { + cir::GlobalOp var = info.var; + if (info.flags != cir::CUDADeviceVarKind::Variable || + !info.d->hasAttr<HIPManagedAttr>()) + continue; + + cir::PointerType varPtrTy = + builder.getPointerTo(var.getSymType(), var.getAddrSpaceAttr()); + mlir::ptr::MemorySpaceAttrInterface managedAS = cir::toCIRAddressSpaceAttr( + cgm.getMLIRContext(), + cgm.getLangOpts().CUDAIsDevice ? LangAS::cuda_device : LangAS::Default); + std::optional<mlir::SymbolTable::UseRange> uses = var.getSymbolUses(module); + + std::string name = var.getSymName().str(); + // An empty comdat would follow the rename. + std::optional<llvm::StringRef> comdat = var.getComdat(); + if (comdat && comdat->empty()) + var.setComdat(llvm::StringRef(name)); + cgm.eraseGlobalSymbol(var); + var.setSymName(name + ".managed"); + cgm.insertGlobalSymbol(var); + + cir::GlobalOp managedVar = + cgm.createGlobalOp(var.getLoc(), name, varPtrTy, + /*isConstant=*/false, managedAS); + managedVar.setLinkage(var.getLinkage()); + if (!var.isDeclaration()) + managedVar.setInitialValueAttr(builder.getConstNullPtrAttr(varPtrTy)); + managedVar.setDsoLocal(var.getDsoLocal()); + managedVar.setGlobalVisibility(var.getGlobalVisibility()); + mlir::SymbolTable::setSymbolVisibility( + managedVar, mlir::SymbolTable::getSymbolVisibility(var)); + managedVar->setAttr( + cir::CUDAExternallyInitializedAttr::getMnemonic(), + cir::CUDAExternallyInitializedAttr::get(&cgm.getMLIRContext())); + + // The uses now name the pointer global. + cir::PointerType managedPtrTy = + builder.getPointerTo(varPtrTy, managedVar.getAddrSpaceAttr()); + for (mlir::SymbolTable::SymbolUse use : *uses) { + auto getGlobalOp = dyn_cast<cir::GetGlobalOp>(use.getUser()); + if (!getGlobalOp) { + cgm.errorNYI(info.d->getSourceRange(), + "transformManagedVars: non-instruction use"); + continue; + } + mlir::Value addr = getGlobalOp.getAddr(); + addr.setType(managedPtrTy); + mlir::OpBuilder::InsertionGuard guard(builder); + builder.setInsertionPointAfter(getGlobalOp); + mlir::Value ld = builder.createAlignedLoad( + getGlobalOp.getLoc(), varPtrTy, addr, + CharUnits::fromQuantity(var.getAlignment().value_or(1))); + addr.replaceAllUsesExcept(ld, ld.getDefiningOp()); + } + + // The runtime allocates them even if device code never uses them. + if (cgm.getLangOpts().CUDAIsDevice && !var.isDeclaration()) { + cgm.addCompilerUsedGlobal(var); + cgm.addCompilerUsedGlobal(managedVar); + } + } +} + void CIRGenNVCUDARuntime::finalizeModule() { + transformManagedVars(); if (!cgm.getLangOpts().CUDAIsDevice) { recordDeviceBinary(); return; diff --git a/clang/lib/CIR/CodeGen/CIRGenModule.cpp b/clang/lib/CIR/CodeGen/CIRGenModule.cpp index e49b77f7daa9d..245af3dbdd98b 100644 --- a/clang/lib/CIR/CodeGen/CIRGenModule.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenModule.cpp @@ -1417,8 +1417,7 @@ CIRGenModule::getOrCreateCIRGlobal(StringRef mangledName, mlir::Type ty, // in both device and host compilations. if (getLangOpts().CUDA && d && d->hasAttr<HIPManagedAttr>() && d->hasExternalStorage()) - errorNYI(d->getSourceRange(), - "getOrCreateCIRGlobal: HIP managed attribute"); + getCUDARuntime().handleVarRegistration(d, gv); } assert(!cir::MissingFeatures::addressSpace()); diff --git a/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp b/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp index 9334bf65210c4..9eb0a2c542ef8 100644 --- a/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp +++ b/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp @@ -3251,17 +3251,9 @@ void LoweringPreparePass::buildCUDARegisterVars(cir::CIRBaseBuilderTy &builder, cir::CIRBaseBuilderTy globalBuilder(getContext()); globalBuilder.setInsertionPointToStart(mlirModule.getBody()); - // void __{cuda|hip}RegisterVar(void **fatbinHandle, - // char *hostVar, char *deviceAddress, - // const char *deviceName, int ext, - // size_t size, int constant, int normalized); - // OG ignores parameter types, treating pointers as void*. cir::VoidType voidTy = builder.getVoidTy(); - FuncOp cudaRegisterVar = buildRuntimeFunction( - globalBuilder, addUnderscoredPrefix(cudaPrefix, "RegisterVar"), loc, - FuncType::get({voidPtrPtrTy, voidPtrTy, voidPtrTy, voidPtrTy, intTy, - sizeTy, intTy, intTy}, - voidTy)); + FuncOp cudaRegisterVar; + FuncOp cudaRegisterManagedVar; auto makeConstantString = [&](llvm::StringRef str) -> GlobalOp { auto strType = ArrayType::get(&getContext(), charTy, 1 + str.size()); @@ -3287,8 +3279,9 @@ void LoweringPreparePass::buildCUDARegisterVars(cir::CIRBaseBuilderTy &builder, llvm_unreachable("Texture registration NYI"); } - if (regAttr.getIsManaged()) - llvm_unreachable("Managed variable registration NYI"); + // External managed variables are registered by their defining TU. + if (regAttr.getIsManaged() && global.isDeclaration()) + continue; GlobalOp deviceNameStr = makeConstantString(regAttr.getDeviceSideName()); mlir::Value deviceName = builder.createBitcast( @@ -3296,17 +3289,58 @@ void LoweringPreparePass::buildCUDARegisterVars(cir::CIRBaseBuilderTy &builder, mlir::Value hostVar = builder.createBitcast(builder.createGetGlobal(global), voidPtrTy); - auto isExtern = ConstantOp::create( - builder, loc, IntAttr::get(intTy, regAttr.getIsExtern() ? 1 : 0)); llvm::TypeSize size = dataLayout.getTypeAllocSize(global.getSymType()); - auto varSize = ConstantOp::create( - builder, loc, IntAttr::get(sizeTy, size.getFixedValue())); - auto isConstant = ConstantOp::create( - builder, loc, IntAttr::get(intTy, regAttr.getIsConstant() ? 1 : 0)); - auto normalized = ConstantOp::create(builder, loc, IntAttr::get(intTy, 0)); - builder.createCallOp(loc, cudaRegisterVar, - {fatbinHandle, hostVar, deviceName, deviceName, - isExtern, varSize, isConstant, normalized}); + + if (regAttr.getIsManaged()) { + // void __hipRegisterManagedVar(void **fatbinHandle, char *managedVar, + // char *var, const char *deviceName, + // size_t size, unsigned align); + if (!cudaRegisterManagedVar) + cudaRegisterManagedVar = buildRuntimeFunction( + globalBuilder, + addUnderscoredPrefix(cudaPrefix, "RegisterManagedVar"), loc, + FuncType::get( + {voidPtrPtrTy, voidPtrTy, voidPtrTy, voidPtrTy, sizeTy, intTy}, + voidTy)); + llvm::StringRef name = global.getSymName(); + assert(name.ends_with(".managed") && + "HIP managed variables not transformed"); + auto managedVar = mlirModule.lookupSymbol<GlobalOp>( + name.drop_back(llvm::StringRef(".managed").size())); + mlir::Value managedPtr = + builder.createBitcast(builder.createGetGlobal(managedVar), voidPtrTy); + auto varSize = ConstantOp::create( + builder, loc, IntAttr::get(sizeTy, size.getFixedValue())); + auto align = ConstantOp::create( + builder, loc, IntAttr::get(intTy, global.getAlignment().value_or(1))); + builder.createCallOp( + loc, cudaRegisterManagedVar, + {fatbinHandle, managedPtr, hostVar, deviceName, varSize, align}); + } else { + // void __{cuda|hip}RegisterVar(void **fatbinHandle, + // char *hostVar, char *deviceAddress, + // const char *deviceName, int ext, + // size_t size, int constant, + // int normalized); + // OG ignores parameter types, treating pointers as void*. + if (!cudaRegisterVar) + cudaRegisterVar = buildRuntimeFunction( + globalBuilder, addUnderscoredPrefix(cudaPrefix, "RegisterVar"), loc, + FuncType::get({voidPtrPtrTy, voidPtrTy, voidPtrTy, voidPtrTy, intTy, + sizeTy, intTy, intTy}, + voidTy)); + auto isExtern = ConstantOp::create( + builder, loc, IntAttr::get(intTy, regAttr.getIsExtern() ? 1 : 0)); + auto varSize = ConstantOp::create( + builder, loc, IntAttr::get(sizeTy, size.getFixedValue())); + auto isConstant = ConstantOp::create( + builder, loc, IntAttr::get(intTy, regAttr.getIsConstant() ? 1 : 0)); + auto normalized = + ConstantOp::create(builder, loc, IntAttr::get(intTy, 0)); + builder.createCallOp(loc, cudaRegisterVar, + {fatbinHandle, hostVar, deviceName, deviceName, + isExtern, varSize, isConstant, normalized}); + } } } diff --git a/clang/test/CIR/CodeGenCUDA/template-class-static-member.cu b/clang/test/CIR/CodeGenCUDA/template-class-static-member.cu index fccfddc2b8018..32a8f62a8de30 100644 --- a/clang/test/CIR/CodeGenCUDA/template-class-static-member.cu +++ b/clang/test/CIR/CodeGenCUDA/template-class-static-member.cu @@ -56,7 +56,8 @@ template class A<int>; // CIR-DEV-DAG: cir.global "private" internal comdat dso_local target_address_space(1) @_ZN1AIiE8d_memberE = #cir.int<0> // CIR-DEV-DAG: cir.global "private" constant internal comdat dso_local target_address_space(4) @_ZN1AIiE8c_memberE = #cir.int<0> -// CIR-DEV-DAG: cir.global "private" internal comdat dso_local target_address_space(1) @_ZN1AIiE8m_memberE = #cir.int<0> +// CIR-DEV-DAG: cir.global "private" internal comdat("_ZN1AIiE8m_memberE") dso_local target_address_space(1) @_ZN1AIiE8m_memberE.managed = #cir.int<0> +// CIR-DEV-DAG: cir.global "private" internal dso_local target_address_space(1) @_ZN1AIiE8m_memberE = #cir.ptr<null> : !cir.ptr<!s32i, target_address_space(1)> {cu.externally_initialized = #cir.cu.externally_initialized} // CIR-DEV-DAG: cir.global "private" constant internal comdat dso_local target_address_space(4) @_ZN1AIiE12const_memberE = #cir.int<0> // CIR-DEV-NEG-NOT: @_ZN1AIiE8h_memberE @@ -69,7 +70,8 @@ template class A<int>; // CIR-HOST-DAG: cir.global weak_odr comdat @_ZN1AIiE8h_memberE = #cir.int<0> // CIR-HOST-DAG: cir.global "private" internal comdat dso_local @_ZN1AIiE8d_memberE = #cir.undef // CIR-HOST-DAG: cir.global "private" internal comdat dso_local @_ZN1AIiE8c_memberE = #cir.undef -// CIR-HOST-DAG: cir.global "private" internal comdat dso_local @_ZN1AIiE8m_memberE = #cir.int<0> +// CIR-HOST-DAG: cir.global "private" internal comdat("_ZN1AIiE8m_memberE") dso_local @_ZN1AIiE8m_memberE.managed = #cir.int<0> +// CIR-HOST-DAG: cir.global "private" internal dso_local @_ZN1AIiE8m_memberE = #cir.ptr<null> : !cir.ptr<!s32i> {cu.externally_initialized = #cir.cu.externally_initialized} // CIR-HOST-DAG: cir.global constant weak_odr comdat @_ZN1AIiE12const_memberE = #cir.int<0> // HOST-DAG: @_ZN1AIiE8h_memberE = weak_odr global i32 0, comdat, align 4 diff --git a/clang/test/CIR/CodeGenHIP/managed-var.hip b/clang/test/CIR/CodeGenHIP/managed-var.hip new file mode 100644 index 0000000000000..4df2044e30dfd --- /dev/null +++ b/clang/test/CIR/CodeGenHIP/managed-var.hip @@ -0,0 +1,97 @@ +// RUN: echo -n "GPU binary would be here." > %t + +// RUN: %clang_cc1 -triple amdgpu-amd-amdhsa -fcuda-is-device -fclangir \ +// RUN: -x hip -emit-cir %s -o %t.dev.cir +// RUN: FileCheck --check-prefix=CIR-DEV --input-file=%t.dev.cir %s + +// RUN: %clang_cc1 -triple amdgpu-amd-amdhsa -fcuda-is-device -fclangir \ +// RUN: -x hip -emit-llvm %s -o %t.dev.cir.ll +// RUN: FileCheck --check-prefix=DEV --input-file=%t.dev.cir.ll %s + +// RUN: %clang_cc1 -triple amdgpu-amd-amdhsa -fcuda-is-device \ +// RUN: -x hip -emit-llvm %s -o %t.dev.ll +// RUN: FileCheck --check-prefix=DEV --input-file=%t.dev.ll %s + +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir \ +// RUN: -x hip -fhip-new-launch-api -fcuda-include-gpubinary %t \ +// RUN: -emit-cir %s -o %t.host.cir +// RUN: FileCheck --check-prefix=CIR-HOST --input-file=%t.host.cir %s + +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir \ +// RUN: -x hip -fhip-new-launch-api -fcuda-include-gpubinary %t \ +// RUN: -emit-llvm %s -o %t.host.cir.ll +// RUN: FileCheck --check-prefix=HOST --input-file=%t.host.cir.ll %s + +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu \ +// RUN: -x hip -fhip-new-launch-api -fcuda-include-gpubinary %t \ +// RUN: -emit-llvm %s -o %t.host.ll +// RUN: FileCheck --check-prefix=HOST --input-file=%t.host.ll %s + +#include "../CodeGenCUDA/Inputs/cuda.h" + +__managed__ int x = 1; +__managed__ float arr[8]; +extern __managed__ int ex; +static __managed__ int sx = 2; + +template <typename T> __managed__ T tv = 3; + +// CIR-DEV-DAG: cir.global external target_address_space(1) @x.managed = #cir.int<1> : !s32i align(4) {cu.externally_initialized = #cir.cu.externally_initialized, cu.var_registration = #cir.cu.var_registration<x, Variable, managed>} +// CIR-DEV-DAG: cir.global external target_address_space(1) @x = #cir.ptr<null> : !cir.ptr<!s32i, target_address_space(1)> {cu.externally_initialized = #cir.cu.externally_initialized} +// CIR-DEV-DAG: cir.global linkonce_odr comdat("_Z2tvIiE") target_address_space(1) @_Z2tvIiE.managed = #cir.int<3> + +// DEV-DAG: @x.managed = addrspace(1) externally_initialized global i32 1, align 4 +// DEV-DAG: @x = addrspace(1) externally_initialized global ptr addrspace(1) null +// DEV-DAG: @arr.managed = addrspace(1) externally_initialized global [8 x float] zeroinitializer, align 4 +// DEV-DAG: @arr = addrspace(1) externally_initialized global ptr addrspace(1) null +// DEV-DAG: @ex.managed = external addrspace(1) global i32, align 4 +// DEV-DAG: @ex = external addrspace(1) externally_initialized global ptr addrspace(1){{$}} +// DEV-DAG: @_ZL2sx.managed = addrspace(1) externally_initialized global i32 2, align 4 +// DEV-DAG: @_ZL2sx = addrspace(1) externally_initialized global ptr addrspace(1) null +// DEV-DAG: @_Z2tvIiE.managed = linkonce_odr addrspace(1) externally_initialized global i32 3, comdat($_Z2tvIiE), align 4 +// DEV-DAG: @_Z2tvIiE = linkonce_odr addrspace(1) externally_initialized global ptr addrspace(1) null{{$}} +// DEV-DAG: @llvm.compiler.used = {{.*}}@x.managed{{.*}}@x to{{.*}}@arr.managed{{.*}}@arr to{{.*}}@_ZL2sx.managed{{.*}}@_ZL2sx to{{.*}}@_Z2tvIiE.managed{{.*}}@_Z2tvIiE to + +// HOST-DAG: @x.managed = internal global i32 1, align 4 +// HOST-DAG: @x = internal externally_initialized global ptr null +// HOST-DAG: @arr.managed = internal global [8 x float] zeroinitializer, align 16 +// HOST-DAG: @arr = internal externally_initialized global ptr null +// HOST-DAG: @ex.managed = external global i32, align 4 +// HOST-DAG: @ex = external externally_initialized global ptr{{$}} +// HOST-DAG: @_ZL2sx.managed = internal global i32 2, align 4 +// HOST-DAG: @_ZL2sx = internal externally_initialized global ptr null +// HOST-DAG: @_Z2tvIiE.managed = internal global i32 3, comdat($_Z2tvIiE), align 4 +// HOST-DAG: @_Z2tvIiE = internal externally_initialized global ptr null + +// CIR-DEV-LABEL: cir.func{{.*}} @_Z4kernPi +// CIR-DEV: %[[XADDR:.*]] = cir.get_global @x : !cir.ptr<!cir.ptr<!s32i, target_address_space(1)>, target_address_space(1)> +// CIR-DEV: %{{.*}} = cir.load align(4) %[[XADDR]] : !cir.ptr<!cir.ptr<!s32i, target_address_space(1)>, target_address_space(1)>, !cir.ptr<!s32i, target_address_space(1)> + +// DEV-LABEL: define {{.*}}@_Z4kernPi +// DEV: %[[LD_X:.*]] = load ptr addrspace(1), ptr addrspace(1) @x, align 4 +// DEV: store i32 2, ptr %{{.*}}, align 4 +// DEV: load ptr addrspace(1), ptr addrspace(1) @arr, align 4 +// DEV: load ptr addrspace(1), ptr addrspace(1) @ex, align 4 +// DEV: load ptr addrspace(1), ptr addrspace(1) @_ZL2sx, align 4 +// DEV: load ptr addrspace(1), ptr addrspace(1) @_Z2tvIiE, align 4 +__attribute__((global)) void kern(int *out) { + x = 2; + arr[1] = 3.0f; + *out = ex + sx + tv<int>; +} + +// HOST-DAG: load ptr, ptr @x, align 4 +// HOST-DAG: load ptr, ptr @ex, align 4 +// HOST-DAG: load ptr, ptr @_ZL2sx, align 4 +// HOST-DAG: load ptr, ptr @_Z2tvIiE, align 4 +int host_use() { return x + ex + sx + tv<int>; } + +// External managed variables are registered by their defining TU. +// CIR-HOST-LABEL: cir.func internal private @__hip_register_globals +// CIR-HOST-COUNT-4: cir.call @__hipRegisterManagedVar +// CIR-HOST-NEXT: cir.return + +// HOST-DAG: call void @__hipRegisterManagedVar(ptr %{{.*}}, ptr @x, ptr @x.managed, ptr @{{.*}}, i64 4, i32 4) +// HOST-DAG: call void @__hipRegisterManagedVar(ptr %{{.*}}, ptr @arr, ptr @arr.managed, ptr @{{.*}}, i64 32, i32 16) +// HOST-DAG: call void @__hipRegisterManagedVar(ptr %{{.*}}, ptr @_ZL2sx, ptr @_ZL2sx.managed, ptr @{{.*}}, i64 4, i32 4) +// HOST-DAG: call void @__hipRegisterManagedVar(ptr %{{.*}}, ptr @_Z2tvIiE, ptr @_Z2tvIiE.managed, ptr @{{.*}}, i64 4, i32 4) _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
