Author: Sivapriya Date: 2026-10-24T14:19:31-07:00 New Revision: 78a9c8d0d021c2e64826098488be92b983da6001
URL: https://github.com/llvm/llvm-project/commit/78a9c8d0d021c2e64826098488be92b983da6001 DIFF: https://github.com/llvm/llvm-project/commit/78a9c8d0d021c2e64826098488be92b983da6001.diff LOG: [CIR][CUDA] Add host-side surface registration support (#225146) Related: #179278. This patch adds host-side registration support for CUDA built-in surface references in CIR. Previously, CIR recognized CUDA surface variables during registration, but host-side surface handling was not implemented. This change preserves the surface type information in CIR registration metadata and emits the corresponding `__cudaRegisterSurface` runtime registration call. ## Changes - Internalize CUDA surface and texture host shadow variables to match original CodeGen behavior - Extend `CUDAVarRegistrationInfoAttr` with the CUDA surface type - Extract the surface type from the CUDA surface template arguments during CIRGen - Add `__cudaRegisterSurface` runtime declaration and call generation in `LoweringPrepare` - Keep texture runtime registration explicitly NYI - Add host-side CIR and LLVM test coverage for surface registration - Check generated LLVM registration against original CodeGen behavior ## Testing - `git diff --check` - `clang/test/CIR/CodeGenCUDA/surface.cu` - `ninja -C build -j1 clang` Added: Modified: clang/include/clang/CIR/Dialect/IR/CIRCUDAAttrs.td clang/lib/CIR/CodeGen/CIRGenCUDANV.cpp clang/lib/CIR/Dialect/IR/CIRAttrs.cpp clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp clang/test/CIR/CodeGenCUDA/surface.cu Removed: ################################################################################ diff --git a/clang/include/clang/CIR/Dialect/IR/CIRCUDAAttrs.td b/clang/include/clang/CIR/Dialect/IR/CIRCUDAAttrs.td index ebe8eaf60cc432..8d6cb1bdbdd86d 100644 --- a/clang/include/clang/CIR/Dialect/IR/CIRCUDAAttrs.td +++ b/clang/include/clang/CIR/Dialect/IR/CIRCUDAAttrs.td @@ -71,7 +71,8 @@ def CIR_CUDAVarRegistrationInfoAttr : CIR_Attr<"CUDAVarRegistrationInfo", "cu.va // handleVarRegistration via hasAttr<HIPManagedAttr>(). "bool":$isExtern, "bool":$isConstant, - "bool":$isManaged + "bool":$isManaged, + "int32_t":$surfTexType ); let hasCustomAssemblyFormat = 1; diff --git a/clang/lib/CIR/CodeGen/CIRGenCUDANV.cpp b/clang/lib/CIR/CodeGen/CIRGenCUDANV.cpp index 680486411a479f..03dfdfba65c7a4 100644 --- a/clang/lib/CIR/CodeGen/CIRGenCUDANV.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenCUDANV.cpp @@ -100,7 +100,8 @@ class CIRGenNVCUDARuntime : public CIRGenCUDARuntime { builder.getContext(), getDeviceSideName(cast<NamedDecl>(vd)), cir::CUDADeviceVarKind::Variable, isExtern, isConstant, - vd->hasAttr<HIPManagedAttr>())); + vd->hasAttr<HIPManagedAttr>(), + /*surfTexType=*/0)); deviceVars.push_back({ var, vd, @@ -108,8 +109,8 @@ class CIRGenNVCUDARuntime : public CIRGenCUDARuntime { }); } - void registerDeviceSurf(const VarDecl *vd, cir::GlobalOp &var, - bool isExtern) { + void registerDeviceSurf(const VarDecl *vd, cir::GlobalOp &var, bool isExtern, + int32_t surfaceType) { auto &builder = cgm.getBuilder(); var->setAttr(cir::CUDAVarRegistrationInfoAttr::getMnemonic(), @@ -118,7 +119,7 @@ class CIRGenNVCUDARuntime : public CIRGenCUDARuntime { getDeviceSideName(cast<NamedDecl>(vd)), cir::CUDADeviceVarKind::Surface, isExtern, /*isConstant=*/false, - /*isManaged=*/false)); + /*isManaged=*/false, surfaceType)); deviceVars.push_back({ var, @@ -136,7 +137,8 @@ class CIRGenNVCUDARuntime : public CIRGenCUDARuntime { getDeviceSideName(cast<NamedDecl>(vd)), cir::CUDADeviceVarKind::Texture, isExtern, /*isConstant=*/false, - /*isManaged=*/false)); + /*isManaged=*/false, + /*surfTexType=*/0)); deviceVars.push_back({ var, @@ -450,14 +452,15 @@ void CIRGenNVCUDARuntime::internalizeDeviceSideVar( // counterparts. It's not clear yet whether it's nvcc's bug or // a feature, but we've got to do the same for compatibility. if (d->hasAttr<CUDADeviceAttr>() || d->hasAttr<CUDAConstantAttr>() || - d->hasAttr<CUDASharedAttr>()) { + d->hasAttr<CUDASharedAttr>() || + d->getType()->isCUDADeviceBuiltinSurfaceType() || + d->getType()->isCUDADeviceBuiltinTextureType()) { linkage = cir::GlobalLinkageKind::InternalLinkage; } - if (d->getType()->isCUDADeviceBuiltinSurfaceType() || - d->getType()->isCUDADeviceBuiltinTextureType()) + if (d->getType()->isCUDADeviceBuiltinTextureType()) cgm.errorNYI(d->getSourceRange(), - "internalizeDeviceSideVar: CUDA Surface/Texture support"); + "internalizeDeviceSideVar: CUDA Texture support"); } std::string CIRGenNVCUDARuntime::getDeviceSideName(const NamedDecl *nd) { @@ -518,8 +521,19 @@ void CIRGenNVCUDARuntime::handleVarRegistration(const VarDecl *vd, } else if (vd->getType()->isCUDADeviceBuiltinSurfaceType()) { // Builtin surfaces and their template arguments are also registered // with CUDA runtime. + const auto *td = cast<ClassTemplateSpecializationDecl>( + vd->getType()->castAsCXXRecordDecl()); + const TemplateArgumentList &args = td->getTemplateArgs(); + + assert(args.size() == 2 && + "Unexpected number of template arguments of CUDA device " + "builtin surface type."); + + llvm::APSInt surfaceType = args[1].getAsIntegral(); + if (!vd->hasExternalStorage()) - registerDeviceSurf(vd, var, !vd->hasDefinition()); + registerDeviceSurf(vd, var, !vd->hasDefinition(), + surfaceType.getSExtValue()); } else if (vd->getType()->isCUDADeviceBuiltinTextureType()) { // Builtin textures and their template arguments are also registered diff --git a/clang/lib/CIR/Dialect/IR/CIRAttrs.cpp b/clang/lib/CIR/Dialect/IR/CIRAttrs.cpp index 023ac73178152a..0c92e222fe2c5b 100644 --- a/clang/lib/CIR/Dialect/IR/CIRAttrs.cpp +++ b/clang/lib/CIR/Dialect/IR/CIRAttrs.cpp @@ -616,6 +616,8 @@ void CUDAVarRegistrationInfoAttr::print(AsmPrinter &p) const { p << ", constant"; if (getIsManaged()) p << ", managed"; + if (getKind() == CUDADeviceVarKind::Surface) + p << ", surface_type = " << getSurfTexType(); p << ">"; } @@ -650,6 +652,7 @@ Attribute CUDAVarRegistrationInfoAttr::parse(AsmParser &parser, Type odsType) { bool isExtern = false; bool isConstant = false; bool isManaged = false; + int32_t surfTexType = 0; while (parser.parseOptionalGreater().failed()) { if (parser.parseComma()) @@ -665,14 +668,17 @@ Attribute CUDAVarRegistrationInfoAttr::parse(AsmParser &parser, Type odsType) { isConstant = true; else if (flag == "managed") isManaged = true; - else { + else if (flag == "surface_type") { + if (parser.parseEqual() || parser.parseInteger(surfTexType)) + return {}; + } else { parser.emitError(parser.getCurrentLocation(), "unknown flag: ") << flag; return {}; } } return get(parser.getContext(), deviceSideName, *kind, isExtern, isConstant, - isManaged); + isManaged, surfTexType); } //===----------------------------------------------------------------------===// diff --git a/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp b/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp index 10f36c5ec8f0e2..0af8f16f190f4e 100644 --- a/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp +++ b/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp @@ -3345,6 +3345,17 @@ void LoweringPreparePass::buildCUDARegisterVars(cir::CIRBaseBuilderTy &builder, FuncOp cudaRegisterVar; FuncOp cudaRegisterManagedVar; + // void __cudaRegisterSurface(void **fatbinHandle, + // const struct surfaceReference *hostVar, + // const void **deviceAddress, + // const char *deviceName, + // int type, int ext); + FuncOp cudaRegisterSurface = buildRuntimeFunction( + globalBuilder, addUnderscoredPrefix(cudaPrefix, "RegisterSurface"), loc, + FuncType::get( + {voidPtrPtrTy, voidPtrTy, voidPtrTy, voidPtrTy, intTy, intTy}, + voidTy)); + auto makeConstantString = [&](llvm::StringRef str) -> GlobalOp { auto strType = ArrayType::get(&getContext(), charTy, 1 + str.size()); auto tmpString = cir::GlobalOp::create( @@ -3360,15 +3371,6 @@ void LoweringPreparePass::buildCUDARegisterVars(cir::CIRBaseBuilderTy &builder, mlir::Value fatbinHandle = *regGlobalFunc.args_begin(); for (auto &[global, regAttr] : cudaDeviceVars) { - switch (regAttr.getKind()) { - case cir::CUDADeviceVarKind::Variable: - break; - case cir::CUDADeviceVarKind::Surface: - llvm_unreachable("Surface registration NYI"); - case cir::CUDADeviceVarKind::Texture: - llvm_unreachable("Texture registration NYI"); - } - // External managed variables are registered by their defining TU. if (regAttr.getIsManaged() && global.isDeclaration()) continue; @@ -3392,6 +3394,7 @@ void LoweringPreparePass::buildCUDARegisterVars(cir::CIRBaseBuilderTy &builder, FuncType::get( {voidPtrPtrTy, voidPtrTy, voidPtrTy, voidPtrTy, sizeTy, intTy}, voidTy)); + llvm::StringRef name = global.getSymName(); assert(name.ends_with(".managed") && "HIP managed variables not transformed"); @@ -3407,29 +3410,50 @@ void LoweringPreparePass::buildCUDARegisterVars(cir::CIRBaseBuilderTy &builder, loc, cudaRegisterManagedVar, {fatbinHandle, managedPtr, hostVar, deviceName, varSize, align}); } else { - // void __{cuda|hip}RegisterVar(void **fatbinHandle, - // char *hostVar, char *deviceAddress, - // const char *deviceName, int ext, - // size_t size, int constant, - // int normalized); - // OG ignores parameter types, treating pointers as void*. - if (!cudaRegisterVar) - cudaRegisterVar = buildRuntimeFunction( - globalBuilder, addUnderscoredPrefix(cudaPrefix, "RegisterVar"), loc, - FuncType::get({voidPtrPtrTy, voidPtrTy, voidPtrTy, voidPtrTy, intTy, - sizeTy, intTy, intTy}, - voidTy)); auto isExtern = ConstantOp::create( builder, loc, IntAttr::get(intTy, regAttr.getIsExtern() ? 1 : 0)); - auto varSize = ConstantOp::create( - builder, loc, IntAttr::get(sizeTy, size.getFixedValue())); - auto isConstant = ConstantOp::create( - builder, loc, IntAttr::get(intTy, regAttr.getIsConstant() ? 1 : 0)); - auto normalized = - ConstantOp::create(builder, loc, IntAttr::get(intTy, 0)); - builder.createCallOp(loc, cudaRegisterVar, - {fatbinHandle, hostVar, deviceName, deviceName, - isExtern, varSize, isConstant, normalized}); + + switch (regAttr.getKind()) { + case cir::CUDADeviceVarKind::Variable: { + // void __{cuda|hip}RegisterVar(void **fatbinHandle, + // char *hostVar, char *deviceAddress, + // const char *deviceName, int ext, + // size_t size, int constant, + // int normalized); + // OG ignores parameter types, treating pointers as void*. + if (!cudaRegisterVar) + cudaRegisterVar = buildRuntimeFunction( + globalBuilder, addUnderscoredPrefix(cudaPrefix, "RegisterVar"), + loc, + FuncType::get({voidPtrPtrTy, voidPtrTy, voidPtrTy, voidPtrTy, + intTy, sizeTy, intTy, intTy}, + voidTy)); + + auto varSize = ConstantOp::create( + builder, loc, IntAttr::get(sizeTy, size.getFixedValue())); + auto isConstant = ConstantOp::create( + builder, loc, IntAttr::get(intTy, regAttr.getIsConstant() ? 1 : 0)); + auto normalized = + ConstantOp::create(builder, loc, IntAttr::get(intTy, 0)); + + builder.createCallOp(loc, cudaRegisterVar, + {fatbinHandle, hostVar, deviceName, deviceName, + isExtern, varSize, isConstant, normalized}); + break; + } + + case cir::CUDADeviceVarKind::Surface: { + cir::ConstantOp surfaceType = ConstantOp::create( + builder, loc, IntAttr::get(intTy, regAttr.getSurfTexType())); + builder.createCallOp(loc, cudaRegisterSurface, + {fatbinHandle, hostVar, deviceName, deviceName, + surfaceType, isExtern}); + break; + } + + case cir::CUDADeviceVarKind::Texture: + llvm_unreachable("Texture registration NYI"); + } } } } diff --git a/clang/test/CIR/CodeGenCUDA/surface.cu b/clang/test/CIR/CodeGenCUDA/surface.cu index 67d7257c07c6e5..b4bb5ea0f623ee 100644 --- a/clang/test/CIR/CodeGenCUDA/surface.cu +++ b/clang/test/CIR/CodeGenCUDA/surface.cu @@ -1,9 +1,21 @@ // REQUIRES: x86-registered-target // REQUIRES: nvptx-registered-target + // RUN: %clang_cc1 -fclangir -std=c++11 -fcuda-is-device -triple nvptx64-nvidia-cuda -emit-cir -o - %s | FileCheck --check-prefix=CIR-DEVICE %s // RUN: %clang_cc1 -fclangir -std=c++11 -fcuda-is-device -triple nvptx64-nvidia-cuda -emit-llvm -o - %s | FileCheck --check-prefix=LLVM-DEVICE %s // RUN: %clang_cc1 -std=c++11 -fcuda-is-device -triple nvptx64-nvidia-cuda -emit-llvm -o - %s | FileCheck --check-prefix=OGCG-DEVICE %s +// RUN: echo -n "GPU binary would be here." > %t +// RUN: %clang_cc1 -fclangir -std=c++11 -triple x86_64-unknown-linux-gnu \ +// RUN: -target-sdk-version=12.3 -fcuda-include-gpubinary %t \ +// RUN: -emit-cir -o - %s | FileCheck --check-prefix=CIR-HOST %s +// RUN: %clang_cc1 -fclangir -std=c++11 -triple x86_64-unknown-linux-gnu \ +// RUN: -target-sdk-version=12.3 -fcuda-include-gpubinary %t \ +// RUN: -emit-llvm -o - %s | FileCheck --check-prefix=LLVM-HOST %s +// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-linux-gnu \ +// RUN: -target-sdk-version=12.3 -fcuda-include-gpubinary %t \ +// RUN: -emit-llvm -o - %s | FileCheck --check-prefix=LLVM-HOST %s + struct surfaceReference { int desc; }; @@ -18,8 +30,41 @@ struct __attribute__((device_builtin_surface_type)) surface<void, dim> surface<void, 2> surf; +//===----------------------------------------------------------------------===// +// Device-side checks +//===----------------------------------------------------------------------===// + // CIR-DEVICE: cir.global external target_address_space(1) @surf = #cir.undef : !cir.cuda_surface // CIR now matches OG CodeGen and emits undef for CUDA shadow variables. // LLVM-DEVICE: @surf ={{.*}} addrspace(1) externally_initialized global i64 undef // OGCG-DEVICE: @surf ={{.*}} addrspace(1) externally_initialized global i64 undef + +//===----------------------------------------------------------------------===// +// Host-side checks +//===----------------------------------------------------------------------===// + +// Check the CUDA surface registration runtime declaration. +// CIR-HOST: cir.func private @__cudaRegisterSurface(!cir.ptr<!cir.ptr<!void>>, !cir.ptr<!void>, !cir.ptr<!void>, !cir.ptr<!void>, !s32i, !s32i) + +// Check that __cuda_register_globals registers the surface using the host +// shadow, device-side name, surface type, and extern flag. +// CIR-HOST-LABEL: cir.func internal private @__cuda_register_globals +// CIR-HOST-SAME: (%[[FATBIN:.*]]: !cir.ptr<!cir.ptr<!void>> +// CIR-HOST: %[[NAME_RAW:.*]] = cir.get_global @".strsurf" +// CIR-HOST-NEXT: %[[NAME:.*]] = cir.cast bitcast %[[NAME_RAW]] +// CIR-HOST-NEXT: %[[HOST_RAW:.*]] = cir.get_global @surf +// CIR-HOST-NEXT: %[[HOST:.*]] = cir.cast bitcast %[[HOST_RAW]] +// CIR-HOST-NEXT: %[[EXTERN:.*]] = cir.const #cir.int<0> : !s32i +// CIR-HOST-NEXT: %[[SURFACE_TYPE:.*]] = cir.const #cir.int<2> : !s32i +// CIR-HOST-NEXT: cir.call @__cudaRegisterSurface(%[[FATBIN]], %[[HOST]], %[[NAME]], %[[NAME]], %[[SURFACE_TYPE]], %[[EXTERN]]) + +// Check that the host-side shadow is internalized and carries the surface +// registration metadata, including the surface type from surface<void, 2>. +// CIR-HOST: cir.global "private" internal dso_local @surf = {{.*}}cu.var_registration = #cir.cu.var_registration<surf, Surface, surface_type = 2> +// LLVM-HOST: @surf = internal global %{{.*}} zeroinitializer + +// Check both CIR-lowered LLVM and original CodeGen registration. +// LLVM-HOST-LABEL: define internal void @__cuda_register_globals +// LLVM-HOST-SAME: (ptr %[[FATBIN:.*]]) +// LLVM-HOST: call void @__cudaRegisterSurface(ptr %[[FATBIN]], ptr @surf, ptr @[[NAME:.*]], ptr @[[NAME]], i32 2, i32 0) \ No newline at end of file _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
