https://github.com/bob80905 created https://github.com/llvm/llvm-project/pull/208128
This PR completes the implementation of the InterlockedAdd functions in HLSL, by adding these methods to all existing resource classes. Fixes: https://github.com/llvm/llvm-project/issues/99122 Assisted by: Github Copilot >From c2e34e072d1e76d164397fae6b45de750d779913 Mon Sep 17 00:00:00 2001 From: Joshua Batista <[email protected]> Date: Tue, 7 Jul 2026 18:00:56 -0700 Subject: [PATCH] first attempt, add interlockedadd resource methods --- clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp | 68 +++++++++ clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h | 4 + clang/lib/Sema/HLSLExternalSemaSource.cpp | 2 + clang/lib/Sema/SemaHLSL.cpp | 23 +++ .../ByteAddressBuffer-InterlockedAdd.hlsl | 80 ++++++++++ ...teAddressBuffer-InterlockedAdd-errors.hlsl | 63 ++++++++ llvm/include/llvm/IR/IntrinsicsDirectX.td | 8 + llvm/lib/Target/DirectX/DXIL.td | 11 ++ llvm/lib/Target/DirectX/DXILOpLowering.cpp | 30 ++++ .../lib/Target/DirectX/DXILResourceAccess.cpp | 141 ++++++++++++++++++ .../DirectX/ResourceAtomicBinOp-i64-sm65.ll | 16 ++ .../CodeGen/DirectX/ResourceAtomicBinOp.ll | 59 ++++++++ 12 files changed, 505 insertions(+) create mode 100644 clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl create mode 100644 clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl create mode 100644 llvm/test/CodeGen/DirectX/ResourceAtomicBinOp-i64-sm65.ll create mode 100644 llvm/test/CodeGen/DirectX/ResourceAtomicBinOp.ll diff --git a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp index f8018729b4644..245487b71875f 100644 --- a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp +++ b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp @@ -1635,6 +1635,38 @@ BuiltinTypeDeclBuilder::addByteAddressBufferStoreMethods() { return *this; } +BuiltinTypeDeclBuilder & +BuiltinTypeDeclBuilder::addByteAddressBufferInterlockedMethods() { + assert(!Record->isCompleteDefinition() && "record is already complete"); + ASTContext &AST = SemaRef.getASTContext(); + + // Each entry declares two overloads (with and without an out original-value + // parameter). Adding a new atomic here only requires a new line — the shared + // helper takes care of the composition. + addByteAddressBufferInterlockedMethod("InterlockedAdd", AST.UnsignedIntTy, + "__builtin_hlsl_interlocked_add"); + + // 64-bit typed atomics on UAVs require SM 6.6 (DXIL 1.6 introduces the + // int64 overload of the atomicBinOp op). Skip synthesizing the *64 methods + // on older DXIL targets so callers get "no matching member function" from + // overload resolution — this matches DXC and mirrors how other HLSL SM-gated + // features are handled (see hlsl_intrinsics.h `_HLSL_AVAILABILITY` + // annotations). Non-DXIL targets (e.g., SPIR-V) always get the method: their + // 64-bit atomic support is gated by device extensions, not shader model. + const llvm::Triple &TT = AST.getTargetInfo().getTriple(); + bool DXILNeedsSM66 = + TT.getArch() == llvm::Triple::dxil && + AST.getTargetInfo().getPlatformMinVersion() < VersionTuple(6, 6); + if (!DXILNeedsSM66) { + // HLSL's uint64_t is `unsigned long`. + addByteAddressBufferInterlockedMethod("InterlockedAdd64", + AST.UnsignedLongTy, + "__builtin_hlsl_interlocked_add"); + } + + return *this; +} + BuiltinTypeDeclBuilder & BuiltinTypeDeclBuilder::addSampleMethods(ResourceDimension Dim, bool IsArray) { assert(!Record->isCompleteDefinition() && "record is already complete"); @@ -2356,6 +2388,42 @@ BuiltinTypeDeclBuilder::addStoreFunction(DeclarationName &Name, bool IsConst, .finalize(); } +BuiltinTypeDeclBuilder & +BuiltinTypeDeclBuilder::addByteAddressBufferInterlockedMethod( + StringRef MethodName, QualType ValueTy, StringRef BuiltinName) { + assert(!Record->isCompleteDefinition() && "record is already complete"); + ASTContext &AST = SemaRef.getASTContext(); + using PH = BuiltinTypeMethodBuilder::PlaceHolder; + + // Interlocked atomics operate on a typed slot in the buffer. Compose + // `resource_getpointer_typed` with the scalar `__builtin_hlsl_interlocked_*` + // builtin so backend lowering (DXIL and SPIR-V) can pattern-match a + // resource-pointer atomicrmw. + QualType AddrSpaceElemTy = + AST.getAddrSpaceQualType(ValueTy, LangAS::hlsl_device); + QualType ElemPtrTy = AST.getPointerType(AddrSpaceElemTy); + + auto BuildOverload = [&](bool WithOriginalValue) { + BuiltinTypeMethodBuilder MMB(*this, MethodName, AST.VoidTy); + MMB.addParam("Offset", AST.UnsignedIntTy).addParam("Value", ValueTy); + if (WithOriginalValue) + MMB.addParam("OriginalValue", ValueTy, + HLSLParamModifierAttr::Keyword_out); + MMB.callBuiltin("__builtin_hlsl_resource_getpointer_typed", ElemPtrTy, + PH::Handle, PH::_0, ValueTy) + .dereference(PH::LastStmt); + if (WithOriginalValue) + MMB.callBuiltin(BuiltinName, AST.VoidTy, PH::LastStmt, PH::_1, PH::_2); + else + MMB.callBuiltin(BuiltinName, AST.VoidTy, PH::LastStmt, PH::_1); + MMB.finalize(); + }; + + BuildOverload(/*WithOriginalValue=*/false); + BuildOverload(/*WithOriginalValue=*/true); + return *this; +} + BuiltinTypeDeclBuilder &BuiltinTypeDeclBuilder::addAppendMethod() { using PH = BuiltinTypeMethodBuilder::PlaceHolder; ASTContext &AST = SemaRef.getASTContext(); diff --git a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h index 09cf1fceca116..e809ef264198c 100644 --- a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h +++ b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h @@ -106,6 +106,7 @@ class BuiltinTypeDeclBuilder { bool IsArray = false); BuiltinTypeDeclBuilder &addByteAddressBufferLoadMethods(); BuiltinTypeDeclBuilder &addByteAddressBufferStoreMethods(); + BuiltinTypeDeclBuilder &addByteAddressBufferInterlockedMethods(); BuiltinTypeDeclBuilder &addSampleMethods(ResourceDimension Dim, bool IsArray = false); BuiltinTypeDeclBuilder &addSampleBiasMethods(ResourceDimension Dim, @@ -134,6 +135,9 @@ class BuiltinTypeDeclBuilder { QualType ReturnTy = QualType()); BuiltinTypeDeclBuilder &addStoreFunction(DeclarationName &Name, bool IsConst, QualType ValueType); + BuiltinTypeDeclBuilder & + addByteAddressBufferInterlockedMethod(StringRef MethodName, QualType ValueTy, + StringRef BuiltinName); BuiltinTypeDeclBuilder &addAppendMethod(); BuiltinTypeDeclBuilder &addConsumeMethod(); diff --git a/clang/lib/Sema/HLSLExternalSemaSource.cpp b/clang/lib/Sema/HLSLExternalSemaSource.cpp index 1d2a3983a639d..4c1894d6761d2 100644 --- a/clang/lib/Sema/HLSLExternalSemaSource.cpp +++ b/clang/lib/Sema/HLSLExternalSemaSource.cpp @@ -645,6 +645,7 @@ void HLSLExternalSemaSource::defineHLSLTypesWithForwardDeclarations() { /*RawBuffer=*/true, /*HasCounter=*/false) .addByteAddressBufferLoadMethods() .addByteAddressBufferStoreMethods() + .addByteAddressBufferInterlockedMethods() .addGetDimensionsMethodForBuffer() .completeDefinition(); }); @@ -654,6 +655,7 @@ void HLSLExternalSemaSource::defineHLSLTypesWithForwardDeclarations() { onCompletion(Decl, [this](CXXRecordDecl *Decl) { setupBufferType(Decl, *SemaPtr, ResourceClass::UAV, /*IsROV=*/true, /*RawBuffer=*/true, /*HasCounter=*/false) + .addByteAddressBufferInterlockedMethods() .addGetDimensionsMethodForBuffer() .completeDefinition(); }); diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp index ae6cbce2f890c..1ffddeac080fa 100644 --- a/clang/lib/Sema/SemaHLSL.cpp +++ b/clang/lib/Sema/SemaHLSL.cpp @@ -4568,6 +4568,29 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { return true; } + // 64-bit interlocked ops require SM 6.6 on DXIL — the DXIL 1.6 int64 + // overloads of atomicBinOp/cmpXchg are what enable them. The synthesized + // wrapper methods (e.g. RWByteAddressBuffer::InterlockedAdd64) that call + // this builtin are themselves only declared when the target supports it + // (see HLSLBuiltinTypeDeclBuilder), so pre-SM6.6 usage is caught by + // overload resolution. This defensive check catches direct + // `__builtin_hlsl_interlocked_add` calls from HLSL code with a 64-bit + // dest on pre-SM6.6 DXIL targets. Skip synthetic invocations (invalid + // source location) built while composing wrapper method bodies. + const TargetInfo &TI = SemaRef.Context.getTargetInfo(); + if (TheCall->getBeginLoc().isValid() && + TI.getTriple().getArch() == llvm::Triple::dxil && + SemaRef.Context.getTypeSize(DestTy) == 64 && + TI.getPlatformMinVersion() < VersionTuple(6, 6)) { + llvm::StringRef PlatformName( + AvailabilityAttr::getPrettyPlatformName(TI.getPlatformName())); + SemaRef.Diag(TheCall->getBeginLoc(), diag::warn_hlsl_availability) + << TheCall->getDirectCallee() << PlatformName + << VersionTuple(6, 6).getAsString() << /*UseEnvironment=*/false + << /*EnvName=*/""; + return true; + } + if (CheckModifiableLValue(&SemaRef, TheCall, 0)) return true; diff --git a/clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl b/clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl new file mode 100644 index 0000000000000..ae6179dc4bc99 --- /dev/null +++ b/clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl @@ -0,0 +1,80 @@ +// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header -triple \ +// RUN: dxil-pc-shadermodel6.6-compute %s -emit-llvm -disable-llvm-passes -o - | \ +// RUN: FileCheck %s --check-prefixes=CHECK,DXCHECK + +// Test that the RWByteAddressBuffer::InterlockedAdd and +// RasterizerOrderedByteAddressBuffer::InterlockedAdd member methods lower to +// `dx.resource.getpointer.typed -> dx.interlocked.add`, and that the +// 3-argument overload stores the returned original value through the out +// parameter. + +RWByteAddressBuffer BAB : register(u0); +RasterizerOrderedByteAddressBuffer ROVB : register(u1); + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_int_2arg +// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}} +// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}}) +// DXCHECK: call i32 @llvm.dx.interlocked.add.i32.p0(ptr %[[PTR]], i32 %{{.*}}) +export void test_bab_int_2arg(uint off, int v) { + BAB.InterlockedAdd(off, v); +} + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_uint_3arg +// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}} +// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}}) +// DXCHECK: %[[R:.*]] = call i32 @llvm.dx.interlocked.add.i32.p0(ptr %[[PTR]], i32 %{{.*}}) +// DXCHECK: store i32 %[[R]], ptr {{.*}} +export void test_bab_uint_3arg(uint off, uint v, out uint orig) { + BAB.InterlockedAdd(off, v, orig); +} + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_rovb_int_2arg +// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}} +// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}}) +// DXCHECK: call i32 @llvm.dx.interlocked.add.i32.p0(ptr %[[PTR]], i32 %{{.*}}) +export void test_rovb_int_2arg(uint off, int v) { + ROVB.InterlockedAdd(off, v); +} + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_rovb_uint_3arg +// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}} +// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}}) +// DXCHECK: %[[R:.*]] = call i32 @llvm.dx.interlocked.add.i32.p0(ptr %[[PTR]], i32 %{{.*}}) +// DXCHECK: store i32 %[[R]], ptr {{.*}} +export void test_rovb_uint_3arg(uint off, uint v, out uint orig) { + ROVB.InterlockedAdd(off, v, orig); +} + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_int64_2arg +// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}} +// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}}) +// DXCHECK: call i64 @llvm.dx.interlocked.add.i64.p0(ptr %[[PTR]], i64 %{{.*}}) +export void test_bab_int64_2arg(uint off, int64_t v) { + BAB.InterlockedAdd64(off, v); +} + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_uint64_3arg +// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}} +// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}}) +// DXCHECK: %[[R:.*]] = call i64 @llvm.dx.interlocked.add.i64.p0(ptr %[[PTR]], i64 %{{.*}}) +// DXCHECK: store i64 %[[R]], ptr {{.*}} +export void test_bab_uint64_3arg(uint off, uint64_t v, out uint64_t orig) { + BAB.InterlockedAdd64(off, v, orig); +} + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_rovb_int64_2arg +// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}} +// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}}) +// DXCHECK: call i64 @llvm.dx.interlocked.add.i64.p0(ptr %[[PTR]], i64 %{{.*}}) +export void test_rovb_int64_2arg(uint off, int64_t v) { + ROVB.InterlockedAdd64(off, v); +} + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_rovb_uint64_3arg +// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}} +// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}}) +// DXCHECK: %[[R:.*]] = call i64 @llvm.dx.interlocked.add.i64.p0(ptr %[[PTR]], i64 %{{.*}}) +// DXCHECK: store i64 %[[R]], ptr {{.*}} +export void test_rovb_uint64_3arg(uint off, uint64_t v, out uint64_t orig) { + ROVB.InterlockedAdd64(off, v, orig); +} diff --git a/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl new file mode 100644 index 0000000000000..13fc70d700421 --- /dev/null +++ b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl @@ -0,0 +1,63 @@ +// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header \ +// RUN: -triple dxil-pc-shadermodel6.6-compute %s -fsyntax-only -verify \ +// RUN: -verify-ignore-unexpected=note,warning + +// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header \ +// RUN: -triple dxil-pc-shadermodel6.5-compute -DTEST_SM65 %s -fsyntax-only \ +// RUN: -verify -verify-ignore-unexpected=note,warning + +RWByteAddressBuffer BAB : register(u0); +RasterizerOrderedByteAddressBuffer ROVB : register(u1); + +struct S { int x; }; + +#ifdef TEST_SM65 + +// InterlockedAdd64 is only synthesized on DXIL when the shader model is at +// least 6.6 (matches DXC). On SM 6.5 the member is not declared at all, so +// the reference must fail with "no member named". +void sm65_no_bab_add64(uint off, int64_t v) { + BAB.InterlockedAdd64(off, v); + // expected-error@-1 {{no member named 'InterlockedAdd64' in 'hlsl::RWByteAddressBuffer'}} +} + +void sm65_no_rovb_add64(uint off, int64_t v) { + ROVB.InterlockedAdd64(off, v); + // expected-error@-1 {{no member named 'InterlockedAdd64' in 'hlsl::RasterizerOrderedByteAddressBuffer'}} +} + +// 32-bit InterlockedAdd is always available. +void sm65_bab_add32_ok(uint off, int v) { + BAB.InterlockedAdd(off, v); +} + +#else + +void too_few(uint off) { + BAB.InterlockedAdd(off); + // expected-error@-1 {{no matching member function for call to 'InterlockedAdd'}} +} + +void too_many(uint off, int v, int extra) { + int orig; + BAB.InterlockedAdd(off, v, orig, extra); + // expected-error@-1 {{no matching member function for call to 'InterlockedAdd'}} +} + +void struct_value(uint off, S v) { + BAB.InterlockedAdd(off, v); + // expected-error@-1 {{no matching member function for call to 'InterlockedAdd'}} +} + +// Same shape of errors on RasterizerOrderedByteAddressBuffer. +void rovb_too_few(uint off) { + ROVB.InterlockedAdd(off); + // expected-error@-1 {{no matching member function for call to 'InterlockedAdd'}} +} + +void rovb_struct_value(uint off, S v) { + ROVB.InterlockedAdd(off, v); + // expected-error@-1 {{no matching member function for call to 'InterlockedAdd'}} +} + +#endif diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td index 63186ab4d7191..6b88d5c633621 100644 --- a/llvm/include/llvm/IR/IntrinsicsDirectX.td +++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td @@ -61,6 +61,14 @@ def int_dx_resource_store_rawbuffer : DefaultAttrsIntrinsic< [], [llvm_any_ty, llvm_i32_ty, llvm_i32_ty, llvm_any_ty], [IntrWriteMem]>; +// Resource atomic binary op: performs an atomic read-modify-write on a UAV +// resource element and returns the original value. The i32 operation code +// matches DXIL's AtomicBinOpCode enum. +def int_dx_resource_atomicbinop + : DefaultAttrsIntrinsic<[llvm_anyint_ty], + [llvm_any_ty, llvm_i32_ty, llvm_i32_ty, + llvm_i32_ty, LLVMMatchType<0>], + [IntrArgMemOnly]>; // dx.resource.load.cbufferrow encodes the number of elements returned in the // function name. The total size of the return should always be 128 bits. diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td index 299d2d113b6bf..637fc05b3b2da 100644 --- a/llvm/lib/Target/DirectX/DXIL.td +++ b/llvm/lib/Target/DirectX/DXIL.td @@ -983,6 +983,17 @@ def BufferStore : DXILOp<69, bufferStore> { let stages = [Stages<DXIL1_0, [all_stages]>]; } +def AtomicBinOp : DXILOp<78, atomicBinOp> { + let Doc = "performs an atomic read-modify-write on a UAV resource " + "element, returning the original value"; + // Handle, AtomicBinOpCode, Coord0, Coord1, Coord2, NewValue + let arguments = [HandleTy, Int32Ty, Int32Ty, Int32Ty, Int32Ty, OverloadTy]; + let result = OverloadTy; + let overloads = [Overloads<DXIL1_0, [Int32Ty]>, + Overloads<DXIL1_6, [Int32Ty, Int64Ty]>]; + let stages = [Stages<DXIL1_0, [all_stages]>]; +} + def UpdateCounter : DXILOp<70, bufferUpdateCounter> { let Doc = "increments/decrements a buffer counter"; let arguments = [HandleTy, Int8Ty]; diff --git a/llvm/lib/Target/DirectX/DXILOpLowering.cpp b/llvm/lib/Target/DirectX/DXILOpLowering.cpp index 93d5a08a6e0a2..64e0de8e55bdd 100644 --- a/llvm/lib/Target/DirectX/DXILOpLowering.cpp +++ b/llvm/lib/Target/DirectX/DXILOpLowering.cpp @@ -849,6 +849,33 @@ class OpLowerer { }); } + [[nodiscard]] bool lowerResourceAtomicBinOp(Function &F) { + IRBuilder<> &IRB = OpBuilder.getIRB(); + + return replaceFunction(F, [&](CallInst *CI) -> Error { + IRB.SetInsertPoint(CI); + Value *Handle = + createTmpHandleCast(CI->getArgOperand(0), OpBuilder.getHandleType()); + Value *Index = CI->getArgOperand(1); + Value *Offset = CI->getArgOperand(2); + Value *BinOp = CI->getArgOperand(3); + Value *NewValue = CI->getArgOperand(4); + + std::array<Value *, 6> Args{Handle, BinOp, Index, + Offset, IRB.getInt32(0), NewValue}; + + Expected<CallInst *> OpCall = OpBuilder.tryCreateOp( + OpCode::AtomicBinOp, Args, CI->getName(), CI->getType()); + + if (Error E = OpCall.takeError()) + return E; + + CI->replaceAllUsesWith(*OpCall); + CI->eraseFromParent(); + return Error::success(); + }); + } + [[nodiscard]] bool lowerGetDimensionsX(Function &F) { IRBuilder<> &IRB = OpBuilder.getIRB(); Type *Int32Ty = IRB.getInt32Ty(); @@ -1210,6 +1237,9 @@ class OpLowerer { case Intrinsic::dx_resource_updatecounter: HasErrors |= lowerUpdateCounter(F); break; + case Intrinsic::dx_resource_atomicbinop: + HasErrors |= lowerResourceAtomicBinOp(F); + break; case Intrinsic::dx_resource_getdimensions_x: HasErrors |= lowerGetDimensionsX(F); break; diff --git a/llvm/lib/Target/DirectX/DXILResourceAccess.cpp b/llvm/lib/Target/DirectX/DXILResourceAccess.cpp index 25d860e615c17..5f5c4660e744b 100644 --- a/llvm/lib/Target/DirectX/DXILResourceAccess.cpp +++ b/llvm/lib/Target/DirectX/DXILResourceAccess.cpp @@ -26,6 +26,7 @@ #include "llvm/InitializePasses.h" #include "llvm/Support/FormatVariadic.h" #include "llvm/Transforms/Utils/ValueMapper.h" +#include <optional> #define DEBUG_TYPE "dxil-resource-access" @@ -230,6 +231,114 @@ static void createStoreIntrinsic(IntrinsicInst *II, StoreInst *SI, llvm_unreachable("Unhandled case in switch"); } +static std::optional<unsigned> getAtomicBinOpCode(AtomicRMWInst::BinOp BinOp) { + switch (BinOp) { + case AtomicRMWInst::Add: + return 0; + case AtomicRMWInst::And: + return 1; + case AtomicRMWInst::Or: + return 2; + case AtomicRMWInst::Xor: + return 3; + case AtomicRMWInst::Min: + return 4; + case AtomicRMWInst::Max: + return 5; + case AtomicRMWInst::UMin: + return 6; + case AtomicRMWInst::UMax: + return 7; + case AtomicRMWInst::Xchg: + return 8; + case AtomicRMWInst::Sub: + case AtomicRMWInst::Nand: + case AtomicRMWInst::FAdd: + case AtomicRMWInst::FSub: + case AtomicRMWInst::FMax: + case AtomicRMWInst::FMin: + case AtomicRMWInst::FMaximum: + case AtomicRMWInst::FMinimum: + case AtomicRMWInst::FMaximumNum: + case AtomicRMWInst::FMinimumNum: + case AtomicRMWInst::UIncWrap: + case AtomicRMWInst::UDecWrap: + case AtomicRMWInst::USubCond: + case AtomicRMWInst::USubSat: + case AtomicRMWInst::BAD_BINOP: + return std::nullopt; + } + llvm_unreachable("Unhandled atomicrmw operation"); +} + +static void createAtomicBinOp(IntrinsicInst *II, AtomicRMWInst *AI, + dxil::ResourceTypeInfo &RTI) { + std::optional<unsigned> BinOpCode = getAtomicBinOpCode(AI->getOperation()); + if (!BinOpCode) { + reportFatalUsageError("DXIL resource atomicrmw operation not implemented"); + return; + } + + const DataLayout &DL = AI->getDataLayout(); + IRBuilder<> Builder(AI); + Value *Index = II->getOperand(1); + + // The offset for the rawbuffer load/store/atomic ops is always in bytes. + uint64_t AccessSize = 1; + Value *Offset = + traverseGEPOffsets(DL, Builder, AI->getPointerOperand(), AccessSize); + + // For raw buffer (ie, HLSL's ByteAddressBuffer), we need to fold the access + // entirely into the index. + if (!RTI.isStruct()) { + auto *ConstantOffset = dyn_cast<ConstantInt>(Offset); + if (!ConstantOffset || !ConstantOffset->isZero()) + Index = Builder.CreateAdd(Index, Offset); + Offset = llvm::PoisonValue::get(Builder.getInt32Ty()); + } + + auto *BinOp = Builder.getInt32(*BinOpCode); + Value *V = Builder.CreateIntrinsic( + AI->getType(), Intrinsic::dx_resource_atomicbinop, + {II->getOperand(0), Index, Offset, BinOp, AI->getValOperand()}); + AI->replaceAllUsesWith(V); +} + +static void createAtomicBinOpIntrinsic(IntrinsicInst *II, AtomicRMWInst *AI, + dxil::ResourceTypeInfo &RTI) { + switch (RTI.getResourceKind()) { + case dxil::ResourceKind::TypedBuffer: + case dxil::ResourceKind::RawBuffer: + case dxil::ResourceKind::StructuredBuffer: + return createAtomicBinOp(II, AI, RTI); + case dxil::ResourceKind::Texture1D: + case dxil::ResourceKind::Texture2D: + case dxil::ResourceKind::Texture2DMS: + case dxil::ResourceKind::Texture3D: + case dxil::ResourceKind::TextureCube: + case dxil::ResourceKind::Texture1DArray: + case dxil::ResourceKind::Texture2DArray: + case dxil::ResourceKind::Texture2DMSArray: + case dxil::ResourceKind::TextureCubeArray: + case dxil::ResourceKind::FeedbackTexture2D: + case dxil::ResourceKind::FeedbackTexture2DArray: + reportFatalUsageError( + "DXIL atomicrmw not implemented for texture resources"); + return; + case dxil::ResourceKind::CBuffer: + case dxil::ResourceKind::Sampler: + case dxil::ResourceKind::TBuffer: + reportFatalUsageError( + "DXIL atomicrmw not implemented for this resource type"); + return; + case dxil::ResourceKind::RTAccelerationStructure: + case dxil::ResourceKind::Invalid: + case dxil::ResourceKind::NumEntries: + llvm_unreachable("Invalid resource kind for atomicrmw"); + } + llvm_unreachable("Unhandled case in switch"); +} + static void createTypedBufferLoad(IntrinsicInst *II, LoadInst *LI, dxil::ResourceTypeInfo &RTI) { const DataLayout &DL = LI->getDataLayout(); @@ -550,6 +659,8 @@ static Instruction *getStoreLoadPointerOperand(Instruction *AI) { return dyn_cast<Instruction>(LI->getPointerOperand()); if (auto *SI = dyn_cast<StoreInst>(AI)) return dyn_cast<Instruction>(SI->getPointerOperand()); + if (auto *RMWI = dyn_cast<AtomicRMWInst>(AI)) + return dyn_cast<Instruction>(RMWI->getPointerOperand()); return nullptr; } @@ -786,6 +897,36 @@ static void replaceAccess(IntrinsicInst *II, dxil::ResourceTypeInfo &RTI) { } else if (auto *LI = dyn_cast<LoadInst>(U)) { createLoadIntrinsic(II, LI, RTI); DeadInsts.push_back(LI); + } else if (auto *AI = dyn_cast<AtomicRMWInst>(U)) { + createAtomicBinOpIntrinsic(II, AI, RTI); + DeadInsts.push_back(AI); + } else if (auto *CI = dyn_cast<CallInst>(U)) { + // `dx.interlocked.*` intrinsics wrap an atomicrmw and are expanded to + // one by DXILIntrinsicExpansion — but that pass runs after this one, so + // when the source of the pointer is a resource we must expand them here + // (and immediately process the resulting atomicrmw) instead of letting + // the pointer escape. + auto *IntrinCall = dyn_cast<IntrinsicInst>(CI); + std::optional<AtomicRMWInst::BinOp> Op; + if (IntrinCall) { + switch (IntrinCall->getIntrinsicID()) { + case Intrinsic::dx_interlocked_add: + Op = AtomicRMWInst::Add; + break; + default: + break; + } + } + if (!Op) + llvm_unreachable("Unhandled instruction - pointer escaped?"); + IRBuilder<> Builder(IntrinCall); + auto *AI = Builder.CreateAtomicRMW( + *Op, IntrinCall->getArgOperand(0), IntrinCall->getArgOperand(1), + MaybeAlign(), AtomicOrdering::Monotonic); + IntrinCall->replaceAllUsesWith(AI); + createAtomicBinOpIntrinsic(II, AI, RTI); + DeadInsts.push_back(AI); + DeadInsts.push_back(IntrinCall); } else llvm_unreachable("Unhandled instruction - pointer escaped?"); } diff --git a/llvm/test/CodeGen/DirectX/ResourceAtomicBinOp-i64-sm65.ll b/llvm/test/CodeGen/DirectX/ResourceAtomicBinOp-i64-sm65.ll new file mode 100644 index 0000000000000..6f5a40e0b6c2a --- /dev/null +++ b/llvm/test/CodeGen/DirectX/ResourceAtomicBinOp-i64-sm65.ll @@ -0,0 +1,16 @@ +; RUN: not opt -S -dxil-resource-access -dxil-op-lower -mtriple=dxil-pc-shadermodel6.5-compute %s 2>&1 | FileCheck %s + +; Verify resource i64 atomicrmw rejects shader models before SM 6.6, where +; dx.op.atomicBinOp gained i64 overload support. + +target triple = "dxil-pc-shadermodel6.5-compute" + +define i64 @atomic_i64(i32 %index, i64 %value) { + %buffer = call target("dx.RawBuffer", i64, 1, 0, 0) + @llvm.dx.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, ptr null) + %ptr = call ptr @llvm.dx.resource.getpointer( + target("dx.RawBuffer", i64, 1, 0, 0) %buffer, i32 %index) + ; CHECK: Cannot create AtomicBinOp operation: Invalid overload type + %old = atomicrmw add ptr %ptr, i64 %value monotonic + ret i64 %old +} diff --git a/llvm/test/CodeGen/DirectX/ResourceAtomicBinOp.ll b/llvm/test/CodeGen/DirectX/ResourceAtomicBinOp.ll new file mode 100644 index 0000000000000..bc852c8a2c81f --- /dev/null +++ b/llvm/test/CodeGen/DirectX/ResourceAtomicBinOp.ll @@ -0,0 +1,59 @@ +; RUN: opt -S -dxil-resource-access -dxil-op-lower %s | FileCheck %s --check-prefixes=CHECK,I32 +; RUN: opt -S -dxil-resource-access -dxil-op-lower -mtriple=dxil-pc-shadermodel6.6-compute %s | FileCheck %s --check-prefixes=CHECK,I32,I64 + +; Verify atomicrmw through a dx.resource.getpointer is lowered to +; dx.op.atomicBinOp for UAV resources. + +target triple = "dxil-pc-shadermodel6.6-compute" + +; CHECK-LABEL: define i32 @atomic_i32( +define i32 @atomic_i32(i32 %index, i32 %value) { + %buffer = call target("dx.RawBuffer", i32, 1, 0, 0) + @llvm.dx.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, ptr null) + %ptr = call ptr @llvm.dx.resource.getpointer( + target("dx.RawBuffer", i32, 1, 0, 0) %buffer, i32 %index) + + ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 0, i32 %index, i32 0, i32 0, i32 %value) + %add = atomicrmw add ptr %ptr, i32 %value monotonic + ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 1, i32 %index, i32 0, i32 0, i32 %value) + %and = atomicrmw and ptr %ptr, i32 %value monotonic + ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 2, i32 %index, i32 0, i32 0, i32 %value) + %or = atomicrmw or ptr %ptr, i32 %value monotonic + ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 3, i32 %index, i32 0, i32 0, i32 %value) + %xor = atomicrmw xor ptr %ptr, i32 %value monotonic + ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 4, i32 %index, i32 0, i32 0, i32 %value) + %min = atomicrmw min ptr %ptr, i32 %value monotonic + ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 5, i32 %index, i32 0, i32 0, i32 %value) + %max = atomicrmw max ptr %ptr, i32 %value monotonic + ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 6, i32 %index, i32 0, i32 0, i32 %value) + %umin = atomicrmw umin ptr %ptr, i32 %value monotonic + ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 7, i32 %index, i32 0, i32 0, i32 %value) + %umax = atomicrmw umax ptr %ptr, i32 %value monotonic + ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 8, i32 %index, i32 0, i32 0, i32 %value) + %xchg = atomicrmw xchg ptr %ptr, i32 %value monotonic + ret i32 %xchg +} + +; CHECK-LABEL: define i32 @atomic_i32_byteaddress( +define i32 @atomic_i32_byteaddress(i32 %offset, i32 %value) { + %buffer = call target("dx.RawBuffer", i8, 1, 0, 0) + @llvm.dx.resource.handlefrombinding(i32 0, i32 1, i32 1, i32 0, ptr null) + %ptr = call ptr @llvm.dx.resource.getpointer( + target("dx.RawBuffer", i8, 1, 0, 0) %buffer, i32 %offset) + + ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 0, i32 %offset, i32 poison, i32 0, i32 %value) + %old = atomicrmw add ptr %ptr, i32 %value monotonic + ret i32 %old +} + +; CHECK-LABEL: define i64 @atomic_i64( +define i64 @atomic_i64(i32 %index, i64 %value) { + %buffer = call target("dx.RawBuffer", i64, 1, 0, 0) + @llvm.dx.resource.handlefrombinding(i32 0, i32 2, i32 1, i32 0, ptr null) + %ptr = call ptr @llvm.dx.resource.getpointer( + target("dx.RawBuffer", i64, 1, 0, 0) %buffer, i32 %index) + + ; I64: call i64 @dx.op.atomicBinOp.i64(i32 78, %dx.types.Handle %{{.*}}, i32 0, i32 %index, i32 0, i32 0, i64 %value) + %old = atomicrmw add ptr %ptr, i64 %value monotonic + ret i64 %old +} _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
