https://github.com/bob80905 created https://github.com/llvm/llvm-project/pull/222162
This PR is a backend change alone. It teaches DXIL lowering to handle a `cmpxchg` on a buffer resource pointer, which the `InterlockedCompareStore` and `InterlockedCompareExchange` PRs above it in this stack need. The PR adds: - the `llvm.dx.resource.atomic.compare.exchange` intrinsic, - its DXIL operation and op lowering, - the `DXILResourceAccess` handling that turns a `cmpxchg` on a value returned by `llvm.dx.resource.getpointer` into that intrinsic. DXIL has no atomic compare exchange for texture resources, so the pass reports an error for them. A test covers that path. There is no Clang change here, and no HLSL function becomes available. Assisted by: Github Copilot >From 679fb2ba3dca5d57f22ee084562cd221bcbdcfe3 Mon Sep 17 00:00:00 2001 From: Joshua Batista <[email protected]> Date: Fri, 4 Sep 2026 13:33:38 -0700 Subject: [PATCH] First attempt implementing DXIL compare exchange infrastructure --- llvm/include/llvm/IR/IntrinsicsDirectX.td | 12 ++ llvm/lib/Target/DirectX/DXIL.td | 11 ++ llvm/lib/Target/DirectX/DXILOpLowering.cpp | 41 +++++++ .../lib/Target/DirectX/DXILResourceAccess.cpp | 104 ++++++++++++++---- .../ResourceAtomicCompareExchange-i64-sm65.ll | 17 +++ .../DirectX/ResourceAtomicCompareExchange.ll | 57 ++++++++++ ...urceAtomicCompareExchangeTexture-errors.ll | 17 +++ .../DirectX/ResourceAtomicTexture-errors.ll | 17 +++ 8 files changed, 252 insertions(+), 24 deletions(-) create mode 100644 llvm/test/CodeGen/DirectX/ResourceAtomicCompareExchange-i64-sm65.ll create mode 100644 llvm/test/CodeGen/DirectX/ResourceAtomicCompareExchange.ll create mode 100644 llvm/test/CodeGen/DirectX/ResourceAtomicCompareExchangeTexture-errors.ll create mode 100644 llvm/test/CodeGen/DirectX/ResourceAtomicTexture-errors.ll diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td index a8927f83ee2f8..fb8e9eb51b96e 100644 --- a/llvm/include/llvm/IR/IntrinsicsDirectX.td +++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td @@ -79,6 +79,18 @@ def int_dx_resource_atomic_binop [llvm_any_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, LLVMMatchType<0>], [IntrArgMemOnly]>; + +// Atomic compare-exchange on a UAV resource element. Emitted by +// DXILResourceAccess from a `cmpxchg` on a `dx.resource.getpointer` result, +// and lowered by DXILOpLowering to the DXIL `AtomicCompareExchange` op (79). +// Args: handle, coord0 (index), coord1 (offset or poison), compare value, +// new value. Returns the original value. +def int_dx_resource_atomic_compare_exchange + : DefaultAttrsIntrinsic< + [llvm_any_ty], + [llvm_any_ty, llvm_i32_ty, llvm_i32_ty, LLVMMatchType<0>, + LLVMMatchType<0>], + [IntrArgMemOnly]>; // dx.resource.load.cbufferrow encodes the number of elements returned in the // function name. The total size of the return should always be 128 bits. def int_dx_resource_load_cbufferrow_8 diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td index 4beafd0c619b0..cde3a2ca8db31 100644 --- a/llvm/lib/Target/DirectX/DXIL.td +++ b/llvm/lib/Target/DirectX/DXIL.td @@ -1074,6 +1074,17 @@ def AtomicBinOp : DXILOp<78, atomicBinOp> { let stages = [Stages<DXIL1_0, [all_stages]>]; } +def AtomicCompareExchange : DXILOp<79, atomicCompareExchange> { + let Doc = "performs an atomic compare-exchange on a UAV resource element, " + "returning the original value"; + // Handle, Coord0, Coord1, Coord2, CompareValue, NewValue + let arguments = [HandleTy, Int32Ty, Int32Ty, Int32Ty, OverloadTy, OverloadTy]; + let result = OverloadTy; + let overloads = [Overloads<DXIL1_0, [Int32Ty]>, + Overloads<DXIL1_6, [Int32Ty, Int64Ty]>]; + let stages = [Stages<DXIL1_0, [all_stages]>]; +} + def Barrier : DXILOp<80, barrier> { let Doc = "inserts a memory barrier in the shader"; let intrinsics = [ diff --git a/llvm/lib/Target/DirectX/DXILOpLowering.cpp b/llvm/lib/Target/DirectX/DXILOpLowering.cpp index 20c2f93a17c10..4b069cc23fd16 100644 --- a/llvm/lib/Target/DirectX/DXILOpLowering.cpp +++ b/llvm/lib/Target/DirectX/DXILOpLowering.cpp @@ -1144,6 +1144,44 @@ class OpLowerer { }); } + [[nodiscard]] bool lowerResourceAtomicCompareExchange(Function &F) { + IRBuilder<> &IRB = OpBuilder.getIRB(); + + return replaceFunction(F, [&](CallInst *CI) -> Error { + IRB.SetInsertPoint(CI); + + // Cast the target-extension typed handle to `%dx.types.Handle`, tracked + // via CleanupCasts so the pair is reconciled by `cleanupHandleCasts`. + Value *Handle = + createTmpHandleCast(CI->getArgOperand(0), OpBuilder.getHandleType()); + Value *Coord0 = CI->getArgOperand(1); + Value *Coord1 = CI->getArgOperand(2); + Value *CompareValue = CI->getArgOperand(3); + Value *NewValue = CI->getArgOperand(4); + + std::array<Value *, 6> Args{ + Handle, Coord0, Coord1, ConstantInt::get(IRB.getInt32Ty(), 0), + CompareValue, NewValue}; + Expected<CallInst *> OpCall = + OpBuilder.tryCreateOp(dxil::OpCode::AtomicCompareExchange, Args, + CI->getName(), CI->getType()); + if (Error E = OpCall.takeError()) { + // Preserve the DXIL op error text but attach it as a + // DiagnosticInfoUnsupported so we don't crash with a dangling call. + std::string Message(toString(std::move(E))); + CI->getContext().diagnose(DiagnosticInfoUnsupported( + *CI->getFunction(), Message, CI->getDebugLoc())); + CI->replaceAllUsesWith(PoisonValue::get(CI->getType())); + CI->eraseFromParent(); + return Error::success(); + } + + CI->replaceAllUsesWith(*OpCall); + CI->eraseFromParent(); + return Error::success(); + }); + } + [[nodiscard]] bool lowerCtpopToCountBits(Function &F) { IRBuilder<> &IRB = OpBuilder.getIRB(); Type *Int32Ty = IRB.getInt32Ty(); @@ -1386,6 +1424,9 @@ class OpLowerer { case Intrinsic::dx_resource_atomic_binop: HasErrors |= lowerResourceAtomicBinOp(F); break; + case Intrinsic::dx_resource_atomic_compare_exchange: + HasErrors |= lowerResourceAtomicCompareExchange(F); + break; case Intrinsic::dx_resource_getdimensions_x: HasErrors |= lowerGetDimensionsX(F); break; diff --git a/llvm/lib/Target/DirectX/DXILResourceAccess.cpp b/llvm/lib/Target/DirectX/DXILResourceAccess.cpp index 8c4d2b6c14042..e8ac9bfb0c99e 100644 --- a/llvm/lib/Target/DirectX/DXILResourceAccess.cpp +++ b/llvm/lib/Target/DirectX/DXILResourceAccess.cpp @@ -330,27 +330,20 @@ getAtomicBinOpCode(AtomicRMWInst::BinOp BinOp) { llvm_unreachable("Unhandled atomicrmw operation"); } -static void createAtomicBinOp(IntrinsicInst *II, AtomicRMWInst *AI, - dxil::ResourceTypeInfo &RTI) { - std::optional<dxil::AtomicBinOpCode> BinOpCode = - getAtomicBinOpCode(AI->getOperation()); - if (!BinOpCode) { - reportFatalUsageError("DXIL resource atomicrmw operation not implemented"); - return; - } - - const DataLayout &DL = AI->getDataLayout(); - IRBuilder<> Builder(AI); +// Compute the (coord0, coord1) pair for a resource atomic operation. For +// non-struct buffers (RawBuffer or TypedBuffer), the byte offset is folded +// into the index and coord1 is poison — only StructuredBuffer atomics use both +// a struct index and a byte offset. +static std::pair<Value *, Value *> +getAtomicResourceCoords(IntrinsicInst *II, Value *PointerOperand, + dxil::ResourceTypeInfo &RTI, IRBuilder<> &Builder, + const DataLayout &DL) { Value *Index = II->getOperand(1); // The offset for the rawbuffer load/store/atomic ops is always in bytes. uint64_t AccessSize = 1; - Value *Offset = - traverseGEPOffsets(DL, Builder, AI->getPointerOperand(), AccessSize); + Value *Offset = traverseGEPOffsets(DL, Builder, PointerOperand, AccessSize); - // For non-struct buffers (RawBuffer or TypedBuffer), fold the byte offset - // into the index and mark the coord1 arg as poison — only StructuredBuffer - // atomics use both a struct index and a byte offset. if (!RTI.isStruct()) { auto *ConstantOffset = dyn_cast<ConstantInt>(Offset); if (!ConstantOffset || !ConstantOffset->isZero()) @@ -358,6 +351,23 @@ static void createAtomicBinOp(IntrinsicInst *II, AtomicRMWInst *AI, Offset = llvm::PoisonValue::get(Builder.getInt32Ty()); } + return {Index, Offset}; +} + +static void createAtomicBinOp(IntrinsicInst *II, AtomicRMWInst *AI, + dxil::ResourceTypeInfo &RTI) { + std::optional<dxil::AtomicBinOpCode> BinOpCode = + getAtomicBinOpCode(AI->getOperation()); + if (!BinOpCode) { + reportFatalUsageError("DXIL resource atomicrmw operation not implemented"); + return; + } + + const DataLayout &DL = AI->getDataLayout(); + IRBuilder<> Builder(AI); + auto [Index, Offset] = + getAtomicResourceCoords(II, AI->getPointerOperand(), RTI, Builder, DL); + Value *BinOp = Builder.getInt32(static_cast<uint32_t>(*BinOpCode)); // Emit the target-independent intrinsic; DXILOpLowering lowers it to the @@ -370,13 +380,41 @@ static void createAtomicBinOp(IntrinsicInst *II, AtomicRMWInst *AI, AI->replaceAllUsesWith(Result); } -static void createAtomicBinOpIntrinsic(IntrinsicInst *II, AtomicRMWInst *AI, - dxil::ResourceTypeInfo &RTI) { +static void createAtomicCompareExchange(IntrinsicInst *II, + AtomicCmpXchgInst *AI, + dxil::ResourceTypeInfo &RTI) { + const DataLayout &DL = AI->getDataLayout(); + IRBuilder<> Builder(AI); + auto [Index, Offset] = + getAtomicResourceCoords(II, AI->getPointerOperand(), RTI, Builder, DL); + + Value *Compare = AI->getCompareOperand(); + Value *NewValue = AI->getNewValOperand(); + + Value *Original = Builder.CreateIntrinsic( + NewValue->getType(), Intrinsic::dx_resource_atomic_compare_exchange, + {II->getOperand(0), Index, Offset, Compare, NewValue}); + + // `cmpxchg` yields a { original, success } pair, but the DXIL op returns + // only the original value. Recover the success flag by comparing the + // returned value against the expected one. + Value *Success = Builder.CreateICmpEQ(Original, Compare); + Value *Result = + Builder.CreateInsertValue(PoisonValue::get(AI->getType()), Original, 0); + Result = Builder.CreateInsertValue(Result, Success, 1); + + AI->replaceAllUsesWith(Result); +} + +// Diagnose resource kinds that cannot carry atomic operations. `OpName` names +// the LLVM instruction being lowered so the diagnostic matches the source. +static void checkAtomicResourceKind(dxil::ResourceTypeInfo &RTI, + StringRef OpName) { switch (RTI.getResourceKind()) { case dxil::ResourceKind::TypedBuffer: case dxil::ResourceKind::RawBuffer: case dxil::ResourceKind::StructuredBuffer: - return createAtomicBinOp(II, AI, RTI); + return; case dxil::ResourceKind::Texture1D: case dxil::ResourceKind::Texture2D: case dxil::ResourceKind::Texture2DMS: @@ -388,23 +426,36 @@ static void createAtomicBinOpIntrinsic(IntrinsicInst *II, AtomicRMWInst *AI, case dxil::ResourceKind::TextureCubeArray: case dxil::ResourceKind::FeedbackTexture2D: case dxil::ResourceKind::FeedbackTexture2DArray: - reportFatalUsageError( - "DXIL atomicrmw not implemented for texture resources"); + reportFatalUsageError(Twine("DXIL ") + OpName + + " not implemented for texture resources"); return; case dxil::ResourceKind::CBuffer: case dxil::ResourceKind::Sampler: case dxil::ResourceKind::TBuffer: - reportFatalUsageError( - "DXIL atomicrmw not implemented for this resource type"); + reportFatalUsageError(Twine("DXIL ") + OpName + + " not implemented for this resource type"); return; case dxil::ResourceKind::RTAccelerationStructure: case dxil::ResourceKind::Invalid: case dxil::ResourceKind::NumEntries: - llvm_unreachable("Invalid resource kind for atomicrmw"); + llvm_unreachable("Invalid resource kind for atomic operation"); } llvm_unreachable("Unhandled case in switch"); } +static void createAtomicBinOpIntrinsic(IntrinsicInst *II, AtomicRMWInst *AI, + dxil::ResourceTypeInfo &RTI) { + checkAtomicResourceKind(RTI, "atomicrmw"); + createAtomicBinOp(II, AI, RTI); +} + +static void createAtomicCompareExchangeIntrinsic(IntrinsicInst *II, + AtomicCmpXchgInst *AI, + dxil::ResourceTypeInfo &RTI) { + checkAtomicResourceKind(RTI, "cmpxchg"); + createAtomicCompareExchange(II, AI, RTI); +} + static void createTypedBufferLoad(IntrinsicInst *II, LoadInst *LI, dxil::ResourceTypeInfo &RTI) { const DataLayout &DL = LI->getDataLayout(); @@ -720,6 +771,8 @@ static Instruction *getStoreLoadPointerOperand(Instruction *AI) { return dyn_cast<Instruction>(SI->getPointerOperand()); if (auto *RMWI = dyn_cast<AtomicRMWInst>(AI)) return dyn_cast<Instruction>(RMWI->getPointerOperand()); + if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(AI)) + return dyn_cast<Instruction>(CXI->getPointerOperand()); return nullptr; } @@ -1001,6 +1054,9 @@ static void replaceAccess(IntrinsicInst *II, dxil::ResourceTypeInfo &RTI) { } else if (auto *AI = dyn_cast<AtomicRMWInst>(U)) { createAtomicBinOpIntrinsic(II, AI, RTI); DeadInsts.push_back(AI); + } else if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(U)) { + createAtomicCompareExchangeIntrinsic(II, CXI, RTI); + DeadInsts.push_back(CXI); } else llvm_unreachable("Unhandled instruction - pointer escaped?"); } diff --git a/llvm/test/CodeGen/DirectX/ResourceAtomicCompareExchange-i64-sm65.ll b/llvm/test/CodeGen/DirectX/ResourceAtomicCompareExchange-i64-sm65.ll new file mode 100644 index 0000000000000..0673bb13bc114 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/ResourceAtomicCompareExchange-i64-sm65.ll @@ -0,0 +1,17 @@ +; RUN: not opt -S -dxil-resource-access -dxil-op-lower -mtriple=dxil-pc-shadermodel6.5-compute %s 2>&1 | FileCheck %s + +; Verify resource i64 cmpxchg rejects shader models before SM 6.6, where +; dx.op.atomicCompareExchange gained i64 overload support. + +target triple = "dxil-pc-shadermodel6.5-compute" + +define i64 @cmpxchg_i64(i32 %index, i64 %cmp, i64 %value) { + %buffer = call target("dx.RawBuffer", i64, 1, 0, 0) + @llvm.dx.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, ptr null) + %ptr = call ptr @llvm.dx.resource.getpointer( + target("dx.RawBuffer", i64, 1, 0, 0) %buffer, i32 %index) + ; CHECK: Cannot create AtomicCompareExchange operation: Invalid overload type + %pair = cmpxchg ptr %ptr, i64 %cmp, i64 %value monotonic monotonic + %old = extractvalue { i64, i1 } %pair, 0 + ret i64 %old +} diff --git a/llvm/test/CodeGen/DirectX/ResourceAtomicCompareExchange.ll b/llvm/test/CodeGen/DirectX/ResourceAtomicCompareExchange.ll new file mode 100644 index 0000000000000..dacd4732faa49 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/ResourceAtomicCompareExchange.ll @@ -0,0 +1,57 @@ +; RUN: opt -S -dxil-resource-access -dxil-op-lower %s | FileCheck %s --check-prefixes=CHECK,I32 +; RUN: opt -S -dxil-resource-access -dxil-op-lower -mtriple=dxil-pc-shadermodel6.6-compute %s | FileCheck %s --check-prefixes=CHECK,I32,I64 + +; Verify cmpxchg through a dx.resource.getpointer is lowered to +; dx.op.atomicCompareExchange for UAV resources. The DXIL op returns only the +; original value, so the { value, success } pair that cmpxchg produces is +; rebuilt by comparing the returned value against the expected one. + +target triple = "dxil-pc-shadermodel6.6-compute" + +; CHECK-LABEL: define i32 @cmpxchg_i32( +define i32 @cmpxchg_i32(i32 %index, i32 %cmp, i32 %value) { + %buffer = call target("dx.RawBuffer", i32, 1, 0, 0) + @llvm.dx.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, ptr null) + %ptr = call ptr @llvm.dx.resource.getpointer( + target("dx.RawBuffer", i32, 1, 0, 0) %buffer, i32 %index) + + ; I32: [[ORIG:%.*]] = call i32 @dx.op.atomicCompareExchange.i32(i32 79, %dx.types.Handle %{{.*}}, i32 %index, i32 0, i32 0, i32 %cmp, i32 %value) + ; I32: [[OK:%.*]] = icmp eq i32 [[ORIG]], %cmp + ; I32: [[AGG:%.*]] = insertvalue { i32, i1 } poison, i32 [[ORIG]], 0 + ; I32: insertvalue { i32, i1 } [[AGG]], i1 [[OK]], 1 + %pair = cmpxchg ptr %ptr, i32 %cmp, i32 %value monotonic monotonic + %old = extractvalue { i32, i1 } %pair, 0 + ret i32 %old +} + +; A ByteAddressBuffer is not a struct, so the byte offset is folded into the +; index and coord1 is poison. +; CHECK-LABEL: define i32 @cmpxchg_i32_byteaddress( +define i32 @cmpxchg_i32_byteaddress(i32 %offset, i32 %cmp, i32 %value) { + %buffer = call target("dx.RawBuffer", i8, 1, 0, 0) + @llvm.dx.resource.handlefrombinding(i32 0, i32 1, i32 1, i32 0, ptr null) + %ptr = call ptr @llvm.dx.resource.getpointer( + target("dx.RawBuffer", i8, 1, 0, 0) %buffer, i32 %offset) + + ; I32: [[ORIG:%.*]] = call i32 @dx.op.atomicCompareExchange.i32(i32 79, %dx.types.Handle %{{.*}}, i32 %offset, i32 poison, i32 0, i32 %cmp, i32 %value) + ; I32: [[OK:%.*]] = icmp eq i32 [[ORIG]], %cmp + %pair = cmpxchg ptr %ptr, i32 %cmp, i32 %value monotonic monotonic + %old = extractvalue { i32, i1 } %pair, 0 + %ok = extractvalue { i32, i1 } %pair, 1 + %sel = select i1 %ok, i32 %old, i32 0 + ret i32 %sel +} + +; CHECK-LABEL: define i64 @cmpxchg_i64( +define i64 @cmpxchg_i64(i32 %index, i64 %cmp, i64 %value) { + %buffer = call target("dx.RawBuffer", i64, 1, 0, 0) + @llvm.dx.resource.handlefrombinding(i32 0, i32 2, i32 1, i32 0, ptr null) + %ptr = call ptr @llvm.dx.resource.getpointer( + target("dx.RawBuffer", i64, 1, 0, 0) %buffer, i32 %index) + + ; I64: [[ORIG:%.*]] = call i64 @dx.op.atomicCompareExchange.i64(i32 79, %dx.types.Handle %{{.*}}, i32 %index, i32 0, i32 0, i64 %cmp, i64 %value) + ; I64: icmp eq i64 [[ORIG]], %cmp + %pair = cmpxchg ptr %ptr, i64 %cmp, i64 %value monotonic monotonic + %old = extractvalue { i64, i1 } %pair, 0 + ret i64 %old +} diff --git a/llvm/test/CodeGen/DirectX/ResourceAtomicCompareExchangeTexture-errors.ll b/llvm/test/CodeGen/DirectX/ResourceAtomicCompareExchangeTexture-errors.ll new file mode 100644 index 0000000000000..810e903d247ff --- /dev/null +++ b/llvm/test/CodeGen/DirectX/ResourceAtomicCompareExchangeTexture-errors.ll @@ -0,0 +1,17 @@ +; RUN: not opt -S -dxil-resource-access %s 2>&1 | FileCheck %s + +; Verify that cmpxchg on a texture resource is rejected. DXIL has no texture +; atomic op, so the compare-exchange lowering must report an error. + +target triple = "dxil-pc-shadermodel6.6-compute" + +; CHECK: DXIL cmpxchg not implemented for texture resources + +define void @cmpxchg_texture(i32 %index, i32 %cmp, i32 %value) { + %texture = call target("dx.Texture", i32, 1, 0, 0, 2) + @llvm.dx.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, ptr null) + %ptr = call ptr @llvm.dx.resource.getpointer( + target("dx.Texture", i32, 1, 0, 0, 2) %texture, i32 %index) + %pair = cmpxchg ptr %ptr, i32 %cmp, i32 %value acq_rel monotonic + ret void +} diff --git a/llvm/test/CodeGen/DirectX/ResourceAtomicTexture-errors.ll b/llvm/test/CodeGen/DirectX/ResourceAtomicTexture-errors.ll new file mode 100644 index 0000000000000..ec0b958cf3155 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/ResourceAtomicTexture-errors.ll @@ -0,0 +1,17 @@ +; RUN: not opt -S -dxil-resource-access %s 2>&1 | FileCheck %s + +; Verify that atomic operations on texture resources are rejected. DXIL has no +; texture atomic op, so both atomicrmw and cmpxchg must report an error. + +target triple = "dxil-pc-shadermodel6.6-compute" + +; CHECK: DXIL atomicrmw not implemented for texture resources + +define void @atomicrmw_texture(i32 %index, i32 %value) { + %texture = call target("dx.Texture", i32, 1, 0, 0, 2) + @llvm.dx.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, ptr null) + %ptr = call ptr @llvm.dx.resource.getpointer( + target("dx.Texture", i32, 1, 0, 0, 2) %texture, i32 %index) + %old = atomicrmw add ptr %ptr, i32 %value acq_rel + ret void +} _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
