https://github.com/bob80905 created https://github.com/llvm/llvm-project/pull/222164
This PR adds the `InterlockedCompareStore` standalone function and resource methods. The operation lowers to `cmpxchg monotonic`. `InterlockedCompareStore` reports nothing, so the `cmpxchg` result stays unused. The PR adds a compare overload shape to `HLSLExternalSemaSource` and a matching resource method helper, which the `InterlockedCompareExchange` PR above it reuses. The PR also adds the 64-bit `InterlockedCompareStore64` methods, which DXIL gates on shader model 6.6. Fixes: https://github.com/llvm/llvm-project/issues/99128 Assisted by: Github Copilot >From 8208ec4531be02d7a15c50735b7520e927b4b811 Mon Sep 17 00:00:00 2001 From: Joshua Batista <[email protected]> Date: Tue, 8 Sep 2026 12:34:23 -0700 Subject: [PATCH] First attempt implementing InterlockedCompareStore --- clang/include/clang/Basic/Builtins.td | 9 ++ clang/lib/CodeGen/CGHLSLBuiltins.cpp | 36 ++++++-- clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp | 31 +++++++ clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h | 2 + clang/lib/Sema/HLSLExternalSemaSource.cpp | 53 +++++++++-- clang/lib/Sema/SemaHLSL.cpp | 12 ++- .../builtins/InterlockedCompareStore.hlsl | 54 +++++++++++ .../builtins/RWBuffer-Interlocked.hlsl | 5 ++ ...AddressBuffer-InterlockedCompareStore.hlsl | 36 ++++++++ ...AddressBuffer-InterlockedCompareStore.hlsl | 27 ++++++ ...r-InterlockedCompareStore-sm65-errors.hlsl | 26 ++++++ .../InterlockedCompareStore-errors.hlsl | 89 +++++++++++++++++++ .../DirectX/ResourceAtomicCompareStore.ll | 37 ++++++++ 13 files changed, 401 insertions(+), 16 deletions(-) create mode 100644 clang/test/CodeGenHLSL/builtins/InterlockedCompareStore.hlsl create mode 100644 clang/test/CodeGenHLSL/builtins/RWByteAddressBuffer-InterlockedCompareStore.hlsl create mode 100644 clang/test/CodeGenHLSL/builtins/RasterizerOrderedByteAddressBuffer-InterlockedCompareStore.hlsl create mode 100644 clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedCompareStore-sm65-errors.hlsl create mode 100644 clang/test/SemaHLSL/BuiltIns/InterlockedCompareStore-errors.hlsl create mode 100644 llvm/test/CodeGen/DirectX/ResourceAtomicCompareStore.ll diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td index e9a8a73afbd3c..de8b600db6763 100644 --- a/clang/include/clang/Basic/Builtins.td +++ b/clang/include/clang/Basic/Builtins.td @@ -5557,6 +5557,15 @@ def HLSLInterlockedAnd : LangBuiltin<"HLSL_LANG"> { let Prototype = "void (...)"; } +def HLSLInterlockedCompareStore : LangBuiltin<"HLSL_LANG"> { + let Spellings = ["__builtin_hlsl_interlocked_compare_store"]; + // SemaHLSL checks these arguments itself. Custom type checking also stops + // the default variadic promotion, which would otherwise widen a float + // `dest` to double and misreport its type in the diagnostic. + let Attributes = [NoThrow, CustomTypeChecking]; + let Prototype = "void (...)"; +} + def HLSLInterlockedExchange : LangBuiltin<"HLSL_LANG"> { let Spellings = ["__builtin_hlsl_interlocked_exchange"]; // SemaHLSL checks these arguments itself. Custom type checking also stops diff --git a/clang/lib/CodeGen/CGHLSLBuiltins.cpp b/clang/lib/CodeGen/CGHLSLBuiltins.cpp index d6c501a20f900..ef379aca49a5f 100644 --- a/clang/lib/CodeGen/CGHLSLBuiltins.cpp +++ b/clang/lib/CodeGen/CGHLSLBuiltins.cpp @@ -310,6 +310,16 @@ static Value *handleElementwiseF32ToF16(CodeGenFunction &CGF, llvm_unreachable("Intrinsic F32ToF16 not supported by target architecture"); } +// Scopeless atomics will default to CrossDevice, which is illegal in Vulkan. +// Set the memory scope: Workgroup for groupshared, otherwise Device. +static llvm::SyncScope::ID getHLSLAtomicScope(CodeGenFunction &CGF, + const LValue &DestLV) { + StringRef ScopeName = DestLV.getAddressSpace() == LangAS::hlsl_groupshared + ? "workgroup" + : "device"; + return CGF.getLLVMContext().getOrInsertSyncScopeID(ScopeName); +} + static Value *handleInterlockedOp(CodeGenFunction &CGF, const CallExpr *E, llvm::AtomicRMWInst::BinOp Op) { // Emit `atomicrmw <op>` directly — no intermediate intrinsic needed on @@ -323,13 +333,7 @@ static Value *handleInterlockedOp(CodeGenFunction &CGF, const CallExpr *E, "Intrinsic InterlockedOp value operand must be an integer, or a " "float for InterlockedExchange"); - // Scopeless atomics will default to CrossDevice, which is illegal in Vulkan. - // Set the memory scope: Workgroup for groupshared, otherwise Device. - StringRef ScopeName = DestLV.getAddressSpace() == LangAS::hlsl_groupshared - ? "workgroup" - : "device"; - llvm::SyncScope::ID SSID = - CGF.getLLVMContext().getOrInsertSyncScopeID(ScopeName); + llvm::SyncScope::ID SSID = getHLSLAtomicScope(CGF, DestLV); llvm::AtomicRMWInst *Call = CGF.Builder.CreateAtomicRMW( Op, DestAddr, Val, llvm::AtomicOrdering::Monotonic, SSID); @@ -343,6 +347,21 @@ static Value *handleInterlockedOp(CodeGenFunction &CGF, const CallExpr *E, return Call; } +// InterlockedCompareStore(dest, compare_value, value) stores `value` only when +// `dest` holds `compare_value`. It reports nothing, so the `cmpxchg` result is +// unused. DXILResourceAccess and the SPIR-V selector both match `cmpxchg`. +static Value *handleInterlockedCompareStore(CodeGenFunction &CGF, + const CallExpr *E) { + LValue DestLV = CGF.EmitLValue(E->getArg(0)); + Address DestAddr = DestLV.getAddress(); + Value *Compare = CGF.EmitScalarExpr(E->getArg(1)); + Value *Val = CGF.EmitScalarExpr(E->getArg(2)); + + return CGF.Builder.CreateAtomicCmpXchg( + DestAddr, Compare, Val, llvm::AtomicOrdering::Monotonic, + llvm::AtomicOrdering::Monotonic, getHLSLAtomicScope(CGF, DestLV)); +} + static Value *emitBufferStride(CodeGenFunction *CGF, const Expr *HandleExpr, LValue &Stride) { // Figure out the stride of the buffer elements from the handle type. @@ -1484,6 +1503,9 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID, case Builtin::BI__builtin_hlsl_interlocked_and: { return handleInterlockedOp(*this, E, llvm::AtomicRMWInst::And); } + case Builtin::BI__builtin_hlsl_interlocked_compare_store: { + return handleInterlockedCompareStore(*this, E); + } case Builtin::BI__builtin_hlsl_interlocked_exchange: { return handleInterlockedOp(*this, E, llvm::AtomicRMWInst::Xchg); } diff --git a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp index 63f1dbb518edd..16bc77c5ea127 100644 --- a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp +++ b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp @@ -1753,6 +1753,9 @@ BuiltinTypeDeclBuilder::addByteAddressBufferInterlockedMethods() { "__builtin_hlsl_interlocked_add"); addByteAddressBufferInterlockedMethod("InterlockedAnd", AST.UnsignedIntTy, "__builtin_hlsl_interlocked_and"); + addByteAddressBufferInterlockedCompareStoreMethod( + "InterlockedCompareStore", AST.UnsignedIntTy, + "__builtin_hlsl_interlocked_compare_store"); addByteAddressBufferInterlockedMethod( "InterlockedExchange", AST.UnsignedIntTy, "__builtin_hlsl_interlocked_exchange", /*RequiresOriginalValue=*/true); @@ -1788,6 +1791,9 @@ BuiltinTypeDeclBuilder::addByteAddressBufferInterlockedMethods() { addByteAddressBufferInterlockedMethod("InterlockedAnd64", AST.UnsignedLongTy, "__builtin_hlsl_interlocked_and"); + addByteAddressBufferInterlockedCompareStoreMethod( + "InterlockedCompareStore64", AST.UnsignedLongTy, + "__builtin_hlsl_interlocked_compare_store"); addByteAddressBufferInterlockedMethod( "InterlockedExchange64", AST.UnsignedLongTy, "__builtin_hlsl_interlocked_exchange", /*RequiresOriginalValue=*/true); @@ -2715,6 +2721,31 @@ BuiltinTypeDeclBuilder::addByteAddressBufferInterlockedMethod( return *this; } +BuiltinTypeDeclBuilder & +BuiltinTypeDeclBuilder::addByteAddressBufferInterlockedCompareStoreMethod( + StringRef MethodName, QualType ValueTy, StringRef BuiltinName) { + assert(!Record->isCompleteDefinition() && "record is already complete"); + ASTContext &AST = SemaRef.getASTContext(); + using PH = BuiltinTypeMethodBuilder::PlaceHolder; + + // Compare-store reports nothing, so it has a single overload. It reaches the + // buffer slot the same way as the other interlocked methods. + QualType AddrSpaceElemTy = + AST.getAddrSpaceQualType(ValueTy, LangAS::hlsl_device); + QualType ElemPtrTy = AST.getPointerType(AddrSpaceElemTy); + + BuiltinTypeMethodBuilder MMB(*this, MethodName, AST.VoidTy); + MMB.addParam("Offset", AST.UnsignedIntTy) + .addParam("CompareValue", ValueTy) + .addParam("Value", ValueTy); + MMB.callBuiltin("__builtin_hlsl_resource_getpointer_typed", ElemPtrTy, + PH::Handle, PH::_0, ValueTy) + .dereference(PH::LastStmt) + .callBuiltin(BuiltinName, AST.VoidTy, PH::LastStmt, PH::_1, PH::_2); + MMB.finalize(); + return *this; +} + BuiltinTypeDeclBuilder &BuiltinTypeDeclBuilder::addAppendMethod() { using PH = BuiltinTypeMethodBuilder::PlaceHolder; ASTContext &AST = SemaRef.getASTContext(); diff --git a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h index 6d96b1df4cfea..92260868c2719 100644 --- a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h +++ b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h @@ -151,6 +151,8 @@ class BuiltinTypeDeclBuilder { addByteAddressBufferInterlockedMethod(StringRef MethodName, QualType ValueTy, StringRef BuiltinName, bool RequiresOriginalValue = false); + BuiltinTypeDeclBuilder &addByteAddressBufferInterlockedCompareStoreMethod( + StringRef MethodName, QualType ValueTy, StringRef BuiltinName); BuiltinTypeDeclBuilder &addAppendMethod(); BuiltinTypeDeclBuilder &addConsumeMethod(); diff --git a/clang/lib/Sema/HLSLExternalSemaSource.cpp b/clang/lib/Sema/HLSLExternalSemaSource.cpp index b1c64f99629b2..443b1d94757fc 100644 --- a/clang/lib/Sema/HLSLExternalSemaSource.cpp +++ b/clang/lib/Sema/HLSLExternalSemaSource.cpp @@ -812,24 +812,42 @@ void HLSLExternalSemaSource::defineHLSLTypesWithForwardDeclarations() { } } +// Shapes of the synthesized atomic overloads. The read-modify-write operations +// can report the previous value through a trailing reference. Every argument +// of compare-store is an input. +enum class AtomicOverloadShape { + Binary, // (dest, value) + BinaryWithOriginal, // (dest, value, original_value) + CompareStore, // (dest, compare_value, value) +}; + // Build a single overload of an HLSL atomic intrinsic in the hlsl namespace. // `dest` is an address-space-qualified reference; `original_value` (when // present) is a plain reference. The synthesized FunctionDecl aliases the // underlying clang builtin via BuiltinAliasAttr. static void buildAtomicOverload(Sema &S, NamespaceDecl *NS, StringRef FuncName, StringRef BuiltinName, QualType ElemTy, - LangAS DestAS, bool ThreeArg) { + LangAS DestAS, AtomicOverloadShape Shape) { ASTContext &AST = S.getASTContext(); QualType DestTy = AST.getLValueReferenceType(AST.getAddrSpaceQualType(ElemTy, DestAS)); QualType OrigRefTy = AST.getLValueReferenceType(ElemTy); - SmallVector<QualType, 3> ParamTypes; - ParamTypes.push_back(DestTy); - ParamTypes.push_back(ElemTy); - if (ThreeArg) + SmallVector<QualType, 3> ParamTypes = {DestTy, ElemTy}; + if (Shape == AtomicOverloadShape::BinaryWithOriginal) ParamTypes.push_back(OrigRefTy); + else if (Shape == AtomicOverloadShape::CompareStore) + ParamTypes.push_back(ElemTy); + + // The loop below stops at the end of ParamTypes, so a two-argument overload + // ignores the trailing name. + constexpr const char *BinaryNames[] = {"dest", "value", "original_value"}; + constexpr const char *CompareStoreNames[] = {"dest", "compare_value", + "value"}; + ArrayRef<const char *> ParamNames = Shape == AtomicOverloadShape::CompareStore + ? CompareStoreNames + : BinaryNames; FunctionProtoType::ExtProtoInfo EPI; QualType FuncTy = AST.getFunctionType(AST.VoidTy, ParamTypes, EPI); @@ -843,7 +861,6 @@ static void buildAtomicOverload(Sema &S, NamespaceDecl *NS, StringRef FuncName, SC_Extern, /*UsesFPIntrin=*/false, /*isInlineSpecified=*/false, /*hasWrittenPrototype=*/true); - constexpr const char *ParamNames[] = {"dest", "value", "original_value"}; SmallVector<ParmVarDecl *, 3> ParmDecls; unsigned I = 0; for (auto [ParamType, ParamName] : llvm::zip(ParamTypes, ParamNames)) { @@ -886,15 +903,37 @@ static void defineHLSLInterlockedFunc(Sema &S, NamespaceDecl *NS, for (bool ThreeArg : {false, true}) { if (RequiresOriginalValue && !ThreeArg) continue; - buildAtomicOverload(S, NS, FuncName, BuiltinName, ElemTy, AS, ThreeArg); + buildAtomicOverload(S, NS, FuncName, BuiltinName, ElemTy, AS, + ThreeArg ? AtomicOverloadShape::BinaryWithOriginal + : AtomicOverloadShape::Binary); } } +// Synthesize the InterlockedCompareStore overload set: {int, uint, int64_t, +// uint64_t} x {groupshared, device}. The operation reports nothing, so it has +// a single arity. Float compare-store is a separate intrinsic. +static void defineHLSLInterlockedCompareStoreFunc(Sema &S, NamespaceDecl *NS, + StringRef FuncName, + StringRef BuiltinName) { + ASTContext &AST = S.getASTContext(); + // HLSL: int64_t == long, uint64_t == unsigned long (see hlsl_basic_types.h). + QualType Elems[] = {AST.IntTy, AST.UnsignedIntTy, AST.LongTy, + AST.UnsignedLongTy}; + + for (QualType ElemTy : Elems) + for (LangAS AS : {LangAS::hlsl_groupshared, LangAS::hlsl_device}) + buildAtomicOverload(S, NS, FuncName, BuiltinName, ElemTy, AS, + AtomicOverloadShape::CompareStore); +} + void HLSLExternalSemaSource::defineHLSLAtomicIntrinsics() { defineHLSLInterlockedFunc(*SemaPtr, HLSLNamespace, "InterlockedAdd", "__builtin_hlsl_interlocked_add"); defineHLSLInterlockedFunc(*SemaPtr, HLSLNamespace, "InterlockedAnd", "__builtin_hlsl_interlocked_and"); + defineHLSLInterlockedCompareStoreFunc( + *SemaPtr, HLSLNamespace, "InterlockedCompareStore", + "__builtin_hlsl_interlocked_compare_store"); defineHLSLInterlockedFunc(*SemaPtr, HLSLNamespace, "InterlockedExchange", "__builtin_hlsl_interlocked_exchange", /*RequiresOriginalValue=*/true, diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp index 50735771f52a3..2694ddcc2e306 100644 --- a/clang/lib/Sema/SemaHLSL.cpp +++ b/clang/lib/Sema/SemaHLSL.cpp @@ -4712,6 +4712,7 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { } case Builtin::BI__builtin_hlsl_interlocked_add: case Builtin::BI__builtin_hlsl_interlocked_and: + case Builtin::BI__builtin_hlsl_interlocked_compare_store: case Builtin::BI__builtin_hlsl_interlocked_exchange: case Builtin::BI__builtin_hlsl_interlocked_max: case Builtin::BI__builtin_hlsl_interlocked_min: @@ -4724,9 +4725,14 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { // argument count, integer-type matching, and the address-space requirement // on `dest`. The checks below are a safety net for callers that invoke the // builtin by its mangled name and would otherwise reach CodeGen unchecked. + // InterlockedCompareStore takes `compare_value` and `value`, so its third + // argument is an input rather than an output. + const bool IsCompareStore = + BuiltinID == Builtin::BI__builtin_hlsl_interlocked_compare_store; // InterlockedExchange always reports the previous value, so it requires // `original_value` instead of accepting it as an optional argument. - if (BuiltinID == Builtin::BI__builtin_hlsl_interlocked_exchange) { + if (IsCompareStore || + BuiltinID == Builtin::BI__builtin_hlsl_interlocked_exchange) { if (SemaRef.checkArgCount(TheCall, 3)) return true; } else { @@ -4783,7 +4789,9 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { if (TheCall->getNumArgs() == 3) { if (CheckArgTypeMatches(&SemaRef, TheCall->getArg(2), DestTy)) return true; - if (CheckModifiableLValue(&SemaRef, TheCall, 2)) + // Only the read-modify-write operations write the previous value back + // through the third argument. For compare-store it is the new value. + if (!IsCompareStore && CheckModifiableLValue(&SemaRef, TheCall, 2)) return true; } diff --git a/clang/test/CodeGenHLSL/builtins/InterlockedCompareStore.hlsl b/clang/test/CodeGenHLSL/builtins/InterlockedCompareStore.hlsl new file mode 100644 index 0000000000000..05cad81f2c78a --- /dev/null +++ b/clang/test/CodeGenHLSL/builtins/InterlockedCompareStore.hlsl @@ -0,0 +1,54 @@ +// RUN: %clang_cc1 -std=hlsl2021 -finclude-default-header -triple \ +// RUN: dxil-pc-shadermodel6.6-library %s -emit-llvm -disable-llvm-passes -o - | \ +// RUN: FileCheck %s --check-prefixes=CHECK,DXCHECK + +// RUN: %clang_cc1 -std=hlsl2021 -finclude-default-header -triple \ +// RUN: spirv-pc-vulkan-library %s -emit-llvm -disable-llvm-passes -o - | \ +// RUN: FileCheck %s --check-prefixes=CHECK,SPVCHECK + +// Test basic lowering of HLSL InterlockedCompareStore to `cmpxchg monotonic`. +// The operation reports nothing, so it has a single 3-argument form and the +// `cmpxchg` result stays unused. + +groupshared int gs_i32; +groupshared uint gs_u32; +groupshared int64_t gs_i64; +groupshared uint64_t gs_u64; + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_int +// DXCHECK: cmpxchg ptr addrspace(3) {{.*}}@gs_i32{{.*}}, i32 %{{.*}}, i32 %{{.*}} syncscope("workgroup") monotonic monotonic +// SPVCHECK: cmpxchg ptr addrspace(3) {{.*}}@gs_i32{{.*}}, i32 %{{.*}}, i32 %{{.*}} syncscope("workgroup") monotonic monotonic +export void test_int(int cmp, int v) { + InterlockedCompareStore(gs_i32, cmp, v); +} + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_uint +// DXCHECK: cmpxchg ptr addrspace(3) {{.*}}@gs_u32{{.*}}, i32 %{{.*}}, i32 %{{.*}} syncscope("workgroup") monotonic monotonic +// SPVCHECK: cmpxchg ptr addrspace(3) {{.*}}@gs_u32{{.*}}, i32 %{{.*}}, i32 %{{.*}} syncscope("workgroup") monotonic monotonic +export void test_uint(uint cmp, uint v) { + InterlockedCompareStore(gs_u32, cmp, v); +} + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_int64 +// DXCHECK: cmpxchg ptr addrspace(3) {{.*}}@gs_i64{{.*}}, i64 %{{.*}}, i64 %{{.*}} syncscope("workgroup") monotonic monotonic +// SPVCHECK: cmpxchg ptr addrspace(3) {{.*}}@gs_i64{{.*}}, i64 %{{.*}}, i64 %{{.*}} syncscope("workgroup") monotonic monotonic +export void test_int64(int64_t cmp, int64_t v) { + InterlockedCompareStore(gs_i64, cmp, v); +} + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_uint64 +// DXCHECK: cmpxchg ptr addrspace(3) {{.*}}@gs_u64{{.*}}, i64 %{{.*}}, i64 %{{.*}} syncscope("workgroup") monotonic monotonic +// SPVCHECK: cmpxchg ptr addrspace(3) {{.*}}@gs_u64{{.*}}, i64 %{{.*}}, i64 %{{.*}} syncscope("workgroup") monotonic monotonic +export void test_uint64(uint64_t cmp, uint64_t v) { + InterlockedCompareStore(gs_u64, cmp, v); +} + +// A device-address-space destination uses the "device" scope instead. +RWBuffer<uint> Buf : register(u0); + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_device +// DXCHECK: cmpxchg ptr %{{.*}}, i32 %{{.*}}, i32 %{{.*}} syncscope("device") monotonic monotonic +// SPVCHECK: cmpxchg ptr addrspace(11) %{{.*}}, i32 %{{.*}}, i32 %{{.*}} syncscope("device") monotonic monotonic +export void test_device(uint cmp, uint v) { + InterlockedCompareStore(Buf[0], cmp, v); +} diff --git a/clang/test/CodeGenHLSL/builtins/RWBuffer-Interlocked.hlsl b/clang/test/CodeGenHLSL/builtins/RWBuffer-Interlocked.hlsl index 42b1813dc2dc3..3af09218eec62 100644 --- a/clang/test/CodeGenHLSL/builtins/RWBuffer-Interlocked.hlsl +++ b/clang/test/CodeGenHLSL/builtins/RWBuffer-Interlocked.hlsl @@ -39,6 +39,8 @@ RWBuffer<uint> UOut : register(u1); // DXCHECK: atomicrmw umax ptr %[[PTR8]], i32 1 syncscope("device") monotonic // DXCHECK: %[[PTR9:.*]] = call {{.*}} @llvm.dx.resource.getpointer.p0.tdx.TypedBuffer_i32_1_0_1t.i32(target("dx.TypedBuffer", i32, 1, 0, 1) %{{.*}}, i32 %{{.*}}) // DXCHECK: atomicrmw xchg ptr %[[PTR9]], i32 1 syncscope("device") monotonic +// DXCHECK: %[[PTR10:.*]] = call {{.*}} @llvm.dx.resource.getpointer.p0.tdx.TypedBuffer_i32_1_0_1t.i32(target("dx.TypedBuffer", i32, 1, 0, 1) %{{.*}}, i32 %{{.*}}) +// DXCHECK: cmpxchg ptr %[[PTR10]], i32 1, i32 2 syncscope("device") monotonic monotonic // SPVCHECK: %[[PTR1:.*]] = call {{.*}} @llvm.spv.resource.getpointer.{{.*}}(target("spirv.SignedImage", i32, {{.*}}) %{{.*}}, i32 %{{.*}}) // SPVCHECK: atomicrmw add ptr addrspace(11) %[[PTR1]], i32 1 syncscope("device") monotonic // SPVCHECK: %[[PTR2:.*]] = call {{.*}} @llvm.spv.resource.getpointer.{{.*}}(target("spirv.SignedImage", i32, {{.*}}) %{{.*}}, i32 %{{.*}}) @@ -57,6 +59,8 @@ RWBuffer<uint> UOut : register(u1); // SPVCHECK: atomicrmw umax ptr addrspace(11) %[[PTR8]], i32 1 syncscope("device") monotonic // SPVCHECK: %[[PTR9:.*]] = call {{.*}} @llvm.spv.resource.getpointer.{{.*}}(target("spirv.SignedImage", i32, {{.*}}) %{{.*}}, i32 %{{.*}}) // SPVCHECK: atomicrmw xchg ptr addrspace(11) %[[PTR9]], i32 1 syncscope("device") monotonic +// SPVCHECK: %[[PTR10:.*]] = call {{.*}} @llvm.spv.resource.getpointer.{{.*}}(target("spirv.SignedImage", i32, {{.*}}) %{{.*}}, i32 %{{.*}}) +// SPVCHECK: cmpxchg ptr addrspace(11) %[[PTR10]], i32 1, i32 2 syncscope("device") monotonic monotonic [shader("compute")] [numthreads(1,1,1)] void main(uint3 id : SV_DispatchThreadID) { @@ -70,4 +74,5 @@ void main(uint3 id : SV_DispatchThreadID) { InterlockedMax(UOut[id.x], 1u); int orig; InterlockedExchange(Out[id.x], 1, orig); + InterlockedCompareStore(Out[id.x], 1, 2); } diff --git a/clang/test/CodeGenHLSL/builtins/RWByteAddressBuffer-InterlockedCompareStore.hlsl b/clang/test/CodeGenHLSL/builtins/RWByteAddressBuffer-InterlockedCompareStore.hlsl new file mode 100644 index 0000000000000..39c380c1bdccc --- /dev/null +++ b/clang/test/CodeGenHLSL/builtins/RWByteAddressBuffer-InterlockedCompareStore.hlsl @@ -0,0 +1,36 @@ +// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header -triple \ +// RUN: dxil-pc-shadermodel6.6-library %s -emit-llvm -disable-llvm-passes -o - | \ +// RUN: FileCheck %s --check-prefixes=CHECK,DXCHECK + +// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header -triple \ +// RUN: spirv-pc-vulkan1.3-library %s -emit-llvm -disable-llvm-passes -o - | \ +// RUN: FileCheck %s --check-prefixes=CHECK,SPVCHECK + +// Test that the RWByteAddressBuffer::InterlockedCompareStore and +// InterlockedCompareStore64 member methods lower to `resource_getpointer -> +// cmpxchg`, for both DXIL and SPIR-V targets. Compare-store reports nothing, +// so there is no out parameter and the `cmpxchg` result stays unused. + +RWByteAddressBuffer BAB : register(u0); + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_uint +// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}} +// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}}) +// DXCHECK: cmpxchg ptr %[[PTR]], i32 %{{.*}}, i32 %{{.*}} syncscope("device") monotonic monotonic +// SPVCHECK: %[[HANDLE:.*]] = load target("spirv.VulkanBuffer", [0 x i8], 12, 1), ptr {{.*}} +// SPVCHECK: %[[PTR:.*]] = call ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.VulkanBuffer_a0i8_12_1t.i32(target("spirv.VulkanBuffer", [0 x i8], 12, 1) %[[HANDLE]], i32 %{{.*}}) +// SPVCHECK: cmpxchg ptr addrspace(11) %[[PTR]], i32 %{{.*}}, i32 %{{.*}} syncscope("device") monotonic monotonic +export void test_bab_uint(uint off, uint cmp, uint v) { + BAB.InterlockedCompareStore(off, cmp, v); +} + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_uint64 +// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}} +// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}}) +// DXCHECK: cmpxchg ptr %[[PTR]], i64 %{{.*}}, i64 %{{.*}} syncscope("device") monotonic monotonic +// SPVCHECK: %[[HANDLE:.*]] = load target("spirv.VulkanBuffer", [0 x i8], 12, 1), ptr {{.*}} +// SPVCHECK: %[[PTR:.*]] = call ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.VulkanBuffer_a0i8_12_1t.i32(target("spirv.VulkanBuffer", [0 x i8], 12, 1) %[[HANDLE]], i32 %{{.*}}) +// SPVCHECK: cmpxchg ptr addrspace(11) %[[PTR]], i64 %{{.*}}, i64 %{{.*}} syncscope("device") monotonic monotonic +export void test_bab_uint64(uint off, uint64_t cmp, uint64_t v) { + BAB.InterlockedCompareStore64(off, cmp, v); +} diff --git a/clang/test/CodeGenHLSL/builtins/RasterizerOrderedByteAddressBuffer-InterlockedCompareStore.hlsl b/clang/test/CodeGenHLSL/builtins/RasterizerOrderedByteAddressBuffer-InterlockedCompareStore.hlsl new file mode 100644 index 0000000000000..6e99265a720f8 --- /dev/null +++ b/clang/test/CodeGenHLSL/builtins/RasterizerOrderedByteAddressBuffer-InterlockedCompareStore.hlsl @@ -0,0 +1,27 @@ +// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header -triple \ +// RUN: dxil-pc-shadermodel6.6-library %s -emit-llvm -disable-llvm-passes -o - | \ +// RUN: FileCheck %s --check-prefixes=CHECK,DXCHECK + +// SPIR-V codegen for RasterizerOrderedByteAddressBuffer is not implemented +// yet (asserts in clang/lib/CodeGen/Targets/SPIR.cpp on +// `!ResAttrs.IsROV && "Rasterizer order views not implemented for SPIR-V yet"`). +// Add a `spirv-pc-vulkan1.3-library` RUN line here when SPIR-V ROV support +// lands. + +RasterizerOrderedByteAddressBuffer ROVB : register(u1); + +// CHECK-LABEL: define void @{{.*}}test_rovb_uint +// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}} +// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}}) +// DXCHECK: cmpxchg ptr %[[PTR]], i32 %{{.*}}, i32 %{{.*}} syncscope("device") monotonic monotonic +export void test_rovb_uint(uint off, uint cmp, uint v) { + ROVB.InterlockedCompareStore(off, cmp, v); +} + +// CHECK-LABEL: define void @{{.*}}test_rovb_uint64 +// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}} +// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}}) +// DXCHECK: cmpxchg ptr %[[PTR]], i64 %{{.*}}, i64 %{{.*}} syncscope("device") monotonic monotonic +export void test_rovb_uint64(uint off, uint64_t cmp, uint64_t v) { + ROVB.InterlockedCompareStore64(off, cmp, v); +} diff --git a/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedCompareStore-sm65-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedCompareStore-sm65-errors.hlsl new file mode 100644 index 0000000000000..ff4a438e64d21 --- /dev/null +++ b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedCompareStore-sm65-errors.hlsl @@ -0,0 +1,26 @@ +// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header \ +// RUN: -triple dxil-pc-shadermodel6.5-library %s -fsyntax-only -verify \ +// RUN: -verify-ignore-unexpected=warning + +RWByteAddressBuffer BAB : register(u0); +RasterizerOrderedByteAddressBuffer ROVB : register(u1); + +void sm65_no_bab_compare_store64(uint off, uint64_t cmp, uint64_t v) { + BAB.InterlockedCompareStore64(off, cmp, v); + // expected-error@-1 {{no member named 'InterlockedCompareStore64' in 'hlsl::RWByteAddressBuffer'}} +} + +void sm65_no_rovb_compare_store64(uint off, uint64_t cmp, uint64_t v) { + ROVB.InterlockedCompareStore64(off, cmp, v); + // expected-error@-1 {{no member named 'InterlockedCompareStore64' in 'hlsl::RasterizerOrderedByteAddressBuffer'}} +} + +void sm65_bab_compare_store32_ok(uint off, uint cmp, uint v) { + BAB.InterlockedCompareStore(off, cmp, v); +} + +groupshared int64_t gs_i64; +void sm65_direct_builtin(int64_t cmp, int64_t v) { + __builtin_hlsl_interlocked_compare_store(gs_i64, cmp, v); + // expected-error@-1 {{'__builtin_hlsl_interlocked_compare_store' requires shader model 6.6 or newer}} +} diff --git a/clang/test/SemaHLSL/BuiltIns/InterlockedCompareStore-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/InterlockedCompareStore-errors.hlsl new file mode 100644 index 0000000000000..a44df5527cb8e --- /dev/null +++ b/clang/test/SemaHLSL/BuiltIns/InterlockedCompareStore-errors.hlsl @@ -0,0 +1,89 @@ +// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header \ +// RUN: -triple dxil-pc-shadermodel6.6-library %s -emit-llvm-only \ +// RUN: -disable-llvm-passes -verify + +// InterlockedCompareStore is provided as a set of address-space-qualified +// overloads (groupshared/device, {int,uint,int64_t,uint64_t}). It reports +// nothing, so it has a single 3-argument form and no out parameter. Float +// compare-store is a separate intrinsic, so float and double have no overload. + +groupshared int gs_i32; +groupshared float gs_f32; +struct S { int x; }; +groupshared S gs_s; + +void too_few(int cmp) { + InterlockedCompareStore(gs_i32, cmp); // expected-error{{no matching function for call to 'InterlockedCompareStore'}} + // expected-note@*:* 8 {{candidate function}} +} + +void too_many(int cmp, int v, int extra) { + InterlockedCompareStore(gs_i32, cmp, v, extra); // expected-error{{no matching function for call to 'InterlockedCompareStore'}} + // expected-note@*:* 8 {{candidate function}} +} + +void local_dest(int cmp, int v) { + int dest; + InterlockedCompareStore(dest, cmp, v); // expected-error{{no matching function for call to 'InterlockedCompareStore'}} + // expected-note@*:* 8 {{candidate function}} +} + +void float_dest(float cmp, float v) { + InterlockedCompareStore(gs_f32, cmp, v); // expected-error{{no matching function for call to 'InterlockedCompareStore'}} + // expected-note@*:* 8 {{candidate function}} +} + +void struct_dest(int cmp, int v) { + InterlockedCompareStore(gs_s, cmp, v); // expected-error{{no matching function for call to 'InterlockedCompareStore'}} + // expected-note@*:* 8 {{candidate function}} +} + +void direct_too_few(int cmp) { + __builtin_hlsl_interlocked_compare_store(gs_i32, cmp); + // expected-error@-1 {{too few arguments to function call, expected 3, have 2}} +} + +void direct_too_many(int cmp, int v, int extra) { + __builtin_hlsl_interlocked_compare_store(gs_i32, cmp, v, extra); + // expected-error@-1 {{too many arguments to function call, expected 3, have 4}} +} + +void direct_non_integer_dest() { + S local_s; + __builtin_hlsl_interlocked_compare_store(local_s, 1, 2); + // expected-error@-1 {{1st argument must be a scalar integer type (was 'S')}} +} + +void direct_float_dest(float cmp, float v) { + __builtin_hlsl_interlocked_compare_store(gs_f32, cmp, v); + // expected-error@-1 {{1st argument must be a scalar integer type (was 'float')}} +} + +void direct_nonlvalue_dest(int cmp, int v) { + __builtin_hlsl_interlocked_compare_store(1, cmp, v); + // expected-error@-1 {{cannot bind non-lvalue argument '1' to out parameter}} +} + +void direct_mismatched_compare() { + uint cmp = 1; + __builtin_hlsl_interlocked_compare_store(gs_i32, cmp, 2); + // expected-error@-1 {{passing 'uint' (aka 'unsigned int') to parameter of incompatible type 'int'}} +} + +void direct_mismatched_value() { + uint v = 1; + __builtin_hlsl_interlocked_compare_store(gs_i32, 1, v); + // expected-error@-1 {{passing 'uint' (aka 'unsigned int') to parameter of incompatible type 'int'}} +} + +void direct_default_as_dest(int cmp, int v) { + int local; + __builtin_hlsl_interlocked_compare_store(local, cmp, v); + // expected-error@-1 {{1st argument to atomic builtin must reference groupshared or device memory (was 'int')}} +} + +// Unlike the read-modify-write operations, the third argument is the new value +// rather than an out parameter, so an rvalue is accepted here. +void direct_rvalue_value_ok() { + __builtin_hlsl_interlocked_compare_store(gs_i32, 1, 2); +} diff --git a/llvm/test/CodeGen/DirectX/ResourceAtomicCompareStore.ll b/llvm/test/CodeGen/DirectX/ResourceAtomicCompareStore.ll new file mode 100644 index 0000000000000..aa8e9dfc67b1d --- /dev/null +++ b/llvm/test/CodeGen/DirectX/ResourceAtomicCompareStore.ll @@ -0,0 +1,37 @@ +; RUN: opt -S -dxil-resource-access -dxil-op-lower -mtriple=dxil-pc-shadermodel6.6-compute %s | FileCheck %s + +; InterlockedCompareStore reports nothing, so it emits a `cmpxchg` whose result +; is unused. Lowering must still produce the DXIL AtomicCompareExchange op. + +target triple = "dxil-pc-shadermodel6.6-compute" + +; CHECK-LABEL: define void @bab_compare_store +define void @bab_compare_store(i32 %offset, i32 %cmp, i32 %val) { + %buffer = call target("dx.RawBuffer", i8, 1, 0, 0) + @llvm.dx.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, ptr null) + %ptr = call ptr @llvm.dx.resource.getpointer( + target("dx.RawBuffer", i8, 1, 0, 0) %buffer, i32 %offset) + ; CHECK: call i32 @dx.op.atomicCompareExchange.i32(i32 79, %dx.types.Handle %{{.*}}, i32 %offset, i32 poison, i32 0, i32 %cmp, i32 %val) + %old = cmpxchg ptr %ptr, i32 %cmp, i32 %val monotonic monotonic + ret void +} + +; The same call keeping the result proves the unused case above is not the only +; shape that lowers. `cmpxchg` yields a { value, success } pair, so the pass +; rebuilds that pair from the single value the DXIL op returns. +; CHECK-LABEL: define i32 @bab_compare_exchange +define i32 @bab_compare_exchange(i32 %offset, i32 %cmp, i32 %val) { + %buffer = call target("dx.RawBuffer", i8, 1, 0, 0) + @llvm.dx.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, ptr null) + %ptr = call ptr @llvm.dx.resource.getpointer( + target("dx.RawBuffer", i8, 1, 0, 0) %buffer, i32 %offset) + ; CHECK: [[OLD:%.*]] = call i32 @dx.op.atomicCompareExchange.i32(i32 79, %dx.types.Handle %{{.*}}, i32 %offset, i32 poison, i32 0, i32 %cmp, i32 %val) + ; CHECK: [[OK:%.*]] = icmp eq i32 [[OLD]], %cmp + ; CHECK: [[P0:%.*]] = insertvalue { i32, i1 } poison, i32 [[OLD]], 0 + ; CHECK: [[P1:%.*]] = insertvalue { i32, i1 } [[P0]], i1 [[OK]], 1 + %pair = cmpxchg ptr %ptr, i32 %cmp, i32 %val monotonic monotonic + ; CHECK: [[RES:%.*]] = extractvalue { i32, i1 } [[P1]], 0 + %old = extractvalue { i32, i1 } %pair, 0 + ; CHECK: ret i32 [[RES]] + ret i32 %old +} _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
