https://github.com/bob80905 created https://github.com/llvm/llvm-project/pull/222165
This PR adds the `InterlockedCompareExchange` standalone function and resource methods. The operation lowers to the same `cmpxchg monotonic` as `InterlockedCompareStore`. `InterlockedCompareExchange` reports the previous value, so an `extractvalue` reads it from the `cmpxchg` result and stores it through the `original_value` reference parameter. The PR also adds the 64-bit `InterlockedCompareExchange64` methods, which DXIL gates on shader model 6.6. Fixes: https://github.com/llvm/llvm-project/issues/99130 Assisted by: Github Copilot >From 83fd470639b181bc436c67b7f5e716de72c3032f Mon Sep 17 00:00:00 2001 From: Joshua Batista <[email protected]> Date: Tue, 8 Sep 2026 13:06:24 -0700 Subject: [PATCH] First attempt implementing InterlockedCompareExchange --- clang/include/clang/Basic/Builtins.td | 9 ++ clang/lib/CodeGen/CGHLSLBuiltins.cpp | 25 +++- clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp | 32 +++-- clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h | 5 +- clang/lib/Sema/HLSLExternalSemaSource.cpp | 67 ++++++---- clang/lib/Sema/SemaHLSL.cpp | 36 ++++-- .../builtins/InterlockedCompareExchange.hlsl | 70 +++++++++++ .../builtins/RWBuffer-Interlocked.hlsl | 5 + ...ressBuffer-InterlockedCompareExchange.hlsl | 44 +++++++ ...ressBuffer-InterlockedCompareExchange.hlsl | 35 ++++++ ...nterlockedCompareExchange-sm65-errors.hlsl | 30 +++++ .../InterlockedCompareExchange-errors.hlsl | 114 ++++++++++++++++++ 12 files changed, 420 insertions(+), 52 deletions(-) create mode 100644 clang/test/CodeGenHLSL/builtins/InterlockedCompareExchange.hlsl create mode 100644 clang/test/CodeGenHLSL/builtins/RWByteAddressBuffer-InterlockedCompareExchange.hlsl create mode 100644 clang/test/CodeGenHLSL/builtins/RasterizerOrderedByteAddressBuffer-InterlockedCompareExchange.hlsl create mode 100644 clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedCompareExchange-sm65-errors.hlsl create mode 100644 clang/test/SemaHLSL/BuiltIns/InterlockedCompareExchange-errors.hlsl diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td index de8b600db6763..c07cffbe4ebae 100644 --- a/clang/include/clang/Basic/Builtins.td +++ b/clang/include/clang/Basic/Builtins.td @@ -5557,6 +5557,15 @@ def HLSLInterlockedAnd : LangBuiltin<"HLSL_LANG"> { let Prototype = "void (...)"; } +def HLSLInterlockedCompareExchange : LangBuiltin<"HLSL_LANG"> { + let Spellings = ["__builtin_hlsl_interlocked_compare_exchange"]; + // SemaHLSL checks these arguments itself. Custom type checking also stops + // the default variadic promotion, which would otherwise widen a float + // `dest` to double and misreport its type in the diagnostic. + let Attributes = [NoThrow, CustomTypeChecking]; + let Prototype = "void (...)"; +} + def HLSLInterlockedCompareStore : LangBuiltin<"HLSL_LANG"> { let Spellings = ["__builtin_hlsl_interlocked_compare_store"]; // SemaHLSL checks these arguments itself. Custom type checking also stops diff --git a/clang/lib/CodeGen/CGHLSLBuiltins.cpp b/clang/lib/CodeGen/CGHLSLBuiltins.cpp index ef379aca49a5f..80835f6f27f2f 100644 --- a/clang/lib/CodeGen/CGHLSLBuiltins.cpp +++ b/clang/lib/CodeGen/CGHLSLBuiltins.cpp @@ -348,18 +348,30 @@ static Value *handleInterlockedOp(CodeGenFunction &CGF, const CallExpr *E, } // InterlockedCompareStore(dest, compare_value, value) stores `value` only when -// `dest` holds `compare_value`. It reports nothing, so the `cmpxchg` result is -// unused. DXILResourceAccess and the SPIR-V selector both match `cmpxchg`. -static Value *handleInterlockedCompareStore(CodeGenFunction &CGF, - const CallExpr *E) { +// `dest` holds `compare_value`. InterlockedCompareExchange takes the same +// arguments and additionally reports the previous value. DXILResourceAccess +// and the SPIR-V selector both match `cmpxchg`. +static Value *handleInterlockedCompareOp(CodeGenFunction &CGF, + const CallExpr *E) { LValue DestLV = CGF.EmitLValue(E->getArg(0)); Address DestAddr = DestLV.getAddress(); Value *Compare = CGF.EmitScalarExpr(E->getArg(1)); Value *Val = CGF.EmitScalarExpr(E->getArg(2)); - return CGF.Builder.CreateAtomicCmpXchg( + Value *Pair = CGF.Builder.CreateAtomicCmpXchg( DestAddr, Compare, Val, llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Monotonic, getHLSLAtomicScope(CGF, DestLV)); + + // Compare-store reports nothing, so it leaves the `cmpxchg` result unused. + if (E->getNumArgs() < 4) + return Pair; + + // `cmpxchg` yields a { previous value, success } pair. HLSL reports only the + // previous value, through the `original_value` reference parameter. + Value *Original = CGF.Builder.CreateExtractValue(Pair, 0); + LValue OrigLV = CGF.EmitLValue(E->getArg(3)); + CGF.EmitStoreThroughLValue(RValue::get(Original), OrigLV); + return Original; } static Value *emitBufferStride(CodeGenFunction *CGF, const Expr *HandleExpr, @@ -1503,8 +1515,9 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID, case Builtin::BI__builtin_hlsl_interlocked_and: { return handleInterlockedOp(*this, E, llvm::AtomicRMWInst::And); } + case Builtin::BI__builtin_hlsl_interlocked_compare_exchange: case Builtin::BI__builtin_hlsl_interlocked_compare_store: { - return handleInterlockedCompareStore(*this, E); + return handleInterlockedCompareOp(*this, E); } case Builtin::BI__builtin_hlsl_interlocked_exchange: { return handleInterlockedOp(*this, E, llvm::AtomicRMWInst::Xchg); diff --git a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp index 16bc77c5ea127..3c374f8649c55 100644 --- a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp +++ b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp @@ -1753,7 +1753,11 @@ BuiltinTypeDeclBuilder::addByteAddressBufferInterlockedMethods() { "__builtin_hlsl_interlocked_add"); addByteAddressBufferInterlockedMethod("InterlockedAnd", AST.UnsignedIntTy, "__builtin_hlsl_interlocked_and"); - addByteAddressBufferInterlockedCompareStoreMethod( + addByteAddressBufferInterlockedCompareMethod( + "InterlockedCompareExchange", AST.UnsignedIntTy, + "__builtin_hlsl_interlocked_compare_exchange", + /*WithOriginalValue=*/true); + addByteAddressBufferInterlockedCompareMethod( "InterlockedCompareStore", AST.UnsignedIntTy, "__builtin_hlsl_interlocked_compare_store"); addByteAddressBufferInterlockedMethod( @@ -1791,7 +1795,11 @@ BuiltinTypeDeclBuilder::addByteAddressBufferInterlockedMethods() { addByteAddressBufferInterlockedMethod("InterlockedAnd64", AST.UnsignedLongTy, "__builtin_hlsl_interlocked_and"); - addByteAddressBufferInterlockedCompareStoreMethod( + addByteAddressBufferInterlockedCompareMethod( + "InterlockedCompareExchange64", AST.UnsignedLongTy, + "__builtin_hlsl_interlocked_compare_exchange", + /*WithOriginalValue=*/true); + addByteAddressBufferInterlockedCompareMethod( "InterlockedCompareStore64", AST.UnsignedLongTy, "__builtin_hlsl_interlocked_compare_store"); addByteAddressBufferInterlockedMethod( @@ -2722,14 +2730,16 @@ BuiltinTypeDeclBuilder::addByteAddressBufferInterlockedMethod( } BuiltinTypeDeclBuilder & -BuiltinTypeDeclBuilder::addByteAddressBufferInterlockedCompareStoreMethod( - StringRef MethodName, QualType ValueTy, StringRef BuiltinName) { +BuiltinTypeDeclBuilder::addByteAddressBufferInterlockedCompareMethod( + StringRef MethodName, QualType ValueTy, StringRef BuiltinName, + bool WithOriginalValue) { assert(!Record->isCompleteDefinition() && "record is already complete"); ASTContext &AST = SemaRef.getASTContext(); using PH = BuiltinTypeMethodBuilder::PlaceHolder; - // Compare-store reports nothing, so it has a single overload. It reaches the - // buffer slot the same way as the other interlocked methods. + // Compare-store reports nothing and compare-exchange always reports the + // previous value, so each has a single overload. Both reach the buffer slot + // the same way as the other interlocked methods. QualType AddrSpaceElemTy = AST.getAddrSpaceQualType(ValueTy, LangAS::hlsl_device); QualType ElemPtrTy = AST.getPointerType(AddrSpaceElemTy); @@ -2738,10 +2748,16 @@ BuiltinTypeDeclBuilder::addByteAddressBufferInterlockedCompareStoreMethod( MMB.addParam("Offset", AST.UnsignedIntTy) .addParam("CompareValue", ValueTy) .addParam("Value", ValueTy); + if (WithOriginalValue) + MMB.addParam("OriginalValue", ValueTy, HLSLParamModifierAttr::Keyword_out); MMB.callBuiltin("__builtin_hlsl_resource_getpointer_typed", ElemPtrTy, PH::Handle, PH::_0, ValueTy) - .dereference(PH::LastStmt) - .callBuiltin(BuiltinName, AST.VoidTy, PH::LastStmt, PH::_1, PH::_2); + .dereference(PH::LastStmt); + if (WithOriginalValue) + MMB.callBuiltin(BuiltinName, AST.VoidTy, PH::LastStmt, PH::_1, PH::_2, + PH::_3); + else + MMB.callBuiltin(BuiltinName, AST.VoidTy, PH::LastStmt, PH::_1, PH::_2); MMB.finalize(); return *this; } diff --git a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h index 92260868c2719..aa77dc7aa7164 100644 --- a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h +++ b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h @@ -151,8 +151,9 @@ class BuiltinTypeDeclBuilder { addByteAddressBufferInterlockedMethod(StringRef MethodName, QualType ValueTy, StringRef BuiltinName, bool RequiresOriginalValue = false); - BuiltinTypeDeclBuilder &addByteAddressBufferInterlockedCompareStoreMethod( - StringRef MethodName, QualType ValueTy, StringRef BuiltinName); + BuiltinTypeDeclBuilder &addByteAddressBufferInterlockedCompareMethod( + StringRef MethodName, QualType ValueTy, StringRef BuiltinName, + bool WithOriginalValue = false); BuiltinTypeDeclBuilder &addAppendMethod(); BuiltinTypeDeclBuilder &addConsumeMethod(); diff --git a/clang/lib/Sema/HLSLExternalSemaSource.cpp b/clang/lib/Sema/HLSLExternalSemaSource.cpp index 443b1d94757fc..13539c421b8e0 100644 --- a/clang/lib/Sema/HLSLExternalSemaSource.cpp +++ b/clang/lib/Sema/HLSLExternalSemaSource.cpp @@ -813,12 +813,13 @@ void HLSLExternalSemaSource::defineHLSLTypesWithForwardDeclarations() { } // Shapes of the synthesized atomic overloads. The read-modify-write operations -// can report the previous value through a trailing reference. Every argument -// of compare-store is an input. +// can report the previous value through a trailing reference. Compare-store +// takes only inputs; compare-exchange adds the trailing reference. enum class AtomicOverloadShape { Binary, // (dest, value) BinaryWithOriginal, // (dest, value, original_value) CompareStore, // (dest, compare_value, value) + CompareExchange, // (dest, compare_value, value, original_value) }; // Build a single overload of an HLSL atomic intrinsic in the hlsl namespace. @@ -834,20 +835,32 @@ static void buildAtomicOverload(Sema &S, NamespaceDecl *NS, StringRef FuncName, AST.getLValueReferenceType(AST.getAddrSpaceQualType(ElemTy, DestAS)); QualType OrigRefTy = AST.getLValueReferenceType(ElemTy); - SmallVector<QualType, 3> ParamTypes = {DestTy, ElemTy}; - if (Shape == AtomicOverloadShape::BinaryWithOriginal) + SmallVector<QualType, 4> ParamTypes = {DestTy, ElemTy}; + switch (Shape) { + case AtomicOverloadShape::Binary: + break; + case AtomicOverloadShape::BinaryWithOriginal: ParamTypes.push_back(OrigRefTy); - else if (Shape == AtomicOverloadShape::CompareStore) + break; + case AtomicOverloadShape::CompareStore: + ParamTypes.push_back(ElemTy); + break; + case AtomicOverloadShape::CompareExchange: ParamTypes.push_back(ElemTy); + ParamTypes.push_back(OrigRefTy); + break; + } - // The loop below stops at the end of ParamTypes, so a two-argument overload - // ignores the trailing name. + // The loop below stops at the end of ParamTypes, so a shorter overload + // ignores the trailing names. constexpr const char *BinaryNames[] = {"dest", "value", "original_value"}; - constexpr const char *CompareStoreNames[] = {"dest", "compare_value", - "value"}; - ArrayRef<const char *> ParamNames = Shape == AtomicOverloadShape::CompareStore - ? CompareStoreNames - : BinaryNames; + constexpr const char *CompareNames[] = {"dest", "compare_value", "value", + "original_value"}; + const bool IsCompare = Shape == AtomicOverloadShape::CompareStore || + Shape == AtomicOverloadShape::CompareExchange; + ArrayRef<const char *> ParamNames = IsCompare + ? ArrayRef<const char *>(CompareNames) + : ArrayRef<const char *>(BinaryNames); FunctionProtoType::ExtProtoInfo EPI; QualType FuncTy = AST.getFunctionType(AST.VoidTy, ParamTypes, EPI); @@ -861,7 +874,7 @@ static void buildAtomicOverload(Sema &S, NamespaceDecl *NS, StringRef FuncName, SC_Extern, /*UsesFPIntrin=*/false, /*isInlineSpecified=*/false, /*hasWrittenPrototype=*/true); - SmallVector<ParmVarDecl *, 3> ParmDecls; + SmallVector<ParmVarDecl *, 4> ParmDecls; unsigned I = 0; for (auto [ParamType, ParamName] : llvm::zip(ParamTypes, ParamNames)) { IdentifierInfo &PII = AST.Idents.get(ParamName, tok::TokenKind::identifier); @@ -909,12 +922,14 @@ static void defineHLSLInterlockedFunc(Sema &S, NamespaceDecl *NS, } } -// Synthesize the InterlockedCompareStore overload set: {int, uint, int64_t, -// uint64_t} x {groupshared, device}. The operation reports nothing, so it has -// a single arity. Float compare-store is a separate intrinsic. -static void defineHLSLInterlockedCompareStoreFunc(Sema &S, NamespaceDecl *NS, - StringRef FuncName, - StringRef BuiltinName) { +// Synthesize the compare-and-swap overload sets: {int, uint, int64_t, +// uint64_t} x {groupshared, device}. Compare-store reports nothing and +// compare-exchange always reports the previous value, so each has a single +// arity. The float forms are separate intrinsics. +static void defineHLSLInterlockedCompareFunc(Sema &S, NamespaceDecl *NS, + StringRef FuncName, + StringRef BuiltinName, + AtomicOverloadShape Shape) { ASTContext &AST = S.getASTContext(); // HLSL: int64_t == long, uint64_t == unsigned long (see hlsl_basic_types.h). QualType Elems[] = {AST.IntTy, AST.UnsignedIntTy, AST.LongTy, @@ -922,8 +937,7 @@ static void defineHLSLInterlockedCompareStoreFunc(Sema &S, NamespaceDecl *NS, for (QualType ElemTy : Elems) for (LangAS AS : {LangAS::hlsl_groupshared, LangAS::hlsl_device}) - buildAtomicOverload(S, NS, FuncName, BuiltinName, ElemTy, AS, - AtomicOverloadShape::CompareStore); + buildAtomicOverload(S, NS, FuncName, BuiltinName, ElemTy, AS, Shape); } void HLSLExternalSemaSource::defineHLSLAtomicIntrinsics() { @@ -931,9 +945,14 @@ void HLSLExternalSemaSource::defineHLSLAtomicIntrinsics() { "__builtin_hlsl_interlocked_add"); defineHLSLInterlockedFunc(*SemaPtr, HLSLNamespace, "InterlockedAnd", "__builtin_hlsl_interlocked_and"); - defineHLSLInterlockedCompareStoreFunc( - *SemaPtr, HLSLNamespace, "InterlockedCompareStore", - "__builtin_hlsl_interlocked_compare_store"); + defineHLSLInterlockedCompareFunc( + *SemaPtr, HLSLNamespace, "InterlockedCompareExchange", + "__builtin_hlsl_interlocked_compare_exchange", + AtomicOverloadShape::CompareExchange); + defineHLSLInterlockedCompareFunc(*SemaPtr, HLSLNamespace, + "InterlockedCompareStore", + "__builtin_hlsl_interlocked_compare_store", + AtomicOverloadShape::CompareStore); defineHLSLInterlockedFunc(*SemaPtr, HLSLNamespace, "InterlockedExchange", "__builtin_hlsl_interlocked_exchange", /*RequiresOriginalValue=*/true, diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp index 2694ddcc2e306..bde8e806b5b82 100644 --- a/clang/lib/Sema/SemaHLSL.cpp +++ b/clang/lib/Sema/SemaHLSL.cpp @@ -4712,6 +4712,7 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { } case Builtin::BI__builtin_hlsl_interlocked_add: case Builtin::BI__builtin_hlsl_interlocked_and: + case Builtin::BI__builtin_hlsl_interlocked_compare_exchange: case Builtin::BI__builtin_hlsl_interlocked_compare_store: case Builtin::BI__builtin_hlsl_interlocked_exchange: case Builtin::BI__builtin_hlsl_interlocked_max: @@ -4729,10 +4730,17 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { // argument is an input rather than an output. const bool IsCompareStore = BuiltinID == Builtin::BI__builtin_hlsl_interlocked_compare_store; + // InterlockedCompareExchange adds `compare_value` before `value` and + // always reports the previous value, so it takes four arguments. + const bool IsCompareExchange = + BuiltinID == Builtin::BI__builtin_hlsl_interlocked_compare_exchange; // InterlockedExchange always reports the previous value, so it requires // `original_value` instead of accepting it as an optional argument. - if (IsCompareStore || - BuiltinID == Builtin::BI__builtin_hlsl_interlocked_exchange) { + if (IsCompareExchange) { + if (SemaRef.checkArgCount(TheCall, 4)) + return true; + } else if (IsCompareStore || + BuiltinID == Builtin::BI__builtin_hlsl_interlocked_exchange) { if (SemaRef.checkArgCount(TheCall, 3)) return true; } else { @@ -4783,17 +4791,21 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { {LangAS::hlsl_groupshared, LangAS::hlsl_device})) return true; - if (CheckArgTypeMatches(&SemaRef, TheCall->getArg(1), DestTy)) - return true; - - if (TheCall->getNumArgs() == 3) { - if (CheckArgTypeMatches(&SemaRef, TheCall->getArg(2), DestTy)) - return true; - // Only the read-modify-write operations write the previous value back - // through the third argument. For compare-store it is the new value. - if (!IsCompareStore && CheckModifiableLValue(&SemaRef, TheCall, 2)) + // Every argument after `dest` has the destination's type. + for (unsigned I = 1, E = TheCall->getNumArgs(); I != E; ++I) + if (CheckArgTypeMatches(&SemaRef, TheCall->getArg(I), DestTy)) return true; - } + + // Operations that report the previous value write it back through their + // last argument. Compare-store reports nothing, so its last argument is + // the new value, and the two-argument read-modify-write forms have no + // such argument at all. + const unsigned NumArgs = TheCall->getNumArgs(); + const bool HasOriginalValue = + !IsCompareStore && NumArgs == (IsCompareExchange ? 4u : 3u); + if (HasOriginalValue && + CheckModifiableLValue(&SemaRef, TheCall, NumArgs - 1)) + return true; TheCall->setType(SemaRef.Context.VoidTy); break; diff --git a/clang/test/CodeGenHLSL/builtins/InterlockedCompareExchange.hlsl b/clang/test/CodeGenHLSL/builtins/InterlockedCompareExchange.hlsl new file mode 100644 index 0000000000000..2ecb60edbcf4a --- /dev/null +++ b/clang/test/CodeGenHLSL/builtins/InterlockedCompareExchange.hlsl @@ -0,0 +1,70 @@ +// RUN: %clang_cc1 -std=hlsl2021 -finclude-default-header -triple \ +// RUN: dxil-pc-shadermodel6.6-library %s -emit-llvm -disable-llvm-passes -o - | \ +// RUN: FileCheck %s --check-prefixes=CHECK,DXCHECK + +// RUN: %clang_cc1 -std=hlsl2021 -finclude-default-header -triple \ +// RUN: spirv-pc-vulkan-library %s -emit-llvm -disable-llvm-passes -o - | \ +// RUN: FileCheck %s --check-prefixes=CHECK,SPVCHECK + +// Test basic lowering of HLSL InterlockedCompareExchange to `cmpxchg +// monotonic`. The operation reports the value that was in the destination +// before the operation, so the first element of the `cmpxchg` result goes to +// the out parameter. + +groupshared int gs_i32; +groupshared uint gs_u32; +groupshared int64_t gs_i64; +groupshared uint64_t gs_u64; + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_int +// DXCHECK: [[PAIR:%.*]] = cmpxchg ptr addrspace(3) {{.*}}@gs_i32{{.*}}, i32 %{{.*}}, i32 %{{.*}} syncscope("workgroup") monotonic monotonic +// SPVCHECK: [[PAIR:%.*]] = cmpxchg ptr addrspace(3) {{.*}}@gs_i32{{.*}}, i32 %{{.*}}, i32 %{{.*}} syncscope("workgroup") monotonic monotonic +// CHECK-NEXT: [[OLD:%.*]] = extractvalue { i32, i1 } [[PAIR]], 0 +// CHECK-NEXT: store i32 [[OLD]], ptr {{.*}}%orig +export void test_int(int cmp, int v) { + int orig; + InterlockedCompareExchange(gs_i32, cmp, v, orig); +} + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_uint +// DXCHECK: [[PAIR:%.*]] = cmpxchg ptr addrspace(3) {{.*}}@gs_u32{{.*}}, i32 %{{.*}}, i32 %{{.*}} syncscope("workgroup") monotonic monotonic +// SPVCHECK: [[PAIR:%.*]] = cmpxchg ptr addrspace(3) {{.*}}@gs_u32{{.*}}, i32 %{{.*}}, i32 %{{.*}} syncscope("workgroup") monotonic monotonic +// CHECK-NEXT: [[OLD:%.*]] = extractvalue { i32, i1 } [[PAIR]], 0 +// CHECK-NEXT: store i32 [[OLD]], ptr {{.*}}%orig +export void test_uint(uint cmp, uint v) { + uint orig; + InterlockedCompareExchange(gs_u32, cmp, v, orig); +} + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_int64 +// DXCHECK: [[PAIR:%.*]] = cmpxchg ptr addrspace(3) {{.*}}@gs_i64{{.*}}, i64 %{{.*}}, i64 %{{.*}} syncscope("workgroup") monotonic monotonic +// SPVCHECK: [[PAIR:%.*]] = cmpxchg ptr addrspace(3) {{.*}}@gs_i64{{.*}}, i64 %{{.*}}, i64 %{{.*}} syncscope("workgroup") monotonic monotonic +// CHECK-NEXT: [[OLD:%.*]] = extractvalue { i64, i1 } [[PAIR]], 0 +// CHECK-NEXT: store i64 [[OLD]], ptr {{.*}}%orig +export void test_int64(int64_t cmp, int64_t v) { + int64_t orig; + InterlockedCompareExchange(gs_i64, cmp, v, orig); +} + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_uint64 +// DXCHECK: [[PAIR:%.*]] = cmpxchg ptr addrspace(3) {{.*}}@gs_u64{{.*}}, i64 %{{.*}}, i64 %{{.*}} syncscope("workgroup") monotonic monotonic +// SPVCHECK: [[PAIR:%.*]] = cmpxchg ptr addrspace(3) {{.*}}@gs_u64{{.*}}, i64 %{{.*}}, i64 %{{.*}} syncscope("workgroup") monotonic monotonic +// CHECK-NEXT: [[OLD:%.*]] = extractvalue { i64, i1 } [[PAIR]], 0 +// CHECK-NEXT: store i64 [[OLD]], ptr {{.*}}%orig +export void test_uint64(uint64_t cmp, uint64_t v) { + uint64_t orig; + InterlockedCompareExchange(gs_u64, cmp, v, orig); +} + +// A device-address-space destination uses the "device" scope instead. +RWBuffer<uint> Buf : register(u0); + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_device +// DXCHECK: [[PAIR:%.*]] = cmpxchg ptr %{{.*}}, i32 %{{.*}}, i32 %{{.*}} syncscope("device") monotonic monotonic +// SPVCHECK: [[PAIR:%.*]] = cmpxchg ptr addrspace(11) %{{.*}}, i32 %{{.*}}, i32 %{{.*}} syncscope("device") monotonic monotonic +// CHECK-NEXT: [[OLD:%.*]] = extractvalue { i32, i1 } [[PAIR]], 0 +// CHECK-NEXT: store i32 [[OLD]], ptr {{.*}}%orig +export void test_device(uint cmp, uint v) { + uint orig; + InterlockedCompareExchange(Buf[0], cmp, v, orig); +} diff --git a/clang/test/CodeGenHLSL/builtins/RWBuffer-Interlocked.hlsl b/clang/test/CodeGenHLSL/builtins/RWBuffer-Interlocked.hlsl index 3af09218eec62..47cff714a7df7 100644 --- a/clang/test/CodeGenHLSL/builtins/RWBuffer-Interlocked.hlsl +++ b/clang/test/CodeGenHLSL/builtins/RWBuffer-Interlocked.hlsl @@ -41,6 +41,8 @@ RWBuffer<uint> UOut : register(u1); // DXCHECK: atomicrmw xchg ptr %[[PTR9]], i32 1 syncscope("device") monotonic // DXCHECK: %[[PTR10:.*]] = call {{.*}} @llvm.dx.resource.getpointer.p0.tdx.TypedBuffer_i32_1_0_1t.i32(target("dx.TypedBuffer", i32, 1, 0, 1) %{{.*}}, i32 %{{.*}}) // DXCHECK: cmpxchg ptr %[[PTR10]], i32 1, i32 2 syncscope("device") monotonic monotonic +// DXCHECK: %[[PTR11:.*]] = call {{.*}} @llvm.dx.resource.getpointer.p0.tdx.TypedBuffer_i32_1_0_1t.i32(target("dx.TypedBuffer", i32, 1, 0, 1) %{{.*}}, i32 %{{.*}}) +// DXCHECK: cmpxchg ptr %[[PTR11]], i32 1, i32 2 syncscope("device") monotonic monotonic // SPVCHECK: %[[PTR1:.*]] = call {{.*}} @llvm.spv.resource.getpointer.{{.*}}(target("spirv.SignedImage", i32, {{.*}}) %{{.*}}, i32 %{{.*}}) // SPVCHECK: atomicrmw add ptr addrspace(11) %[[PTR1]], i32 1 syncscope("device") monotonic // SPVCHECK: %[[PTR2:.*]] = call {{.*}} @llvm.spv.resource.getpointer.{{.*}}(target("spirv.SignedImage", i32, {{.*}}) %{{.*}}, i32 %{{.*}}) @@ -61,6 +63,8 @@ RWBuffer<uint> UOut : register(u1); // SPVCHECK: atomicrmw xchg ptr addrspace(11) %[[PTR9]], i32 1 syncscope("device") monotonic // SPVCHECK: %[[PTR10:.*]] = call {{.*}} @llvm.spv.resource.getpointer.{{.*}}(target("spirv.SignedImage", i32, {{.*}}) %{{.*}}, i32 %{{.*}}) // SPVCHECK: cmpxchg ptr addrspace(11) %[[PTR10]], i32 1, i32 2 syncscope("device") monotonic monotonic +// SPVCHECK: %[[PTR11:.*]] = call {{.*}} @llvm.spv.resource.getpointer.{{.*}}(target("spirv.SignedImage", i32, {{.*}}) %{{.*}}, i32 %{{.*}}) +// SPVCHECK: cmpxchg ptr addrspace(11) %[[PTR11]], i32 1, i32 2 syncscope("device") monotonic monotonic [shader("compute")] [numthreads(1,1,1)] void main(uint3 id : SV_DispatchThreadID) { @@ -75,4 +79,5 @@ void main(uint3 id : SV_DispatchThreadID) { int orig; InterlockedExchange(Out[id.x], 1, orig); InterlockedCompareStore(Out[id.x], 1, 2); + InterlockedCompareExchange(Out[id.x], 1, 2, orig); } diff --git a/clang/test/CodeGenHLSL/builtins/RWByteAddressBuffer-InterlockedCompareExchange.hlsl b/clang/test/CodeGenHLSL/builtins/RWByteAddressBuffer-InterlockedCompareExchange.hlsl new file mode 100644 index 0000000000000..40d391da06cbe --- /dev/null +++ b/clang/test/CodeGenHLSL/builtins/RWByteAddressBuffer-InterlockedCompareExchange.hlsl @@ -0,0 +1,44 @@ +// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header -triple \ +// RUN: dxil-pc-shadermodel6.6-library %s -emit-llvm -disable-llvm-passes -o - | \ +// RUN: FileCheck %s --check-prefixes=CHECK,DXCHECK + +// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header -triple \ +// RUN: spirv-pc-vulkan1.3-library %s -emit-llvm -disable-llvm-passes -o - | \ +// RUN: FileCheck %s --check-prefixes=CHECK,SPVCHECK + +// Test that the RWByteAddressBuffer::InterlockedCompareExchange and +// InterlockedCompareExchange64 member methods lower to `resource_getpointer -> +// cmpxchg`, for both DXIL and SPIR-V targets. The value that was in the +// destination before the operation goes to the out parameter. + +RWByteAddressBuffer BAB : register(u0); + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_uint +// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}} +// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}}) +// DXCHECK: %[[PAIR:.*]] = cmpxchg ptr %[[PTR]], i32 %{{.*}}, i32 %{{.*}} syncscope("device") monotonic monotonic +// SPVCHECK: %[[HANDLE:.*]] = load target("spirv.VulkanBuffer", [0 x i8], 12, 1), ptr {{.*}} +// SPVCHECK: %[[PTR:.*]] = call ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.VulkanBuffer_a0i8_12_1t.i32(target("spirv.VulkanBuffer", [0 x i8], 12, 1) %[[HANDLE]], i32 %{{.*}}) +// SPVCHECK: %[[PAIR:.*]] = cmpxchg ptr addrspace(11) %[[PTR]], i32 %{{.*}}, i32 %{{.*}} syncscope("device") monotonic monotonic +// CHECK-NEXT: %[[OLD:.*]] = extractvalue { i32, i1 } %[[PAIR]], 0 +// CHECK-NEXT: %[[OUT:.*]] = load ptr{{.*}}, ptr {{.*}}%OriginalValue.addr +// CHECK-NEXT: store i32 %[[OLD]], ptr{{.*}} %[[OUT]] +export void test_bab_uint(uint off, uint cmp, uint v) { + uint orig; + BAB.InterlockedCompareExchange(off, cmp, v, orig); +} + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_uint64 +// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}} +// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}}) +// DXCHECK: %[[PAIR:.*]] = cmpxchg ptr %[[PTR]], i64 %{{.*}}, i64 %{{.*}} syncscope("device") monotonic monotonic +// SPVCHECK: %[[HANDLE:.*]] = load target("spirv.VulkanBuffer", [0 x i8], 12, 1), ptr {{.*}} +// SPVCHECK: %[[PTR:.*]] = call ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.VulkanBuffer_a0i8_12_1t.i32(target("spirv.VulkanBuffer", [0 x i8], 12, 1) %[[HANDLE]], i32 %{{.*}}) +// SPVCHECK: %[[PAIR:.*]] = cmpxchg ptr addrspace(11) %[[PTR]], i64 %{{.*}}, i64 %{{.*}} syncscope("device") monotonic monotonic +// CHECK-NEXT: %[[OLD:.*]] = extractvalue { i64, i1 } %[[PAIR]], 0 +// CHECK-NEXT: %[[OUT:.*]] = load ptr{{.*}}, ptr {{.*}}%OriginalValue.addr +// CHECK-NEXT: store i64 %[[OLD]], ptr{{.*}} %[[OUT]] +export void test_bab_uint64(uint off, uint64_t cmp, uint64_t v) { + uint64_t orig; + BAB.InterlockedCompareExchange64(off, cmp, v, orig); +} diff --git a/clang/test/CodeGenHLSL/builtins/RasterizerOrderedByteAddressBuffer-InterlockedCompareExchange.hlsl b/clang/test/CodeGenHLSL/builtins/RasterizerOrderedByteAddressBuffer-InterlockedCompareExchange.hlsl new file mode 100644 index 0000000000000..e69b12cde0586 --- /dev/null +++ b/clang/test/CodeGenHLSL/builtins/RasterizerOrderedByteAddressBuffer-InterlockedCompareExchange.hlsl @@ -0,0 +1,35 @@ +// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header -triple \ +// RUN: dxil-pc-shadermodel6.6-library %s -emit-llvm -disable-llvm-passes -o - | \ +// RUN: FileCheck %s --check-prefixes=CHECK,DXCHECK + +// SPIR-V codegen for RasterizerOrderedByteAddressBuffer is not implemented +// yet (asserts in clang/lib/CodeGen/Targets/SPIR.cpp on +// `!ResAttrs.IsROV && "Rasterizer order views not implemented for SPIR-V yet"`). +// Add a `spirv-pc-vulkan1.3-library` RUN line here when SPIR-V ROV support +// lands. + +RasterizerOrderedByteAddressBuffer ROVB : register(u1); + +// CHECK-LABEL: define void @{{.*}}test_rovb_uint +// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}} +// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}}) +// DXCHECK: %[[PAIR:.*]] = cmpxchg ptr %[[PTR]], i32 %{{.*}}, i32 %{{.*}} syncscope("device") monotonic monotonic +// DXCHECK-NEXT: %[[OLD:.*]] = extractvalue { i32, i1 } %[[PAIR]], 0 +// DXCHECK-NEXT: %[[OUT:.*]] = load ptr, ptr {{.*}}%OriginalValue.addr +// DXCHECK-NEXT: store i32 %[[OLD]], ptr %[[OUT]] +export void test_rovb_uint(uint off, uint cmp, uint v) { + uint orig; + ROVB.InterlockedCompareExchange(off, cmp, v, orig); +} + +// CHECK-LABEL: define void @{{.*}}test_rovb_uint64 +// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}} +// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}}) +// DXCHECK: %[[PAIR:.*]] = cmpxchg ptr %[[PTR]], i64 %{{.*}}, i64 %{{.*}} syncscope("device") monotonic monotonic +// DXCHECK-NEXT: %[[OLD:.*]] = extractvalue { i64, i1 } %[[PAIR]], 0 +// DXCHECK-NEXT: %[[OUT:.*]] = load ptr, ptr {{.*}}%OriginalValue.addr +// DXCHECK-NEXT: store i64 %[[OLD]], ptr %[[OUT]] +export void test_rovb_uint64(uint off, uint64_t cmp, uint64_t v) { + uint64_t orig; + ROVB.InterlockedCompareExchange64(off, cmp, v, orig); +} diff --git a/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedCompareExchange-sm65-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedCompareExchange-sm65-errors.hlsl new file mode 100644 index 0000000000000..cdb6a8c9969bf --- /dev/null +++ b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedCompareExchange-sm65-errors.hlsl @@ -0,0 +1,30 @@ +// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header \ +// RUN: -triple dxil-pc-shadermodel6.5-library %s -fsyntax-only -verify \ +// RUN: -verify-ignore-unexpected=warning + +RWByteAddressBuffer BAB : register(u0); +RasterizerOrderedByteAddressBuffer ROVB : register(u1); + +void sm65_no_bab_compare_exchange64(uint off, uint64_t cmp, uint64_t v) { + uint64_t orig; + BAB.InterlockedCompareExchange64(off, cmp, v, orig); + // expected-error@-1 {{no member named 'InterlockedCompareExchange64' in 'hlsl::RWByteAddressBuffer'}} +} + +void sm65_no_rovb_compare_exchange64(uint off, uint64_t cmp, uint64_t v) { + uint64_t orig; + ROVB.InterlockedCompareExchange64(off, cmp, v, orig); + // expected-error@-1 {{no member named 'InterlockedCompareExchange64' in 'hlsl::RasterizerOrderedByteAddressBuffer'}} +} + +void sm65_bab_compare_exchange32_ok(uint off, uint cmp, uint v) { + uint orig; + BAB.InterlockedCompareExchange(off, cmp, v, orig); +} + +groupshared int64_t gs_i64; +void sm65_direct_builtin(int64_t cmp, int64_t v) { + int64_t orig; + __builtin_hlsl_interlocked_compare_exchange(gs_i64, cmp, v, orig); + // expected-error@-1 {{'__builtin_hlsl_interlocked_compare_exchange' requires shader model 6.6 or newer}} +} diff --git a/clang/test/SemaHLSL/BuiltIns/InterlockedCompareExchange-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/InterlockedCompareExchange-errors.hlsl new file mode 100644 index 0000000000000..468131f43d01e --- /dev/null +++ b/clang/test/SemaHLSL/BuiltIns/InterlockedCompareExchange-errors.hlsl @@ -0,0 +1,114 @@ +// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header \ +// RUN: -triple dxil-pc-shadermodel6.6-library %s -emit-llvm-only \ +// RUN: -disable-llvm-passes -verify + +// InterlockedCompareExchange is provided as a set of address-space-qualified +// overloads (groupshared/device, {int,uint,int64_t,uint64_t}). It reports the +// value that was in the destination, so it has a single 4-argument form with a +// trailing out parameter. Float compare-exchange is a separate intrinsic, so +// float and double have no overload. + +groupshared int gs_i32; +groupshared float gs_f32; +struct S { int x; }; +groupshared S gs_s; + +void too_few(int cmp, int v) { + InterlockedCompareExchange(gs_i32, cmp, v); // expected-error{{no matching function for call to 'InterlockedCompareExchange'}} + // expected-note@*:* 8 {{candidate function}} +} + +void too_many(int cmp, int v, int extra) { + int orig; + InterlockedCompareExchange(gs_i32, cmp, v, orig, extra); // expected-error{{no matching function for call to 'InterlockedCompareExchange'}} + // expected-note@*:* 8 {{candidate function}} +} + +void local_dest(int cmp, int v) { + int dest; + int orig; + InterlockedCompareExchange(dest, cmp, v, orig); // expected-error{{no matching function for call to 'InterlockedCompareExchange'}} + // expected-note@*:* 8 {{candidate function}} +} + +void float_dest(float cmp, float v) { + float orig; + InterlockedCompareExchange(gs_f32, cmp, v, orig); // expected-error{{no matching function for call to 'InterlockedCompareExchange'}} + // expected-note@*:* 8 {{candidate function}} +} + +void struct_dest(int cmp, int v) { + S orig; + InterlockedCompareExchange(gs_s, cmp, v, orig); // expected-error{{no matching function for call to 'InterlockedCompareExchange'}} + // expected-note@*:* 8 {{candidate function}} +} + +// The out parameter is a reference, so it cannot bind across types. +void mismatched_orig_type(int cmp, int v) { + float orig; + InterlockedCompareExchange(gs_i32, cmp, v, orig); // expected-error{{no matching function for call to 'InterlockedCompareExchange'}} + // expected-note@*:* 8 {{candidate function}} +} + +void direct_too_few(int cmp, int v) { + __builtin_hlsl_interlocked_compare_exchange(gs_i32, cmp, v); + // expected-error@-1 {{too few arguments to function call, expected 4, have 3}} +} + +void direct_too_many(int cmp, int v, int extra) { + int orig; + __builtin_hlsl_interlocked_compare_exchange(gs_i32, cmp, v, orig, extra); + // expected-error@-1 {{too many arguments to function call, expected 4, have 5}} +} + +void direct_non_integer_dest() { + S local_s; + int orig; + __builtin_hlsl_interlocked_compare_exchange(local_s, 1, 2, orig); + // expected-error@-1 {{1st argument must be a scalar integer type (was 'S')}} +} + +void direct_float_dest(float cmp, float v) { + float orig; + __builtin_hlsl_interlocked_compare_exchange(gs_f32, cmp, v, orig); + // expected-error@-1 {{1st argument must be a scalar integer type (was 'float')}} +} + +void direct_nonlvalue_dest(int cmp, int v) { + int orig; + __builtin_hlsl_interlocked_compare_exchange(1, cmp, v, orig); + // expected-error@-1 {{cannot bind non-lvalue argument '1' to out parameter}} +} + +// The last argument is an out parameter, so an rvalue is rejected there. +void direct_nonlvalue_original_value(int cmp, int v) { + __builtin_hlsl_interlocked_compare_exchange(gs_i32, cmp, v, 0); + // expected-error@-1 {{cannot bind non-lvalue argument '0' to out parameter}} +} + +void direct_mismatched_compare() { + uint cmp = 1; + int orig; + __builtin_hlsl_interlocked_compare_exchange(gs_i32, cmp, 2, orig); + // expected-error@-1 {{passing 'uint' (aka 'unsigned int') to parameter of incompatible type 'int'}} +} + +void direct_mismatched_value() { + uint v = 1; + int orig; + __builtin_hlsl_interlocked_compare_exchange(gs_i32, 1, v, orig); + // expected-error@-1 {{passing 'uint' (aka 'unsigned int') to parameter of incompatible type 'int'}} +} + +void direct_mismatched_original_value() { + uint orig; + __builtin_hlsl_interlocked_compare_exchange(gs_i32, 1, 2, orig); + // expected-error@-1 {{passing 'uint' (aka 'unsigned int') to parameter of incompatible type 'int'}} +} + +void direct_default_as_dest(int cmp, int v) { + int local; + int orig; + __builtin_hlsl_interlocked_compare_exchange(local, cmp, v, orig); + // expected-error@-1 {{1st argument to atomic builtin must reference groupshared or device memory (was 'int')}} +} _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
