https://github.com/KungFuDonkey created https://github.com/llvm/llvm-project/pull/190633
>From issue #99105, #99076, #99090, #99106 and adds the implementation of >DeviceMemoryBarrier(WithGroupSync) and AllMemoryBarrier(WithGroupSync) to DXIL >and SPIRV. >From 87d9da61a0f90334e08972432354f7263aa70d6f Mon Sep 17 00:00:00 2001 From: KungFuDonkey <[email protected]> Date: Mon, 6 Apr 2026 18:52:36 +0200 Subject: [PATCH] Added Device and AllMemoryBarriers --- clang/include/clang/Basic/Builtins.td | 24 ++++++++++ clang/lib/CodeGen/CGHLSLBuiltins.cpp | 22 +++++++++ clang/lib/CodeGen/CGHLSLRuntime.h | 6 +++ .../lib/Headers/hlsl/hlsl_alias_intrinsics.h | 48 ++++++++++++++++++- .../builtins/AllMemoryBarrier.hlsl | 20 ++++++++ .../AllMemoryBarrierWithGroupSync.hlsl | 20 ++++++++ .../builtins/DeviceMemoryBarrier.hlsl | 20 ++++++++ .../DeviceMemoryBarrierWithGroupSync.hlsl | 20 ++++++++ .../BuiltIns/AllMemoryBarrier-errors.hlsl | 6 +++ .../AllMemoryBarrierWithGroupSync-errors.hlsl | 6 +++ .../BuiltIns/DeviceMemoryBarrier-errors.hlsl | 6 +++ ...viceMemoryBarrierWithGroupSync-errors.hlsl | 6 +++ llvm/include/llvm/IR/IntrinsicsDirectX.td | 12 +++++ llvm/include/llvm/IR/IntrinsicsSPIRV.td | 4 ++ llvm/lib/Target/DirectX/DXIL.td | 8 ++++ .../Target/SPIRV/SPIRVInstructionSelector.cpp | 35 +++++++++++--- .../CodeGen/DirectX/all_memory_barrier.ll | 8 ++++ .../all_memory_barrier_with_group_sync.ll | 8 ++++ .../CodeGen/DirectX/device_memory_barrier.ll | 8 ++++ .../device_memory_barrier_with_group_sync.ll | 8 ++++ .../hlsl-intrinsics/all_memory_barrier.ll | 16 +++++++ .../all_memory_barrier_with_group_sync.ll | 17 +++++++ .../hlsl-intrinsics/device_memory_barrier.ll | 16 +++++++ .../device_memory_barrier_with_group_sync.ll | 17 +++++++ .../hlsl-intrinsics/group_memory_barrier.ll | 4 +- .../group_memory_barrier_with_group_sync.ll | 2 +- 26 files changed, 357 insertions(+), 10 deletions(-) create mode 100644 clang/test/CodeGenHLSL/builtins/AllMemoryBarrier.hlsl create mode 100644 clang/test/CodeGenHLSL/builtins/AllMemoryBarrierWithGroupSync.hlsl create mode 100644 clang/test/CodeGenHLSL/builtins/DeviceMemoryBarrier.hlsl create mode 100644 clang/test/CodeGenHLSL/builtins/DeviceMemoryBarrierWithGroupSync.hlsl create mode 100644 clang/test/SemaHLSL/BuiltIns/AllMemoryBarrier-errors.hlsl create mode 100644 clang/test/SemaHLSL/BuiltIns/AllMemoryBarrierWithGroupSync-errors.hlsl create mode 100644 clang/test/SemaHLSL/BuiltIns/DeviceMemoryBarrier-errors.hlsl create mode 100644 clang/test/SemaHLSL/BuiltIns/DeviceMemoryBarrierWithGroupSync-errors.hlsl create mode 100644 llvm/test/CodeGen/DirectX/all_memory_barrier.ll create mode 100644 llvm/test/CodeGen/DirectX/all_memory_barrier_with_group_sync.ll create mode 100644 llvm/test/CodeGen/DirectX/device_memory_barrier.ll create mode 100644 llvm/test/CodeGen/DirectX/device_memory_barrier_with_group_sync.ll create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-intrinsics/all_memory_barrier.ll create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-intrinsics/all_memory_barrier_with_group_sync.ll create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-intrinsics/device_memory_barrier.ll create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-intrinsics/device_memory_barrier_with_group_sync.ll diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td index b8bbc544595e2..33c176261c9e1 100644 --- a/clang/include/clang/Basic/Builtins.td +++ b/clang/include/clang/Basic/Builtins.td @@ -5492,6 +5492,30 @@ def HLSLClip: LangBuiltin<"HLSL_LANG"> { let Prototype = "void(...)"; } +def HLSLAllMemoryBarrier : LangBuiltin<"HLSL_LANG"> { + let Spellings = ["__builtin_hlsl_all_memory_barrier"]; + let Attributes = [NoThrow, Const]; + let Prototype = "void()"; +} + +def HLSLAllMemoryBarrierWithGroupSync: LangBuiltin<"HLSL_LANG"> { + let Spellings = ["__builtin_hlsl_all_memory_barrier_with_group_sync"]; + let Attributes = [NoThrow, Const]; + let Prototype = "void()"; +} + +def HLSLDeviceMemoryBarrier : LangBuiltin<"HLSL_LANG"> { + let Spellings = ["__builtin_hlsl_device_memory_barrier"]; + let Attributes = [NoThrow, Const]; + let Prototype = "void()"; +} + +def HLSLDeviceMemoryBarrierWithGroupSync: LangBuiltin<"HLSL_LANG"> { + let Spellings = ["__builtin_hlsl_device_memory_barrier_with_group_sync"]; + let Attributes = [NoThrow, Const]; + let Prototype = "void()"; +} + def HLSLGroupMemoryBarrier : LangBuiltin<"HLSL_LANG"> { let Spellings = ["__builtin_hlsl_group_memory_barrier"]; let Attributes = [NoThrow, Const]; diff --git a/clang/lib/CodeGen/CGHLSLBuiltins.cpp b/clang/lib/CodeGen/CGHLSLBuiltins.cpp index f510195dbd6cb..b82a237ecefca 100644 --- a/clang/lib/CodeGen/CGHLSLBuiltins.cpp +++ b/clang/lib/CodeGen/CGHLSLBuiltins.cpp @@ -1566,6 +1566,28 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID, assert(E->getArg(0)->getType()->hasFloatingRepresentation() && "clip operands types mismatch"); return handleHlslClip(E, this); + case Builtin::BI__builtin_hlsl_all_memory_barrier: { + Intrinsic::ID ID = CGM.getHLSLRuntime().getAllMemoryBarrierIntrinsic(); + return EmitRuntimeCall( + Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID)); + } + case Builtin::BI__builtin_hlsl_all_memory_barrier_with_group_sync: { + Intrinsic::ID ID = + CGM.getHLSLRuntime().getAllMemoryBarrierWithGroupSyncIntrinsic(); + return EmitRuntimeCall( + Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID)); + } + case Builtin::BI__builtin_hlsl_device_memory_barrier: { + Intrinsic::ID ID = CGM.getHLSLRuntime().getDeviceMemoryBarrierIntrinsic(); + return EmitRuntimeCall( + Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID)); + } + case Builtin::BI__builtin_hlsl_device_memory_barrier_with_group_sync: { + Intrinsic::ID ID = + CGM.getHLSLRuntime().getDeviceMemoryBarrierWithGroupSyncIntrinsic(); + return EmitRuntimeCall( + Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID)); + } case Builtin::BI__builtin_hlsl_group_memory_barrier: { Intrinsic::ID ID = CGM.getHLSLRuntime().getGroupMemoryBarrierIntrinsic(); return EmitRuntimeCall( diff --git a/clang/lib/CodeGen/CGHLSLRuntime.h b/clang/lib/CodeGen/CGHLSLRuntime.h index b1c5b3318a11e..21e7ddf394bbd 100644 --- a/clang/lib/CodeGen/CGHLSLRuntime.h +++ b/clang/lib/CodeGen/CGHLSLRuntime.h @@ -189,6 +189,12 @@ class CGHLSLRuntime { GENERATE_HLSL_INTRINSIC_FUNCTION(NonUniformResourceIndex, resource_nonuniformindex) GENERATE_HLSL_INTRINSIC_FUNCTION(BufferUpdateCounter, resource_updatecounter) + GENERATE_HLSL_INTRINSIC_FUNCTION(AllMemoryBarrier, all_memory_barrier) + GENERATE_HLSL_INTRINSIC_FUNCTION(AllMemoryBarrierWithGroupSync, + all_memory_barrier_with_group_sync) + GENERATE_HLSL_INTRINSIC_FUNCTION(DeviceMemoryBarrier, device_memory_barrier) + GENERATE_HLSL_INTRINSIC_FUNCTION(DeviceMemoryBarrierWithGroupSync, + device_memory_barrier_with_group_sync) GENERATE_HLSL_INTRINSIC_FUNCTION(GroupMemoryBarrier, group_memory_barrier) GENERATE_HLSL_INTRINSIC_FUNCTION(GroupMemoryBarrierWithGroupSync, group_memory_barrier_with_group_sync) diff --git a/clang/lib/Headers/hlsl/hlsl_alias_intrinsics.h b/clang/lib/Headers/hlsl/hlsl_alias_intrinsics.h index 80c415ef66644..9581c1543f468 100644 --- a/clang/lib/Headers/hlsl/hlsl_alias_intrinsics.h +++ b/clang/lib/Headers/hlsl/hlsl_alias_intrinsics.h @@ -3927,7 +3927,53 @@ _HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_radians) float4 radians(float4); //===----------------------------------------------------------------------===// -// GroupMemoryBarrierbuiltins +// AllMemoryBarrier builtins +//===----------------------------------------------------------------------===// + +/// \fn void AllMemoryBarrier(void) +/// \brief Blocks execution of all threads in a group until all memory +/// accesses have been completed. + +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_all_memory_barrier) +__attribute__((convergent)) void AllMemoryBarrier(void); + +//===----------------------------------------------------------------------===// +// AllMemoryBarrierWithGroupSync builtins +//===----------------------------------------------------------------------===// + +/// \fn void AllMemoryBarrierWithGroupSync(void) +/// \brief Blocks execution of all threads in a group until all memory +/// accesses have been completed and all threads in the group have reached this +/// call. + +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_all_memory_barrier_with_group_sync) +__attribute__((convergent)) void AllMemoryBarrierWithGroupSync(void); + +//===----------------------------------------------------------------------===// +// DeviceMemoryBarrier builtins +//===----------------------------------------------------------------------===// + +/// \fn void DeviceMemoryBarrier(void) +/// \brief Blocks execution of all threads in a group until all device memory +/// accesses have been completed. + +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_device_memory_barrier) +__attribute__((convergent)) void DeviceMemoryBarrier(void); + +//===----------------------------------------------------------------------===// +// DeviceMemoryBarrierWithGroupSync builtins +//===----------------------------------------------------------------------===// + +/// \fn void DeviceMemoryBarrierWithGroupSync(void) +/// \brief Blocks execution of all threads in a group until all device memory +/// accesses have been completed and all threads in the group have reached this +/// call. + +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_device_memory_barrier_with_group_sync) +__attribute__((convergent)) void DeviceMemoryBarrierWithGroupSync(void); + +//===----------------------------------------------------------------------===// +// GroupMemoryBarrier builtins //===----------------------------------------------------------------------===// /// \fn void GroupMemoryBarrier(void) diff --git a/clang/test/CodeGenHLSL/builtins/AllMemoryBarrier.hlsl b/clang/test/CodeGenHLSL/builtins/AllMemoryBarrier.hlsl new file mode 100644 index 0000000000000..90d51c716c771 --- /dev/null +++ b/clang/test/CodeGenHLSL/builtins/AllMemoryBarrier.hlsl @@ -0,0 +1,20 @@ +// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \ +// RUN: dxil-pc-shadermodel6.3-library %s \ +// RUN: -emit-llvm -disable-llvm-passes -o - | FileCheck %s \ +// RUN: -DTARGET=dx -check-prefixes=CHECK,CHECK-DXIL +// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \ +// RUN: spirv-unknown-vulkan-compute %s \ +// RUN: -emit-llvm -disable-llvm-passes -o - | FileCheck %s \ +// RUN: -DTARGET=spv -check-prefixes=CHECK,CHECK-SPIRV + +// CHECK-DXIL: define hidden void @ +// CHECK-SPIRV: define hidden spir_func void @ +void test_AllMemoryBarrier() { +// CHECK-DXIL: call void @llvm.[[TARGET]].all.memory.barrier() +// CHECK-SPIRV: call spir_func void @llvm.[[TARGET]].all.memory.barrier() + AllMemoryBarrier(); +} + +// CHECK: declare void @llvm.[[TARGET]].all.memory.barrier() #[[ATTRS:[0-9]+]] +// CHECK-NOT: attributes #[[ATTRS]] = {{.+}}memory(none){{.+}} +// CHECK: attributes #[[ATTRS]] = {{.+}}convergent{{.+}} diff --git a/clang/test/CodeGenHLSL/builtins/AllMemoryBarrierWithGroupSync.hlsl b/clang/test/CodeGenHLSL/builtins/AllMemoryBarrierWithGroupSync.hlsl new file mode 100644 index 0000000000000..6ddb69671e094 --- /dev/null +++ b/clang/test/CodeGenHLSL/builtins/AllMemoryBarrierWithGroupSync.hlsl @@ -0,0 +1,20 @@ +// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \ +// RUN: dxil-pc-shadermodel6.3-library %s \ +// RUN: -emit-llvm -disable-llvm-passes -o - | FileCheck %s \ +// RUN: -DTARGET=dx -check-prefixes=CHECK,CHECK-DXIL +// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \ +// RUN: spirv-unknown-vulkan-compute %s \ +// RUN: -emit-llvm -disable-llvm-passes -o - | FileCheck %s \ +// RUN: -DTARGET=spv -check-prefixes=CHECK,CHECK-SPIRV + +// CHECK-DXIL: define hidden void @ +// CHECK-SPIRV: define hidden spir_func void @ +void test_AllMemoryBarrierWithGroupSync() { +// CHECK-DXIL: call void @llvm.[[TARGET]].all.memory.barrier.with.group.sync() +// CHECK-SPIRV: call spir_func void @llvm.[[TARGET]].all.memory.barrier.with.group.sync() + AllMemoryBarrierWithGroupSync(); +} + +// CHECK: declare void @llvm.[[TARGET]].all.memory.barrier.with.group.sync() #[[ATTRS:[0-9]+]] +// CHECK-NOT: attributes #[[ATTRS]] = {{.+}}memory(none){{.+}} +// CHECK: attributes #[[ATTRS]] = {{.+}}convergent{{.+}} diff --git a/clang/test/CodeGenHLSL/builtins/DeviceMemoryBarrier.hlsl b/clang/test/CodeGenHLSL/builtins/DeviceMemoryBarrier.hlsl new file mode 100644 index 0000000000000..e2c08f7775c8c --- /dev/null +++ b/clang/test/CodeGenHLSL/builtins/DeviceMemoryBarrier.hlsl @@ -0,0 +1,20 @@ +// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \ +// RUN: dxil-pc-shadermodel6.3-library %s \ +// RUN: -emit-llvm -disable-llvm-passes -o - | FileCheck %s \ +// RUN: -DTARGET=dx -check-prefixes=CHECK,CHECK-DXIL +// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \ +// RUN: spirv-unknown-vulkan-compute %s \ +// RUN: -emit-llvm -disable-llvm-passes -o - | FileCheck %s \ +// RUN: -DTARGET=spv -check-prefixes=CHECK,CHECK-SPIRV + +// CHECK-DXIL: define hidden void @ +// CHECK-SPIRV: define hidden spir_func void @ +void test_DeviceMemoryBarrier() { +// CHECK-DXIL: call void @llvm.[[TARGET]].device.memory.barrier() +// CHECK-SPIRV: call spir_func void @llvm.[[TARGET]].device.memory.barrier() + DeviceMemoryBarrier(); +} + +// CHECK: declare void @llvm.[[TARGET]].device.memory.barrier() #[[ATTRS:[0-9]+]] +// CHECK-NOT: attributes #[[ATTRS]] = {{.+}}memory(none){{.+}} +// CHECK: attributes #[[ATTRS]] = {{.+}}convergent{{.+}} diff --git a/clang/test/CodeGenHLSL/builtins/DeviceMemoryBarrierWithGroupSync.hlsl b/clang/test/CodeGenHLSL/builtins/DeviceMemoryBarrierWithGroupSync.hlsl new file mode 100644 index 0000000000000..fa455f5f8338b --- /dev/null +++ b/clang/test/CodeGenHLSL/builtins/DeviceMemoryBarrierWithGroupSync.hlsl @@ -0,0 +1,20 @@ +// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \ +// RUN: dxil-pc-shadermodel6.3-library %s \ +// RUN: -emit-llvm -disable-llvm-passes -o - | FileCheck %s \ +// RUN: -DTARGET=dx -check-prefixes=CHECK,CHECK-DXIL +// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \ +// RUN: spirv-unknown-vulkan-compute %s \ +// RUN: -emit-llvm -disable-llvm-passes -o - | FileCheck %s \ +// RUN: -DTARGET=spv -check-prefixes=CHECK,CHECK-SPIRV + +// CHECK-DXIL: define hidden void @ +// CHECK-SPIRV: define hidden spir_func void @ +void test_DeviceMemoryBarrierWithGroupSync() { +// CHECK-DXIL: call void @llvm.[[TARGET]].device.memory.barrier.with.group.sync() +// CHECK-SPIRV: call spir_func void @llvm.[[TARGET]].device.memory.barrier.with.group.sync() + DeviceMemoryBarrierWithGroupSync(); +} + +// CHECK: declare void @llvm.[[TARGET]].device.memory.barrier.with.group.sync() #[[ATTRS:[0-9]+]] +// CHECK-NOT: attributes #[[ATTRS]] = {{.+}}memory(none){{.+}} +// CHECK: attributes #[[ATTRS]] = {{.+}}convergent{{.+}} diff --git a/clang/test/SemaHLSL/BuiltIns/AllMemoryBarrier-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/AllMemoryBarrier-errors.hlsl new file mode 100644 index 0000000000000..63fd11a3095d8 --- /dev/null +++ b/clang/test/SemaHLSL/BuiltIns/AllMemoryBarrier-errors.hlsl @@ -0,0 +1,6 @@ +// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -emit-llvm-only -disable-llvm-passes -verify + +void test_too_many_arg() { + __builtin_hlsl_all_memory_barrier(0); + // expected-error@-1 {{too many arguments to function call, expected 0, have 1}} +} diff --git a/clang/test/SemaHLSL/BuiltIns/AllMemoryBarrierWithGroupSync-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/AllMemoryBarrierWithGroupSync-errors.hlsl new file mode 100644 index 0000000000000..7a50be66ebb8c --- /dev/null +++ b/clang/test/SemaHLSL/BuiltIns/AllMemoryBarrierWithGroupSync-errors.hlsl @@ -0,0 +1,6 @@ +// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -emit-llvm-only -disable-llvm-passes -verify + +void test_too_many_arg() { + __builtin_hlsl_all_memory_barrier_with_group_sync(0); + // expected-error@-1 {{too many arguments to function call, expected 0, have 1}} +} diff --git a/clang/test/SemaHLSL/BuiltIns/DeviceMemoryBarrier-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/DeviceMemoryBarrier-errors.hlsl new file mode 100644 index 0000000000000..16a57a36fc399 --- /dev/null +++ b/clang/test/SemaHLSL/BuiltIns/DeviceMemoryBarrier-errors.hlsl @@ -0,0 +1,6 @@ +// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -emit-llvm-only -disable-llvm-passes -verify + +void test_too_many_arg() { + __builtin_hlsl_device_memory_barrier(0); + // expected-error@-1 {{too many arguments to function call, expected 0, have 1}} +} diff --git a/clang/test/SemaHLSL/BuiltIns/DeviceMemoryBarrierWithGroupSync-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/DeviceMemoryBarrierWithGroupSync-errors.hlsl new file mode 100644 index 0000000000000..ddbd7ba768009 --- /dev/null +++ b/clang/test/SemaHLSL/BuiltIns/DeviceMemoryBarrierWithGroupSync-errors.hlsl @@ -0,0 +1,6 @@ +// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -emit-llvm-only -disable-llvm-passes -verify + +void test_too_many_arg() { + __builtin_hlsl_device_memory_barrier_with_group_sync(0); + // expected-error@-1 {{too many arguments to function call, expected 0, have 1}} +} diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td index 728bf47a17516..fecbff82e3638 100644 --- a/llvm/include/llvm/IR/IntrinsicsDirectX.td +++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td @@ -285,6 +285,18 @@ def int_dx_firstbituhigh : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, def int_dx_firstbitshigh : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>], [llvm_anyint_ty], [IntrNoMem]>; def int_dx_firstbitlow : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>], [llvm_anyint_ty], [IntrNoMem]>; +def int_dx_all_memory_barrier + : DefaultAttrsIntrinsic<[], [], [IntrConvergent]>; + +def int_dx_all_memory_barrier_with_group_sync + : DefaultAttrsIntrinsic<[], [], [IntrConvergent]>; + +def int_dx_device_memory_barrier + : DefaultAttrsIntrinsic<[], [], [IntrConvergent]>; + +def int_dx_device_memory_barrier_with_group_sync + : DefaultAttrsIntrinsic<[], [], [IntrConvergent]>; + def int_dx_group_memory_barrier : DefaultAttrsIntrinsic<[], [], [IntrConvergent]>; diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td index d2a5fa1f08724..e0c64277ae342 100644 --- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td +++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td @@ -157,6 +157,10 @@ def int_spv_rsqrt : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty] def int_spv_quad_read_across_y : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem]>; def int_spv_sign : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>], [llvm_any_ty], [IntrNoMem]>; def int_spv_radians : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty], [IntrNoMem]>; + def int_spv_all_memory_barrier : DefaultAttrsIntrinsic<[], [], [IntrConvergent]>; + def int_spv_all_memory_barrier_with_group_sync : DefaultAttrsIntrinsic<[], [], [IntrConvergent]>; + def int_spv_device_memory_barrier : DefaultAttrsIntrinsic<[], [], [IntrConvergent]>; + def int_spv_device_memory_barrier_with_group_sync : DefaultAttrsIntrinsic<[], [], [IntrConvergent]>; def int_spv_group_memory_barrier : DefaultAttrsIntrinsic<[], [], [IntrConvergent]>; def int_spv_group_memory_barrier_with_group_sync : ClangBuiltin<"__builtin_spirv_group_barrier">, DefaultAttrsIntrinsic<[], [], [IntrConvergent]>; diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td index 0a1e0114aa3bb..41f09f9db1017 100644 --- a/llvm/lib/Target/DirectX/DXIL.td +++ b/llvm/lib/Target/DirectX/DXIL.td @@ -931,6 +931,14 @@ def GetDimensions : DXILOp<72, getDimensions> { def Barrier : DXILOp<80, barrier> { let Doc = "inserts a memory barrier in the shader"; let intrinsics = [ + IntrinSelect<int_dx_all_memory_barrier, + [IntrinArgI32<BarrierMode_AllMemoryBarrier>]>, + IntrinSelect<int_dx_all_memory_barrier_with_group_sync, + [IntrinArgI32<BarrierMode_AllMemoryBarrierWithGroupSync>]>, + IntrinSelect<int_dx_device_memory_barrier, + [IntrinArgI32<BarrierMode_DeviceMemoryBarrier>]>, + IntrinSelect<int_dx_device_memory_barrier_with_group_sync, + [IntrinArgI32<BarrierMode_DeviceMemoryBarrierWithGroupSync>]>, IntrinSelect<int_dx_group_memory_barrier, [IntrinArgI32<BarrierMode_GroupMemoryBarrier>]>, IntrinSelect<int_dx_group_memory_barrier_with_group_sync, diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index 9da8397efc22f..8417e591d6128 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -2928,18 +2928,19 @@ bool SPIRVInstructionSelector::selectBarrierInst(MachineInstr &I, "Device Scope must set UniformMemory and ImageMemory semantic " "in Barrier instruction"); - Register MemSemReg = buildI32Constant(MemSem, I); - Register ScopeReg = buildI32Constant(Scope, I); MachineBasicBlock &BB = *I.getParent(); - auto MI = - BuildMI(BB, I, I.getDebugLoc(), TII.get(BarrierType)).addUse(ScopeReg); + auto MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(BarrierType)); // OpControlBarrier needs to also set Execution Scope if (WithGroupSync) { - MI.addUse(ScopeReg); + Register ExecReg = buildI32Constant(SPIRV::Scope::Workgroup, I); + MI.addUse(ExecReg); } - MI.addUse(MemSemReg).constrainAllUses(TII, TRI, RBI); + Register ScopeReg = buildI32Constant(Scope, I); + Register MemSemReg = buildI32Constant(MemSem, I); + + MI.addUse(ScopeReg).addUse(MemSemReg).constrainAllUses(TII, TRI, RBI); return true; } @@ -4521,6 +4522,28 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/true); case Intrinsic::spv_firstbitlow: // There is no CL equivlent of FindILsb return selectFirstBitLow(ResVReg, ResType, I); + case Intrinsic::spv_all_memory_barrier: + return selectBarrierInst(I, SPIRV::Scope::Device, + SPIRV::MemorySemantics::UniformMemory | + SPIRV::MemorySemantics::ImageMemory | + SPIRV::MemorySemantics::WorkgroupMemory, + /*WithGroupSync*/ false); + case Intrinsic::spv_all_memory_barrier_with_group_sync: + return selectBarrierInst(I, SPIRV::Scope::Device, + SPIRV::MemorySemantics::UniformMemory | + SPIRV::MemorySemantics::ImageMemory | + SPIRV::MemorySemantics::WorkgroupMemory, + /*WithGroupSync*/ true); + case Intrinsic::spv_device_memory_barrier: + return selectBarrierInst(I, SPIRV::Scope::Device, + SPIRV::MemorySemantics::UniformMemory | + SPIRV::MemorySemantics::ImageMemory, + /*WithGroupSync*/ false); + case Intrinsic::spv_device_memory_barrier_with_group_sync: + return selectBarrierInst(I, SPIRV::Scope::Device, + SPIRV::MemorySemantics::UniformMemory | + SPIRV::MemorySemantics::ImageMemory, + /*WithGroupSync*/ true); case Intrinsic::spv_group_memory_barrier: return selectBarrierInst(I, SPIRV::Scope::Workgroup, SPIRV::MemorySemantics::WorkgroupMemory, diff --git a/llvm/test/CodeGen/DirectX/all_memory_barrier.ll b/llvm/test/CodeGen/DirectX/all_memory_barrier.ll new file mode 100644 index 0000000000000..be2af03238187 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/all_memory_barrier.ll @@ -0,0 +1,8 @@ +; RUN: opt -S -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library < %s | FileCheck %s + +define void @test_all_memory_barrier() { +entry: + ; CHECK: call void @dx.op.barrier(i32 80, i32 10) + call void @llvm.dx.all.memory.barrier() + ret void +} diff --git a/llvm/test/CodeGen/DirectX/all_memory_barrier_with_group_sync.ll b/llvm/test/CodeGen/DirectX/all_memory_barrier_with_group_sync.ll new file mode 100644 index 0000000000000..48ed1b6d36b1f --- /dev/null +++ b/llvm/test/CodeGen/DirectX/all_memory_barrier_with_group_sync.ll @@ -0,0 +1,8 @@ +; RUN: opt -S -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library < %s | FileCheck %s + +define void @test_all_memory_barrier_with_group_sync() { +entry: + ; CHECK: call void @dx.op.barrier(i32 80, i32 11) + call void @llvm.dx.all.memory.barrier.with.group.sync() + ret void +} diff --git a/llvm/test/CodeGen/DirectX/device_memory_barrier.ll b/llvm/test/CodeGen/DirectX/device_memory_barrier.ll new file mode 100644 index 0000000000000..dde51af9262d0 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/device_memory_barrier.ll @@ -0,0 +1,8 @@ +; RUN: opt -S -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library < %s | FileCheck %s + +define void @test_device_memory_barrier() { +entry: + ; CHECK: call void @dx.op.barrier(i32 80, i32 2) + call void @llvm.dx.device.memory.barrier() + ret void +} diff --git a/llvm/test/CodeGen/DirectX/device_memory_barrier_with_group_sync.ll b/llvm/test/CodeGen/DirectX/device_memory_barrier_with_group_sync.ll new file mode 100644 index 0000000000000..cb321d5cb798d --- /dev/null +++ b/llvm/test/CodeGen/DirectX/device_memory_barrier_with_group_sync.ll @@ -0,0 +1,8 @@ +; RUN: opt -S -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library < %s | FileCheck %s + +define void @test_device_memory_barrier_with_group_sync() { +entry: + ; CHECK: call void @dx.op.barrier(i32 80, i32 3) + call void @llvm.dx.device.memory.barrier.with.group.sync() + ret void +} diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/all_memory_barrier.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/all_memory_barrier.ll new file mode 100644 index 0000000000000..f33724c9e6af8 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/all_memory_barrier.ll @@ -0,0 +1,16 @@ +; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %} + +; CHECK: OpMemoryModel Logical GLSL450 + +define void @test_all_memory_barrier() #0 { +entry: + ; CHECK: %[[#TY:]] = OpTypeInt 32 0 + ; CHECK-DAG: %[[#MEM_SCOPE:]] = OpConstant %[[#TY]] 1 + ; CHECK-DAG: %[[#MEM_SEM:]] = OpConstant %[[#TY]] 2376 + ; CHECK: OpMemoryBarrier %[[#MEM_SCOPE]] %[[#MEM_SEM]] + call void @llvm.spv.all.memory.barrier() + ret void +} + +attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" } diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/all_memory_barrier_with_group_sync.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/all_memory_barrier_with_group_sync.ll new file mode 100644 index 0000000000000..b39c305b45a07 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/all_memory_barrier_with_group_sync.ll @@ -0,0 +1,17 @@ +; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %} + +; CHECK: OpMemoryModel Logical GLSL450 + +define void @test_all_memory_barrier_with_group_sync() #0 { +entry: + ; CHECK: %[[#TY:]] = OpTypeInt 32 0 + ; CHECK-DAG: %[[#EXEC_SCOPE:]] = OpConstant %[[#TY]] 2 + ; CHECK-DAG: %[[#MEM_SCOPE:]] = OpConstant %[[#TY]] 1 + ; CHECK-DAG: %[[#MEM_SEM:]] = OpConstant %[[#TY]] 2376 + ; CHECK: OpControlBarrier %[[#EXEC_SCOPE]] %[[#MEM_SCOPE]] %[[#MEM_SEM]] + call void @llvm.spv.all.memory.barrier.with.group.sync() + ret void +} + +attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" } diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/device_memory_barrier.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/device_memory_barrier.ll new file mode 100644 index 0000000000000..87b9e23ded50a --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/device_memory_barrier.ll @@ -0,0 +1,16 @@ +; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %} + +; CHECK: OpMemoryModel Logical GLSL450 + +define void @test_device_memory_barrier() #0 { +entry: + ; CHECK: %[[#TY:]] = OpTypeInt 32 0 + ; CHECK-DAG: %[[#MEM_SCOPE:]] = OpConstant %[[#TY]] 1 + ; CHECK-DAG: %[[#MEM_SEM:]] = OpConstant %[[#TY]] 2120 + ; CHECK: OpMemoryBarrier %[[#MEM_SCOPE]] %[[#MEM_SEM]] + call void @llvm.spv.device.memory.barrier() + ret void +} + +attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" } diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/device_memory_barrier_with_group_sync.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/device_memory_barrier_with_group_sync.ll new file mode 100644 index 0000000000000..08eb0ed5706a5 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/device_memory_barrier_with_group_sync.ll @@ -0,0 +1,17 @@ +; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %} + +; CHECK: OpMemoryModel Logical GLSL450 + +define void @test_device_memory_barrier_with_group_sync() #0 { +entry: + ; CHECK: %[[#TY:]] = OpTypeInt 32 0 + ; CHECK-DAG: %[[#EXEC_SCOPE:]] = OpConstant %[[#TY]] 2 + ; CHECK-DAG: %[[#MEM_SCOPE:]] = OpConstant %[[#TY]] 1 + ; CHECK-DAG: %[[#MEM_SEM:]] = OpConstant %[[#TY]] 2120 + ; CHECK: OpControlBarrier %[[#EXEC_SCOPE]] %[[#MEM_SCOPE]] %[[#MEM_SEM]] + call void @llvm.spv.device.memory.barrier.with.group.sync() + ret void +} + +attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" } diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/group_memory_barrier.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/group_memory_barrier.ll index 7e8ca90fb51ff..987e7b57662c9 100644 --- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/group_memory_barrier.ll +++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/group_memory_barrier.ll @@ -6,9 +6,9 @@ define void @test_group_memory_barrier() #0 { entry: ; CHECK: %[[#TY:]] = OpTypeInt 32 0 + ; CHECK-DAG: %[[#MEM_SCOPE:]] = OpConstant %[[#TY]] 2 ; CHECK-DAG: %[[#MEM_SEM:]] = OpConstant %[[#TY]] 264 - ; CHECK-DAG: %[[#EXEC_AND_MEM_SCOPE:]] = OpConstant %[[#TY]] 2 - ; CHECK: OpMemoryBarrier %[[#EXEC_AND_MEM_SCOPE]] %[[#MEM_SEM]] + ; CHECK: OpMemoryBarrier %[[#MEM_SCOPE]] %[[#MEM_SEM]] call void @llvm.spv.group.memory.barrier() ret void } diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/group_memory_barrier_with_group_sync.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/group_memory_barrier_with_group_sync.ll index 3dcd21ef64be7..0ecd0a053ae3b 100644 --- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/group_memory_barrier_with_group_sync.ll +++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/group_memory_barrier_with_group_sync.ll @@ -6,8 +6,8 @@ define void @test_group_memory_barrier_with_group_sync() #0 { entry: ; CHECK: %[[#TY:]] = OpTypeInt 32 0 - ; CHECK-DAG: %[[#MEM_SEM:]] = OpConstant %[[#TY]] 264 ; CHECK-DAG: %[[#EXEC_AND_MEM_SCOPE:]] = OpConstant %[[#TY]] 2 + ; CHECK-DAG: %[[#MEM_SEM:]] = OpConstant %[[#TY]] 264 ; CHECK: OpControlBarrier %[[#EXEC_AND_MEM_SCOPE]] %[[#EXEC_AND_MEM_SCOPE]] %[[#MEM_SEM]] call void @llvm.spv.group.memory.barrier.with.group.sync() ret void _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
