https://github.com/bob80905 created https://github.com/llvm/llvm-project/pull/238097
Adds WaveMultiPrefixBitXor for Shader Model 6.5 and newer. Adds Clang checks and code generation, DXIL lowering, and SPIR-V lowering. Uses `llvm.spv.subgroup.multi.prefix.bit.xor` for SPIR-V. Accepts either partition extension and prefers EXT when both EXT and NV are enabled. Reuses the BitAnd mask checks and SPIR-V lowering code. This is the second PR in the five-PR WaveMultiPrefix stack. It builds on #238083. The existing SPIR-V limit for long-vector function parameters remains. The backend test uses memory loads and stores for a long vector. Fixes #99210. >From e6d56733761e1bc60557f2ef60578340ef8fa7ac Mon Sep 17 00:00:00 2001 From: Joshua Batista <[email protected]> Date: Mon, 2 Nov 2026 13:00:31 -0800 Subject: [PATCH] first attempt Co-authored-by: Copilot <[email protected]> Copilot-Session: 40997c2f-5aaa-41c6-8d1f-3e2e9ada315f --- clang/include/clang/Basic/Builtins.td | 6 ++ clang/include/clang/Basic/HLSLIntrinsics.td | 10 ++ clang/lib/CodeGen/CGHLSLBuiltins.cpp | 11 ++- clang/lib/CodeGen/CGHLSLRuntime.h | 3 + clang/lib/Sema/SemaHLSL.cpp | 3 +- .../builtins/WaveMultiPrefixBitXor.hlsl | 97 ++++++++++++++++++ .../WaveMultiPrefixBitXor-errors.hlsl | 64 ++++++++++++ llvm/include/llvm/IR/IntrinsicsDirectX.td | 7 ++ llvm/include/llvm/IR/IntrinsicsSPIRV.td | 7 ++ llvm/lib/Target/DirectX/DXIL.td | 8 ++ llvm/lib/Target/DirectX/DXILShaderFlags.cpp | 1 + .../DirectX/DirectXTargetTransformInfo.cpp | 1 + .../Target/SPIRV/SPIRVInstructionSelector.cpp | 26 +++-- .../CodeGen/DirectX/ShaderFlags/wave-ops.ll | 7 ++ .../CodeGen/DirectX/WaveMultiPrefixBitXor.ll | 48 +++++++++ .../DirectX/WaveMultiPrefixBitXor_errors.ll | 14 +++ .../hlsl-intrinsics/WaveMultiPrefixBitXor.ll | 98 +++++++++++++++++++ 17 files changed, 398 insertions(+), 13 deletions(-) create mode 100644 clang/test/CodeGenHLSL/builtins/WaveMultiPrefixBitXor.hlsl create mode 100644 clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixBitXor-errors.hlsl create mode 100644 llvm/test/CodeGen/DirectX/WaveMultiPrefixBitXor.ll create mode 100644 llvm/test/CodeGen/DirectX/WaveMultiPrefixBitXor_errors.ll create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveMultiPrefixBitXor.ll diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td index 5ae3edcc6601c..ab4b7f4b1187a 100644 --- a/clang/include/clang/Basic/Builtins.td +++ b/clang/include/clang/Basic/Builtins.td @@ -5795,6 +5795,12 @@ def HLSLWaveMultiPrefixBitAnd : LangBuiltin<"HLSL_LANG"> { let Prototype = "void(...)"; } +def HLSLWaveMultiPrefixBitXor : LangBuiltin<"HLSL_LANG"> { + let Spellings = ["__builtin_hlsl_wave_multi_prefix_bit_xor"]; + let Attributes = [NoThrow, Const]; + let Prototype = "void(...)"; +} + def HLSLWavePrefixProduct : LangBuiltin<"HLSL_LANG"> { let Spellings = ["__builtin_hlsl_wave_prefix_product"]; let Attributes = [NoThrow, Const]; diff --git a/clang/include/clang/Basic/HLSLIntrinsics.td b/clang/include/clang/Basic/HLSLIntrinsics.td index 902a31d8278b3..300d46a7fd324 100644 --- a/clang/include/clang/Basic/HLSLIntrinsics.td +++ b/clang/include/clang/Basic/HLSLIntrinsics.td @@ -1893,6 +1893,16 @@ def hlsl_wave_multi_prefix_bit_and : let Availability = SM6_5; } +def hlsl_wave_multi_prefix_bit_xor : + HLSLTwoArgBuiltin<"WaveMultiPrefixBitXor", + "__builtin_hlsl_wave_multi_prefix_bit_xor"> { + let Args = [Varying, VectorType<UIntTy, 4>]; + let VaryingTypes = AllIntTypes; + let VaryingLongVector = 1; + let IsConvergent = 1; + let Availability = SM6_5; +} + // Returns the count of bits set to 1 on prior lanes. def hlsl_wave_prefix_count_bits : HLSLBuiltin<"WavePrefixCountBits", diff --git a/clang/lib/CodeGen/CGHLSLBuiltins.cpp b/clang/lib/CodeGen/CGHLSLBuiltins.cpp index d4f5f47d9b24e..3afc8a0668f27 100644 --- a/clang/lib/CodeGen/CGHLSLBuiltins.cpp +++ b/clang/lib/CodeGen/CGHLSLBuiltins.cpp @@ -1721,15 +1721,20 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID, return EmitIntrinsicCall(IID, {OpExpr->getType()}, ArrayRef{OpExpr}, "hlsl.wave.prefix.sum"); } - case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and: { + case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and: + case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_xor: { Value *ValueOp = EmitScalarExpr(E->getArg(0)); Value *Mask = EmitScalarExpr(E->getArg(1)); SmallVector<Value *, 5> Args{ValueOp}; for (unsigned I = 0; I != 4; ++I) Args.push_back(Builder.CreateExtractElement(Mask, I)); + if (BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and) + return EmitIntrinsicCall( + CGM.getHLSLRuntime().getWaveMultiPrefixBitAndIntrinsic(), + {ValueOp->getType()}, Args, "hlsl.wave.multi.prefix.bit.and"); return EmitIntrinsicCall( - CGM.getHLSLRuntime().getWaveMultiPrefixBitAndIntrinsic(), - {ValueOp->getType()}, Args, "hlsl.wave.multi.prefix.bit.and"); + CGM.getHLSLRuntime().getWaveMultiPrefixBitXorIntrinsic(), + {ValueOp->getType()}, Args, "hlsl.wave.multi.prefix.bit.xor"); } case Builtin::BI__builtin_hlsl_wave_prefix_product: { Value *OpExpr = EmitScalarExpr(E->getArg(0)); diff --git a/clang/lib/CodeGen/CGHLSLRuntime.h b/clang/lib/CodeGen/CGHLSLRuntime.h index 89b52f287d9ed..71d8255ab3279 100644 --- a/clang/lib/CodeGen/CGHLSLRuntime.h +++ b/clang/lib/CodeGen/CGHLSLRuntime.h @@ -152,6 +152,9 @@ class CGHLSLRuntime { GENERATE_HLSL_INTRINSIC_FUNCTION_BASE(WaveMultiPrefixBitAnd, wave_multi_prefix_bit_and, subgroup_multi_prefix_bit_and) + GENERATE_HLSL_INTRINSIC_FUNCTION_BASE(WaveMultiPrefixBitXor, + wave_multi_prefix_bit_xor, + subgroup_multi_prefix_bit_xor) GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveMax, wave_reduce_max) GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveUMax, wave_reduce_umax) GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveMin, wave_reduce_min) diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp index 6147a0446a838..d0da34ef79f89 100644 --- a/clang/lib/Sema/SemaHLSL.cpp +++ b/clang/lib/Sema/SemaHLSL.cpp @@ -5377,7 +5377,8 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { TheCall->setType(ArgTyExpr); break; } - case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and: { + case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and: + case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_xor: { if (SemaRef.checkArgCount(TheCall, 2)) return true; diff --git a/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixBitXor.hlsl b/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixBitXor.hlsl new file mode 100644 index 0000000000000..b903fe8a168cc --- /dev/null +++ b/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixBitXor.hlsl @@ -0,0 +1,97 @@ +// RUN: %clang_cc1 -finclude-default-header \ +// RUN: -fnative-half-type -fnative-int16-type \ +// RUN: -triple dxil-pc-shadermodel6.5-library %s -emit-llvm \ +// RUN: -disable-llvm-passes -o - | \ +// RUN: FileCheck %s -DINTRINSIC=dx.wave.multi.prefix.bit.xor +// RUN: %clang_cc1 -finclude-default-header \ +// RUN: -fnative-half-type -fnative-int16-type \ +// RUN: -triple spirv-pc-vulkan-library %s -emit-llvm \ +// RUN: -disable-llvm-passes -o - | \ +// RUN: FileCheck %s -DINTRINSIC=spv.subgroup.multi.prefix.bit.xor + +// CHECK-LABEL: test_uint +uint test_uint(uint value, uint4 mask) { + // CHECK: %[[TOKEN:.*]] = call token @llvm.experimental.convergence.entry() + // CHECK: %[[MASK:.*]] = load <4 x i32> + // CHECK: %[[M0:.*]] = extractelement <4 x i32> %[[MASK]], i64 0 + // CHECK: %[[M1:.*]] = extractelement <4 x i32> %[[MASK]], i64 1 + // CHECK: %[[M2:.*]] = extractelement <4 x i32> %[[MASK]], i64 2 + // CHECK: %[[M3:.*]] = extractelement <4 x i32> %[[MASK]], i64 3 + // CHECK: %[[RET:.*]] = call i32 @llvm.[[INTRINSIC]].i32(i32 %{{.*}}, i32 %[[M0]], i32 %[[M1]], i32 %[[M2]], i32 %[[M3]]) [ "convergencectrl"(token %[[TOKEN]]) ] + // CHECK: ret i32 %[[RET]] + return WaveMultiPrefixBitXor(value, mask); +} + +// CHECK: declare i32 @llvm.[[INTRINSIC]].i32(i32, i32, i32, i32, i32) #[[ATTR:[0-9]+]] + +// CHECK-LABEL: test_int +int test_int(int value) { + // CHECK: %[[RET:.*]] = call i32 @llvm.[[INTRINSIC]].i32(i32 %{{.*}}, i32 1, i32 2, i32 4, i32 8) + // CHECK: ret i32 %[[RET]] + return WaveMultiPrefixBitXor(value, uint4(1, 2, 4, 8)); +} + +// CHECK-LABEL: test_int16 +int16_t test_int16(int16_t value) { + // CHECK: %[[RET:.*]] = call i16 @llvm.[[INTRINSIC]].i16(i16 %{{.*}}, i32 1, i32 2, i32 4, i32 8) + // CHECK: ret i16 %[[RET]] + return WaveMultiPrefixBitXor(value, uint4(1, 2, 4, 8)); +} + +// CHECK-LABEL: test_uint16 +uint16_t test_uint16(uint16_t value) { + // CHECK: %[[RET:.*]] = call i16 @llvm.[[INTRINSIC]].i16(i16 %{{.*}}, i32 1, i32 2, i32 4, i32 8) + // CHECK: ret i16 %[[RET]] + return WaveMultiPrefixBitXor(value, uint4(1, 2, 4, 8)); +} + +// CHECK-LABEL: test_int64 +int64_t test_int64(int64_t value) { + // CHECK: %[[RET:.*]] = call i64 @llvm.[[INTRINSIC]].i64(i64 %{{.*}}, i32 1, i32 2, i32 4, i32 8) + // CHECK: ret i64 %[[RET]] + return WaveMultiPrefixBitXor(value, uint4(1, 2, 4, 8)); +} + +// CHECK-LABEL: test_uint64 +uint64_t test_uint64(uint64_t value) { + // CHECK: %[[RET:.*]] = call i64 @llvm.[[INTRINSIC]].i64(i64 %{{.*}}, i32 1, i32 2, i32 4, i32 8) + // CHECK: ret i64 %[[RET]] + return WaveMultiPrefixBitXor(value, uint4(1, 2, 4, 8)); +} + +// CHECK-LABEL: test_uint2 +uint2 test_uint2(uint2 value) { + // CHECK: %[[RET:.*]] = call <2 x i32> @llvm.[[INTRINSIC]].v2i32(<2 x i32> %{{.*}}, i32 1, i32 2, i32 4, i32 8) + // CHECK: ret <2 x i32> %[[RET]] + return WaveMultiPrefixBitXor(value, uint4(1, 2, 4, 8)); +} + +// CHECK-LABEL: test_int3 +int3 test_int3(int3 value) { + // CHECK: %[[RET:.*]] = call <3 x i32> @llvm.[[INTRINSIC]].v3i32(<3 x i32> %{{.*}}, i32 1, i32 2, i32 4, i32 8) + // CHECK: ret <3 x i32> %[[RET]] + return WaveMultiPrefixBitXor(value, uint4(1, 2, 4, 8)); +} + +// CHECK-LABEL: test_uint64_t4 +uint64_t4 test_uint64_t4(uint64_t4 value) { + // CHECK: %[[RET:.*]] = call <4 x i64> @llvm.[[INTRINSIC]].v4i64(<4 x i64> %{{.*}}, i32 1, i32 2, i32 4, i32 8) + // CHECK: ret <4 x i64> %[[RET]] + return WaveMultiPrefixBitXor(value, uint4(1, 2, 4, 8)); +} + +// CHECK-LABEL: test_uint5 +vector<uint, 5> test_uint5(vector<uint, 5> value) { + // CHECK: %[[RET:.*]] = call <5 x i32> @llvm.[[INTRINSIC]].v5i32(<5 x i32> %{{.*}}, i32 1, i32 2, i32 4, i32 8) + // CHECK: ret <5 x i32> %[[RET]] + return WaveMultiPrefixBitXor(value, uint4(1, 2, 4, 8)); +} + +// CHECK-LABEL: test_int2x3 +int2x3 test_int2x3(int2x3 value) { + // CHECK: %[[RET:.*]] = call <6 x i32> @llvm.[[INTRINSIC]].v6i32(<6 x i32> %{{.*}}, i32 1, i32 2, i32 4, i32 8) + // CHECK: ret <6 x i32> %[[RET]] + return WaveMultiPrefixBitXor(value, uint4(1, 2, 4, 8)); +} + +// CHECK: attributes #[[ATTR]] = { {{.*}}convergent {{.*}} } diff --git a/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixBitXor-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixBitXor-errors.hlsl new file mode 100644 index 0000000000000..6c1365c92accd --- /dev/null +++ b/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixBitXor-errors.hlsl @@ -0,0 +1,64 @@ +// RUN: %clang_cc1 -finclude-default-header \ +// RUN: -fnative-half-type -fnative-int16-type \ +// RUN: -triple dxil-pc-shadermodel6.5-library -verify %s +// RUN: %clang_cc1 -finclude-default-header \ +// RUN: -fnative-half-type -fnative-int16-type \ +// RUN: -triple spirv-pc-vulkan-library -verify %s +// RUN: %clang_cc1 -finclude-default-header -DTEST_AVAILABILITY \ +// RUN: -triple dxil-pc-shadermodel6.4-library -verify=availability %s + +#ifdef TEST_AVAILABILITY +[shader("compute")] +[numthreads(1, 1, 1)] +void main() { + (void)WaveMultiPrefixBitXor(1u, uint4(1, 2, 4, 8)); + // availability-error@-1 {{only available on Shader Model 6.5 or newer}} + // availability-note@*:* {{introduced in Shader Model 6.5}} +} +#else +void test_arg_count(uint value, uint4 mask) { + __builtin_hlsl_wave_multi_prefix_bit_xor(value); + // expected-error@-1 {{too few arguments to function call, expected 2}} + __builtin_hlsl_wave_multi_prefix_bit_xor(value, mask, mask); + // expected-error@-1 {{too many arguments to function call, expected 2}} +} + +struct S { uint value; }; + +void test_value_types(float f, float2 fv, bool b, bool2 bv, S s, + float2x2 fm, bool2x2 bm, uint4 mask) { + __builtin_hlsl_wave_multi_prefix_bit_xor(f, mask); + // expected-error@-1 {{1st argument must be a scalar or vector of integer}} + __builtin_hlsl_wave_multi_prefix_bit_xor(fv, mask); + // expected-error@-1 {{1st argument must be a scalar or vector of integer}} + __builtin_hlsl_wave_multi_prefix_bit_xor(b, mask); + // expected-error@-1 {{invalid operand of type 'bool'}} + __builtin_hlsl_wave_multi_prefix_bit_xor(bv, mask); + // expected-error@-1 {{invalid operand of type 'bool2'}} + __builtin_hlsl_wave_multi_prefix_bit_xor(s, mask); + // expected-error@-1 {{invalid operand of type 'S'}} + __builtin_hlsl_wave_multi_prefix_bit_xor(fm, mask); + // expected-error@-1 {{1st argument must be a matrix integer type}} + // expected-error@-2 {{cannot pass object of non-trivial type 'float2x2'}} + __builtin_hlsl_wave_multi_prefix_bit_xor(bm, mask); + // expected-error@-1 {{1st argument must be a matrix integer type}} + // expected-error@-2 {{cannot pass object of non-trivial type 'bool2x2'}} +} + +void test_mask_types(uint value, uint scalar, uint3 short_mask, + int4 signed_mask, float4 float_mask, + uint64_t4 wide_mask, S struct_mask) { + __builtin_hlsl_wave_multi_prefix_bit_xor(value, scalar); + // expected-error@-1 {{passing 'uint' (aka 'unsigned int')}} + __builtin_hlsl_wave_multi_prefix_bit_xor(value, short_mask); + // expected-error@-1 {{passing 'uint3' (aka 'vector<uint, 3>')}} + __builtin_hlsl_wave_multi_prefix_bit_xor(value, signed_mask); + // expected-error@-1 {{passing 'int4' (aka 'vector<int, 4>')}} + __builtin_hlsl_wave_multi_prefix_bit_xor(value, float_mask); + // expected-error@-1 {{passing 'float4' (aka 'vector<float, 4>')}} + __builtin_hlsl_wave_multi_prefix_bit_xor(value, wide_mask); + // expected-error@-1 {{passing 'uint64_t4'}} + __builtin_hlsl_wave_multi_prefix_bit_xor(value, struct_mask); + // expected-error@-1 {{passing 'S' to parameter of incompatible type}} +} +#endif diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td index 4f23541e7eeed..c292a0a46b2ae 100644 --- a/llvm/include/llvm/IR/IntrinsicsDirectX.td +++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td @@ -481,6 +481,13 @@ let TargetPrefix = "dx" in { [IntrConvergent, IntrNoMem, IntrTriviallyScalarizable]>; + def int_dx_wave_multi_prefix_bit_xor + : DefaultAttrsIntrinsic<[llvm_anyint_ty], + [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty, + llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem, + IntrTriviallyScalarizable]>; + def int_dx_wave_prefix_usum : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem, diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td index 2b1127445f799..0c6dc262c3b09 100644 --- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td +++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td @@ -390,6 +390,13 @@ let TargetPrefix = "spv" in { [IntrConvergent, IntrNoMem, IntrTriviallyScalarizable]>; + def int_spv_subgroup_multi_prefix_bit_xor + : DefaultAttrsIntrinsic<[llvm_anyint_ty], + [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty, + llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem, + IntrTriviallyScalarizable]>; + def int_spv_wave_prefix_product : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem, diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td index c2165c9664a3d..cf3d6404f6210 100644 --- a/llvm/lib/Target/DirectX/DXIL.td +++ b/llvm/lib/Target/DirectX/DXIL.td @@ -325,6 +325,7 @@ defvar WaveBitOpKind_Or = 1; defvar WaveBitOpKind_Xor = 2; defvar WaveMultiPrefixOpKind_And = 1; +defvar WaveMultiPrefixOpKind_Xor = 3; defvar SignedOpKind_Signed = 0; defvar SignedOpKind_Unsigned = 1; @@ -1570,6 +1571,13 @@ def WaveMultiPrefixOp : DXILOp<166, waveMultiPrefixOp> { IntrinArgIndex<3>, IntrinArgIndex<4>, IntrinArgI8<WaveMultiPrefixOpKind_And>, IntrinArgI8<SignedOpKind_Signed> + ]>, + IntrinSelect<int_dx_wave_multi_prefix_bit_xor, + [ + IntrinArgIndex<0>, IntrinArgIndex<1>, IntrinArgIndex<2>, + IntrinArgIndex<3>, IntrinArgIndex<4>, + IntrinArgI8<WaveMultiPrefixOpKind_Xor>, + IntrinArgI8<SignedOpKind_Signed> ]> ]; let arguments = [ diff --git a/llvm/lib/Target/DirectX/DXILShaderFlags.cpp b/llvm/lib/Target/DirectX/DXILShaderFlags.cpp index 1c27d16e05979..1f37844dc5f19 100644 --- a/llvm/lib/Target/DirectX/DXILShaderFlags.cpp +++ b/llvm/lib/Target/DirectX/DXILShaderFlags.cpp @@ -112,6 +112,7 @@ static bool checkWaveOps(Intrinsic::ID IID) { case Intrinsic::dx_wave_prefix_product: case Intrinsic::dx_wave_prefix_uproduct: case Intrinsic::dx_wave_multi_prefix_bit_and: + case Intrinsic::dx_wave_multi_prefix_bit_xor: // Quad Op Variants case Intrinsic::dx_quad_read_across_x: case Intrinsic::dx_quad_read_across_y: diff --git a/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp b/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp index 6fa1d27533442..00ef806735eff 100644 --- a/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp +++ b/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp @@ -21,6 +21,7 @@ bool DirectXTTIImpl::isTargetIntrinsicWithScalarOpAtArg( case Intrinsic::dx_wave_readlane: return ScalarOpdIdx == 1; case Intrinsic::dx_wave_multi_prefix_bit_and: + case Intrinsic::dx_wave_multi_prefix_bit_xor: return ScalarOpdIdx >= 1 && ScalarOpdIdx <= 4; default: return false; diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index a1ae8d2edce95..ec9e548d319b2 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -322,8 +322,9 @@ class SPIRVInstructionSelector : public InstructionSelector { bool selectWaveExclusiveScanProduct(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const; - bool selectWaveMultiPrefixBitAnd(Register ResVReg, SPIRVTypeInst ResType, - MachineInstr &I) const; + bool selectWaveMultiPrefixOp(Register ResVReg, SPIRVTypeInst ResType, + MachineInstr &I, unsigned Opcode, + StringRef BuiltinName) const; bool selectQuadSwap(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, unsigned Direction) const; @@ -4014,15 +4015,17 @@ bool SPIRVInstructionSelector::selectWaveExclusiveScan( return true; } -bool SPIRVInstructionSelector::selectWaveMultiPrefixBitAnd( - Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const { +bool SPIRVInstructionSelector::selectWaveMultiPrefixOp( + Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, unsigned Opcode, + StringRef BuiltinName) const { if (!STI.canUseExtension( SPIRV::Extension::SPV_EXT_shader_subgroup_partitioned) && !STI.canUseExtension( SPIRV::Extension::SPV_NV_shader_subgroup_partitioned)) return diagnoseUnsupported( - I, "WaveMultiPrefixBitAnd requires SPV_EXT_shader_subgroup_partitioned " - "or SPV_NV_shader_subgroup_partitioned"); + I, Twine(BuiltinName) + + " requires SPV_EXT_shader_subgroup_partitioned or " + "SPV_NV_shader_subgroup_partitioned"); assert(I.getNumOperands() == 7); SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); @@ -4037,8 +4040,7 @@ bool SPIRVInstructionSelector::selectWaveMultiPrefixBitAnd( MaskMIB.addUse(I.getOperand(Op).getReg()); MaskMIB.constrainAllUses(TII, TRI, RBI); - BuildMI(*I.getParent(), I, I.getDebugLoc(), - TII.get(SPIRV::OpGroupNonUniformBitwiseAnd)) + BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII, @@ -5829,7 +5831,13 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, case Intrinsic::spv_wave_prefix_sum: return selectWaveExclusiveScanSum(ResVReg, ResType, I); case Intrinsic::spv_subgroup_multi_prefix_bit_and: - return selectWaveMultiPrefixBitAnd(ResVReg, ResType, I); + return selectWaveMultiPrefixOp(ResVReg, ResType, I, + SPIRV::OpGroupNonUniformBitwiseAnd, + "WaveMultiPrefixBitAnd"); + case Intrinsic::spv_subgroup_multi_prefix_bit_xor: + return selectWaveMultiPrefixOp(ResVReg, ResType, I, + SPIRV::OpGroupNonUniformBitwiseXor, + "WaveMultiPrefixBitXor"); case Intrinsic::spv_wave_prefix_product: return selectWaveExclusiveScanProduct(ResVReg, ResType, I); case Intrinsic::spv_quad_read_across_x: { diff --git a/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll b/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll index 55dd18c81f420..833b5fee3d946 100644 --- a/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll +++ b/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll @@ -77,6 +77,13 @@ define i32 @wave_multi_prefix_bit_and(i32 %x) { ret i32 %ret } +define i32 @wave_multi_prefix_bit_xor(i32 %x) { + ; CHECK: Function wave_multi_prefix_bit_xor : [[WAVE_FLAG]] + %ret = call i32 @llvm.dx.wave.multi.prefix.bit.xor.i32( + i32 %x, i32 1, i32 2, i32 4, i32 8) + ret i32 %ret +} + define noundef i1 @wave_all_equal(i1 %x) { entry: ; CHECK: Function wave_all_equal : [[WAVE_FLAG]] diff --git a/llvm/test/CodeGen/DirectX/WaveMultiPrefixBitXor.ll b/llvm/test/CodeGen/DirectX/WaveMultiPrefixBitXor.ll new file mode 100644 index 0000000000000..d5475630a6f7d --- /dev/null +++ b/llvm/test/CodeGen/DirectX/WaveMultiPrefixBitXor.ll @@ -0,0 +1,48 @@ +; RUN: opt -S -scalarizer -dxil-op-lower \ +; RUN: -mtriple=dxil-pc-shadermodel6.5-library < %s | FileCheck %s + +; CHECK-LABEL: define i16 @test_i16( +; CHECK: call i16 @dx.op.waveMultiPrefixOp.i16(i32 166, i16 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3, i8 3, i8 0) +define i16 @test_i16(i16 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) { + %ret = call i16 @llvm.dx.wave.multi.prefix.bit.xor.i16( + i16 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) + ret i16 %ret +} + +; CHECK-LABEL: define i32 @test_i32( +; CHECK: call i32 @dx.op.waveMultiPrefixOp.i32(i32 166, i32 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3, i8 3, i8 0) +define i32 @test_i32(i32 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) { + %ret = call i32 @llvm.dx.wave.multi.prefix.bit.xor.i32( + i32 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) + ret i32 %ret +} + +; CHECK-LABEL: define i64 @test_i64( +; CHECK: call i64 @dx.op.waveMultiPrefixOp.i64(i32 166, i64 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3, i8 3, i8 0) +define i64 @test_i64(i64 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) { + %ret = call i64 @llvm.dx.wave.multi.prefix.bit.xor.i64( + i64 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) + ret i64 %ret +} + +; CHECK-LABEL: define <2 x i32> @test_vector( +; CHECK: call i32 @dx.op.waveMultiPrefixOp.i32(i32 166, i32 %value.i0, i32 %m0, i32 %m1, i32 %m2, i32 %m3, i8 3, i8 0) +; CHECK: call i32 @dx.op.waveMultiPrefixOp.i32(i32 166, i32 %value.i1, i32 %m0, i32 %m1, i32 %m2, i32 %m3, i8 3, i8 0) +define <2 x i32> @test_vector(<2 x i32> %value, i32 %m0, i32 %m1, + i32 %m2, i32 %m3) { + %ret = call <2 x i32> @llvm.dx.wave.multi.prefix.bit.xor.v2i32( + <2 x i32> %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) + ret <2 x i32> %ret +} + +; CHECK-LABEL: define <5 x i32> @test_long_vector( +; CHECK: call i32 @dx.op.waveMultiPrefixOp.i32(i32 166, i32 %value.i0, i32 1, i32 2, i32 4, i32 8, i8 3, i8 0) +; CHECK: call i32 @dx.op.waveMultiPrefixOp.i32(i32 166, i32 %value.i1, i32 1, i32 2, i32 4, i32 8, i8 3, i8 0) +; CHECK: call i32 @dx.op.waveMultiPrefixOp.i32(i32 166, i32 %value.i2, i32 1, i32 2, i32 4, i32 8, i8 3, i8 0) +; CHECK: call i32 @dx.op.waveMultiPrefixOp.i32(i32 166, i32 %value.i3, i32 1, i32 2, i32 4, i32 8, i8 3, i8 0) +; CHECK: call i32 @dx.op.waveMultiPrefixOp.i32(i32 166, i32 %value.i4, i32 1, i32 2, i32 4, i32 8, i8 3, i8 0) +define <5 x i32> @test_long_vector(<5 x i32> %value) { + %ret = call <5 x i32> @llvm.dx.wave.multi.prefix.bit.xor.v5i32( + <5 x i32> %value, i32 1, i32 2, i32 4, i32 8) + ret <5 x i32> %ret +} diff --git a/llvm/test/CodeGen/DirectX/WaveMultiPrefixBitXor_errors.ll b/llvm/test/CodeGen/DirectX/WaveMultiPrefixBitXor_errors.ll new file mode 100644 index 0000000000000..757c64940e02e --- /dev/null +++ b/llvm/test/CodeGen/DirectX/WaveMultiPrefixBitXor_errors.ll @@ -0,0 +1,14 @@ +; RUN: not opt -S -dxil-op-lower \ +; RUN: -mtriple=dxil-pc-shadermodel6.4-library %s 2>&1 | \ +; RUN: FileCheck %s --check-prefix=VERSION +; RUN: not opt -S -dxil-op-lower \ +; RUN: -mtriple=dxil-pc-shadermodel6.5-library %s 2>&1 | \ +; RUN: FileCheck %s --check-prefix=TYPE + +; VERSION: Cannot create WaveMultiPrefixOp operation: No valid overloads for DXIL version 1.4 +; TYPE: Cannot create WaveMultiPrefixOp operation: Invalid overload type +define i8 @test_i8(i8 %value) { + %ret = call i8 @llvm.dx.wave.multi.prefix.bit.xor.i8( + i8 %value, i32 1, i32 2, i32 4, i32 8) + ret i8 %ret +} diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveMultiPrefixBitXor.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveMultiPrefixBitXor.ll new file mode 100644 index 0000000000000..876fb0c25a1d6 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveMultiPrefixBitXor.ll @@ -0,0 +1,98 @@ +; DEFINE: %{nv} = SPV_NV_shader_subgroup_partitioned +; DEFINE: %{ext} = SPV_EXT_shader_subgroup_partitioned +; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-vulkan-unknown \ +; RUN: --spirv-ext=+%{nv} %s -o - | FileCheck %s -DEXTENSION=NV \ +; RUN: --implicit-check-not=%{ext} +; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-vulkan-unknown \ +; RUN: --spirv-ext=+%{ext} %s -o - | FileCheck %s -DEXTENSION=EXT \ +; RUN: --implicit-check-not=%{nv} +; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-vulkan-unknown \ +; RUN: --spirv-ext=+%{nv},+%{ext} %s -o - | FileCheck %s -DEXTENSION=EXT \ +; RUN: --implicit-check-not=%{nv} +; RUN: not llc -O0 -mtriple=spirv-vulkan-unknown %s -o - 2>&1 | \ +; RUN: FileCheck %s --check-prefix=NO-EXT +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-vulkan-unknown \ +; RUN: --spirv-ext=+%{nv} %s -o - -filetype=obj | spirv-val %} +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-vulkan-unknown \ +; RUN: --spirv-ext=+%{ext} %s -o - -filetype=obj | spirv-val %} +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-vulkan-unknown \ +; RUN: --spirv-ext=+%{nv},+%{ext} %s -o - \ +; RUN: -filetype=obj | spirv-val %} + +; NO-EXT: WaveMultiPrefixBitXor requires SPV_EXT_shader_subgroup_partitioned or SPV_NV_shader_subgroup_partitioned + +; CHECK-DAG: OpCapability GroupNonUniformPartitionedNV +; CHECK-DAG: OpExtension "SPV_[[EXTENSION]]_shader_subgroup_partitioned" +; CHECK-DAG: %[[#I16:]] = OpTypeInt 16 0 +; CHECK-DAG: %[[#I32:]] = OpTypeInt 32 0 +; CHECK-DAG: %[[#I64:]] = OpTypeInt 64 0 +; CHECK-DAG: %[[#V2I32:]] = OpTypeVector %[[#I32]] 2 +; CHECK-DAG: %[[#V4I32:]] = OpTypeVector %[[#I32]] 4 +; CHECK-DAG: %[[#SCOPE:]] = OpConstant %[[#I32]] 3 +; CHECK-DAG: %[[#C1:]] = OpConstant %[[#I32]] 1 +; CHECK-DAG: %[[#C2:]] = OpConstant %[[#I32]] 2 +; CHECK-DAG: %[[#C4:]] = OpConstant %[[#I32]] 4 +; CHECK-DAG: %[[#C8:]] = OpConstant %[[#I32]] 8 + +@wide_i32 = internal addrspace(10) global [5 x i32] zeroinitializer + +; CHECK-LABEL: Begin function test_i32 +; CHECK: %[[#VALUE:]] = OpFunctionParameter %[[#I32]] +; CHECK: %[[#M0:]] = OpFunctionParameter %[[#I32]] +; CHECK: %[[#M1:]] = OpFunctionParameter %[[#I32]] +; CHECK: %[[#M2:]] = OpFunctionParameter %[[#I32]] +; CHECK: %[[#M3:]] = OpFunctionParameter %[[#I32]] +; CHECK: %[[#MASK:]] = OpCompositeConstruct %[[#V4I32]] %[[#M0]] %[[#M1]] %[[#M2]] %[[#M3]] +; CHECK: %[[#RET:]] = OpGroupNonUniformBitwiseXor %[[#I32]] %[[#SCOPE]] PartitionedExclusiveScanNV %[[#VALUE]] %[[#MASK]] +; CHECK: OpReturnValue %[[#RET]] +define i32 @test_i32(i32 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) { + %ret = call i32 @llvm.spv.subgroup.multi.prefix.bit.xor.i32( + i32 %value, i32 %m0, i32 %m1, i32 %m2, i32 %m3) + ret i32 %ret +} + +; CHECK-LABEL: Begin function test_i16 +; CHECK: %[[#VALUE:]] = OpFunctionParameter %[[#I16]] +; CHECK: %[[#MASK:]] = OpCompositeConstruct %[[#V4I32]] %[[#C1]] %[[#C2]] %[[#C4]] %[[#C8]] +; CHECK: %[[#RET:]] = OpGroupNonUniformBitwiseXor %[[#I16]] %[[#SCOPE]] PartitionedExclusiveScanNV %[[#VALUE]] %[[#MASK]] +; CHECK: OpReturnValue %[[#RET]] +define i16 @test_i16(i16 %value) { + %ret = call i16 @llvm.spv.subgroup.multi.prefix.bit.xor.i16( + i16 %value, i32 1, i32 2, i32 4, i32 8) + ret i16 %ret +} + +; CHECK-LABEL: Begin function test_i64 +; CHECK: %[[#VALUE:]] = OpFunctionParameter %[[#I64]] +; CHECK: %[[#MASK:]] = OpCompositeConstruct %[[#V4I32]] %[[#C1]] %[[#C2]] %[[#C4]] %[[#C8]] +; CHECK: %[[#RET:]] = OpGroupNonUniformBitwiseXor %[[#I64]] %[[#SCOPE]] PartitionedExclusiveScanNV %[[#VALUE]] %[[#MASK]] +; CHECK: OpReturnValue %[[#RET]] +define i64 @test_i64(i64 %value) { + %ret = call i64 @llvm.spv.subgroup.multi.prefix.bit.xor.i64( + i64 %value, i32 1, i32 2, i32 4, i32 8) + ret i64 %ret +} + +; CHECK-LABEL: Begin function test_vector +; CHECK: %[[#VALUE:]] = OpFunctionParameter %[[#V2I32]] +; CHECK: %[[#MASK:]] = OpCompositeConstruct %[[#V4I32]] %[[#C1]] %[[#C2]] %[[#C4]] %[[#C8]] +; CHECK: %[[#RET:]] = OpGroupNonUniformBitwiseXor %[[#V2I32]] %[[#SCOPE]] PartitionedExclusiveScanNV %[[#VALUE]] %[[#MASK]] +; CHECK: OpReturnValue %[[#RET]] +define <2 x i32> @test_vector(<2 x i32> %value) { + %ret = call <2 x i32> @llvm.spv.subgroup.multi.prefix.bit.xor.v2i32( + <2 x i32> %value, i32 1, i32 2, i32 4, i32 8) + ret <2 x i32> %ret +} + +; CHECK-LABEL: Begin function test_long_vector +; CHECK: %[[#MASK0:]] = OpCompositeConstruct %[[#V4I32]] %[[#C1]] %[[#C2]] %[[#C4]] %[[#C8]] +; CHECK: OpGroupNonUniformBitwiseXor %[[#V4I32]] %[[#SCOPE]] PartitionedExclusiveScanNV %{{[0-9]+}} %[[#MASK0]] +; CHECK: %[[#MASK1:]] = OpCompositeConstruct %[[#V4I32]] %[[#C1]] %[[#C2]] %[[#C4]] %[[#C8]] +; CHECK: OpGroupNonUniformBitwiseXor %[[#I32]] %[[#SCOPE]] PartitionedExclusiveScanNV %{{[0-9]+}} %[[#MASK1]] +define void @test_long_vector() { + %value = load <5 x i32>, ptr addrspace(10) @wide_i32 + %ret = call <5 x i32> @llvm.spv.subgroup.multi.prefix.bit.xor.v5i32( + <5 x i32> %value, i32 1, i32 2, i32 4, i32 8) + store <5 x i32> %ret, ptr addrspace(10) @wide_i32 + ret void +} _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
