llvmorg-github-actions[bot] wrote:

<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-clang-codegen

Author: Joshua Batista (bob80905)

<details>
<summary>Changes</summary>

Adds WaveMultiPrefixSum for Shader Model 6.5 and newer.

Supports numeric scalars, vectors, long vectors, and matrices in Clang. Adds 
DXIL lowering with the Sum operation value 0 and the correct signedness. Adds 
SPIR-V subgroup lowering with either the EXT or NV partition extension. Prefers 
EXT when both are enabled.

The existing SPIR-V limit on long-vector function parameters remains. The 
backend test uses memory for the long-vector case.

The build and all 33 focused tests pass. SPIR-V binaries pass spirv-val with 
NV, EXT, and both extensions.

Stacked on #<!-- -->238164.

Fixes #<!-- -->99213

---

Patch is 32.53 KiB, truncated to 20.00 KiB below, full version: 
https://github.com/llvm/llvm-project/pull/238175.diff


17 Files Affected:

- (modified) clang/include/clang/Basic/Builtins.td (+6) 
- (modified) clang/include/clang/Basic/HLSLIntrinsics.td (+10) 
- (modified) clang/lib/CodeGen/CGHLSLBuiltins.cpp (+9) 
- (modified) clang/lib/CodeGen/CGHLSLRuntime.h (+6) 
- (modified) clang/lib/Sema/SemaHLSL.cpp (+7-5) 
- (added) clang/test/CodeGenHLSL/builtins/WaveMultiPrefixSum.hlsl (+113) 
- (added) clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixSum-errors.hlsl (+56) 
- (modified) llvm/include/llvm/IR/IntrinsicsDirectX.td (+14) 
- (modified) llvm/include/llvm/IR/IntrinsicsSPIRV.td (+7) 
- (modified) llvm/lib/Target/DirectX/DXIL.td (+15) 
- (modified) llvm/lib/Target/DirectX/DXILShaderFlags.cpp (+2) 
- (modified) llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp (+2) 
- (modified) llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp (+8) 
- (modified) llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll (+14) 
- (added) llvm/test/CodeGen/DirectX/WaveMultiPrefixSum.ll (+81) 
- (added) llvm/test/CodeGen/DirectX/WaveMultiPrefixSum_errors.ll (+14) 
- (added) llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveMultiPrefixSum.ll (+147) 


``````````diff
diff --git a/clang/include/clang/Basic/Builtins.td 
b/clang/include/clang/Basic/Builtins.td
index 0d0bc61907fbb..ad5f1e10f34b1 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -5813,6 +5813,12 @@ def HLSLWaveMultiPrefixProduct : 
LangBuiltin<"HLSL_LANG"> {
   let Prototype = "void(...)";
 }
 
+def HLSLWaveMultiPrefixSum : LangBuiltin<"HLSL_LANG"> {
+  let Spellings = ["__builtin_hlsl_wave_multi_prefix_sum"];
+  let Attributes = [NoThrow, Const];
+  let Prototype = "void(...)";
+}
+
 def HLSLWavePrefixProduct : LangBuiltin<"HLSL_LANG"> {
   let Spellings = ["__builtin_hlsl_wave_prefix_product"];
   let Attributes = [NoThrow, Const];
diff --git a/clang/include/clang/Basic/HLSLIntrinsics.td 
b/clang/include/clang/Basic/HLSLIntrinsics.td
index 7009127c444b6..e87cabb5c865b 100644
--- a/clang/include/clang/Basic/HLSLIntrinsics.td
+++ b/clang/include/clang/Basic/HLSLIntrinsics.td
@@ -1922,6 +1922,16 @@ def hlsl_wave_multi_prefix_product :
   let Availability = SM6_5;
 }
 
+def hlsl_wave_multi_prefix_sum :
+    HLSLTwoArgBuiltin<"WaveMultiPrefixSum",
+                      "__builtin_hlsl_wave_multi_prefix_sum"> {
+  let Args = [Varying, VectorType<UIntTy, 4>];
+  let VaryingTypes = AllNumericTypes;
+  let VaryingLongVector = 1;
+  let IsConvergent = 1;
+  let Availability = SM6_5;
+}
+
 // Returns the count of bits set to 1 on prior lanes.
 def hlsl_wave_prefix_count_bits :
     HLSLBuiltin<"WavePrefixCountBits",
diff --git a/clang/lib/CodeGen/CGHLSLBuiltins.cpp 
b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
index d301a16f5b2e1..38893d841d922 100644
--- a/clang/lib/CodeGen/CGHLSLBuiltins.cpp
+++ b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
@@ -1723,6 +1723,7 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned 
BuiltinID,
   }
   case Builtin::BI__builtin_hlsl_wave_multi_prefix_count_bits:
   case Builtin::BI__builtin_hlsl_wave_multi_prefix_product:
+  case Builtin::BI__builtin_hlsl_wave_multi_prefix_sum:
   case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and:
   case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_xor: {
     Value *ValueOp = EmitScalarExpr(E->getArg(0));
@@ -1741,6 +1742,14 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned 
BuiltinID,
       return EmitIntrinsicCall(IID, {ValueOp->getType()}, Args,
                                "hlsl.wave.multi.prefix.product");
     }
+    if (BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_sum) {
+      Intrinsic::ID IID =
+          E->getArg(0)->getType()->hasUnsignedIntegerRepresentation()
+              ? CGM.getHLSLRuntime().getWaveMultiPrefixUSumIntrinsic()
+              : CGM.getHLSLRuntime().getWaveMultiPrefixSumIntrinsic();
+      return EmitIntrinsicCall(IID, {ValueOp->getType()}, Args,
+                               "hlsl.wave.multi.prefix.sum");
+    }
     if (BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and)
       return EmitIntrinsicCall(
           CGM.getHLSLRuntime().getWaveMultiPrefixBitAndIntrinsic(),
diff --git a/clang/lib/CodeGen/CGHLSLRuntime.h 
b/clang/lib/CodeGen/CGHLSLRuntime.h
index 50a7729be6875..35237123713c9 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.h
+++ b/clang/lib/CodeGen/CGHLSLRuntime.h
@@ -161,6 +161,12 @@ class CGHLSLRuntime {
   GENERATE_HLSL_INTRINSIC_FUNCTION_BASE(WaveMultiPrefixUProduct,
                                         wave_multi_prefix_uproduct,
                                         subgroup_multi_prefix_product)
+  GENERATE_HLSL_INTRINSIC_FUNCTION_BASE(WaveMultiPrefixSum,
+                                        wave_multi_prefix_sum,
+                                        subgroup_multi_prefix_sum)
+  GENERATE_HLSL_INTRINSIC_FUNCTION_BASE(WaveMultiPrefixUSum,
+                                        wave_multi_prefix_usum,
+                                        subgroup_multi_prefix_sum)
   GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveMax, wave_reduce_max)
   GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveUMax, wave_reduce_umax)
   GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveMin, wave_reduce_min)
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index d6a280e1fb6cb..7b8d911d8309d 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -5408,7 +5408,8 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned 
BuiltinID, CallExpr *TheCall) {
   }
   case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_and:
   case Builtin::BI__builtin_hlsl_wave_multi_prefix_bit_xor:
-  case Builtin::BI__builtin_hlsl_wave_multi_prefix_product: {
+  case Builtin::BI__builtin_hlsl_wave_multi_prefix_product:
+  case Builtin::BI__builtin_hlsl_wave_multi_prefix_sum: {
     if (SemaRef.checkArgCount(TheCall, 2))
       return true;
 
@@ -5418,14 +5419,15 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned 
BuiltinID, CallExpr *TheCall) {
 
     QualType ValueTy = TheCall->getArg(0)->getType();
     QualType ElementTy = getScalarComponentType(ValueTy);
-    bool IsProduct =
-        BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_product;
+    bool IsArithmetic =
+        BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_product ||
+        BuiltinID == Builtin::BI__builtin_hlsl_wave_multi_prefix_sum;
     if ((!ElementTy->isIntegerType() &&
-         !(IsProduct && ElementTy->hasFloatingRepresentation())) ||
+         !(IsArithmetic && ElementTy->hasFloatingRepresentation())) ||
         ElementTy->isBooleanType()) {
       SemaRef.Diag(TheCall->getArg(0)->getBeginLoc(),
                    diag::err_builtin_invalid_arg_type)
-          << 1 << (ValueTy->isMatrixType() ? 3 : 5) << 1 << IsProduct
+          << 1 << (ValueTy->isMatrixType() ? 3 : 5) << 1 << IsArithmetic
           << ValueTy;
       return true;
     }
diff --git a/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixSum.hlsl 
b/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixSum.hlsl
new file mode 100644
index 0000000000000..943a90884521d
--- /dev/null
+++ b/clang/test/CodeGenHLSL/builtins/WaveMultiPrefixSum.hlsl
@@ -0,0 +1,113 @@
+// RUN: %clang_cc1 -finclude-default-header \
+// RUN:   -fnative-half-type -fnative-int16-type \
+// RUN:   -triple dxil-pc-shadermodel6.5-library %s -emit-llvm \
+// RUN:   -disable-llvm-passes -o - | FileCheck %s \
+// RUN:   -DINTRINSIC=dx.wave.multi.prefix.sum \
+// RUN:   -DUINTRINSIC=dx.wave.multi.prefix.usum
+// RUN: %clang_cc1 -finclude-default-header \
+// RUN:   -fnative-half-type -fnative-int16-type \
+// RUN:   -triple spirv-pc-vulkan-library %s -emit-llvm \
+// RUN:   -disable-llvm-passes -o - | FileCheck %s \
+// RUN:   -DINTRINSIC=spv.subgroup.multi.prefix.sum \
+// RUN:   -DUINTRINSIC=spv.subgroup.multi.prefix.sum
+
+// CHECK-LABEL: test_float
+float test_float(float value, uint4 mask) {
+  // CHECK: %[[TOKEN:.*]] = call token @llvm.experimental.convergence.entry()
+  // CHECK: %[[MASK:.*]] = load <4 x i32>
+  // CHECK: %[[M0:.*]] = extractelement <4 x i32> %[[MASK]], i64 0
+  // CHECK: %[[M1:.*]] = extractelement <4 x i32> %[[MASK]], i64 1
+  // CHECK: %[[M2:.*]] = extractelement <4 x i32> %[[MASK]], i64 2
+  // CHECK: %[[M3:.*]] = extractelement <4 x i32> %[[MASK]], i64 3
+  // CHECK: %[[RET:.*]] = call {{.*}}float @llvm.[[INTRINSIC]].f32(float 
%{{.*}}, i32 %[[M0]], i32 %[[M1]], i32 %[[M2]], i32 %[[M3]]) [ 
"convergencectrl"(token %[[TOKEN]]) ]
+  // CHECK: ret float %[[RET]]
+  return WaveMultiPrefixSum(value, mask);
+}
+
+// CHECK: declare float @llvm.[[INTRINSIC]].f32(float, i32, i32, i32, i32) 
#[[ATTR:[0-9]+]]
+
+// CHECK-LABEL: test_half
+float16_t test_half(float16_t value) {
+  // CHECK: call {{.*}}half @llvm.[[INTRINSIC]].f16(half %{{.*}}, i32 1, i32 
2, i32 4, i32 8)
+  return WaveMultiPrefixSum(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_double
+double test_double(double value) {
+  // CHECK: call {{.*}}double @llvm.[[INTRINSIC]].f64(double %{{.*}}, i32 1, 
i32 2, i32 4, i32 8)
+  return WaveMultiPrefixSum(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_int
+int test_int(int value) {
+  // CHECK: call i32 @llvm.[[INTRINSIC]].i32(i32 %{{.*}}, i32 1, i32 2, i32 4, 
i32 8)
+  return __builtin_hlsl_wave_multi_prefix_sum(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_int16
+int16_t test_int16(int16_t value) {
+  // CHECK: call i16 @llvm.[[INTRINSIC]].i16(i16 %{{.*}}, i32 1, i32 2, i32 4, 
i32 8)
+  return WaveMultiPrefixSum(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_int64
+int64_t test_int64(int64_t value) {
+  // CHECK: call i64 @llvm.[[INTRINSIC]].i64(i64 %{{.*}}, i32 1, i32 2, i32 4, 
i32 8)
+  return WaveMultiPrefixSum(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_uint
+uint test_uint(uint value) {
+  // CHECK: call i32 @llvm.[[UINTRINSIC]].i32(i32 %{{.*}}, i32 1, i32 2, i32 
4, i32 8)
+  return WaveMultiPrefixSum(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_uint16
+uint16_t test_uint16(uint16_t value) {
+  // CHECK: call i16 @llvm.[[UINTRINSIC]].i16(i16 %{{.*}}, i32 1, i32 2, i32 
4, i32 8)
+  return WaveMultiPrefixSum(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_uint64
+uint64_t test_uint64(uint64_t value) {
+  // CHECK: call i64 @llvm.[[UINTRINSIC]].i64(i64 %{{.*}}, i32 1, i32 2, i32 
4, i32 8)
+  return WaveMultiPrefixSum(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_uint2
+uint2 test_uint2(uint2 value, uint4 mask) {
+  // CHECK: call <2 x i32> @llvm.[[UINTRINSIC]].v2i32(<2 x i32> %{{.*}}, i32 
%{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
+  return WaveMultiPrefixSum(value, mask);
+}
+
+// CHECK-LABEL: test_float4
+float4 test_float4(float4 value, uint4 mask) {
+  // CHECK: call {{.*}}<4 x float> @llvm.[[INTRINSIC]].v4f32(<4 x float> 
%{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
+  return WaveMultiPrefixSum(value, mask);
+}
+
+// CHECK-LABEL: test_uint_matrix
+uint2x2 test_uint_matrix(uint2x2 value, uint4 mask) {
+  // CHECK: call <4 x i32> @llvm.[[UINTRINSIC]].v4i32(<4 x i32> %{{.*}}, i32 
%{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
+  return WaveMultiPrefixSum(value, mask);
+}
+
+// CHECK-LABEL: test_float_matrix
+float2x2 test_float_matrix(float2x2 value, uint4 mask) {
+  // CHECK: call {{.*}}<4 x float> @llvm.[[INTRINSIC]].v4f32(<4 x float> 
%{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
+  return WaveMultiPrefixSum(value, mask);
+}
+
+// CHECK-LABEL: test_uint5
+vector<uint, 5> test_uint5(vector<uint, 5> value) {
+  // CHECK: call <5 x i32> @llvm.[[UINTRINSIC]].v5i32(<5 x i32> %{{.*}}, i32 
1, i32 2, i32 4, i32 8)
+  return WaveMultiPrefixSum(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK-LABEL: test_float5
+vector<float, 5> test_float5(vector<float, 5> value) {
+  // CHECK: call {{.*}}<5 x float> @llvm.[[INTRINSIC]].v5f32(<5 x float> 
%{{.*}}, i32 1, i32 2, i32 4, i32 8)
+  return WaveMultiPrefixSum(value, uint4(1, 2, 4, 8));
+}
+
+// CHECK: attributes #[[ATTR]] = { convergent {{.*}}memory(none) }
diff --git a/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixSum-errors.hlsl 
b/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixSum-errors.hlsl
new file mode 100644
index 0000000000000..f6ffeee2e1eaf
--- /dev/null
+++ b/clang/test/SemaHLSL/BuiltIns/WaveMultiPrefixSum-errors.hlsl
@@ -0,0 +1,56 @@
+// RUN: %clang_cc1 -finclude-default-header \
+// RUN:   -fnative-half-type -fnative-int16-type \
+// RUN:   -triple dxil-pc-shadermodel6.5-library -verify %s
+// RUN: %clang_cc1 -finclude-default-header \
+// RUN:   -fnative-half-type -fnative-int16-type \
+// RUN:   -triple spirv-pc-vulkan-library -verify %s
+// RUN: %clang_cc1 -finclude-default-header -DTEST_AVAILABILITY \
+// RUN:   -triple dxil-pc-shadermodel6.4-library -verify=availability %s
+
+#ifdef TEST_AVAILABILITY
+[shader("compute")]
+[numthreads(1, 1, 1)]
+void main() {
+  (void)WaveMultiPrefixSum(1.0f, uint4(1, 2, 4, 8));
+  // availability-error@-1 {{only available on Shader Model 6.5 or newer}}
+  // availability-note@*:* {{introduced in Shader Model 6.5}}
+}
+#else
+void test_arg_count(float value, uint4 mask) {
+  __builtin_hlsl_wave_multi_prefix_sum(value);
+  // expected-error@-1 {{too few arguments to function call, expected 2}}
+  __builtin_hlsl_wave_multi_prefix_sum(value, mask, mask);
+  // expected-error@-1 {{too many arguments to function call, expected 2}}
+}
+
+struct S { float value; };
+
+void test_value_types(bool b, bool2 bv, bool2x2 bm, S s, uint4 mask) {
+  __builtin_hlsl_wave_multi_prefix_sum(b, mask);
+  // expected-error@-1 {{invalid operand of type 'bool'}}
+  __builtin_hlsl_wave_multi_prefix_sum(bv, mask);
+  // expected-error@-1 {{invalid operand of type 'bool2'}}
+  __builtin_hlsl_wave_multi_prefix_sum(bm, mask);
+  // expected-error@-1 {{matrix, integer or floating-point}}
+  // expected-error@-2 {{cannot pass object of non-trivial type 'bool2x2'}}
+  __builtin_hlsl_wave_multi_prefix_sum(s, mask);
+  // expected-error@-1 {{invalid operand of type 'S'}}
+}
+
+void test_mask_types(float value, uint scalar, uint3 short_mask,
+                     int4 signed_mask, float4 float_mask,
+                     uint64_t4 wide_mask, S struct_mask) {
+  __builtin_hlsl_wave_multi_prefix_sum(value, scalar);
+  // expected-error@-1 {{passing 'uint' (aka 'unsigned int')}}
+  __builtin_hlsl_wave_multi_prefix_sum(value, short_mask);
+  // expected-error@-1 {{passing 'uint3' (aka 'vector<uint, 3>')}}
+  __builtin_hlsl_wave_multi_prefix_sum(value, signed_mask);
+  // expected-error@-1 {{passing 'int4' (aka 'vector<int, 4>')}}
+  __builtin_hlsl_wave_multi_prefix_sum(value, float_mask);
+  // expected-error@-1 {{passing 'float4' (aka 'vector<float, 4>')}}
+  __builtin_hlsl_wave_multi_prefix_sum(value, wide_mask);
+  // expected-error@-1 {{passing 'uint64_t4'}}
+  __builtin_hlsl_wave_multi_prefix_sum(value, struct_mask);
+  // expected-error@-1 {{passing 'S' to parameter of incompatible type}}
+}
+#endif
diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td 
b/llvm/include/llvm/IR/IntrinsicsDirectX.td
index 1aaabaec97309..4a7b49831c9d4 100644
--- a/llvm/include/llvm/IR/IntrinsicsDirectX.td
+++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td
@@ -508,6 +508,20 @@ let TargetPrefix = "dx" in {
                               [IntrConvergent, IntrNoMem,
                                IntrTriviallyScalarizable]>;
 
+  def int_dx_wave_multi_prefix_sum
+      : DefaultAttrsIntrinsic<[llvm_any_ty],
+                              [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty,
+                               llvm_i32_ty, llvm_i32_ty],
+                              [IntrConvergent, IntrNoMem,
+                               IntrTriviallyScalarizable]>;
+
+  def int_dx_wave_multi_prefix_usum
+      : DefaultAttrsIntrinsic<[llvm_anyint_ty],
+                              [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty,
+                               llvm_i32_ty, llvm_i32_ty],
+                              [IntrConvergent, IntrNoMem,
+                               IntrTriviallyScalarizable]>;
+
   def int_dx_wave_prefix_usum
       : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>],
                               [IntrConvergent, IntrNoMem,
diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td 
b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index 926320653ab52..28e4d580bbcdf 100644
--- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -404,6 +404,13 @@ let TargetPrefix = "spv" in {
                               [IntrConvergent, IntrNoMem,
                                IntrTriviallyScalarizable]>;
 
+  def int_spv_subgroup_multi_prefix_sum
+      : DefaultAttrsIntrinsic<[llvm_any_ty],
+                              [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty,
+                               llvm_i32_ty, llvm_i32_ty],
+                              [IntrConvergent, IntrNoMem,
+                               IntrTriviallyScalarizable]>;
+
   def int_spv_wave_prefix_product
       : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>],
                               [IntrConvergent, IntrNoMem,
diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td
index 361ddac0a1b3e..accd5be5801f1 100644
--- a/llvm/lib/Target/DirectX/DXIL.td
+++ b/llvm/lib/Target/DirectX/DXIL.td
@@ -324,6 +324,7 @@ defvar WaveBitOpKind_And = 0;
 defvar WaveBitOpKind_Or = 1;
 defvar WaveBitOpKind_Xor = 2;
 
+defvar WaveMultiPrefixOpKind_Sum = 0;
 defvar WaveMultiPrefixOpKind_And = 1;
 defvar WaveMultiPrefixOpKind_Xor = 3;
 defvar WaveMultiPrefixOpKind_Product = 4;
@@ -1593,6 +1594,20 @@ def WaveMultiPrefixOp : DXILOp<166, waveMultiPrefixOp> {
                    IntrinArgIndex<3>, IntrinArgIndex<4>,
                    IntrinArgI8<WaveMultiPrefixOpKind_Product>,
                    IntrinArgI8<SignedOpKind_Unsigned>
+                 ]>,
+    IntrinSelect<int_dx_wave_multi_prefix_sum,
+                 [
+                   IntrinArgIndex<0>, IntrinArgIndex<1>, IntrinArgIndex<2>,
+                   IntrinArgIndex<3>, IntrinArgIndex<4>,
+                   IntrinArgI8<WaveMultiPrefixOpKind_Sum>,
+                   IntrinArgI8<SignedOpKind_Signed>
+                 ]>,
+    IntrinSelect<int_dx_wave_multi_prefix_usum,
+                 [
+                   IntrinArgIndex<0>, IntrinArgIndex<1>, IntrinArgIndex<2>,
+                   IntrinArgIndex<3>, IntrinArgIndex<4>,
+                   IntrinArgI8<WaveMultiPrefixOpKind_Sum>,
+                   IntrinArgI8<SignedOpKind_Unsigned>
                  ]>
   ];
   let arguments = [
diff --git a/llvm/lib/Target/DirectX/DXILShaderFlags.cpp 
b/llvm/lib/Target/DirectX/DXILShaderFlags.cpp
index ccfe6e9c947c0..cbbc17de32e96 100644
--- a/llvm/lib/Target/DirectX/DXILShaderFlags.cpp
+++ b/llvm/lib/Target/DirectX/DXILShaderFlags.cpp
@@ -116,6 +116,8 @@ static bool checkWaveOps(Intrinsic::ID IID) {
   case Intrinsic::dx_wave_multi_prefix_bit_count:
   case Intrinsic::dx_wave_multi_prefix_product:
   case Intrinsic::dx_wave_multi_prefix_uproduct:
+  case Intrinsic::dx_wave_multi_prefix_sum:
+  case Intrinsic::dx_wave_multi_prefix_usum:
     // Quad Op Variants
   case Intrinsic::dx_quad_read_across_x:
   case Intrinsic::dx_quad_read_across_y:
diff --git a/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp 
b/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp
index 320156ee3d153..8bf0de1c60859 100644
--- a/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp
+++ b/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp
@@ -24,6 +24,8 @@ bool DirectXTTIImpl::isTargetIntrinsicWithScalarOpAtArg(
   case Intrinsic::dx_wave_multi_prefix_bit_xor:
   case Intrinsic::dx_wave_multi_prefix_product:
   case Intrinsic::dx_wave_multi_prefix_uproduct:
+  case Intrinsic::dx_wave_multi_prefix_sum:
+  case Intrinsic::dx_wave_multi_prefix_usum:
     return ScalarOpdIdx >= 1 && ScalarOpdIdx <= 4;
   default:
     return false;
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp 
b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index b72d5517e5ce9..c61ed5c02d8a8 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -5846,6 +5846,14 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register 
ResVReg,
                                              : SPIRV::OpGroupNonUniformIMul,
                                    "WaveMultiPrefixProduct");
   }
+  case Intrinsic::spv_subgroup_multi_prefix_sum: {
+    bool IsFloatTy =
+        GR.isScalarOrVectorOfType(I.getOperand(2).getReg(), 
SPIRV::OpTypeFloat);
+    return selectWaveMultiPrefixOp(ResVReg, ResType, I,
+                                   IsFloatTy ? SPIRV::OpGroupNonUniformFAdd
+                                             : SPIRV::OpGroupNonUniformIAdd,
+                                   "WaveMultiPrefixSum");
+  }
   case Intrinsic::spv_wave_prefix_product:
     return selectWaveExclusiveScanProduct(ResVReg, ResType, I);
   case Intrinsic::spv_quad_read_across_x: {
diff --git a/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll 
b/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll
index 5ee508ab5c49c..f045f0f85b95a 100644
--- a/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll
+++ b/llvm/test/CodeGen/DirectX/ShaderFlags/wave-ops.ll
@@ -105,6 +105,20 @@ define i32 @wave_multi_prefix_uproduct(i32 %x) {
   ret i32 %ret
 }
 
+define float @wave_multi_prefix_sum(float %x) {
+  ; CHECK: Function wave_multi_prefix_sum : [[WAVE_FLAG]]
+  %ret = call float @llvm.dx.wave.multi.prefix.sum.f32(
+      float %x, i32 1, i32 2, i32 4, i32 8)
+  ret float %ret
+}
+
+define i32 @wave_multi_prefix_usum(i32 %x) {
+  ; CHECK: Function wave_multi_prefix_usum : [[WAVE_FLAG]]
+  %ret = cal...
[truncated]

``````````

</details>


https://github.com/llvm/llvm-project/pull/238175
_______________________________________________
llvm-branch-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits

Reply via email to