llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-backend-x86 Author: 陈子昂 (Michael-Chen-NJU) <details> <summary>Changes</summary> Add `__riscv_pm4adda_i8x8`, `__riscv_pm4addau_u8x8`, and `__riscv_pm4addasu_i8x8` from the Packed Multiplication with Horizontal Addition and Accumulate section of the P-extension intrinsic specification. The wrappers reuse the existing packed dot-product intrinsics and generic vector addition. RV64 selects one byte multiply-accumulate instruction; RV32 selects one for each word. --- Full diff: https://github.com/llvm/llvm-project/pull/234484.diff 6 Files Affected: - (modified) clang/lib/Headers/riscv_packed_simd.h (+18) - (modified) clang/test/CodeGen/RISCV/rvp-intrinsics.c (+47) - (modified) cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c (+25) - (modified) llvm/lib/Target/RISCV/RISCVISelLowering.cpp (+35) - (modified) llvm/lib/Target/RISCV/RISCVInstrInfoP.td (+15) - (added) llvm/test/CodeGen/RISCV/rvp-byte-horizontal-acc.ll (+131) ``````````diff diff --git a/clang/lib/Headers/riscv_packed_simd.h b/clang/lib/Headers/riscv_packed_simd.h index ef5708689331c..f45c45d14200a 100644 --- a/clang/lib/Headers/riscv_packed_simd.h +++ b/clang/lib/Headers/riscv_packed_simd.h @@ -1193,6 +1193,24 @@ __packed_binary_builtin_mixed(pm4add_i16x4, int64_t, int16x4_t, int16x4_t, __bui __packed_binary_builtin_mixed(pm4addu_u16x4, uint64_t, uint16x4_t, uint16x4_t, __builtin_riscv_pm4addu_u16x4) __packed_binary_builtin_mixed(pm4addsu_i16x4, int64_t, int16x4_t, uint16x4_t, __builtin_riscv_pm4addsu_i16x4) +/* Packed Multiplication with Horizontal Addition and Accumulate (64-bit) */ +static __inline__ int32x2_t __DEFAULT_FN_ATTRS +__riscv_pm4adda_i8x8(int32x2_t __rd, int8x8_t __rs1, int8x8_t __rs2) { + return (int32x2_t)((uint32x2_t)__rd + + (uint32x2_t)__riscv_pm4add_i8x8(__rs1, __rs2)); +} + +static __inline__ uint32x2_t __DEFAULT_FN_ATTRS +__riscv_pm4addau_u8x8(uint32x2_t __rd, uint8x8_t __rs1, uint8x8_t __rs2) { + return __rd + __riscv_pm4addu_u8x8(__rs1, __rs2); +} + +static __inline__ int32x2_t __DEFAULT_FN_ATTRS +__riscv_pm4addasu_i8x8(int32x2_t __rd, int8x8_t __rs1, uint8x8_t __rs2) { + return (int32x2_t)((uint32x2_t)__rd + + (uint32x2_t)__riscv_pm4addsu_i8x8(__rs1, __rs2)); +} + /* Packed Multiplication with Widening Horizontal Addition */ __packed_binary_builtin_mixed(pm2wadd_i64, int64_t, int16x2_t, int16x2_t, __builtin_riscv_pm2wadd_i64) __packed_binary_builtin_mixed(pm2wadd_x_i64, int64_t, int16x2_t, int16x2_t, __builtin_riscv_pm2wadd_x_i64) diff --git a/clang/test/CodeGen/RISCV/rvp-intrinsics.c b/clang/test/CodeGen/RISCV/rvp-intrinsics.c index 74500ed9d79be..7297a1f4de195 100644 --- a/clang/test/CodeGen/RISCV/rvp-intrinsics.c +++ b/clang/test/CodeGen/RISCV/rvp-intrinsics.c @@ -10152,3 +10152,50 @@ int64_t test_pm2wsub_x_i64(int16x2_t rs1, int16x2_t rs2) { int64_t test_pm2waddsu_i64(int16x2_t rs1, uint16x2_t rs2) { return __riscv_pm2waddsu_i64(rs1, rs2); } + +/* Packed Byte Multiplication with Horizontal Addition and Accumulate */ + +// CHECK-LABEL: define dso_local i64 @test_pm4adda_i8x8( +// CHECK-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[A_COERCE:%.*]], i64 noundef [[B_COERCE:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32> +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[A_COERCE]] to <8 x i8> +// CHECK-NEXT: [[TMP2:%.*]] = bitcast i64 [[B_COERCE]] to <8 x i8> +// CHECK-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pm4add.v2i32.v8i8(<8 x i8> [[TMP1]], <8 x i8> [[TMP2]]) +// CHECK-NEXT: [[ADD_I:%.*]] = add <2 x i32> [[TMP3]], [[TMP0]] +// CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[ADD_I]] to i64 +// CHECK-NEXT: ret i64 [[TMP4]] +// +int32x2_t test_pm4adda_i8x8(int32x2_t rd, int8x8_t a, int8x8_t b) { + return __riscv_pm4adda_i8x8(rd, a, b); +} + +// CHECK-LABEL: define dso_local i64 @test_pm4addau_u8x8( +// CHECK-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[A_COERCE:%.*]], i64 noundef [[B_COERCE:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32> +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[A_COERCE]] to <8 x i8> +// CHECK-NEXT: [[TMP2:%.*]] = bitcast i64 [[B_COERCE]] to <8 x i8> +// CHECK-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pm4addu.v2i32.v8i8(<8 x i8> [[TMP1]], <8 x i8> [[TMP2]]) +// CHECK-NEXT: [[ADD_I:%.*]] = add <2 x i32> [[TMP3]], [[TMP0]] +// CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[ADD_I]] to i64 +// CHECK-NEXT: ret i64 [[TMP4]] +// +uint32x2_t test_pm4addau_u8x8(uint32x2_t rd, uint8x8_t a, uint8x8_t b) { + return __riscv_pm4addau_u8x8(rd, a, b); +} + +// CHECK-LABEL: define dso_local i64 @test_pm4addasu_i8x8( +// CHECK-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[A_COERCE:%.*]], i64 noundef [[B_COERCE:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32> +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[A_COERCE]] to <8 x i8> +// CHECK-NEXT: [[TMP2:%.*]] = bitcast i64 [[B_COERCE]] to <8 x i8> +// CHECK-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pm4addsu.v2i32.v8i8(<8 x i8> [[TMP1]], <8 x i8> [[TMP2]]) +// CHECK-NEXT: [[ADD_I:%.*]] = add <2 x i32> [[TMP3]], [[TMP0]] +// CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[ADD_I]] to i64 +// CHECK-NEXT: ret i64 [[TMP4]] +// +int32x2_t test_pm4addasu_i8x8(int32x2_t rd, int8x8_t a, uint8x8_t b) { + return __riscv_pm4addasu_i8x8(rd, a, b); +} diff --git a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c index 875d2de46a0b3..53cafd8b0f146 100644 --- a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c +++ b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c @@ -4530,6 +4530,31 @@ int64_t test_pm4addsu_i16x4(int16x4_t rs1, uint16x4_t rs2) { return __riscv_pm4addsu_i16x4(rs1, rs2); } +// Packed Byte Multiplication with Horizontal Addition and Accumulate. +// CHECK-LABEL: test_pm4adda_i8x8: +// RV32: pm4adda.b{{[[:space:]]}} +// RV32: pm4adda.b{{[[:space:]]}} +// RV64: pm4adda.b{{[[:space:]]}} +int32x2_t test_pm4adda_i8x8(int32x2_t rd, int8x8_t a, int8x8_t b) { + return __riscv_pm4adda_i8x8(rd, a, b); +} + +// CHECK-LABEL: test_pm4addau_u8x8: +// RV32: pm4addau.b{{[[:space:]]}} +// RV32: pm4addau.b{{[[:space:]]}} +// RV64: pm4addau.b{{[[:space:]]}} +uint32x2_t test_pm4addau_u8x8(uint32x2_t rd, uint8x8_t a, uint8x8_t b) { + return __riscv_pm4addau_u8x8(rd, a, b); +} + +// CHECK-LABEL: test_pm4addasu_i8x8: +// RV32: pm4addasu.b{{[[:space:]]}} +// RV32: pm4addasu.b{{[[:space:]]}} +// RV64: pm4addasu.b{{[[:space:]]}} +int32x2_t test_pm4addasu_i8x8(int32x2_t rd, int8x8_t a, uint8x8_t b) { + return __riscv_pm4addasu_i8x8(rd, a, b); +} + // Packed Multiplication with Widening Horizontal Addition. // CHECK-LABEL: test_pm2wadd_i64: // RV32: pm2wadd.h diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 5aaa11a9b0474..995066d2cb487 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -19414,6 +19414,39 @@ static SDValue combineAddMulParts(SDNode *N, SelectionDAG &DAG, return SDValue(); } +// Expose the two scalar accumulates instead of a dot-product pair and PADD.DW. +static SDValue combineAddPackedByteDot(SDNode *N, SelectionDAG &DAG, + const RISCVSubtarget &Subtarget) { + if (!Subtarget.hasStdExtP() || Subtarget.is64Bit() || + N->getValueType(0) != MVT::v2i32) + return SDValue(); + + using namespace SDPatternMatch; + for (unsigned I = 0; I != 2; ++I) { + SDValue Lo, Hi; + if (!sd_match(N->getOperand(I), m_OneUse(m_Node(ISD::BUILD_VECTOR, + m_Value(Lo), m_Value(Hi))))) + continue; + // Splitting is profitable if at least one dot can fold into its add. + if (!Lo.hasOneUse() && !Hi.hasOneUse()) + continue; + unsigned Opc = Lo.getOpcode(); + if (Opc != Hi.getOpcode() || + (Opc != RISCVISD::PM4ADD && Opc != RISCVISD::PM4ADDU && + Opc != RISCVISD::PM4ADDSU)) + continue; + + SDLoc DL(N); + SDValue Acc = N->getOperand(1 - I); + SDValue AccLo = DAG.getExtractVectorElt(DL, MVT::i32, Acc, 0); + SDValue AccHi = DAG.getExtractVectorElt(DL, MVT::i32, Acc, 1); + SDValue SumLo = DAG.getNode(ISD::ADD, DL, MVT::i32, AccLo, Lo); + SDValue SumHi = DAG.getNode(ISD::ADD, DL, MVT::i32, AccHi, Hi); + return DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v2i32, SumLo, SumHi); + } + return SDValue(); +} + static SDValue combinePExtWideningAddSub(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget) { // Recognize the RV64 decompositions listed for the 32-bit packed widening @@ -19636,6 +19669,8 @@ static SDValue performADDCombine(SDNode *N, return V; if (SDValue V = combineAddOfExts(N, DAG, Subtarget)) return V; + if (SDValue V = combineAddPackedByteDot(N, DAG, Subtarget)) + return V; if (SDValue V = combineAddMulParts(N, DAG, Subtarget)) return V; if (SDValue V = combineAddMulh(N, DAG, Subtarget)) diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td index 99439fe03dc47..ba47136a08810 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td @@ -2210,6 +2210,13 @@ class PatHorizontalMul<SDPatternOperator OpNode, RVInst Inst, : Pat<(ResultVT (OpNode (SrcVT GPR:$rs1), (SrcVT GPR:$rs2))), (Inst GPR:$rs1, GPR:$rs2)>; +class PatHorizontalMulAdd<SDPatternOperator OpNode, RVInst Inst, + ValueType ResultVT, ValueType SrcVT> + : Pat<(ResultVT (add (ResultVT GPR:$rd), + (binop_oneuse<OpNode> (SrcVT GPR:$rs1), + (SrcVT GPR:$rs2)))), + (Inst GPR:$rd, GPR:$rs1, GPR:$rs2)>; + // (rs1 << 1) + rs2 class PatPSh1Add<RVInst Inst, ValueType vt> : Pat<(vt (add (riscv_psll (vt GPR:$rs1), (XLenVT 1)), (vt GPR:$rs2))), @@ -2590,6 +2597,10 @@ let append Predicates = [IsRV32] in { def : PatGprGpr<riscv_mulqr, MULQR, i32>; // Packed multiplication with horizontal addition patterns. + def : PatHorizontalMulAdd<riscv_pm4add, PM4ADDA_B, i32, v4i8>; + def : PatHorizontalMulAdd<riscv_pm4addu, PM4ADDAU_B, i32, v4i8>; + def : PatHorizontalMulAdd<riscv_pm4addsu, PM4ADDASU_B, i32, v4i8>; + def : PatHorizontalMul<riscv_pm4add, PM4ADD_B, i32, v4i8>; def : PatHorizontalMul<riscv_pm2add, PM2ADD_H, i32, v2i16>; def : PatHorizontalMul<riscv_pm2add_x, PM2ADD_HX, i32, v2i16>; @@ -3429,6 +3440,10 @@ let append Predicates = [IsRV64] in { def : PatGprGpr<riscv_mulqr, PMULQR_W, v2i32>; // Packed multiplication with horizontal addition patterns. + def : PatHorizontalMulAdd<riscv_pm4add, PM4ADDA_B, v2i32, v8i8>; + def : PatHorizontalMulAdd<riscv_pm4addu, PM4ADDAU_B, v2i32, v8i8>; + def : PatHorizontalMulAdd<riscv_pm4addsu, PM4ADDASU_B, v2i32, v8i8>; + def : PatHorizontalMul<riscv_pm4add, PM4ADD_B, v2i32, v8i8>; def : PatHorizontalMul<riscv_pm2add, PM2ADD_H, v2i32, v4i16>; def : PatHorizontalMul<riscv_pm2add_x, PM2ADD_HX, v2i32, v4i16>; diff --git a/llvm/test/CodeGen/RISCV/rvp-byte-horizontal-acc.ll b/llvm/test/CodeGen/RISCV/rvp-byte-horizontal-acc.ll new file mode 100644 index 0000000000000..505cfd7ed8475 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/rvp-byte-horizontal-acc.ll @@ -0,0 +1,131 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=riscv32 -mattr=+experimental-p -verify-machineinstrs < %s | FileCheck %s --check-prefix=RV32 +; RUN: llc -mtriple=riscv64 -mattr=+experimental-p -verify-machineinstrs < %s | FileCheck %s --check-prefix=RV64 + +define <2 x i32> @pm4adda(<2 x i32> %rd, <8 x i8> %a, <8 x i8> %b) { +; RV32-LABEL: pm4adda: +; RV32: # %bb.0: +; RV32-NEXT: pm4adda.b a1, a3, a5 +; RV32-NEXT: pm4adda.b a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: pm4adda: +; RV64: # %bb.0: +; RV64-NEXT: pm4adda.b a0, a1, a2 +; RV64-NEXT: ret + %dot = call <2 x i32> @llvm.riscv.pm4add.v2i32.v8i8(<8 x i8> %a, <8 x i8> %b) + %r = add <2 x i32> %rd, %dot + ret <2 x i32> %r +} + +declare <2 x i32> @llvm.riscv.pm4add.v2i32.v8i8(<8 x i8>, <8 x i8>) + +define <2 x i32> @pm4addau(<2 x i32> %rd, <8 x i8> %a, <8 x i8> %b) { +; RV32-LABEL: pm4addau: +; RV32: # %bb.0: +; RV32-NEXT: pm4addau.b a1, a3, a5 +; RV32-NEXT: pm4addau.b a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: pm4addau: +; RV64: # %bb.0: +; RV64-NEXT: pm4addau.b a0, a1, a2 +; RV64-NEXT: ret + %dot = call <2 x i32> @llvm.riscv.pm4addu.v2i32.v8i8(<8 x i8> %a, <8 x i8> %b) + %r = add <2 x i32> %rd, %dot + ret <2 x i32> %r +} + +declare <2 x i32> @llvm.riscv.pm4addu.v2i32.v8i8(<8 x i8>, <8 x i8>) + +define <2 x i32> @pm4addasu(<2 x i32> %rd, <8 x i8> %a, <8 x i8> %b) { +; RV32-LABEL: pm4addasu: +; RV32: # %bb.0: +; RV32-NEXT: pm4addasu.b a1, a3, a5 +; RV32-NEXT: pm4addasu.b a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: pm4addasu: +; RV64: # %bb.0: +; RV64-NEXT: pm4addasu.b a0, a1, a2 +; RV64-NEXT: ret + %dot = call <2 x i32> @llvm.riscv.pm4addsu.v2i32.v8i8(<8 x i8> %a, <8 x i8> %b) + %r = add <2 x i32> %rd, %dot + ret <2 x i32> %r +} + +declare <2 x i32> @llvm.riscv.pm4addsu.v2i32.v8i8(<8 x i8>, <8 x i8>) + +; A shared dot product should retain the paired add instead of duplicating it. +define <2 x i32> @pm4adda_shared(<2 x i32> %rd, <8 x i8> %a, <8 x i8> %b, ptr %out) { +; RV32-LABEL: pm4adda_shared: +; RV32: # %bb.0: +; RV32-NEXT: pm4add.b a3, a3, a5 +; RV32-NEXT: pm4add.b a2, a2, a4 +; RV32-NEXT: padd.dw a0, a0, a2 +; RV32-NEXT: sw a2, 0(a6) +; RV32-NEXT: sw a3, 4(a6) +; RV32-NEXT: ret +; +; RV64-LABEL: pm4adda_shared: +; RV64: # %bb.0: +; RV64-NEXT: pm4add.b a1, a1, a2 +; RV64-NEXT: padd.w a0, a0, a1 +; RV64-NEXT: sd a1, 0(a3) +; RV64-NEXT: ret + %dot = call <2 x i32> @llvm.riscv.pm4add.v2i32.v8i8(<8 x i8> %a, <8 x i8> %b) + store <2 x i32> %dot, ptr %out, align 8 + %r = add <2 x i32> %rd, %dot + ret <2 x i32> %r +} + +; One shared word must not cause its dot product to be duplicated. +define <2 x i32> @pm4adda_shared_word(<2 x i32> %rd, <8 x i8> %a, <8 x i8> %b, ptr %out) { +; RV32-LABEL: pm4adda_shared_word: +; RV32: # %bb.0: +; RV32-NEXT: pm4add.b a3, a3, a5 +; RV32-NEXT: pm4add.b a2, a2, a4 +; RV32-NEXT: padd.dw a0, a0, a2 +; RV32-NEXT: sw a2, 0(a6) +; RV32-NEXT: ret +; +; RV64-LABEL: pm4adda_shared_word: +; RV64: # %bb.0: +; RV64-NEXT: pm4add.b a1, a1, a2 +; RV64-NEXT: padd.w a0, a0, a1 +; RV64-NEXT: sw a1, 0(a3) +; RV64-NEXT: ret + %dot = call <2 x i32> @llvm.riscv.pm4add.v2i32.v8i8(<8 x i8> %a, <8 x i8> %b) + %lo = extractelement <2 x i32> %dot, i32 0 + store i32 %lo, ptr %out, align 4 + %r = add <2 x i32> %rd, %dot + ret <2 x i32> %r +} + +; Keep the paired add when neither word's dot product can be folded. +define <2 x i32> @pm4adda_shared_words(<2 x i32> %rd, <8 x i8> %a, <8 x i8> %b, ptr %out) { +; RV32-LABEL: pm4adda_shared_words: +; RV32: # %bb.0: +; RV32-NEXT: pm4add.b a3, a3, a5 +; RV32-NEXT: pm4add.b a2, a2, a4 +; RV32-NEXT: xor a4, a2, a3 +; RV32-NEXT: padd.dw a0, a0, a2 +; RV32-NEXT: sw a4, 0(a6) +; RV32-NEXT: ret +; +; RV64-LABEL: pm4adda_shared_words: +; RV64: # %bb.0: +; RV64-NEXT: pm4add.b a1, a1, a2 +; RV64-NEXT: srli a2, a1, 32 +; RV64-NEXT: padd.w a0, a0, a1 +; RV64-NEXT: xor a1, a1, a2 +; RV64-NEXT: sw a1, 0(a3) +; RV64-NEXT: ret + %dot = call <2 x i32> @llvm.riscv.pm4add.v2i32.v8i8(<8 x i8> %a, <8 x i8> %b) + %lo = extractelement <2 x i32> %dot, i32 0 + %hi = extractelement <2 x i32> %dot, i32 1 + %x = xor i32 %lo, %hi + store i32 %x, ptr %out, align 4 + %r = add <2 x i32> %rd, %dot + ret <2 x i32> %r +} `````````` </details> https://github.com/llvm/llvm-project/pull/234484 _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
