https://github.com/Michael-Chen-NJU updated https://github.com/llvm/llvm-project/pull/221622
From 3474e8c82f83278368652a394745b0308ca35bc2 Mon Sep 17 00:00:00 2001 From: Michael-Chen-NJU <[email protected]> Date: Mon, 7 Sep 2026 10:32:23 +0800 Subject: [PATCH 1/3] [Clang][RISCV] Add packed widening add accumulate intrinsics --- clang/lib/Headers/riscv_packed_simd.h | 13 ++ clang/test/CodeGen/RISCV/rvp-intrinsics.c | 122 ++++++++++++++++++ .../riscv_packed_simd.c | 39 ++++++ llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 82 ++++++++++++ llvm/lib/Target/RISCV/RISCVInstrInfoP.td | 24 ++++ .../CodeGen/RISCV/rvp-widening-add-acc.ll | 86 ++++++++++++ 6 files changed, 366 insertions(+) create mode 100644 llvm/test/CodeGen/RISCV/rvp-widening-add-acc.ll diff --git a/clang/lib/Headers/riscv_packed_simd.h b/clang/lib/Headers/riscv_packed_simd.h index 5c359337d8b7f5..e04c51b3208f92 100644 --- a/clang/lib/Headers/riscv_packed_simd.h +++ b/clang/lib/Headers/riscv_packed_simd.h @@ -157,6 +157,12 @@ typedef uint32_t uint32x2_t __attribute__((__vector_size__(8))); return __builtin_convertvector(__rs1, rty) \ op __builtin_convertvector(__rs2, rty); \ } +#define __packed_widen_binary_acc_op(name, rty, ty, op) \ + static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(rty __rd, ty __rs1, \ + ty __rs2) { \ + return __rd op __builtin_convertvector(__rs1, rty) \ + op __builtin_convertvector(__rs2, rty); \ + } #define __packed_widen_mul(name, rty, ty) \ static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1, \ ty __rs2) { \ @@ -629,6 +635,12 @@ __packed_widen_binary_op(pwsub_i32x2, int32x2_t, int16x2_t, -) __packed_widen_binary_op(pwsubu_u16x4, uint16x4_t, uint8x4_t, -) __packed_widen_binary_op(pwsubu_u32x2, uint32x2_t, uint16x2_t, -) +/* Packed Widening Addition Accumulate */ +__packed_widen_binary_acc_op(pwadda_i16x4, int16x4_t, int8x4_t, +) +__packed_widen_binary_acc_op(pwadda_i32x2, int32x2_t, int16x2_t, +) +__packed_widen_binary_acc_op(pwaddau_u16x4, uint16x4_t, uint8x4_t, +) +__packed_widen_binary_acc_op(pwaddau_u32x2, uint32x2_t, uint16x2_t, +) + /* Packed Widening Multiply (32-bit) */ __packed_widen_mul(pwmul_i16x4, int16x4_t, int8x4_t) __packed_widen_mul(pwmul_i32x2, int32x2_t, int16x2_t) @@ -1157,6 +1169,7 @@ __packed_reinterpret(u32x2_i32x2, int32x2_t, uint32x2_t) #undef __packed_unary_builtin #undef __packed_widen_convert #undef __packed_widen_binary_op +#undef __packed_widen_binary_acc_op #undef __packed_widen_mul #undef __packed_widen_mulsu #undef __packed_widen_high2 diff --git a/clang/test/CodeGen/RISCV/rvp-intrinsics.c b/clang/test/CodeGen/RISCV/rvp-intrinsics.c index 7589157c38754d..954e8eb77e41ea 100644 --- a/clang/test/CodeGen/RISCV/rvp-intrinsics.c +++ b/clang/test/CodeGen/RISCV/rvp-intrinsics.c @@ -8136,6 +8136,128 @@ uint32x2_t test_pwaddu_u32x2(uint16x2_t rs1, uint16x2_t rs2) { return __riscv_pwaddu_u32x2(rs1, rs2); } +// RV32-LABEL: define dso_local i64 @test_pwadda_i16x4( +// RV32-SAME: i64 noundef [[RD_COERCE:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV32-NEXT: [[ENTRY:.*:]] +// RV32-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <4 x i16> +// RV32-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS1_COERCE]] to <4 x i8> +// RV32-NEXT: [[TMP2:%.*]] = bitcast i32 [[RS2_COERCE]] to <4 x i8> +// RV32-NEXT: [[CONV_I:%.*]] = sext <4 x i8> [[TMP1]] to <4 x i16> +// RV32-NEXT: [[ADD_I:%.*]] = add <4 x i16> [[TMP0]], [[CONV_I]] +// RV32-NEXT: [[CONV4_I:%.*]] = sext <4 x i8> [[TMP2]] to <4 x i16> +// RV32-NEXT: [[ADD5_I:%.*]] = add <4 x i16> [[ADD_I]], [[CONV4_I]] +// RV32-NEXT: [[TMP3:%.*]] = bitcast <4 x i16> [[ADD5_I]] to i64 +// RV32-NEXT: ret i64 [[TMP3]] +// +// RV64-LABEL: define dso_local i64 @test_pwadda_i16x4( +// RV64-SAME: i64 noundef [[RD_COERCE:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV64-NEXT: [[ENTRY:.*:]] +// RV64-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <4 x i16> +// RV64-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS1_COERCE]] to <4 x i8> +// RV64-NEXT: [[TMP2:%.*]] = bitcast i32 [[RS2_COERCE]] to <4 x i8> +// RV64-NEXT: [[CONV_I:%.*]] = sext <4 x i8> [[TMP1]] to <4 x i16> +// RV64-NEXT: [[ADD_I:%.*]] = add <4 x i16> [[TMP0]], [[CONV_I]] +// RV64-NEXT: [[CONV4_I:%.*]] = sext <4 x i8> [[TMP2]] to <4 x i16> +// RV64-NEXT: [[ADD5_I:%.*]] = add <4 x i16> [[ADD_I]], [[CONV4_I]] +// RV64-NEXT: [[TMP3:%.*]] = bitcast <4 x i16> [[ADD5_I]] to i64 +// RV64-NEXT: ret i64 [[TMP3]] +// +int16x4_t test_pwadda_i16x4(int16x4_t rd, int8x4_t rs1, int8x4_t rs2) { + return __riscv_pwadda_i16x4(rd, rs1, rs2); +} + +// RV32-LABEL: define dso_local i64 @test_pwadda_i32x2( +// RV32-SAME: i64 noundef [[RD_COERCE:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV32-NEXT: [[ENTRY:.*:]] +// RV32-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32> +// RV32-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// RV32-NEXT: [[TMP2:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// RV32-NEXT: [[CONV_I:%.*]] = sext <2 x i16> [[TMP1]] to <2 x i32> +// RV32-NEXT: [[ADD_I:%.*]] = add <2 x i32> [[TMP0]], [[CONV_I]] +// RV32-NEXT: [[CONV4_I:%.*]] = sext <2 x i16> [[TMP2]] to <2 x i32> +// RV32-NEXT: [[ADD5_I:%.*]] = add <2 x i32> [[ADD_I]], [[CONV4_I]] +// RV32-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[ADD5_I]] to i64 +// RV32-NEXT: ret i64 [[TMP3]] +// +// RV64-LABEL: define dso_local i64 @test_pwadda_i32x2( +// RV64-SAME: i64 noundef [[RD_COERCE:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV64-NEXT: [[ENTRY:.*:]] +// RV64-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32> +// RV64-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// RV64-NEXT: [[TMP2:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// RV64-NEXT: [[CONV_I:%.*]] = sext <2 x i16> [[TMP1]] to <2 x i32> +// RV64-NEXT: [[ADD_I:%.*]] = add <2 x i32> [[TMP0]], [[CONV_I]] +// RV64-NEXT: [[CONV4_I:%.*]] = sext <2 x i16> [[TMP2]] to <2 x i32> +// RV64-NEXT: [[ADD5_I:%.*]] = add <2 x i32> [[ADD_I]], [[CONV4_I]] +// RV64-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[ADD5_I]] to i64 +// RV64-NEXT: ret i64 [[TMP3]] +// +int32x2_t test_pwadda_i32x2(int32x2_t rd, int16x2_t rs1, int16x2_t rs2) { + return __riscv_pwadda_i32x2(rd, rs1, rs2); +} + +// RV32-LABEL: define dso_local i64 @test_pwaddau_u16x4( +// RV32-SAME: i64 noundef [[RD_COERCE:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV32-NEXT: [[ENTRY:.*:]] +// RV32-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <4 x i16> +// RV32-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS1_COERCE]] to <4 x i8> +// RV32-NEXT: [[TMP2:%.*]] = bitcast i32 [[RS2_COERCE]] to <4 x i8> +// RV32-NEXT: [[CONV_I:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i16> +// RV32-NEXT: [[ADD_I:%.*]] = add <4 x i16> [[TMP0]], [[CONV_I]] +// RV32-NEXT: [[CONV4_I:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i16> +// RV32-NEXT: [[ADD5_I:%.*]] = add <4 x i16> [[ADD_I]], [[CONV4_I]] +// RV32-NEXT: [[TMP3:%.*]] = bitcast <4 x i16> [[ADD5_I]] to i64 +// RV32-NEXT: ret i64 [[TMP3]] +// +// RV64-LABEL: define dso_local i64 @test_pwaddau_u16x4( +// RV64-SAME: i64 noundef [[RD_COERCE:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV64-NEXT: [[ENTRY:.*:]] +// RV64-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <4 x i16> +// RV64-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS1_COERCE]] to <4 x i8> +// RV64-NEXT: [[TMP2:%.*]] = bitcast i32 [[RS2_COERCE]] to <4 x i8> +// RV64-NEXT: [[CONV_I:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i16> +// RV64-NEXT: [[ADD_I:%.*]] = add <4 x i16> [[TMP0]], [[CONV_I]] +// RV64-NEXT: [[CONV4_I:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i16> +// RV64-NEXT: [[ADD5_I:%.*]] = add <4 x i16> [[ADD_I]], [[CONV4_I]] +// RV64-NEXT: [[TMP3:%.*]] = bitcast <4 x i16> [[ADD5_I]] to i64 +// RV64-NEXT: ret i64 [[TMP3]] +// +uint16x4_t test_pwaddau_u16x4(uint16x4_t rd, uint8x4_t rs1, + uint8x4_t rs2) { + return __riscv_pwaddau_u16x4(rd, rs1, rs2); +} + +// RV32-LABEL: define dso_local i64 @test_pwaddau_u32x2( +// RV32-SAME: i64 noundef [[RD_COERCE:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV32-NEXT: [[ENTRY:.*:]] +// RV32-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32> +// RV32-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// RV32-NEXT: [[TMP2:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// RV32-NEXT: [[CONV_I:%.*]] = zext <2 x i16> [[TMP1]] to <2 x i32> +// RV32-NEXT: [[ADD_I:%.*]] = add <2 x i32> [[TMP0]], [[CONV_I]] +// RV32-NEXT: [[CONV4_I:%.*]] = zext <2 x i16> [[TMP2]] to <2 x i32> +// RV32-NEXT: [[ADD5_I:%.*]] = add <2 x i32> [[ADD_I]], [[CONV4_I]] +// RV32-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[ADD5_I]] to i64 +// RV32-NEXT: ret i64 [[TMP3]] +// +// RV64-LABEL: define dso_local i64 @test_pwaddau_u32x2( +// RV64-SAME: i64 noundef [[RD_COERCE:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV64-NEXT: [[ENTRY:.*:]] +// RV64-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32> +// RV64-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// RV64-NEXT: [[TMP2:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// RV64-NEXT: [[CONV_I:%.*]] = zext <2 x i16> [[TMP1]] to <2 x i32> +// RV64-NEXT: [[ADD_I:%.*]] = add <2 x i32> [[TMP0]], [[CONV_I]] +// RV64-NEXT: [[CONV4_I:%.*]] = zext <2 x i16> [[TMP2]] to <2 x i32> +// RV64-NEXT: [[ADD5_I:%.*]] = add <2 x i32> [[ADD_I]], [[CONV4_I]] +// RV64-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[ADD5_I]] to i64 +// RV64-NEXT: ret i64 [[TMP3]] +// +uint32x2_t test_pwaddau_u32x2(uint32x2_t rd, uint16x2_t rs1, + uint16x2_t rs2) { + return __riscv_pwaddau_u32x2(rd, rs1, rs2); +} + // RV32-LABEL: define dso_local i64 @test_pwsub_i16x4( // RV32-SAME: i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { // RV32-NEXT: [[ENTRY:.*:]] diff --git a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c index df9623edfa71ae..f222e66ff1dd5a 100644 --- a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c +++ b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c @@ -2343,6 +2343,45 @@ uint32x2_t test_pwsubu_u32x2(uint16x2_t rs1, uint16x2_t rs2) { return __riscv_pwsubu_u32x2(rs1, rs2); } +// CHECK-LABEL: test_pwadda_i16x4: +// RV32: pwadda.b +// RV64: zip8p +// RV64: psext.h.b +// RV64: padd.h +// RV64: psrai.h +// RV64: padd.h +int16x4_t test_pwadda_i16x4(int16x4_t rd, int8x4_t rs1, int8x4_t rs2) { + return __riscv_pwadda_i16x4(rd, rs1, rs2); +} + +// CHECK-LABEL: test_pwadda_i32x2: +// RV32: pwadda.h +// RV64: zip16p +// RV64: pli.h +// RV64: pm2adda.h +int32x2_t test_pwadda_i32x2(int32x2_t rd, int16x2_t rs1, int16x2_t rs2) { + return __riscv_pwadda_i32x2(rd, rs1, rs2); +} + +// CHECK-LABEL: test_pwaddau_u16x4: +// RV32: pwaddau.b +// RV64: pwcvtu.wb +// RV64: padd.h +// RV64: pwcvtu.wb +// RV64: padd.h +uint16x4_t test_pwaddau_u16x4(uint16x4_t rd, uint8x4_t rs1, uint8x4_t rs2) { + return __riscv_pwaddau_u16x4(rd, rs1, rs2); +} + +// CHECK-LABEL: test_pwaddau_u32x2: +// RV32: pwaddau.h +// RV64: zip16p +// RV64: pli.h +// RV64: pm2addau.h +uint32x2_t test_pwaddau_u32x2(uint32x2_t rd, uint16x2_t rs1, uint16x2_t rs2) { + return __riscv_pwaddau_u32x2(rd, rs1, rs2); +} + // CHECK-LABEL: test_pwcvth_i16x4: // RV32: pwcvth.b // RV64: pwcvth.wb diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 788ae8a57b7792..86d5da06e55f5e 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -18626,6 +18626,86 @@ static SDValue combinePExtWideningAddSub(SDNode *N, SelectionDAG &DAG, return SDValue(); } +static SDValue combinePExtWideningAddAcc(SDNode *N, SelectionDAG &DAG, + const RISCVSubtarget &Subtarget) { + if (!Subtarget.hasStdExtP() || !Subtarget.is64Bit()) + return SDValue(); + + if (N->getOpcode() != ISD::ADD) + return SDValue(); + + MVT VT = N->getSimpleValueType(0); + if (VT != MVT::v4i16 && VT != MVT::v2i32) + return SDValue(); + + auto MatchExtend = [](SDValue V, unsigned ExtendOpcode, MVT SrcVT, + SDValue &Src) -> bool { + if (V.getOpcode() != ExtendOpcode || !V.hasOneUse() || + V.getOperand(0).getSimpleValueType() != SrcVT) + return false; + Src = V.getOperand(0); + return true; + }; + + auto Match = [&](SDValue V, SDValue &Acc, SDValue &A, SDValue &B, + bool &IsSExt) -> bool { + unsigned ExtendOpcode = V.getOpcode(); + if (ExtendOpcode != ISD::SIGN_EXTEND && ExtendOpcode != ISD::ZERO_EXTEND) + return false; + + MVT SrcVT = VT == MVT::v4i16 ? MVT::v4i8 : MVT::v2i16; + if (!MatchExtend(V, ExtendOpcode, SrcVT, B)) + return false; + + SDValue Add = V == N->getOperand(0) ? N->getOperand(1) : N->getOperand(0); + if (Add.getOpcode() != ISD::ADD || !Add.hasOneUse()) + return false; + + if (MatchExtend(Add.getOperand(0), ExtendOpcode, SrcVT, A)) + Acc = Add.getOperand(1); + else if (MatchExtend(Add.getOperand(1), ExtendOpcode, SrcVT, A)) + Acc = Add.getOperand(0); + else + return false; + + if (Acc.getValueType() != VT) + return false; + + IsSExt = ExtendOpcode == ISD::SIGN_EXTEND; + return true; + }; + + SDValue Acc, A, B; + bool IsSExt; + if (!Match(N->getOperand(0), Acc, A, B, IsSExt) && + !Match(N->getOperand(1), Acc, A, B, IsSExt)) + return SDValue(); + + if (VT == MVT::v4i16 && !IsSExt) + return SDValue(); + + SDLoc DL(N); + MVT LegalSrcVT = VT == MVT::v4i16 ? MVT::v8i8 : MVT::v4i16; + MVT SrcVT = VT == MVT::v4i16 ? MVT::v4i8 : MVT::v2i16; + A = DAG.getNode(ISD::CONCAT_VECTORS, DL, LegalSrcVT, A, DAG.getUNDEF(SrcVT)); + B = DAG.getNode(ISD::CONCAT_VECTORS, DL, LegalSrcVT, B, DAG.getUNDEF(SrcVT)); + + SDValue Zip = DAG.getNode(RISCVISD::PZIP, DL, LegalSrcVT, A, B); + if (VT == MVT::v4i16) { + SDValue ZipAsVT = DAG.getBitcast(VT, Zip); + SDValue Low = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, ZipAsVT, + DAG.getValueType(MVT::v4i8)); + SDValue High = DAG.getNode(RISCVISD::PSRA, DL, VT, ZipAsVT, + DAG.getConstant(8, DL, MVT::i64)); + return DAG.getNode(ISD::ADD, DL, VT, + DAG.getNode(ISD::ADD, DL, VT, Acc, Low), High); + } + + SDValue Ones = DAG.getConstant(1, DL, LegalSrcVT); + unsigned Opc = IsSExt ? RISCVISD::PM2ADDA_H : RISCVISD::PM2ADDAU_H; + return DAG.getNode(Opc, DL, VT, Acc, Zip, Ones); +} + static SDValue performADDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget) { @@ -18644,6 +18724,8 @@ static SDValue performADDCombine(SDNode *N, return V; if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget)) return V; + if (SDValue V = combinePExtWideningAddAcc(N, DAG, Subtarget)) + return V; if (SDValue V = combinePExtWideningAddSub(N, DAG, Subtarget)) return V; if (SDValue V = combineBinOpOfZExt(N, DAG)) diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td index dfdfb5435c3c30..7f7d8b47481ea7 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td @@ -1927,6 +1927,12 @@ def riscv_pm2addu_h : RVSDNode<"PM2ADDU_H", SDT_RISCVPM2Halfword, [SDNPCommutative]>; def riscv_pm2sub_h : RVSDNode<"PM2SUB_H", SDT_RISCVPM2Halfword>; +def SDT_RISCVPM2HalfwordAcc + : SDTypeProfile<1, 3, [SDTCisVT<0, v2i32>, SDTCisSameAs<0, 1>, + SDTCisVT<2, v4i16>, SDTCisSameAs<2, 3>]>; +def riscv_pm2adda_h : RVSDNode<"PM2ADDA_H", SDT_RISCVPM2HalfwordAcc>; +def riscv_pm2addau_h : RVSDNode<"PM2ADDAU_H", SDT_RISCVPM2HalfwordAcc>; + def SDT_RISCVWideningMulAccByHalves : SDTypeProfile<1, 3, [SDTCisSameAs<0, 1>, SDTCisSameAs<2, 3>, @@ -2554,6 +2560,18 @@ let append Predicates = [IsRV32] in { (PWADDU_B GPR:$rs1, GPR:$rs2)>; def : Pat<(v2i32 (add (zext (v2i16 GPR:$rs1)), (zext (v2i16 GPR:$rs2)))), (PWADDU_H GPR:$rs1, GPR:$rs2)>; + def : Pat<(v4i16 (add (add GPRPair:$rd, (sext (v4i8 GPR:$rs1))), + (sext (v4i8 GPR:$rs2)))), + (PWADDA_B GPRPair:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(v2i32 (add (add GPRPair:$rd, (sext (v2i16 GPR:$rs1))), + (sext (v2i16 GPR:$rs2)))), + (PWADDA_H GPRPair:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(v4i16 (add (add GPRPair:$rd, (zext (v4i8 GPR:$rs1))), + (zext (v4i8 GPR:$rs2)))), + (PWADDAU_B GPRPair:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(v2i32 (add (add GPRPair:$rd, (zext (v2i16 GPR:$rs1))), + (zext (v2i16 GPR:$rs2)))), + (PWADDAU_H GPRPair:$rd, GPR:$rs1, GPR:$rs2)>; def : Pat<(v4i16 (sub (sext (v4i8 GPR:$rs1)), (sext (v4i8 GPR:$rs2)))), (PWSUB_B GPR:$rs1, GPR:$rs2)>; def : Pat<(v2i32 (sub (sext (v2i16 GPR:$rs1)), (sext (v2i16 GPR:$rs2)))), @@ -3232,6 +3250,12 @@ let append Predicates = [IsRV64] in { (PM2ADDU_H GPR:$rs1, GPR:$rs2)>; def : Pat<(v2i32 (riscv_pm2sub_h (v4i16 GPR:$rs1), (v4i16 GPR:$rs2))), (PM2SUB_H GPR:$rs1, GPR:$rs2)>; + def : Pat<(v2i32 (riscv_pm2adda_h (v2i32 GPR:$rd), (v4i16 GPR:$rs1), + (v4i16 GPR:$rs2))), + (PM2ADDA_H GPR:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(v2i32 (riscv_pm2addau_h (v2i32 GPR:$rd), (v4i16 GPR:$rs1), + (v4i16 GPR:$rs2))), + (PM2ADDAU_H GPR:$rd, GPR:$rs1, GPR:$rs2)>; // 32-bit logical shift left/right patterns def : PatGprImm<riscv_pshl, PSLLI_W, uimm5, v2i32>; diff --git a/llvm/test/CodeGen/RISCV/rvp-widening-add-acc.ll b/llvm/test/CodeGen/RISCV/rvp-widening-add-acc.ll new file mode 100644 index 00000000000000..637ce4d988be01 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/rvp-widening-add-acc.ll @@ -0,0 +1,86 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=riscv32 -mattr=+experimental-p -verify-machineinstrs < %s | \ +; RUN: FileCheck %s --check-prefixes=CHECK,RV32 +; RUN: llc -mtriple=riscv64 -mattr=+experimental-p -verify-machineinstrs < %s | \ +; RUN: FileCheck %s --check-prefixes=CHECK,RV64 + +define <4 x i16> @test_pwadda_v4i8(<4 x i16> %rd, <4 x i8> %a, <4 x i8> %b) { +; RV32-LABEL: test_pwadda_v4i8: +; RV32: # %bb.0: +; RV32-NEXT: pwadda.b a0, a2, a3 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pwadda_v4i8: +; RV64: # %bb.0: +; RV64-NEXT: zip8p a1, a1, a2 +; RV64-NEXT: psext.h.b a2, a1 +; RV64-NEXT: padd.h a0, a0, a2 +; RV64-NEXT: psrai.h a1, a1, 8 +; RV64-NEXT: padd.h a0, a0, a1 +; RV64-NEXT: ret + %ext.a = sext <4 x i8> %a to <4 x i16> + %ext.b = sext <4 x i8> %b to <4 x i16> + %sum.a = add <4 x i16> %rd, %ext.a + %sum.b = add <4 x i16> %sum.a, %ext.b + ret <4 x i16> %sum.b +} + +define <2 x i32> @test_pwadda_v2i16(<2 x i32> %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: test_pwadda_v2i16: +; RV32: # %bb.0: +; RV32-NEXT: pwadda.h a0, a2, a3 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pwadda_v2i16: +; RV64: # %bb.0: +; RV64-NEXT: zip16p a1, a1, a2 +; RV64-NEXT: pli.h a2, 1 +; RV64-NEXT: pm2adda.h a0, a1, a2 +; RV64-NEXT: ret + %ext.a = sext <2 x i16> %a to <2 x i32> + %ext.b = sext <2 x i16> %b to <2 x i32> + %sum.a = add <2 x i32> %rd, %ext.a + %sum.b = add <2 x i32> %sum.a, %ext.b + ret <2 x i32> %sum.b +} + +define <4 x i16> @test_pwaddau_v4i8(<4 x i16> %rd, <4 x i8> %a, <4 x i8> %b) { +; RV32-LABEL: test_pwaddau_v4i8: +; RV32: # %bb.0: +; RV32-NEXT: pwaddau.b a0, a2, a3 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pwaddau_v4i8: +; RV64: # %bb.0: +; RV64-NEXT: pwcvtu.wb a1, a1 +; RV64-NEXT: pwcvtu.wb a2, a2 +; RV64-NEXT: padd.h a0, a0, a1 +; RV64-NEXT: padd.h a0, a0, a2 +; RV64-NEXT: ret + %ext.a = zext <4 x i8> %a to <4 x i16> + %ext.b = zext <4 x i8> %b to <4 x i16> + %sum.a = add <4 x i16> %rd, %ext.a + %sum.b = add <4 x i16> %sum.a, %ext.b + ret <4 x i16> %sum.b +} + +define <2 x i32> @test_pwaddau_v2i16(<2 x i32> %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: test_pwaddau_v2i16: +; RV32: # %bb.0: +; RV32-NEXT: pwaddau.h a0, a2, a3 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pwaddau_v2i16: +; RV64: # %bb.0: +; RV64-NEXT: zip16p a1, a1, a2 +; RV64-NEXT: pli.h a2, 1 +; RV64-NEXT: pm2addau.h a0, a1, a2 +; RV64-NEXT: ret + %ext.a = zext <2 x i16> %a to <2 x i32> + %ext.b = zext <2 x i16> %b to <2 x i32> + %sum.a = add <2 x i32> %rd, %ext.a + %sum.b = add <2 x i32> %sum.a, %ext.b + ret <2 x i32> %sum.b +} +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; CHECK: {{.*}} From 54c0dff3ca2cabcb60b609397a9050f317432973 Mon Sep 17 00:00:00 2001 From: Michael-Chen-NJU <[email protected]> Date: Tue, 8 Sep 2026 09:59:39 +0800 Subject: [PATCH 2/3] [RISCV] Address review comments for widening add accumulate --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 28 ++++++++++----------- llvm/lib/Target/RISCV/RISCVInstrInfoP.td | 4 +-- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 86d5da06e55f5e..b61d1e86451719 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -18628,6 +18628,8 @@ static SDValue combinePExtWideningAddSub(SDNode *N, SelectionDAG &DAG, static SDValue combinePExtWideningAddAcc(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget) { + using namespace SDPatternMatch; + if (!Subtarget.hasStdExtP() || !Subtarget.is64Bit()) return SDValue(); @@ -18639,26 +18641,22 @@ static SDValue combinePExtWideningAddAcc(SDNode *N, SelectionDAG &DAG, return SDValue(); auto MatchExtend = [](SDValue V, unsigned ExtendOpcode, MVT SrcVT, - SDValue &Src) -> bool { - if (V.getOpcode() != ExtendOpcode || !V.hasOneUse() || - V.getOperand(0).getSimpleValueType() != SrcVT) - return false; - Src = V.getOperand(0); - return true; + SDValue &Src) { + return sd_match(V, m_OneUse(m_Node(ExtendOpcode, + m_Value(Src, m_SpecificVT(SrcVT))))); }; - auto Match = [&](SDValue V, SDValue &Acc, SDValue &A, SDValue &B, - bool &IsSExt) -> bool { - unsigned ExtendOpcode = V.getOpcode(); + auto Match = [&](SDValue Ext, SDValue Add, SDValue &Acc, SDValue &A, + SDValue &B, bool &IsSExt) { + MVT SrcVT = VT == MVT::v4i16 ? MVT::v4i8 : MVT::v2i16; + unsigned ExtendOpcode = Ext.getOpcode(); if (ExtendOpcode != ISD::SIGN_EXTEND && ExtendOpcode != ISD::ZERO_EXTEND) return false; - MVT SrcVT = VT == MVT::v4i16 ? MVT::v4i8 : MVT::v2i16; - if (!MatchExtend(V, ExtendOpcode, SrcVT, B)) + if (!MatchExtend(Ext, ExtendOpcode, SrcVT, B)) return false; - SDValue Add = V == N->getOperand(0) ? N->getOperand(1) : N->getOperand(0); - if (Add.getOpcode() != ISD::ADD || !Add.hasOneUse()) + if (!sd_match(Add, m_OneUse(m_Add(m_Value(), m_Value())))) return false; if (MatchExtend(Add.getOperand(0), ExtendOpcode, SrcVT, A)) @@ -18677,8 +18675,8 @@ static SDValue combinePExtWideningAddAcc(SDNode *N, SelectionDAG &DAG, SDValue Acc, A, B; bool IsSExt; - if (!Match(N->getOperand(0), Acc, A, B, IsSExt) && - !Match(N->getOperand(1), Acc, A, B, IsSExt)) + if (!Match(N->getOperand(0), N->getOperand(1), Acc, A, B, IsSExt) && + !Match(N->getOperand(1), N->getOperand(0), Acc, A, B, IsSExt)) return SDValue(); if (VT == MVT::v4i16 && !IsSExt) diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td index 7f7d8b47481ea7..831a591cf62834 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td @@ -3251,10 +3251,10 @@ let append Predicates = [IsRV64] in { def : Pat<(v2i32 (riscv_pm2sub_h (v4i16 GPR:$rs1), (v4i16 GPR:$rs2))), (PM2SUB_H GPR:$rs1, GPR:$rs2)>; def : Pat<(v2i32 (riscv_pm2adda_h (v2i32 GPR:$rd), (v4i16 GPR:$rs1), - (v4i16 GPR:$rs2))), + (v4i16 GPR:$rs2))), (PM2ADDA_H GPR:$rd, GPR:$rs1, GPR:$rs2)>; def : Pat<(v2i32 (riscv_pm2addau_h (v2i32 GPR:$rd), (v4i16 GPR:$rs1), - (v4i16 GPR:$rs2))), + (v4i16 GPR:$rs2))), (PM2ADDAU_H GPR:$rd, GPR:$rs1, GPR:$rs2)>; // 32-bit logical shift left/right patterns From 823c9b96c6b8b4d045c94aa7c723461a507a4a50 Mon Sep 17 00:00:00 2001 From: Michael-Chen-NJU <[email protected]> Date: Tue, 8 Sep 2026 10:29:35 +0800 Subject: [PATCH 3/3] [RISCV] Fix clang-format issue in widening add accumulate --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index b61d1e86451719..7bbb0735128d3e 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -18642,8 +18642,8 @@ static SDValue combinePExtWideningAddAcc(SDNode *N, SelectionDAG &DAG, auto MatchExtend = [](SDValue V, unsigned ExtendOpcode, MVT SrcVT, SDValue &Src) { - return sd_match(V, m_OneUse(m_Node(ExtendOpcode, - m_Value(Src, m_SpecificVT(SrcVT))))); + return sd_match( + V, m_OneUse(m_Node(ExtendOpcode, m_Value(Src, m_SpecificVT(SrcVT))))); }; auto Match = [&](SDValue Ext, SDValue Add, SDValue &Acc, SDValue &A, _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
