https://github.com/TelGome updated https://github.com/llvm/llvm-project/pull/232612
>From 5bec3058d44c213f8818ddf169785e953eafbdab Mon Sep 17 00:00:00 2001 From: Dongyan Chen <[email protected]> Date: Fri, 16 Oct 2026 06:19:28 +0000 Subject: [PATCH 1/3] [RISCV][P-ext] Add Packed Multiplication with Widening Horizontal Addition intrinsics --- clang/include/clang/Basic/BuiltinsRISCV.td | 8 ++ clang/lib/CodeGen/TargetBuiltins/RISCV.cpp | 27 ++++- clang/lib/Headers/riscv_packed_simd.h | 8 ++ clang/test/CodeGen/RISCV/rvp-intrinsics.c | 72 +++++++++++++ .../riscv_packed_simd.c | 50 +++++++++ llvm/include/llvm/IR/IntrinsicsRISCV.td | 8 ++ llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 102 ++++++++++++++++++ llvm/lib/Target/RISCV/RISCVInstrInfoP.td | 12 +++ llvm/test/CodeGen/RISCV/rvp-simd-64.ll | 92 ++++++++++++++++ 9 files changed, 378 insertions(+), 1 deletion(-) diff --git a/clang/include/clang/Basic/BuiltinsRISCV.td b/clang/include/clang/Basic/BuiltinsRISCV.td index d496333952052f..8382d025775ba2 100644 --- a/clang/include/clang/Basic/BuiltinsRISCV.td +++ b/clang/include/clang/Basic/BuiltinsRISCV.td @@ -384,6 +384,14 @@ def pm4add_i16x4 : RISCVBuiltin<"int64_t(_Vector<4, short>, _Vector<4, short>)"> def pm4addu_u16x4 : RISCVBuiltin<"uint64_t(_Vector<4, unsigned short>, _Vector<4, unsigned short>)">; def pm4addsu_i16x4 : RISCVBuiltin<"int64_t(_Vector<4, short>, _Vector<4, unsigned short>)">; +// Packed Multiplication with Widening Horizontal Addition (32-bit) +def pm2wadd_i64 : RISCVBuiltin<"int64_t(_Vector<2, short>, _Vector<2, short>)">; +def pm2wadd_x_i64 : RISCVBuiltin<"int64_t(_Vector<2, short>, _Vector<2, short>)">; +def pm2waddu_u64 : RISCVBuiltin<"uint64_t(_Vector<2, unsigned short>, _Vector<2, unsigned short>)">; +def pm2wsub_i64 : RISCVBuiltin<"int64_t(_Vector<2, short>, _Vector<2, short>)">; +def pm2wsub_x_i64 : RISCVBuiltin<"int64_t(_Vector<2, short>, _Vector<2, short>)">; +def pm2waddsu_i64 : RISCVBuiltin<"int64_t(_Vector<2, short>, _Vector<2, unsigned short>)">; + // Packed Absolute Difference Sum (32-bit) def pabdsumu_u8x4_u32 : RISCVBuiltin<"unsigned int(_Vector<4, unsigned char>, _Vector<4, unsigned char>)">; def pabdsumau_u8x4_u32 : RISCVBuiltin<"unsigned int(unsigned int, _Vector<4, unsigned char>, _Vector<4, unsigned char>)">; diff --git a/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp b/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp index e6f1d3c59b9460..dd93a7da75f83f 100644 --- a/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp +++ b/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp @@ -1765,7 +1765,14 @@ Value *CodeGenFunction::EmitRISCVBuiltinExpr(unsigned BuiltinID, case RISCV::BI__builtin_riscv_pm4addsu_i16x4: case RISCV::BI__builtin_riscv_pm2addsu_i16x2: case RISCV::BI__builtin_riscv_pm2addsu_i16x4: - case RISCV::BI__builtin_riscv_pm2addsu_i32x2: { + case RISCV::BI__builtin_riscv_pm2addsu_i32x2: + // Packed Multiplication with Widening Horizontal Addition + case RISCV::BI__builtin_riscv_pm2wadd_i64: + case RISCV::BI__builtin_riscv_pm2wadd_x_i64: + case RISCV::BI__builtin_riscv_pm2waddu_u64: + case RISCV::BI__builtin_riscv_pm2wsub_i64: + case RISCV::BI__builtin_riscv_pm2wsub_x_i64: + case RISCV::BI__builtin_riscv_pm2waddsu_i64: { switch (BuiltinID) { default: llvm_unreachable("unexpected builtin ID"); @@ -1832,6 +1839,24 @@ Value *CodeGenFunction::EmitRISCVBuiltinExpr(unsigned BuiltinID, case RISCV::BI__builtin_riscv_pm2addsu_i32x2: ID = Intrinsic::riscv_pm2addsu; break; + case RISCV::BI__builtin_riscv_pm2wadd_i64: + ID = Intrinsic::riscv_pm2wadd; + break; + case RISCV::BI__builtin_riscv_pm2wadd_x_i64: + ID = Intrinsic::riscv_pm2wadd_x; + break; + case RISCV::BI__builtin_riscv_pm2waddu_u64: + ID = Intrinsic::riscv_pm2waddu; + break; + case RISCV::BI__builtin_riscv_pm2wsub_i64: + ID = Intrinsic::riscv_pm2wsub; + break; + case RISCV::BI__builtin_riscv_pm2wsub_x_i64: + ID = Intrinsic::riscv_pm2wsub_x; + break; + case RISCV::BI__builtin_riscv_pm2waddsu_i64: + ID = Intrinsic::riscv_pm2waddsu; + break; } IntrinsicTypes = {ResultType, Ops[0]->getType()}; diff --git a/clang/lib/Headers/riscv_packed_simd.h b/clang/lib/Headers/riscv_packed_simd.h index 4c3599e8f192bc..eb16fcfde06eef 100644 --- a/clang/lib/Headers/riscv_packed_simd.h +++ b/clang/lib/Headers/riscv_packed_simd.h @@ -1173,6 +1173,14 @@ __packed_binary_builtin_mixed(pm4add_i16x4, int64_t, int16x4_t, int16x4_t, __bui __packed_binary_builtin_mixed(pm4addu_u16x4, uint64_t, uint16x4_t, uint16x4_t, __builtin_riscv_pm4addu_u16x4) __packed_binary_builtin_mixed(pm4addsu_i16x4, int64_t, int16x4_t, uint16x4_t, __builtin_riscv_pm4addsu_i16x4) +/* Packed Multiplication with Widening Horizontal Addition */ +__packed_binary_builtin_mixed(pm2wadd_i64, int64_t, int16x2_t, int16x2_t, __builtin_riscv_pm2wadd_i64) +__packed_binary_builtin_mixed(pm2wadd_x_i64, int64_t, int16x2_t, int16x2_t, __builtin_riscv_pm2wadd_x_i64) +__packed_binary_builtin_mixed(pm2waddu_u64, uint64_t, uint16x2_t, uint16x2_t, __builtin_riscv_pm2waddu_u64) +__packed_binary_builtin_mixed(pm2wsub_i64, int64_t, int16x2_t, int16x2_t, __builtin_riscv_pm2wsub_i64) +__packed_binary_builtin_mixed(pm2wsub_x_i64, int64_t, int16x2_t, int16x2_t, __builtin_riscv_pm2wsub_x_i64) +__packed_binary_builtin_mixed(pm2waddsu_i64, int64_t, int16x2_t, uint16x2_t, __builtin_riscv_pm2waddsu_i64) + /* Packed Absolute Difference Sum (32-bit) */ __packed_abdsum(pabdsumu_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumu_u8x4_u32) __packed_ternary_builtin_cast(pabdsumau_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumau_u8x4_u32) diff --git a/clang/test/CodeGen/RISCV/rvp-intrinsics.c b/clang/test/CodeGen/RISCV/rvp-intrinsics.c index 9866b4915d2674..6aab6cebde991c 100644 --- a/clang/test/CodeGen/RISCV/rvp-intrinsics.c +++ b/clang/test/CodeGen/RISCV/rvp-intrinsics.c @@ -9892,3 +9892,75 @@ int32x2_t test_pslide1down_i32x2(int32x2_t rd, int32_t rs1) { uint32x2_t test_pslide1down_u32x2(uint32x2_t rd, uint32_t rs1) { return __riscv_pslide1down_u32x2(rd, rs1); } + +// CHECK-LABEL: define dso_local i64 @test_pm2wadd_i64( +// CHECK-SAME: i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2wadd.i64.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// CHECK-NEXT: ret i64 [[TMP2]] +// +int64_t test_pm2wadd_i64(int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2wadd_i64(rs1, rs2); +} + +// CHECK-LABEL: define dso_local i64 @test_pm2wadd_x_i64( +// CHECK-SAME: i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2wadd.x.i64.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// CHECK-NEXT: ret i64 [[TMP2]] +// +int64_t test_pm2wadd_x_i64(int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2wadd_x_i64(rs1, rs2); +} + +// CHECK-LABEL: define dso_local i64 @test_pm2waddu_u64( +// CHECK-SAME: i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2waddu.i64.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// CHECK-NEXT: ret i64 [[TMP2]] +// +uint64_t test_pm2waddu_u64(uint16x2_t rs1, uint16x2_t rs2) { + return __riscv_pm2waddu_u64(rs1, rs2); +} + +// CHECK-LABEL: define dso_local i64 @test_pm2wsub_i64( +// CHECK-SAME: i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2wsub.i64.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// CHECK-NEXT: ret i64 [[TMP2]] +// +int64_t test_pm2wsub_i64(int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2wsub_i64(rs1, rs2); +} + +// CHECK-LABEL: define dso_local i64 @test_pm2wsub_x_i64( +// CHECK-SAME: i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2wsub.x.i64.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// CHECK-NEXT: ret i64 [[TMP2]] +// +int64_t test_pm2wsub_x_i64(int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2wsub_x_i64(rs1, rs2); +} + +// CHECK-LABEL: define dso_local i64 @test_pm2waddsu_i64( +// CHECK-SAME: i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2waddsu.i64.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// CHECK-NEXT: ret i64 [[TMP2]] +// +int64_t test_pm2waddsu_i64(int16x2_t rs1, uint16x2_t rs2) { + return __riscv_pm2waddsu_i64(rs1, rs2); +} diff --git a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c index 507ad0c83f776a..1a17d4a26b3226 100644 --- a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c +++ b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c @@ -4452,6 +4452,56 @@ int64_t test_pm4addsu_i16x4(int16x4_t rs1, uint16x4_t rs2) { return __riscv_pm4addsu_i16x4(rs1, rs2); } +// Packed Multiplication with Widening Horizontal Addition. +// CHECK-LABEL: test_pm2wadd_i64: +// RV32: pm2wadd.h +// RV64: zext.w +// RV64: pm4add.h +int64_t test_pm2wadd_i64(int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2wadd_i64(rs1, rs2); +} + +// CHECK-LABEL: test_pm2wadd_x_i64: +// RV32: pm2wadd.hx +// RV64: ppairoe.h +// RV64: zext.w +// RV64: pm4add.h +int64_t test_pm2wadd_x_i64(int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2wadd_x_i64(rs1, rs2); +} + +// CHECK-LABEL: test_pm2waddu_u64: +// RV32: pm2waddu.h +// RV64: zext.w +// RV64: pm4addu.h +uint64_t test_pm2waddu_u64(uint16x2_t rs1, uint16x2_t rs2) { + return __riscv_pm2waddu_u64(rs1, rs2); +} + +// CHECK-LABEL: test_pm2wsub_i64: +// RV32: pm2wsub.h +// RV64: pm2sub.h +// RV64: sext.w +int64_t test_pm2wsub_i64(int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2wsub_i64(rs1, rs2); +} + +// CHECK-LABEL: test_pm2wsub_x_i64: +// RV32: pm2wsub.hx +// RV64: pm2sub.hx +// RV64: sext.w +int64_t test_pm2wsub_x_i64(int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2wsub_x_i64(rs1, rs2); +} + +// CHECK-LABEL: test_pm2waddsu_i64: +// RV32: pm2waddsu.h +// RV64: zext.w +// RV64: pm4addsu.h +int64_t test_pm2waddsu_i64(int16x2_t rs1, uint16x2_t rs2) { + return __riscv_pm2waddsu_i64(rs1, rs2); +} + // Packed Multiply Parts. // CHECK-LABEL: test_pmul_b00_i16x2: // RV32: pmul.h.b00 diff --git a/llvm/include/llvm/IR/IntrinsicsRISCV.td b/llvm/include/llvm/IR/IntrinsicsRISCV.td index c33f0fa792f0f2..523f3b58145767 100644 --- a/llvm/include/llvm/IR/IntrinsicsRISCV.td +++ b/llvm/include/llvm/IR/IntrinsicsRISCV.td @@ -2294,6 +2294,14 @@ class RVPBinaryIntrinsic def int_riscv_pm4addsu : RVPHorizontalMulIntrinsic; def int_riscv_pm2addsu : RVPHorizontalMulIntrinsic; + // Packed Multiplication with Widening Horizontal Addition. + def int_riscv_pm2wadd : RVPHorizontalMulIntrinsic; + def int_riscv_pm2wadd_x : RVPHorizontalMulIntrinsic; + def int_riscv_pm2waddu : RVPHorizontalMulIntrinsic; + def int_riscv_pm2wsub : RVPHorizontalMulIntrinsic; + def int_riscv_pm2wsub_x : RVPHorizontalMulIntrinsic; + def int_riscv_pm2waddsu : RVPHorizontalMulIntrinsic; + // Packed Multiply Parts. The packed forms multiply the selected element of // every pair, so the operands are the result with each element split in two. class RVPPackedMulPartsIntrinsic diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 668e037415a9d7..1735dc92531e3f 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -12597,6 +12597,18 @@ static unsigned getRVPHorizontalMulOpcode(unsigned IntNo) { return RISCVISD::PM4ADDSU; case Intrinsic::riscv_pm2addsu: return RISCVISD::PM2ADDSU; + case Intrinsic::riscv_pm2wadd: + return RISCVISD::PM2WADD; + case Intrinsic::riscv_pm2wadd_x: + return RISCVISD::PM2WADD_X; + case Intrinsic::riscv_pm2waddu: + return RISCVISD::PM2WADDU; + case Intrinsic::riscv_pm2wsub: + return RISCVISD::PM2WSUB; + case Intrinsic::riscv_pm2wsub_x: + return RISCVISD::PM2WSUB_X; + case Intrinsic::riscv_pm2waddsu: + return RISCVISD::PM2WADDSU; } } @@ -12607,6 +12619,12 @@ static SDValue lowerRV32HorizontalMul64(unsigned IntNo, SDValue Rs1, return DAG.getExtractVectorElt(DL, MVT::i32, V, Idx); }; + if (Rs1.getSimpleValueType() == MVT::v2i16) { + SDValue Res = + DAG.getNode(getRVPHorizontalMulOpcode(IntNo), DL, MVT::v2i32, Rs1, Rs2); + return DAG.getBitcast(MVT::i64, Res); + } + if (Rs1.getSimpleValueType() == MVT::v4i16) { auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL); auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL); @@ -13573,6 +13591,74 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, return DAG.getNode(Opc, DL, VT, Rs1, Rs2); } + case Intrinsic::riscv_pm2wadd: + case Intrinsic::riscv_pm2wadd_x: + case Intrinsic::riscv_pm2waddu: + case Intrinsic::riscv_pm2wsub: + case Intrinsic::riscv_pm2wsub_x: + case Intrinsic::riscv_pm2waddsu: { + // These intrinsics only have a dedicated widening instruction on RV32 + // (handled in ReplaceNodeResults). On RV64 the i64 result type is legal, + // so synthesize the equivalent value directly at i64 width: the add forms + // zero-extend their operands to v4i16 and reuse pm4add(u/su).h, relying on + // zero lanes contributing zero products (the "_x" exchanged form + // additionally swaps rs2's halfwords via ppairoe.h first); the sub forms + // extract and sign-extend each halfword lane and compute the product + // difference with plain i64 arithmetic. A custom RISCVISD opcode must + // never be the direct result of an i32-producing DAG.getNode here: i32 is + // not a legal type on RV64 (only i64 has a register class), and the + // generic legalizer's post-hoc promotion of such a "naked" custom-opcode + // node mis-widens its vector operands instead of leaving the node alone. + assert(Subtarget.is64Bit() && Op.getValueType() == MVT::i64 && + "Unexpected widening horizontal multiply-add legalization"); + SDValue Rs1 = Op.getOperand(1); + SDValue Rs2 = Op.getOperand(2); + + if (IntNo == Intrinsic::riscv_pm2wsub || + IntNo == Intrinsic::riscv_pm2wsub_x) { + // Route through the existing riscv_pm2sub(_x) intrinsic (a plain + // ISD::INTRINSIC_WO_CHAIN, which is registered Custom for i32 on RV64) + // rather than calling RISCVISD::PM2SUB(_X) directly: a "naked" custom + // opcode freshly materialized here with an i32 result hits the same + // legalizer trap described above, but INTRINSIC_WO_CHAIN's i32 action + // is a recognized Custom target that correctly re-enters LowerOperation. + unsigned SubIntNo = IntNo == Intrinsic::riscv_pm2wsub + ? Intrinsic::riscv_pm2sub + : Intrinsic::riscv_pm2sub_x; + SDValue IntrId = DAG.getTargetConstant(SubIntNo, DL, MVT::i64); + SDValue Res = + DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32, IntrId, Rs1, Rs2); + return DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Res); + } + + auto WidenAny = [&](SDValue V) { + SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, + DAG.getBitcast(MVT::i32, V)); + return DAG.getBitcast(MVT::v4i16, Ext); + }; + auto WidenZero = [&](SDValue V32) { + SDValue Ext = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, V32); + return DAG.getBitcast(MVT::v4i16, Ext); + }; + + SDValue A = WidenAny(Rs1); + SDValue B; + if (IntNo == Intrinsic::riscv_pm2wadd_x) { + SDValue Rs2Dup = WidenAny(Rs2); + SDValue Swapped = + DAG.getNode(RISCVISD::PPAIROE, DL, MVT::v4i16, Rs2Dup, Rs2Dup); + SDValue SwappedLo32 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, + DAG.getBitcast(MVT::i64, Swapped)); + B = WidenZero(SwappedLo32); + } else { + B = WidenZero(DAG.getBitcast(MVT::i32, Rs2)); + } + + unsigned MulOpc = IntNo == Intrinsic::riscv_pm2waddu ? RISCVISD::PM4ADDU + : IntNo == Intrinsic::riscv_pm2waddsu ? RISCVISD::PM4ADDSU + : RISCVISD::PM4ADD; + return DAG.getNode(MulOpc, DL, MVT::i64, A, B); + } case Intrinsic::riscv_pmerge: { EVT VT = Op.getValueType(); auto buildMerge = [&](SDValue Rs1, SDValue Rs2, SDValue Mask, @@ -17786,6 +17872,22 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N, return; } + case Intrinsic::riscv_pm2wadd: + case Intrinsic::riscv_pm2wadd_x: + case Intrinsic::riscv_pm2waddu: + case Intrinsic::riscv_pm2wsub: + case Intrinsic::riscv_pm2wsub_x: + case Intrinsic::riscv_pm2waddsu: { + // i64 is only illegal here on RV32; the RV64 (legal i64) case is + // handled directly in LowerOperation. + assert(!Subtarget.is64Bit() && N->getSimpleValueType(0) == MVT::i64 && + "Unexpected widening horizontal multiply-add legalization"); + SDValue Rs1 = N->getOperand(1); + SDValue Rs2 = N->getOperand(2); + Results.push_back(lowerRV32HorizontalMul64(IntNo, Rs1, Rs2, DL, DAG)); + return; + } + case Intrinsic::riscv_pmul_00: case Intrinsic::riscv_pmul_01: case Intrinsic::riscv_pmul_11: diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td index 44050f1b29bb3d..b344638a754c88 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td @@ -1925,6 +1925,12 @@ def riscv_pm2waddu : RVSDNode<"PM2WADDU", SDT_RISCVPackedWideningHorizontalAdd>; def riscv_pm2waddsu : RVSDNode<"PM2WADDSU", SDT_RISCVPackedWideningHorizontalAdd>; +def riscv_pm2wadd_x + : RVSDNode<"PM2WADD_X", SDT_RISCVPackedWideningHorizontalAdd>; +def riscv_pm2wsub + : RVSDNode<"PM2WSUB", SDT_RISCVPackedWideningHorizontalAdd>; +def riscv_pm2wsub_x + : RVSDNode<"PM2WSUB_X", SDT_RISCVPackedWideningHorizontalAdd>; // Packed widening multiply-accumulate operations. def SDT_RISCVPackedWideningAcc @@ -2736,6 +2742,12 @@ let append Predicates = [IsRV32] in { (PM2WADDU_H GPR:$rs1, GPR:$rs2)>; def : Pat<(v2i32 (riscv_pm2waddsu (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))), (PM2WADDSU_H GPR:$rs1, GPR:$rs2)>; + def : Pat<(v2i32 (riscv_pm2wadd_x (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))), + (PM2WADD_HX GPR:$rs1, GPR:$rs2)>; + def : Pat<(v2i32 (riscv_pm2wsub (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))), + (PM2WSUB_H GPR:$rs1, GPR:$rs2)>; + def : Pat<(v2i32 (riscv_pm2wsub_x (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))), + (PM2WSUB_HX GPR:$rs1, GPR:$rs2)>; // Packed multiplication with widening horizontal addition and accumulate // patterns. diff --git a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll index ee26f6ab53dd02..aae7002ea6aa2e 100644 --- a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll +++ b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll @@ -8049,6 +8049,98 @@ define i64 @test_pm4addsu_v4i16_i64(<4 x i16> %a, <4 x i16> %b) { ret i64 %r } +; Packed Multiplication with Widening Horizontal Addition. +define i64 @test_pm2wadd_v2i16_i64(<2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: test_pm2wadd_v2i16_i64: +; RV32: # %bb.0: +; RV32-NEXT: pm2wadd.h a0, a0, a1 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm2wadd_v2i16_i64: +; RV64: # %bb.0: +; RV64-NEXT: zext.w a1, a1 +; RV64-NEXT: pm4add.h a0, a0, a1 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.pm2wadd.i64.v2i16(<2 x i16> %a, <2 x i16> %b) + ret i64 %r +} + +define i64 @test_pm2wadd_x_v2i16_i64(<2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: test_pm2wadd_x_v2i16_i64: +; RV32: # %bb.0: +; RV32-NEXT: pm2wadd.hx a0, a0, a1 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm2wadd_x_v2i16_i64: +; RV64: # %bb.0: +; RV64-NEXT: ppairoe.h a1, a1, a1 +; RV64-NEXT: zext.w a1, a1 +; RV64-NEXT: pm4add.h a0, a0, a1 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.pm2wadd.x.i64.v2i16(<2 x i16> %a, <2 x i16> %b) + ret i64 %r +} + +define i64 @test_pm2waddu_v2i16_i64(<2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: test_pm2waddu_v2i16_i64: +; RV32: # %bb.0: +; RV32-NEXT: pm2waddu.h a0, a0, a1 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm2waddu_v2i16_i64: +; RV64: # %bb.0: +; RV64-NEXT: zext.w a1, a1 +; RV64-NEXT: pm4addu.h a0, a0, a1 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.pm2waddu.i64.v2i16(<2 x i16> %a, <2 x i16> %b) + ret i64 %r +} + +define i64 @test_pm2wsub_v2i16_i64(<2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: test_pm2wsub_v2i16_i64: +; RV32: # %bb.0: +; RV32-NEXT: pm2wsub.h a0, a0, a1 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm2wsub_v2i16_i64: +; RV64: # %bb.0: +; RV64-NEXT: pm2sub.h a0, a0, a1 +; RV64-NEXT: sext.w a0, a0 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.pm2wsub.i64.v2i16(<2 x i16> %a, <2 x i16> %b) + ret i64 %r +} + +define i64 @test_pm2wsub_x_v2i16_i64(<2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: test_pm2wsub_x_v2i16_i64: +; RV32: # %bb.0: +; RV32-NEXT: pm2wsub.hx a0, a0, a1 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm2wsub_x_v2i16_i64: +; RV64: # %bb.0: +; RV64-NEXT: pm2sub.hx a0, a0, a1 +; RV64-NEXT: sext.w a0, a0 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.pm2wsub.x.i64.v2i16(<2 x i16> %a, <2 x i16> %b) + ret i64 %r +} + +define i64 @test_pm2waddsu_v2i16_i64(<2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: test_pm2waddsu_v2i16_i64: +; RV32: # %bb.0: +; RV32-NEXT: pm2waddsu.h a0, a0, a1 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm2waddsu_v2i16_i64: +; RV64: # %bb.0: +; RV64-NEXT: zext.w a1, a1 +; RV64-NEXT: pm4addsu.h a0, a0, a1 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.pm2waddsu.i64.v2i16(<2 x i16> %a, <2 x i16> %b) + ret i64 %r +} + ; Packed Multiply Parts Accumulate. declare <2 x i32> @llvm.riscv.pmacc.00.v2i32(<2 x i32>, <4 x i16>, <4 x i16>) declare <2 x i32> @llvm.riscv.pmacc.01.v2i32(<2 x i32>, <4 x i16>, <4 x i16>) >From 7584f9ab06d9a94eef2e59826e390e57cb0e1b77 Mon Sep 17 00:00:00 2001 From: Dongyan Chen <[email protected]> Date: Sun, 18 Oct 2026 05:29:22 +0000 Subject: [PATCH 2/3] [RISCV][P-ext] Address review comments for pm2wadd/pm2wsub RV64 lowering --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 22 ++++++++------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 1735dc92531e3f..4b09d678b7bfb1 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -12868,6 +12868,9 @@ getRVPWordMulPartsAccOpcodeAndLanes(unsigned IntNo) { } } +static SDValue widenPackedVectorWithZeros(SelectionDAG &DAG, const SDLoc &DL, + SDValue V, MVT WideVT); + SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const { unsigned IntNo = Op.getConstantOperandVal(0); @@ -13632,13 +13635,8 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, } auto WidenAny = [&](SDValue V) { - SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, - DAG.getBitcast(MVT::i32, V)); - return DAG.getBitcast(MVT::v4i16, Ext); - }; - auto WidenZero = [&](SDValue V32) { - SDValue Ext = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, V32); - return DAG.getBitcast(MVT::v4i16, Ext); + return DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i16, V, + DAG.getUNDEF(MVT::v2i16)); }; SDValue A = WidenAny(Rs1); @@ -13647,11 +13645,10 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SDValue Rs2Dup = WidenAny(Rs2); SDValue Swapped = DAG.getNode(RISCVISD::PPAIROE, DL, MVT::v4i16, Rs2Dup, Rs2Dup); - SDValue SwappedLo32 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, - DAG.getBitcast(MVT::i64, Swapped)); - B = WidenZero(SwappedLo32); + SDValue SwappedLo = DAG.getExtractSubvector(DL, MVT::v2i16, Swapped, 0); + B = widenPackedVectorWithZeros(DAG, DL, SwappedLo, MVT::v4i16); } else { - B = WidenZero(DAG.getBitcast(MVT::i32, Rs2)); + B = widenPackedVectorWithZeros(DAG, DL, Rs2, MVT::v4i16); } unsigned MulOpc = IntNo == Intrinsic::riscv_pm2waddu ? RISCVISD::PM4ADDU @@ -14512,9 +14509,6 @@ SDValue RISCVTargetLowering::lowerVPREDUCE(SDValue Op, DAG.getConstantFP(APFloat::getNaN(ResVT.getFltSemantics()), DL, ResVT)); } -static SDValue widenPackedVectorWithZeros(SelectionDAG &DAG, const SDLoc &DL, - SDValue V, MVT WideVT); - SDValue RISCVTargetLowering::lowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const { SDValue Vec = Op.getOperand(0); >From 4fc65ed7b366c45729e3b9514e72846ca46be989 Mon Sep 17 00:00:00 2001 From: Dongyan Chen <[email protected]> Date: Mon, 19 Oct 2026 01:58:33 +0000 Subject: [PATCH 3/3] [RISCV][P-ext] Simplify pm2wadd_x RV64 lowering per review Zero-widen rs2 first and apply PPAIROE directly on the widened vector instead of swapping via an undef-extended duplicate and re-widening afterwards. This matches the zext.w+ppairoe.h+pm4add.h sequence from the intrinsic spec and drops the extra extract/re-widen step. --- .../intrinsic-header-tests/riscv_packed_simd.c | 2 +- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 7 ++----- llvm/test/CodeGen/RISCV/rvp-simd-64.ll | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c index fff93ff31bbbf6..a54807804b9836 100644 --- a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c +++ b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c @@ -4521,8 +4521,8 @@ int64_t test_pm2wadd_i64(int16x2_t rs1, int16x2_t rs2) { // CHECK-LABEL: test_pm2wadd_x_i64: // RV32: pm2wadd.hx -// RV64: ppairoe.h // RV64: pack{{[[:space:]]}} +// RV64: ppairoe.h // RV64: pm4add.h int64_t test_pm2wadd_x_i64(int16x2_t rs1, int16x2_t rs2) { return __riscv_pm2wadd_x_i64(rs1, rs2); diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 6f0cede135e068..c1c2dae429fd3c 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -13695,11 +13695,8 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SDValue A = WidenAny(Rs1); SDValue B; if (IntNo == Intrinsic::riscv_pm2wadd_x) { - SDValue Rs2Dup = WidenAny(Rs2); - SDValue Swapped = - DAG.getNode(RISCVISD::PPAIROE, DL, MVT::v4i16, Rs2Dup, Rs2Dup); - SDValue SwappedLo = DAG.getExtractSubvector(DL, MVT::v2i16, Swapped, 0); - B = widenPackedVectorWithZeros(DAG, DL, SwappedLo, MVT::v4i16); + SDValue Rs2Wide = widenPackedVectorWithZeros(DAG, DL, Rs2, MVT::v4i16); + B = DAG.getNode(RISCVISD::PPAIROE, DL, MVT::v4i16, Rs2Wide, Rs2Wide); } else { B = widenPackedVectorWithZeros(DAG, DL, Rs2, MVT::v4i16); } diff --git a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll index f17965bfcad02c..7cb0eb632f3501 100644 --- a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll +++ b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll @@ -8049,8 +8049,8 @@ define i64 @test_pm2wadd_x_v2i16_i64(<2 x i16> %a, <2 x i16> %b) { ; ; RV64-LABEL: test_pm2wadd_x_v2i16_i64: ; RV64: # %bb.0: -; RV64-NEXT: ppairoe.h a1, a1, a1 ; RV64-NEXT: pack a1, a1, zero +; RV64-NEXT: ppairoe.h a1, a1, a1 ; RV64-NEXT: pm4add.h a0, a0, a1 ; RV64-NEXT: ret %r = call i64 @llvm.riscv.pm2wadd.x.i64.v2i16(<2 x i16> %a, <2 x i16> %b) _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
