https://github.com/XChy updated https://github.com/llvm/llvm-project/pull/218430
>From cb9d1c63130ad55869409db3d225323ed38afb4d Mon Sep 17 00:00:00 2001 From: XChy <[email protected]> Date: Mon, 24 Aug 2026 22:10:31 +0800 Subject: [PATCH 1/4] [RISCV] Support Packed Multiplication with Horizontal Addition --- clang/include/clang/Basic/BuiltinsRISCV.td | 41 ++ clang/lib/CodeGen/TargetBuiltins/RISCV.cpp | 110 +++++ clang/lib/Headers/riscv_packed_simd.h | 41 ++ .../riscv_packed_simd.c | 285 +++++++++++++ llvm/include/llvm/IR/IntrinsicsRISCV.td | 19 + llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 100 +++++ llvm/lib/Target/RISCV/RISCVInstrInfoP.td | 140 +++++++ llvm/test/CodeGen/RISCV/rvp-simd-32.ll | 117 ++++++ llvm/test/CodeGen/RISCV/rvp-simd-64.ll | 375 ++++++++++++++++++ 9 files changed, 1228 insertions(+) diff --git a/clang/include/clang/Basic/BuiltinsRISCV.td b/clang/include/clang/Basic/BuiltinsRISCV.td index b292ffc0140e8..4f648f4fbd873 100644 --- a/clang/include/clang/Basic/BuiltinsRISCV.td +++ b/clang/include/clang/Basic/BuiltinsRISCV.td @@ -291,6 +291,47 @@ def pmhaccsu_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<2, i def pmhraccsu_i16x4 : RISCVBuiltin<"_Vector<4, short>(_Vector<4, short>, _Vector<4, short>, _Vector<4, unsigned short>)">; def pmhraccsu_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<2, int>, _Vector<2, unsigned int>)">; +// Packed Multiplication with Horizontal Addition (32-bit) +def pm4add_i8x4 : RISCVBuiltin<"int(_Vector<4, signed char>, _Vector<4, signed char>)">; +def pm2add_i16x2 : RISCVBuiltin<"int(_Vector<2, short>, _Vector<2, short>)">; +def pm2add_x_i16x2 : RISCVBuiltin<"int(_Vector<2, short>, _Vector<2, short>)">; +def pm4addu_u8x4 : RISCVBuiltin<"unsigned int(_Vector<4, unsigned char>, _Vector<4, unsigned char>)">; +def pm2addu_u16x2 : RISCVBuiltin<"unsigned int(_Vector<2, unsigned short>, _Vector<2, unsigned short>)">; +def pmq2add_i16x2 : RISCVBuiltin<"int(_Vector<2, short>, _Vector<2, short>)">; +def pmqr2add_i16x2 : RISCVBuiltin<"int(_Vector<2, short>, _Vector<2, short>)">; +def pm2sadd_i16x2 : RISCVBuiltin<"int(_Vector<2, short>, _Vector<2, short>)">; +def pm2sadd_x_i16x2 : RISCVBuiltin<"int(_Vector<2, short>, _Vector<2, short>)">; +def pm2sub_i16x2 : RISCVBuiltin<"int(_Vector<2, short>, _Vector<2, short>)">; +def pm2sub_x_i16x2 : RISCVBuiltin<"int(_Vector<2, short>, _Vector<2, short>)">; +def pm4addsu_i8x4 : RISCVBuiltin<"int(_Vector<4, signed char>, _Vector<4, unsigned char>)">; +def pm2addsu_i16x2 : RISCVBuiltin<"int(_Vector<2, short>, _Vector<2, unsigned short>)">; + +// Packed Multiplication with Horizontal Addition (64-bit) +def pm4add_i8x8 : RISCVBuiltin<"_Vector<2, int>(_Vector<8, signed char>, _Vector<8, signed char>)">; +def pm2add_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<4, short>, _Vector<4, short>)">; +def pm2add_x_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<4, short>, _Vector<4, short>)">; +def pm4addu_u8x8 : RISCVBuiltin<"_Vector<2, unsigned int>(_Vector<8, unsigned char>, _Vector<8, unsigned char>)">; +def pm2addu_u16x4 : RISCVBuiltin<"_Vector<2, unsigned int>(_Vector<4, unsigned short>, _Vector<4, unsigned short>)">; +def pmq2add_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<4, short>, _Vector<4, short>)">; +def pmqr2add_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<4, short>, _Vector<4, short>)">; +def pm2sadd_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<4, short>, _Vector<4, short>)">; +def pm2sadd_x_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<4, short>, _Vector<4, short>)">; +def pm2sub_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<4, short>, _Vector<4, short>)">; +def pm2sub_x_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<4, short>, _Vector<4, short>)">; +def pm4addsu_i8x8 : RISCVBuiltin<"_Vector<2, int>(_Vector<8, signed char>, _Vector<8, unsigned char>)">; +def pm2addsu_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<4, short>, _Vector<4, unsigned short>)">; +def pm2add_i32x2 : RISCVBuiltin<"int64_t(_Vector<2, int>, _Vector<2, int>)">; +def pm2add_x_i32x2 : RISCVBuiltin<"int64_t(_Vector<2, int>, _Vector<2, int>)">; +def pm2addu_u32x2 : RISCVBuiltin<"uint64_t(_Vector<2, unsigned int>, _Vector<2, unsigned int>)">; +def pmq2add_i32x2 : RISCVBuiltin<"int64_t(_Vector<2, int>, _Vector<2, int>)">; +def pm2sub_i32x2 : RISCVBuiltin<"int64_t(_Vector<2, int>, _Vector<2, int>)">; +def pm2sub_x_i32x2 : RISCVBuiltin<"int64_t(_Vector<2, int>, _Vector<2, int>)">; +def pm2addsu_i32x2 : RISCVBuiltin<"int64_t(_Vector<2, int>, _Vector<2, unsigned int>)">; +def pmqr2add_i32x2 : RISCVBuiltin<"int64_t(_Vector<2, int>, _Vector<2, int>)">; +def pm4add_i16x4 : RISCVBuiltin<"int64_t(_Vector<4, short>, _Vector<4, short>)">; +def pm4addu_u16x4 : RISCVBuiltin<"uint64_t(_Vector<4, unsigned short>, _Vector<4, unsigned short>)">; +def pm4addsu_i16x4 : RISCVBuiltin<"int64_t(_Vector<4, short>, _Vector<4, unsigned short>)">; + // Packed Absolute Difference Sum (32-bit) def pabdsumu_u8x4_u32 : RISCVBuiltin<"unsigned int(_Vector<4, unsigned char>, _Vector<4, unsigned char>)">; def pabdsumau_u8x4_u32 : RISCVBuiltin<"unsigned int(unsigned int, _Vector<4, unsigned char>, _Vector<4, unsigned char>)">; diff --git a/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp b/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp index 91f451e34c2c9..eee029a513d03 100644 --- a/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp +++ b/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp @@ -1531,6 +1531,116 @@ Value *CodeGenFunction::EmitRISCVBuiltinExpr(unsigned BuiltinID, break; } + // Packed Multiplication with Horizontal Addition + case RISCV::BI__builtin_riscv_pm4add_i8x4: + case RISCV::BI__builtin_riscv_pm4add_i8x8: + case RISCV::BI__builtin_riscv_pm4add_i16x4: + case RISCV::BI__builtin_riscv_pm2add_i16x2: + case RISCV::BI__builtin_riscv_pm2add_i16x4: + case RISCV::BI__builtin_riscv_pm2add_i32x2: + case RISCV::BI__builtin_riscv_pm2add_x_i16x2: + case RISCV::BI__builtin_riscv_pm2add_x_i16x4: + case RISCV::BI__builtin_riscv_pm2add_x_i32x2: + case RISCV::BI__builtin_riscv_pm4addu_u8x4: + case RISCV::BI__builtin_riscv_pm4addu_u8x8: + case RISCV::BI__builtin_riscv_pm4addu_u16x4: + case RISCV::BI__builtin_riscv_pm2addu_u16x2: + case RISCV::BI__builtin_riscv_pm2addu_u16x4: + case RISCV::BI__builtin_riscv_pm2addu_u32x2: + case RISCV::BI__builtin_riscv_pmq2add_i16x2: + case RISCV::BI__builtin_riscv_pmq2add_i16x4: + case RISCV::BI__builtin_riscv_pmq2add_i32x2: + case RISCV::BI__builtin_riscv_pmqr2add_i16x2: + case RISCV::BI__builtin_riscv_pmqr2add_i16x4: + case RISCV::BI__builtin_riscv_pmqr2add_i32x2: + case RISCV::BI__builtin_riscv_pm2sadd_i16x2: + case RISCV::BI__builtin_riscv_pm2sadd_i16x4: + case RISCV::BI__builtin_riscv_pm2sadd_x_i16x2: + case RISCV::BI__builtin_riscv_pm2sadd_x_i16x4: + case RISCV::BI__builtin_riscv_pm2sub_i16x2: + case RISCV::BI__builtin_riscv_pm2sub_i16x4: + case RISCV::BI__builtin_riscv_pm2sub_i32x2: + case RISCV::BI__builtin_riscv_pm2sub_x_i16x2: + case RISCV::BI__builtin_riscv_pm2sub_x_i16x4: + case RISCV::BI__builtin_riscv_pm2sub_x_i32x2: + case RISCV::BI__builtin_riscv_pm4addsu_i8x4: + case RISCV::BI__builtin_riscv_pm4addsu_i8x8: + case RISCV::BI__builtin_riscv_pm4addsu_i16x4: + case RISCV::BI__builtin_riscv_pm2addsu_i16x2: + case RISCV::BI__builtin_riscv_pm2addsu_i16x4: + case RISCV::BI__builtin_riscv_pm2addsu_i32x2: { + switch (BuiltinID) { + default: + llvm_unreachable("unexpected builtin ID"); + case RISCV::BI__builtin_riscv_pm4add_i8x4: + case RISCV::BI__builtin_riscv_pm4add_i8x8: + case RISCV::BI__builtin_riscv_pm4add_i16x4: + ID = Intrinsic::riscv_pm4add; + break; + case RISCV::BI__builtin_riscv_pm2add_i16x2: + case RISCV::BI__builtin_riscv_pm2add_i16x4: + case RISCV::BI__builtin_riscv_pm2add_i32x2: + ID = Intrinsic::riscv_pm2add; + break; + case RISCV::BI__builtin_riscv_pm2add_x_i16x2: + case RISCV::BI__builtin_riscv_pm2add_x_i16x4: + case RISCV::BI__builtin_riscv_pm2add_x_i32x2: + ID = Intrinsic::riscv_pm2add_x; + break; + case RISCV::BI__builtin_riscv_pm4addu_u8x4: + case RISCV::BI__builtin_riscv_pm4addu_u8x8: + case RISCV::BI__builtin_riscv_pm4addu_u16x4: + ID = Intrinsic::riscv_pm4addu; + break; + case RISCV::BI__builtin_riscv_pm2addu_u16x2: + case RISCV::BI__builtin_riscv_pm2addu_u16x4: + case RISCV::BI__builtin_riscv_pm2addu_u32x2: + ID = Intrinsic::riscv_pm2addu; + break; + case RISCV::BI__builtin_riscv_pmq2add_i16x2: + case RISCV::BI__builtin_riscv_pmq2add_i16x4: + case RISCV::BI__builtin_riscv_pmq2add_i32x2: + ID = Intrinsic::riscv_pmq2add; + break; + case RISCV::BI__builtin_riscv_pmqr2add_i16x2: + case RISCV::BI__builtin_riscv_pmqr2add_i16x4: + case RISCV::BI__builtin_riscv_pmqr2add_i32x2: + ID = Intrinsic::riscv_pmqr2add; + break; + case RISCV::BI__builtin_riscv_pm2sadd_i16x2: + case RISCV::BI__builtin_riscv_pm2sadd_i16x4: + ID = Intrinsic::riscv_pm2sadd; + break; + case RISCV::BI__builtin_riscv_pm2sadd_x_i16x2: + case RISCV::BI__builtin_riscv_pm2sadd_x_i16x4: + ID = Intrinsic::riscv_pm2sadd_x; + break; + case RISCV::BI__builtin_riscv_pm2sub_i16x2: + case RISCV::BI__builtin_riscv_pm2sub_i16x4: + case RISCV::BI__builtin_riscv_pm2sub_i32x2: + ID = Intrinsic::riscv_pm2sub; + break; + case RISCV::BI__builtin_riscv_pm2sub_x_i16x2: + case RISCV::BI__builtin_riscv_pm2sub_x_i16x4: + case RISCV::BI__builtin_riscv_pm2sub_x_i32x2: + ID = Intrinsic::riscv_pm2sub_x; + break; + case RISCV::BI__builtin_riscv_pm4addsu_i8x4: + case RISCV::BI__builtin_riscv_pm4addsu_i8x8: + case RISCV::BI__builtin_riscv_pm4addsu_i16x4: + ID = Intrinsic::riscv_pm4addsu; + break; + case RISCV::BI__builtin_riscv_pm2addsu_i16x2: + case RISCV::BI__builtin_riscv_pm2addsu_i16x4: + case RISCV::BI__builtin_riscv_pm2addsu_i32x2: + ID = Intrinsic::riscv_pm2addsu; + break; + } + + IntrinsicTypes = {ResultType, Ops[0]->getType()}; + break; + } + // Packed Reduction Sum case RISCV::BI__builtin_riscv_predsum_i8x4_i32: case RISCV::BI__builtin_riscv_predsum_i16x2_i32: diff --git a/clang/lib/Headers/riscv_packed_simd.h b/clang/lib/Headers/riscv_packed_simd.h index 946dd936d49cd..265d2f71be34c 100644 --- a/clang/lib/Headers/riscv_packed_simd.h +++ b/clang/lib/Headers/riscv_packed_simd.h @@ -813,6 +813,47 @@ __packed_ternary_builtin_mixed(pmhaccsu_i32x2, int32x2_t, int32x2_t, uint32x2_t, __packed_ternary_builtin_mixed(pmhraccsu_i16x4, int16x4_t, int16x4_t, uint16x4_t, __builtin_riscv_pmhraccsu_i16x4) __packed_ternary_builtin_mixed(pmhraccsu_i32x2, int32x2_t, int32x2_t, uint32x2_t, __builtin_riscv_pmhraccsu_i32x2) +/* Packed Multiplication with Horizontal Addition (32-bit) */ +__packed_binary_builtin_mixed(pm4add_i8x4, int32_t, int8x4_t, int8x4_t, __builtin_riscv_pm4add_i8x4) +__packed_binary_builtin_mixed(pm2add_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pm2add_i16x2) +__packed_binary_builtin_mixed(pm2add_x_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pm2add_x_i16x2) +__packed_binary_builtin_mixed(pm4addu_u8x4, uint32_t, uint8x4_t, uint8x4_t, __builtin_riscv_pm4addu_u8x4) +__packed_binary_builtin_mixed(pm2addu_u16x2, uint32_t, uint16x2_t, uint16x2_t, __builtin_riscv_pm2addu_u16x2) +__packed_binary_builtin_mixed(pmq2add_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pmq2add_i16x2) +__packed_binary_builtin_mixed(pmqr2add_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pmqr2add_i16x2) +__packed_binary_builtin_mixed(pm2sadd_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pm2sadd_i16x2) +__packed_binary_builtin_mixed(pm2sadd_x_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pm2sadd_x_i16x2) +__packed_binary_builtin_mixed(pm2sub_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pm2sub_i16x2) +__packed_binary_builtin_mixed(pm2sub_x_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pm2sub_x_i16x2) +__packed_binary_builtin_mixed(pm4addsu_i8x4, int32_t, int8x4_t, uint8x4_t, __builtin_riscv_pm4addsu_i8x4) +__packed_binary_builtin_mixed(pm2addsu_i16x2, int32_t, int16x2_t, uint16x2_t, __builtin_riscv_pm2addsu_i16x2) + +/* Packed Multiplication with Horizontal Addition (64-bit) */ +__packed_binary_builtin_mixed(pm4add_i8x8, int32x2_t, int8x8_t, int8x8_t, __builtin_riscv_pm4add_i8x8) +__packed_binary_builtin_mixed(pm2add_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pm2add_i16x4) +__packed_binary_builtin_mixed(pm2add_x_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pm2add_x_i16x4) +__packed_binary_builtin_mixed(pm4addu_u8x8, uint32x2_t, uint8x8_t, uint8x8_t, __builtin_riscv_pm4addu_u8x8) +__packed_binary_builtin_mixed(pm2addu_u16x4, uint32x2_t, uint16x4_t, uint16x4_t, __builtin_riscv_pm2addu_u16x4) +__packed_binary_builtin_mixed(pmq2add_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pmq2add_i16x4) +__packed_binary_builtin_mixed(pmqr2add_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pmqr2add_i16x4) +__packed_binary_builtin_mixed(pm2sadd_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pm2sadd_i16x4) +__packed_binary_builtin_mixed(pm2sadd_x_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pm2sadd_x_i16x4) +__packed_binary_builtin_mixed(pm2sub_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pm2sub_i16x4) +__packed_binary_builtin_mixed(pm2sub_x_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pm2sub_x_i16x4) +__packed_binary_builtin_mixed(pm4addsu_i8x8, int32x2_t, int8x8_t, uint8x8_t, __builtin_riscv_pm4addsu_i8x8) +__packed_binary_builtin_mixed(pm2addsu_i16x4, int32x2_t, int16x4_t, uint16x4_t, __builtin_riscv_pm2addsu_i16x4) +__packed_binary_builtin_mixed(pm2add_i32x2, int64_t, int32x2_t, int32x2_t, __builtin_riscv_pm2add_i32x2) +__packed_binary_builtin_mixed(pm2add_x_i32x2, int64_t, int32x2_t, int32x2_t, __builtin_riscv_pm2add_x_i32x2) +__packed_binary_builtin_mixed(pm2addu_u32x2, uint64_t, uint32x2_t, uint32x2_t, __builtin_riscv_pm2addu_u32x2) +__packed_binary_builtin_mixed(pmq2add_i32x2, int64_t, int32x2_t, int32x2_t, __builtin_riscv_pmq2add_i32x2) +__packed_binary_builtin_mixed(pm2sub_i32x2, int64_t, int32x2_t, int32x2_t, __builtin_riscv_pm2sub_i32x2) +__packed_binary_builtin_mixed(pm2sub_x_i32x2, int64_t, int32x2_t, int32x2_t, __builtin_riscv_pm2sub_x_i32x2) +__packed_binary_builtin_mixed(pm2addsu_i32x2, int64_t, int32x2_t, uint32x2_t, __builtin_riscv_pm2addsu_i32x2) +__packed_binary_builtin_mixed(pmqr2add_i32x2, int64_t, int32x2_t, int32x2_t, __builtin_riscv_pmqr2add_i32x2) +__packed_binary_builtin_mixed(pm4add_i16x4, int64_t, int16x4_t, int16x4_t, __builtin_riscv_pm4add_i16x4) +__packed_binary_builtin_mixed(pm4addu_u16x4, uint64_t, uint16x4_t, uint16x4_t, __builtin_riscv_pm4addu_u16x4) +__packed_binary_builtin_mixed(pm4addsu_i16x4, int64_t, int16x4_t, uint16x4_t, __builtin_riscv_pm4addsu_i16x4) + /* Packed Absolute Difference Sum (32-bit) */ __packed_abdsum(pabdsumu_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumu_u8x4_u32) __packed_ternary_builtin_cast(pabdsumau_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumau_u8x4_u32) diff --git a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c index 28e3a8911739d..32e4fa767f69e 100644 --- a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c +++ b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c @@ -3532,3 +3532,288 @@ int64_t test_mqracc_w01_i64(int64_t rd, int32x2_t a, int32x2_t b) { int64_t test_mqracc_w11_i64(int64_t rd, int32x2_t a, int32x2_t b) { return __riscv_mqracc_w11_i64(rd, a, b); } + +// CHECK-LABEL: test_pm4add_i8x4: +// RV32: pm4add.b +// RV64: pm4add.b +int32_t test_pm4add_i8x4(int8x4_t rs1, int8x4_t rs2) { + return __riscv_pm4add_i8x4(rs1, rs2); +} + +// CHECK-LABEL: test_pm2add_i16x2: +// RV32: pm2add.h +// RV64: pm2add.h +int32_t test_pm2add_i16x2(int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2add_i16x2(rs1, rs2); +} + +// CHECK-LABEL: test_pm2add_x_i16x2: +// RV32: pm2add.hx +// RV64: pm2add.hx +int32_t test_pm2add_x_i16x2(int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2add_x_i16x2(rs1, rs2); +} + +// CHECK-LABEL: test_pm4addu_u8x4: +// RV32: pm4addu.b +// RV64: pm4addu.b +uint32_t test_pm4addu_u8x4(uint8x4_t rs1, uint8x4_t rs2) { + return __riscv_pm4addu_u8x4(rs1, rs2); +} + +// CHECK-LABEL: test_pm2addu_u16x2: +// RV32: pm2addu.h +// RV64: pm2addu.h +uint32_t test_pm2addu_u16x2(uint16x2_t rs1, uint16x2_t rs2) { + return __riscv_pm2addu_u16x2(rs1, rs2); +} + +// CHECK-LABEL: test_pmq2add_i16x2: +// RV32: pmq2add.h +// RV64: pmq2add.h +int32_t test_pmq2add_i16x2(int16x2_t rs1, int16x2_t rs2) { + return __riscv_pmq2add_i16x2(rs1, rs2); +} + +// CHECK-LABEL: test_pmqr2add_i16x2: +// RV32: pmqr2add.h +// RV64: pmqr2add.h +int32_t test_pmqr2add_i16x2(int16x2_t rs1, int16x2_t rs2) { + return __riscv_pmqr2add_i16x2(rs1, rs2); +} + +// CHECK-LABEL: test_pm2sadd_i16x2: +// RV32: pm2sadd.h +// RV64: pm2sadd.h +int32_t test_pm2sadd_i16x2(int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2sadd_i16x2(rs1, rs2); +} + +// CHECK-LABEL: test_pm2sadd_x_i16x2: +// RV32: pm2sadd.hx +// RV64: pm2sadd.hx +int32_t test_pm2sadd_x_i16x2(int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2sadd_x_i16x2(rs1, rs2); +} + +// CHECK-LABEL: test_pm2sub_i16x2: +// RV32: pm2sub.h +// RV64: pm2sub.h +int32_t test_pm2sub_i16x2(int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2sub_i16x2(rs1, rs2); +} + +// CHECK-LABEL: test_pm2sub_x_i16x2: +// RV32: pm2sub.hx +// RV64: pm2sub.hx +int32_t test_pm2sub_x_i16x2(int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2sub_x_i16x2(rs1, rs2); +} + +// CHECK-LABEL: test_pm4addsu_i8x4: +// RV32: pm4addsu.b +// RV64: pm4addsu.b +int32_t test_pm4addsu_i8x4(int8x4_t rs1, uint8x4_t rs2) { + return __riscv_pm4addsu_i8x4(rs1, rs2); +} + +// CHECK-LABEL: test_pm2addsu_i16x2: +// RV32: pm2addsu.h +// RV64: pm2addsu.h +int32_t test_pm2addsu_i16x2(int16x2_t rs1, uint16x2_t rs2) { + return __riscv_pm2addsu_i16x2(rs1, rs2); +} + +// CHECK-LABEL: test_pm4add_i8x8: +// RV32: pm4add.b +// RV32: pm4add.b +// RV64: pm4add.b +int32x2_t test_pm4add_i8x8(int8x8_t rs1, int8x8_t rs2) { + return __riscv_pm4add_i8x8(rs1, rs2); +} + +// CHECK-LABEL: test_pm2add_i16x4: +// RV32: pm2add.h +// RV32: pm2add.h +// RV64: pm2add.h +int32x2_t test_pm2add_i16x4(int16x4_t rs1, int16x4_t rs2) { + return __riscv_pm2add_i16x4(rs1, rs2); +} + +// CHECK-LABEL: test_pm2add_x_i16x4: +// RV32: pm2add.hx +// RV32: pm2add.hx +// RV64: pm2add.hx +int32x2_t test_pm2add_x_i16x4(int16x4_t rs1, int16x4_t rs2) { + return __riscv_pm2add_x_i16x4(rs1, rs2); +} + +// CHECK-LABEL: test_pm4addu_u8x8: +// RV32: pm4addu.b +// RV32: pm4addu.b +// RV64: pm4addu.b +uint32x2_t test_pm4addu_u8x8(uint8x8_t rs1, uint8x8_t rs2) { + return __riscv_pm4addu_u8x8(rs1, rs2); +} + +// CHECK-LABEL: test_pm2addu_u16x4: +// RV32: pm2addu.h +// RV32: pm2addu.h +// RV64: pm2addu.h +uint32x2_t test_pm2addu_u16x4(uint16x4_t rs1, uint16x4_t rs2) { + return __riscv_pm2addu_u16x4(rs1, rs2); +} + +// CHECK-LABEL: test_pmq2add_i16x4: +// RV32: pmq2add.h +// RV32: pmq2add.h +// RV64: pmq2add.h +int32x2_t test_pmq2add_i16x4(int16x4_t rs1, int16x4_t rs2) { + return __riscv_pmq2add_i16x4(rs1, rs2); +} + +// CHECK-LABEL: test_pmqr2add_i16x4: +// RV32: pmqr2add.h +// RV32: pmqr2add.h +// RV64: pmqr2add.h +int32x2_t test_pmqr2add_i16x4(int16x4_t rs1, int16x4_t rs2) { + return __riscv_pmqr2add_i16x4(rs1, rs2); +} + +// CHECK-LABEL: test_pm2sadd_i16x4: +// RV32: pm2sadd.h +// RV32: pm2sadd.h +// RV64: pm2sadd.h +int32x2_t test_pm2sadd_i16x4(int16x4_t rs1, int16x4_t rs2) { + return __riscv_pm2sadd_i16x4(rs1, rs2); +} + +// CHECK-LABEL: test_pm2sadd_x_i16x4: +// RV32: pm2sadd.hx +// RV32: pm2sadd.hx +// RV64: pm2sadd.hx +int32x2_t test_pm2sadd_x_i16x4(int16x4_t rs1, int16x4_t rs2) { + return __riscv_pm2sadd_x_i16x4(rs1, rs2); +} + +// CHECK-LABEL: test_pm2sub_i16x4: +// RV32: pm2sub.h +// RV32: pm2sub.h +// RV64: pm2sub.h +int32x2_t test_pm2sub_i16x4(int16x4_t rs1, int16x4_t rs2) { + return __riscv_pm2sub_i16x4(rs1, rs2); +} + +// CHECK-LABEL: test_pm2sub_x_i16x4: +// RV32: pm2sub.hx +// RV32: pm2sub.hx +// RV64: pm2sub.hx +int32x2_t test_pm2sub_x_i16x4(int16x4_t rs1, int16x4_t rs2) { + return __riscv_pm2sub_x_i16x4(rs1, rs2); +} + +// CHECK-LABEL: test_pm4addsu_i8x8: +// RV32: pm4addsu.b +// RV32: pm4addsu.b +// RV64: pm4addsu.b +int32x2_t test_pm4addsu_i8x8(int8x8_t rs1, uint8x8_t rs2) { + return __riscv_pm4addsu_i8x8(rs1, rs2); +} + +// CHECK-LABEL: test_pm2addsu_i16x4: +// RV32: pm2addsu.h +// RV32: pm2addsu.h +// RV64: pm2addsu.h +int32x2_t test_pm2addsu_i16x4(int16x4_t rs1, uint16x4_t rs2) { + return __riscv_pm2addsu_i16x4(rs1, rs2); +} + +// CHECK-LABEL: test_pm2add_i32x2: +// RV32: wmul +// RV32: wmacc +// RV64: pm2add.w +int64_t test_pm2add_i32x2(int32x2_t rs1, int32x2_t rs2) { + return __riscv_pm2add_i32x2(rs1, rs2); +} + +// CHECK-LABEL: test_pm2add_x_i32x2: +// RV32: wmul +// RV32: wmacc +// RV64: pm2add.wx +int64_t test_pm2add_x_i32x2(int32x2_t rs1, int32x2_t rs2) { + return __riscv_pm2add_x_i32x2(rs1, rs2); +} + +// CHECK-LABEL: test_pm2addu_u32x2: +// RV32: wmulu +// RV32: wmaccu +// RV64: pm2addu.w +uint64_t test_pm2addu_u32x2(uint32x2_t rs1, uint32x2_t rs2) { + return __riscv_pm2addu_u32x2(rs1, rs2); +} + +// CHECK-LABEL: test_pmq2add_i32x2: +// RV32: mqwacc +// RV32: mqwacc +// RV64: pmq2add.w +int64_t test_pmq2add_i32x2(int32x2_t rs1, int32x2_t rs2) { + return __riscv_pmq2add_i32x2(rs1, rs2); +} + +// CHECK-LABEL: test_pmqr2add_i32x2: +// RV32: mqrwacc +// RV32: mqrwacc +// RV64: pmqr2add.w +int64_t test_pmqr2add_i32x2(int32x2_t rs1, int32x2_t rs2) { + return __riscv_pmqr2add_i32x2(rs1, rs2); +} + +// CHECK-LABEL: test_pm2sub_i32x2: +// RV32: wmul +// RV32: wmul +// RV32: subd +// RV64: pm2sub.w +int64_t test_pm2sub_i32x2(int32x2_t rs1, int32x2_t rs2) { + return __riscv_pm2sub_i32x2(rs1, rs2); +} + +// CHECK-LABEL: test_pm2sub_x_i32x2: +// RV32: wmul +// RV32: wmul +// RV32: subd +// RV64: pm2sub.wx +int64_t test_pm2sub_x_i32x2(int32x2_t rs1, int32x2_t rs2) { + return __riscv_pm2sub_x_i32x2(rs1, rs2); +} + +// CHECK-LABEL: test_pm2addsu_i32x2: +// RV32: wmulsu +// RV32: wmaccsu +// RV64: pm2addsu.w +int64_t test_pm2addsu_i32x2(int32x2_t rs1, uint32x2_t rs2) { + return __riscv_pm2addsu_i32x2(rs1, rs2); +} + +// CHECK-LABEL: test_pm4add_i16x4: +// RV32: pm2wadd.h +// RV32: pm2wadda.h +// RV64: pm4add.h +int64_t test_pm4add_i16x4(int16x4_t rs1, int16x4_t rs2) { + return __riscv_pm4add_i16x4(rs1, rs2); +} + +// CHECK-LABEL: test_pm4addu_u16x4: +// RV32: pm2waddu.h +// RV32: pm2waddau.h +// RV64: pm4addu.h +uint64_t test_pm4addu_u16x4(uint16x4_t rs1, uint16x4_t rs2) { + return __riscv_pm4addu_u16x4(rs1, rs2); +} + +// CHECK-LABEL: test_pm4addsu_i16x4: +// RV32: pm2waddsu.h +// RV32: pm2waddasu.h +// RV64: pm4addsu.h +int64_t test_pm4addsu_i16x4(int16x4_t rs1, uint16x4_t rs2) { + return __riscv_pm4addsu_i16x4(rs1, rs2); +} diff --git a/llvm/include/llvm/IR/IntrinsicsRISCV.td b/llvm/include/llvm/IR/IntrinsicsRISCV.td index 9c810ecc2a624..6320b7710cd18 100644 --- a/llvm/include/llvm/IR/IntrinsicsRISCV.td +++ b/llvm/include/llvm/IR/IntrinsicsRISCV.td @@ -2125,6 +2125,25 @@ class RVPBinaryIntrinsic def int_riscv_pmhaccsu : RVPTernaryIntrinsic; def int_riscv_pmhraccsu : RVPTernaryIntrinsic; + // Packed Multiplication with Horizontal Addition. + class RVPHorizontalMulIntrinsic + : DefaultAttrsIntrinsic<[llvm_any_ty], + [llvm_anyvector_ty, LLVMMatchType<1>], + [IntrNoMem, IntrSpeculatable]>; + def int_riscv_pm4add : RVPHorizontalMulIntrinsic; + def int_riscv_pm2add : RVPHorizontalMulIntrinsic; + def int_riscv_pm2add_x : RVPHorizontalMulIntrinsic; + def int_riscv_pm4addu : RVPHorizontalMulIntrinsic; + def int_riscv_pm2addu : RVPHorizontalMulIntrinsic; + def int_riscv_pmq2add : RVPHorizontalMulIntrinsic; + def int_riscv_pmqr2add : RVPHorizontalMulIntrinsic; + def int_riscv_pm2sadd : RVPHorizontalMulIntrinsic; + def int_riscv_pm2sadd_x : RVPHorizontalMulIntrinsic; + def int_riscv_pm2sub : RVPHorizontalMulIntrinsic; + def int_riscv_pm2sub_x : RVPHorizontalMulIntrinsic; + def int_riscv_pm4addsu : RVPHorizontalMulIntrinsic; + def int_riscv_pm2addsu : RVPHorizontalMulIntrinsic; + // Packed Absolute Difference Sum. def int_riscv_pabdsumu : DefaultAttrsIntrinsic<[llvm_anyint_ty], diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index fb403aa3bc274..8196fbff2ec1a 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -12329,6 +12329,40 @@ static unsigned getRVPQFormatAccOpcode(Intrinsic::ID IntNo) { } } +static unsigned getRVPHorizontalMulOpcode(unsigned IntNo) { + switch (IntNo) { + default: + llvm_unreachable( + "Unexpected RISC-V packed horizontal multiply intrinsic"); + case Intrinsic::riscv_pm4add: + return RISCVISD::PM4ADD; + case Intrinsic::riscv_pm2add: + return RISCVISD::PM2ADD; + case Intrinsic::riscv_pm2add_x: + return RISCVISD::PM2ADD_X; + case Intrinsic::riscv_pm4addu: + return RISCVISD::PM4ADDU; + case Intrinsic::riscv_pm2addu: + return RISCVISD::PM2ADDU; + case Intrinsic::riscv_pmq2add: + return RISCVISD::PMQ2ADD; + case Intrinsic::riscv_pmqr2add: + return RISCVISD::PMQR2ADD; + case Intrinsic::riscv_pm2sadd: + return RISCVISD::PM2SADD; + case Intrinsic::riscv_pm2sadd_x: + return RISCVISD::PM2SADD_X; + case Intrinsic::riscv_pm2sub: + return RISCVISD::PM2SUB; + case Intrinsic::riscv_pm2sub_x: + return RISCVISD::PM2SUB_X; + case Intrinsic::riscv_pm4addsu: + return RISCVISD::PM4ADDSU; + case Intrinsic::riscv_pm2addsu: + return RISCVISD::PM2ADDSU; + } +} + SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const { unsigned IntNo = Op.getConstantOperandVal(0); @@ -12779,6 +12813,36 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, return DAG.getNode(MulOpc, DL, VT, Rd, Rs1, Rs2); } + case Intrinsic::riscv_pm4add: + case Intrinsic::riscv_pm2add: + case Intrinsic::riscv_pm2add_x: + case Intrinsic::riscv_pm4addu: + case Intrinsic::riscv_pm2addu: + case Intrinsic::riscv_pmq2add: + case Intrinsic::riscv_pmqr2add: + case Intrinsic::riscv_pm2sadd: + case Intrinsic::riscv_pm2sadd_x: + case Intrinsic::riscv_pm2sub: + case Intrinsic::riscv_pm2sub_x: + case Intrinsic::riscv_pm4addsu: + case Intrinsic::riscv_pm2addsu: { + EVT VT = Op.getValueType(); + unsigned Opc = getRVPHorizontalMulOpcode(IntNo); + SDValue Rs1 = Op.getOperand(1); + SDValue Rs2 = Op.getOperand(2); + + // RV32 applies the 32-bit instruction independently to both halves of a + // 64-bit packed input. + if (!Subtarget.is64Bit() && VT == MVT::v2i32) { + auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL); + auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL); + SDValue Lo = DAG.getNode(Opc, DL, MVT::i32, Rs1Lo, Rs2Lo); + SDValue Hi = DAG.getNode(Opc, DL, MVT::i32, Rs1Hi, Rs2Hi); + return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Lo, Hi); + } + + return DAG.getNode(Opc, DL, VT, Rs1, Rs2); + } case Intrinsic::riscv_pmerge: { EVT VT = Op.getValueType(); auto buildMerge = [&](SDValue Rs1, SDValue Rs2, SDValue Mask, @@ -16870,6 +16934,42 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N, Results.push_back(DAG.getExtractSubvector(DL, VT, Res, 0)); return; } + case Intrinsic::riscv_pm4add: + case Intrinsic::riscv_pm2add: + case Intrinsic::riscv_pm2add_x: + case Intrinsic::riscv_pm4addu: + case Intrinsic::riscv_pm2addu: + case Intrinsic::riscv_pmq2add: + case Intrinsic::riscv_pmqr2add: + case Intrinsic::riscv_pm2sadd: + case Intrinsic::riscv_pm2sadd_x: + case Intrinsic::riscv_pm2sub: + case Intrinsic::riscv_pm2sub_x: + case Intrinsic::riscv_pm4addsu: + case Intrinsic::riscv_pm2addsu: { + MVT VT = N->getSimpleValueType(0); + unsigned Opc = getRVPHorizontalMulOpcode(IntNo); + SDValue Rs1 = N->getOperand(1); + SDValue Rs2 = N->getOperand(2); + + if (!Subtarget.is64Bit() && VT == MVT::i64) { + SDValue Pair = DAG.getNode(Opc, DL, MVT::v2i32, Rs1, Rs2); + Results.push_back(DAG.getBitcast(MVT::i64, Pair)); + return; + } + + assert(Subtarget.is64Bit() && VT == MVT::i32 && + "Unexpected horizontal multiply legalization"); + MVT SrcVT = Rs1.getSimpleValueType(); + MVT WideSrcVT = SrcVT == MVT::v4i8 ? MVT::v8i8 : MVT::v4i16; + Rs1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideSrcVT, Rs1, + DAG.getUNDEF(SrcVT)); + Rs2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideSrcVT, Rs2, + DAG.getUNDEF(SrcVT)); + SDValue Wide = DAG.getNode(Opc, DL, MVT::v2i32, Rs1, Rs2); + Results.push_back(DAG.getExtractVectorElt(DL, MVT::i32, Wide, 0)); + return; + } case Intrinsic::riscv_paadd: case Intrinsic::riscv_paaddu: case Intrinsic::riscv_pasub: diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td index 06cf949cca56b..807792928a336 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td @@ -1935,6 +1935,23 @@ def riscv_mhraccu : RVSDNode<"MHRACCU", SDT_RISCVSameTernary>; def riscv_mhaccsu : RVSDNode<"MHACCSU", SDT_RISCVSameTernary>; def riscv_mhraccsu : RVSDNode<"MHRACCSU", SDT_RISCVSameTernary>; +// Multiply packed elements and horizontally combine each adjacent group. +def SDT_RISCVHorizontalMul + : SDTypeProfile<1, 2, [SDTCisVec<1>, SDTCisSameAs<1, 2>]>; +def riscv_pm4add : RVSDNode<"PM4ADD", SDT_RISCVHorizontalMul>; +def riscv_pm2add : RVSDNode<"PM2ADD", SDT_RISCVHorizontalMul>; +def riscv_pm2add_x : RVSDNode<"PM2ADD_X", SDT_RISCVHorizontalMul>; +def riscv_pm4addu : RVSDNode<"PM4ADDU", SDT_RISCVHorizontalMul>; +def riscv_pm2addu : RVSDNode<"PM2ADDU", SDT_RISCVHorizontalMul>; +def riscv_pmq2add : RVSDNode<"PMQ2ADD", SDT_RISCVHorizontalMul>; +def riscv_pmqr2add : RVSDNode<"PMQR2ADD", SDT_RISCVHorizontalMul>; +def riscv_pm2sadd : RVSDNode<"PM2SADD", SDT_RISCVHorizontalMul>; +def riscv_pm2sadd_x : RVSDNode<"PM2SADD_X", SDT_RISCVHorizontalMul>; +def riscv_pm2sub : RVSDNode<"PM2SUB", SDT_RISCVHorizontalMul>; +def riscv_pm2sub_x : RVSDNode<"PM2SUB_X", SDT_RISCVHorizontalMul>; +def riscv_pm4addsu : RVSDNode<"PM4ADDSU", SDT_RISCVHorizontalMul>; +def riscv_pm2addsu : RVSDNode<"PM2ADDSU", SDT_RISCVHorizontalMul>; + // "Q-format" multiplication def riscv_mulq : RVSDNode<"MULQ", SDTIntBinOp, [SDNPCommutative]>; def riscv_mulqr : RVSDNode<"MULQR", SDTIntBinOp, [SDNPCommutative]>; @@ -2063,6 +2080,11 @@ def RISCVBuildPairGPRVec : RVSDNode<"BuildPairGPRVec", SDT_RISCVBuildPairGPRVec>; def RISCVSplitGPRVec : RVSDNode<"SplitGPRVec", SDT_RISCVSplitGPRVec>; +class PatHorizontalMul<SDPatternOperator OpNode, RVInst Inst, + ValueType ResultVT, ValueType SrcVT> + : Pat<(ResultVT (OpNode (SrcVT GPR:$rs1), (SrcVT GPR:$rs2))), + (Inst GPR:$rs1, GPR:$rs2)>; + // (rs1 << 1) + rs2 class PatPSh1Add<RVInst Inst, ValueType vt> : Pat<(vt (add (riscv_pshl (vt GPR:$rs1), (XLenVT 1)), (vt GPR:$rs2))), @@ -2340,6 +2362,97 @@ let append Predicates = [IsRV32] in { def : PatGprGpr<riscv_mulq, MULQ, i32>; def : PatGprGpr<riscv_mulqr, MULQR, i32>; + // Packed multiplication with horizontal addition patterns. + def : PatHorizontalMul<riscv_pm4add, PM4ADD_B, i32, v4i8>; + def : PatHorizontalMul<riscv_pm2add, PM2ADD_H, i32, v2i16>; + def : PatHorizontalMul<riscv_pm2add_x, PM2ADD_HX, i32, v2i16>; + def : PatHorizontalMul<riscv_pm4addu, PM4ADDU_B, i32, v4i8>; + def : PatHorizontalMul<riscv_pm2addu, PM2ADDU_H, i32, v2i16>; + def : PatHorizontalMul<riscv_pmq2add, PMQ2ADD_H, i32, v2i16>; + def : PatHorizontalMul<riscv_pmqr2add, PMQR2ADD_H, i32, v2i16>; + def : PatHorizontalMul<riscv_pm2sadd, PM2SADD_H, i32, v2i16>; + def : PatHorizontalMul<riscv_pm2sadd_x, PM2SADD_HX, i32, v2i16>; + def : PatHorizontalMul<riscv_pm2sub, PM2SUB_H, i32, v2i16>; + def : PatHorizontalMul<riscv_pm2sub_x, PM2SUB_HX, i32, v2i16>; + def : PatHorizontalMul<riscv_pm4addsu, PM4ADDSU_B, i32, v4i8>; + def : PatHorizontalMul<riscv_pm2addsu, PM2ADDSU_H, i32, v2i16>; + + // Select the RV32 v2i32 result produced by type legalization into the + // instruction sequences specified for the 64-bit intrinsics. + def : Pat<(v2i32 (riscv_pm2add (v2i32 GPRPair:$rs1), + (v2i32 GPRPair:$rs2))), + (WMACC (WMUL (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even), + (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)), + (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd), + (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>; + def : Pat<(v2i32 (riscv_pm2add_x (v2i32 GPRPair:$rs1), + (v2i32 GPRPair:$rs2))), + (WMACC (WMUL (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even), + (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd)), + (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd), + (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even))>; + def : Pat<(v2i32 (riscv_pm2addu (v2i32 GPRPair:$rs1), + (v2i32 GPRPair:$rs2))), + (WMACCU (WMULU (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even), + (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)), + (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd), + (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>; + def : Pat<(v2i32 (riscv_pmq2add (v2i32 GPRPair:$rs1), + (v2i32 GPRPair:$rs2))), + (MQWACC (v2i32 (MQWACC + (v2i32 (BuildGPRPair (i32 X0), (i32 X0))), + (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even), + (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even))), + (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd), + (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>; + def : Pat<(v2i32 (riscv_pmqr2add (v2i32 GPRPair:$rs1), + (v2i32 GPRPair:$rs2))), + (MQRWACC (v2i32 (MQRWACC + (v2i32 (BuildGPRPair (i32 X0), (i32 X0))), + (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even), + (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even))), + (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd), + (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>; + def : Pat<(v2i32 (riscv_pm2sub (v2i32 GPRPair:$rs1), + (v2i32 GPRPair:$rs2))), + (SUBD (WMUL (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even), + (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)), + (WMUL (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd), + (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd)))>; + def : Pat<(v2i32 (riscv_pm2sub_x (v2i32 GPRPair:$rs1), + (v2i32 GPRPair:$rs2))), + (SUBD (WMUL (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even), + (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd)), + (WMUL (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd), + (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)))>; + def : Pat<(v2i32 (riscv_pm2addsu (v2i32 GPRPair:$rs1), + (v2i32 GPRPair:$rs2))), + (WMACCSU (WMULSU (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even), + (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)), + (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd), + (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>; + def : Pat<(v2i32 (riscv_pm4add (v4i16 GPRPair:$rs1), + (v4i16 GPRPair:$rs2))), + (PM2WADDA_H + (PM2WADD_H (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even), + (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)), + (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd), + (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>; + def : Pat<(v2i32 (riscv_pm4addu (v4i16 GPRPair:$rs1), + (v4i16 GPRPair:$rs2))), + (PM2WADDAU_H + (PM2WADDU_H (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even), + (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)), + (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd), + (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>; + def : Pat<(v2i32 (riscv_pm4addsu (v4i16 GPRPair:$rs1), + (v4i16 GPRPair:$rs2))), + (PM2WADDASU_H + (PM2WADDSU_H (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even), + (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)), + (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd), + (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>; + // Halfword multiply patterns where one operand is a sext.h or zext.h and // the other is a sext.h or zext.h or is known to be sign/zero-extended. We // prefer plain mul when both operands are known to be sign/zero-extended. @@ -2948,6 +3061,33 @@ let append Predicates = [IsRV64] in { def : PatGprGpr<riscv_mulq, PMULQ_W, v2i32>; def : PatGprGpr<riscv_mulqr, PMULQR_W, v2i32>; + // Packed multiplication with horizontal addition patterns. + def : PatHorizontalMul<riscv_pm4add, PM4ADD_B, v2i32, v8i8>; + def : PatHorizontalMul<riscv_pm2add, PM2ADD_H, v2i32, v4i16>; + def : PatHorizontalMul<riscv_pm2add_x, PM2ADD_HX, v2i32, v4i16>; + def : PatHorizontalMul<riscv_pm4addu, PM4ADDU_B, v2i32, v8i8>; + def : PatHorizontalMul<riscv_pm2addu, PM2ADDU_H, v2i32, v4i16>; + def : PatHorizontalMul<riscv_pmq2add, PMQ2ADD_H, v2i32, v4i16>; + def : PatHorizontalMul<riscv_pmqr2add, PMQR2ADD_H, v2i32, v4i16>; + def : PatHorizontalMul<riscv_pm2sadd, PM2SADD_H, v2i32, v4i16>; + def : PatHorizontalMul<riscv_pm2sadd_x, PM2SADD_HX, v2i32, v4i16>; + def : PatHorizontalMul<riscv_pm2sub, PM2SUB_H, v2i32, v4i16>; + def : PatHorizontalMul<riscv_pm2sub_x, PM2SUB_HX, v2i32, v4i16>; + def : PatHorizontalMul<riscv_pm4addsu, PM4ADDSU_B, v2i32, v8i8>; + def : PatHorizontalMul<riscv_pm2addsu, PM2ADDSU_H, v2i32, v4i16>; + + def : PatHorizontalMul<riscv_pm2add, PM2ADD_W, i64, v2i32>; + def : PatHorizontalMul<riscv_pm2add_x, PM2ADD_WX, i64, v2i32>; + def : PatHorizontalMul<riscv_pm2addu, PM2ADDU_W, i64, v2i32>; + def : PatHorizontalMul<riscv_pmq2add, PMQ2ADD_W, i64, v2i32>; + def : PatHorizontalMul<riscv_pmqr2add, PMQR2ADD_W, i64, v2i32>; + def : PatHorizontalMul<riscv_pm2sub, PM2SUB_W, i64, v2i32>; + def : PatHorizontalMul<riscv_pm2sub_x, PM2SUB_WX, i64, v2i32>; + def : PatHorizontalMul<riscv_pm2addsu, PM2ADDSU_W, i64, v2i32>; + def : PatHorizontalMul<riscv_pm4add, PM4ADD_H, i64, v4i16>; + def : PatHorizontalMul<riscv_pm4addu, PM4ADDU_H, i64, v4i16>; + def : PatHorizontalMul<riscv_pm4addsu, PM4ADDSU_H, i64, v4i16>; + // 8/16/32-bit multiply low patterns // FIXME custom lower def : Pat<(v8i8 (mul GPR:$rs1, GPR:$rs2)), diff --git a/llvm/test/CodeGen/RISCV/rvp-simd-32.ll b/llvm/test/CodeGen/RISCV/rvp-simd-32.ll index f1209617b6a90..65294ff5b9b3b 100644 --- a/llvm/test/CodeGen/RISCV/rvp-simd-32.ll +++ b/llvm/test/CodeGen/RISCV/rvp-simd-32.ll @@ -3240,3 +3240,120 @@ define i32 @test_mqracc_h11_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) { %r = call i32 @llvm.riscv.mqracc.h11.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b) ret i32 %r } + +define i32 @test_pm4add_v4i8_i32(<4 x i8> %a, <4 x i8> %b) { +; CHECK-LABEL: test_pm4add_v4i8_i32: +; CHECK: # %bb.0: +; CHECK-NEXT: pm4add.b a0, a0, a1 +; CHECK-NEXT: ret + %r = call i32 @llvm.riscv.pm4add.i32.v4i8(<4 x i8> %a, <4 x i8> %b) + ret i32 %r +} + +define i32 @test_pm2add_v2i16_i32(<2 x i16> %a, <2 x i16> %b) { +; CHECK-LABEL: test_pm2add_v2i16_i32: +; CHECK: # %bb.0: +; CHECK-NEXT: pm2add.h a0, a0, a1 +; CHECK-NEXT: ret + %r = call i32 @llvm.riscv.pm2add.i32.v2i16(<2 x i16> %a, <2 x i16> %b) + ret i32 %r +} + +define i32 @test_pm2add_x_v2i16_i32(<2 x i16> %a, <2 x i16> %b) { +; CHECK-LABEL: test_pm2add_x_v2i16_i32: +; CHECK: # %bb.0: +; CHECK-NEXT: pm2add.hx a0, a0, a1 +; CHECK-NEXT: ret + %r = call i32 @llvm.riscv.pm2add.x.i32.v2i16(<2 x i16> %a, <2 x i16> %b) + ret i32 %r +} + +define i32 @test_pm4addu_v4i8_i32(<4 x i8> %a, <4 x i8> %b) { +; CHECK-LABEL: test_pm4addu_v4i8_i32: +; CHECK: # %bb.0: +; CHECK-NEXT: pm4addu.b a0, a0, a1 +; CHECK-NEXT: ret + %r = call i32 @llvm.riscv.pm4addu.i32.v4i8(<4 x i8> %a, <4 x i8> %b) + ret i32 %r +} + +define i32 @test_pm2addu_v2i16_i32(<2 x i16> %a, <2 x i16> %b) { +; CHECK-LABEL: test_pm2addu_v2i16_i32: +; CHECK: # %bb.0: +; CHECK-NEXT: pm2addu.h a0, a0, a1 +; CHECK-NEXT: ret + %r = call i32 @llvm.riscv.pm2addu.i32.v2i16(<2 x i16> %a, <2 x i16> %b) + ret i32 %r +} + +define i32 @test_pmq2add_v2i16_i32(<2 x i16> %a, <2 x i16> %b) { +; CHECK-LABEL: test_pmq2add_v2i16_i32: +; CHECK: # %bb.0: +; CHECK-NEXT: pmq2add.h a0, a0, a1 +; CHECK-NEXT: ret + %r = call i32 @llvm.riscv.pmq2add.i32.v2i16(<2 x i16> %a, <2 x i16> %b) + ret i32 %r +} + +define i32 @test_pmqr2add_v2i16_i32(<2 x i16> %a, <2 x i16> %b) { +; CHECK-LABEL: test_pmqr2add_v2i16_i32: +; CHECK: # %bb.0: +; CHECK-NEXT: pmqr2add.h a0, a0, a1 +; CHECK-NEXT: ret + %r = call i32 @llvm.riscv.pmqr2add.i32.v2i16(<2 x i16> %a, <2 x i16> %b) + ret i32 %r +} + +define i32 @test_pm2sadd_v2i16_i32(<2 x i16> %a, <2 x i16> %b) { +; CHECK-LABEL: test_pm2sadd_v2i16_i32: +; CHECK: # %bb.0: +; CHECK-NEXT: pm2sadd.h a0, a0, a1 +; CHECK-NEXT: ret + %r = call i32 @llvm.riscv.pm2sadd.i32.v2i16(<2 x i16> %a, <2 x i16> %b) + ret i32 %r +} + +define i32 @test_pm2sadd_x_v2i16_i32(<2 x i16> %a, <2 x i16> %b) { +; CHECK-LABEL: test_pm2sadd_x_v2i16_i32: +; CHECK: # %bb.0: +; CHECK-NEXT: pm2sadd.hx a0, a0, a1 +; CHECK-NEXT: ret + %r = call i32 @llvm.riscv.pm2sadd.x.i32.v2i16(<2 x i16> %a, <2 x i16> %b) + ret i32 %r +} + +define i32 @test_pm2sub_v2i16_i32(<2 x i16> %a, <2 x i16> %b) { +; CHECK-LABEL: test_pm2sub_v2i16_i32: +; CHECK: # %bb.0: +; CHECK-NEXT: pm2sub.h a0, a0, a1 +; CHECK-NEXT: ret + %r = call i32 @llvm.riscv.pm2sub.i32.v2i16(<2 x i16> %a, <2 x i16> %b) + ret i32 %r +} + +define i32 @test_pm2sub_x_v2i16_i32(<2 x i16> %a, <2 x i16> %b) { +; CHECK-LABEL: test_pm2sub_x_v2i16_i32: +; CHECK: # %bb.0: +; CHECK-NEXT: pm2sub.hx a0, a0, a1 +; CHECK-NEXT: ret + %r = call i32 @llvm.riscv.pm2sub.x.i32.v2i16(<2 x i16> %a, <2 x i16> %b) + ret i32 %r +} + +define i32 @test_pm4addsu_v4i8_i32(<4 x i8> %a, <4 x i8> %b) { +; CHECK-LABEL: test_pm4addsu_v4i8_i32: +; CHECK: # %bb.0: +; CHECK-NEXT: pm4addsu.b a0, a0, a1 +; CHECK-NEXT: ret + %r = call i32 @llvm.riscv.pm4addsu.i32.v4i8(<4 x i8> %a, <4 x i8> %b) + ret i32 %r +} + +define i32 @test_pm2addsu_v2i16_i32(<2 x i16> %a, <2 x i16> %b) { +; CHECK-LABEL: test_pm2addsu_v2i16_i32: +; CHECK: # %bb.0: +; CHECK-NEXT: pm2addsu.h a0, a0, a1 +; CHECK-NEXT: ret + %r = call i32 @llvm.riscv.pm2addsu.i32.v2i16(<2 x i16> %a, <2 x i16> %b) + ret i32 %r +} diff --git a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll index 51b86057ccdbe..ad2d4e2dbeb11 100644 --- a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll +++ b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll @@ -7113,3 +7113,378 @@ define i64 @test_mqracc_w11_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) { %r = call i64 @llvm.riscv.mqracc.w11.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b) ret i64 %r } + +define <2 x i32> @test_pm4add_v8i8_v2i32(<8 x i8> %a, <8 x i8> %b) { +; RV32-LABEL: test_pm4add_v8i8_v2i32: +; RV32: # %bb.0: +; RV32-NEXT: pm4add.b a1, a1, a3 +; RV32-NEXT: pm4add.b a0, a0, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm4add_v8i8_v2i32: +; RV64: # %bb.0: +; RV64-NEXT: pm4add.b a0, a0, a1 +; RV64-NEXT: ret + %r = call <2 x i32> @llvm.riscv.pm4add.v2i32.v8i8(<8 x i8> %a, <8 x i8> %b) + ret <2 x i32> %r +} + +define <2 x i32> @test_pm2add_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: test_pm2add_v4i16_v2i32: +; RV32: # %bb.0: +; RV32-NEXT: pm2add.h a1, a1, a3 +; RV32-NEXT: pm2add.h a0, a0, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm2add_v4i16_v2i32: +; RV64: # %bb.0: +; RV64-NEXT: pm2add.h a0, a0, a1 +; RV64-NEXT: ret + %r = call <2 x i32> @llvm.riscv.pm2add.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b) + ret <2 x i32> %r +} + +define <2 x i32> @test_pm2add_x_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: test_pm2add_x_v4i16_v2i32: +; RV32: # %bb.0: +; RV32-NEXT: pm2add.hx a1, a1, a3 +; RV32-NEXT: pm2add.hx a0, a0, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm2add_x_v4i16_v2i32: +; RV64: # %bb.0: +; RV64-NEXT: pm2add.hx a0, a0, a1 +; RV64-NEXT: ret + %r = call <2 x i32> @llvm.riscv.pm2add.x.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b) + ret <2 x i32> %r +} + +define <2 x i32> @test_pm4addu_v8i8_v2i32(<8 x i8> %a, <8 x i8> %b) { +; RV32-LABEL: test_pm4addu_v8i8_v2i32: +; RV32: # %bb.0: +; RV32-NEXT: pm4addu.b a1, a1, a3 +; RV32-NEXT: pm4addu.b a0, a0, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm4addu_v8i8_v2i32: +; RV64: # %bb.0: +; RV64-NEXT: pm4addu.b a0, a0, a1 +; RV64-NEXT: ret + %r = call <2 x i32> @llvm.riscv.pm4addu.v2i32.v8i8(<8 x i8> %a, <8 x i8> %b) + ret <2 x i32> %r +} + +define <2 x i32> @test_pm2addu_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: test_pm2addu_v4i16_v2i32: +; RV32: # %bb.0: +; RV32-NEXT: pm2addu.h a1, a1, a3 +; RV32-NEXT: pm2addu.h a0, a0, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm2addu_v4i16_v2i32: +; RV64: # %bb.0: +; RV64-NEXT: pm2addu.h a0, a0, a1 +; RV64-NEXT: ret + %r = call <2 x i32> @llvm.riscv.pm2addu.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b) + ret <2 x i32> %r +} + +define <2 x i32> @test_pmq2add_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: test_pmq2add_v4i16_v2i32: +; RV32: # %bb.0: +; RV32-NEXT: pmq2add.h a1, a1, a3 +; RV32-NEXT: pmq2add.h a0, a0, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pmq2add_v4i16_v2i32: +; RV64: # %bb.0: +; RV64-NEXT: pmq2add.h a0, a0, a1 +; RV64-NEXT: ret + %r = call <2 x i32> @llvm.riscv.pmq2add.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b) + ret <2 x i32> %r +} + +define <2 x i32> @test_pmqr2add_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: test_pmqr2add_v4i16_v2i32: +; RV32: # %bb.0: +; RV32-NEXT: pmqr2add.h a1, a1, a3 +; RV32-NEXT: pmqr2add.h a0, a0, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pmqr2add_v4i16_v2i32: +; RV64: # %bb.0: +; RV64-NEXT: pmqr2add.h a0, a0, a1 +; RV64-NEXT: ret + %r = call <2 x i32> @llvm.riscv.pmqr2add.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b) + ret <2 x i32> %r +} + +define <2 x i32> @test_pm2sadd_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: test_pm2sadd_v4i16_v2i32: +; RV32: # %bb.0: +; RV32-NEXT: pm2sadd.h a1, a1, a3 +; RV32-NEXT: pm2sadd.h a0, a0, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm2sadd_v4i16_v2i32: +; RV64: # %bb.0: +; RV64-NEXT: pm2sadd.h a0, a0, a1 +; RV64-NEXT: ret + %r = call <2 x i32> @llvm.riscv.pm2sadd.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b) + ret <2 x i32> %r +} + +define <2 x i32> @test_pm2sadd_x_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: test_pm2sadd_x_v4i16_v2i32: +; RV32: # %bb.0: +; RV32-NEXT: pm2sadd.hx a1, a1, a3 +; RV32-NEXT: pm2sadd.hx a0, a0, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm2sadd_x_v4i16_v2i32: +; RV64: # %bb.0: +; RV64-NEXT: pm2sadd.hx a0, a0, a1 +; RV64-NEXT: ret + %r = call <2 x i32> @llvm.riscv.pm2sadd.x.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b) + ret <2 x i32> %r +} + +define <2 x i32> @test_pm2sub_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: test_pm2sub_v4i16_v2i32: +; RV32: # %bb.0: +; RV32-NEXT: pm2sub.h a1, a1, a3 +; RV32-NEXT: pm2sub.h a0, a0, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm2sub_v4i16_v2i32: +; RV64: # %bb.0: +; RV64-NEXT: pm2sub.h a0, a0, a1 +; RV64-NEXT: ret + %r = call <2 x i32> @llvm.riscv.pm2sub.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b) + ret <2 x i32> %r +} + +define <2 x i32> @test_pm2sub_x_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: test_pm2sub_x_v4i16_v2i32: +; RV32: # %bb.0: +; RV32-NEXT: pm2sub.hx a1, a1, a3 +; RV32-NEXT: pm2sub.hx a0, a0, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm2sub_x_v4i16_v2i32: +; RV64: # %bb.0: +; RV64-NEXT: pm2sub.hx a0, a0, a1 +; RV64-NEXT: ret + %r = call <2 x i32> @llvm.riscv.pm2sub.x.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b) + ret <2 x i32> %r +} + +define <2 x i32> @test_pm4addsu_v8i8_v2i32(<8 x i8> %a, <8 x i8> %b) { +; RV32-LABEL: test_pm4addsu_v8i8_v2i32: +; RV32: # %bb.0: +; RV32-NEXT: pm4addsu.b a1, a1, a3 +; RV32-NEXT: pm4addsu.b a0, a0, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm4addsu_v8i8_v2i32: +; RV64: # %bb.0: +; RV64-NEXT: pm4addsu.b a0, a0, a1 +; RV64-NEXT: ret + %r = call <2 x i32> @llvm.riscv.pm4addsu.v2i32.v8i8(<8 x i8> %a, <8 x i8> %b) + ret <2 x i32> %r +} + +define <2 x i32> @test_pm2addsu_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: test_pm2addsu_v4i16_v2i32: +; RV32: # %bb.0: +; RV32-NEXT: pm2addsu.h a1, a1, a3 +; RV32-NEXT: pm2addsu.h a0, a0, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm2addsu_v4i16_v2i32: +; RV64: # %bb.0: +; RV64-NEXT: pm2addsu.h a0, a0, a1 +; RV64-NEXT: ret + %r = call <2 x i32> @llvm.riscv.pm2addsu.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b) + ret <2 x i32> %r +} + +define i64 @test_pm2add_v2i32_i64(<2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: test_pm2add_v2i32_i64: +; RV32: # %bb.0: +; RV32-NEXT: mv a4, a1 +; RV32-NEXT: wmul a0, a0, a2 +; RV32-NEXT: wmacc a0, a4, a3 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm2add_v2i32_i64: +; RV64: # %bb.0: +; RV64-NEXT: pm2add.w a0, a0, a1 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.pm2add.i64.v2i32(<2 x i32> %a, <2 x i32> %b) + ret i64 %r +} + +define i64 @test_pm2add_x_v2i32_i64(<2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: test_pm2add_x_v2i32_i64: +; RV32: # %bb.0: +; RV32-NEXT: mv a4, a1 +; RV32-NEXT: wmul a0, a0, a3 +; RV32-NEXT: wmacc a0, a4, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm2add_x_v2i32_i64: +; RV64: # %bb.0: +; RV64-NEXT: pm2add.wx a0, a0, a1 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.pm2add.x.i64.v2i32(<2 x i32> %a, <2 x i32> %b) + ret i64 %r +} + +define i64 @test_pm2addu_v2i32_i64(<2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: test_pm2addu_v2i32_i64: +; RV32: # %bb.0: +; RV32-NEXT: mv a4, a1 +; RV32-NEXT: wmulu a0, a0, a2 +; RV32-NEXT: wmaccu a0, a4, a3 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm2addu_v2i32_i64: +; RV64: # %bb.0: +; RV64-NEXT: pm2addu.w a0, a0, a1 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.pm2addu.i64.v2i32(<2 x i32> %a, <2 x i32> %b) + ret i64 %r +} + +define i64 @test_pmq2add_v2i32_i64(<2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: test_pmq2add_v2i32_i64: +; RV32: # %bb.0: +; RV32-NEXT: li a4, 0 +; RV32-NEXT: li a5, 0 +; RV32-NEXT: mqwacc a4, a0, a2 +; RV32-NEXT: mqwacc a4, a1, a3 +; RV32-NEXT: mvd a0, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pmq2add_v2i32_i64: +; RV64: # %bb.0: +; RV64-NEXT: pmq2add.w a0, a0, a1 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.pmq2add.i64.v2i32(<2 x i32> %a, <2 x i32> %b) + ret i64 %r +} + +define i64 @test_pmqr2add_v2i32_i64(<2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: test_pmqr2add_v2i32_i64: +; RV32: # %bb.0: +; RV32-NEXT: li a4, 0 +; RV32-NEXT: li a5, 0 +; RV32-NEXT: mqrwacc a4, a0, a2 +; RV32-NEXT: mqrwacc a4, a1, a3 +; RV32-NEXT: mvd a0, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pmqr2add_v2i32_i64: +; RV64: # %bb.0: +; RV64-NEXT: pmqr2add.w a0, a0, a1 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.pmqr2add.i64.v2i32(<2 x i32> %a, <2 x i32> %b) + ret i64 %r +} + +define i64 @test_pm2sub_v2i32_i64(<2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: test_pm2sub_v2i32_i64: +; RV32: # %bb.0: +; RV32-NEXT: wmul a4, a1, a3 +; RV32-NEXT: wmul a0, a0, a2 +; RV32-NEXT: subd a0, a0, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm2sub_v2i32_i64: +; RV64: # %bb.0: +; RV64-NEXT: pm2sub.w a0, a0, a1 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.pm2sub.i64.v2i32(<2 x i32> %a, <2 x i32> %b) + ret i64 %r +} + +define i64 @test_pm2sub_x_v2i32_i64(<2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: test_pm2sub_x_v2i32_i64: +; RV32: # %bb.0: +; RV32-NEXT: wmul a4, a1, a2 +; RV32-NEXT: wmul a0, a0, a3 +; RV32-NEXT: subd a0, a0, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm2sub_x_v2i32_i64: +; RV64: # %bb.0: +; RV64-NEXT: pm2sub.wx a0, a0, a1 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.pm2sub.x.i64.v2i32(<2 x i32> %a, <2 x i32> %b) + ret i64 %r +} + +define i64 @test_pm2addsu_v2i32_i64(<2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: test_pm2addsu_v2i32_i64: +; RV32: # %bb.0: +; RV32-NEXT: mv a4, a1 +; RV32-NEXT: wmulsu a0, a0, a2 +; RV32-NEXT: wmaccsu a0, a4, a3 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm2addsu_v2i32_i64: +; RV64: # %bb.0: +; RV64-NEXT: pm2addsu.w a0, a0, a1 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.pm2addsu.i64.v2i32(<2 x i32> %a, <2 x i32> %b) + ret i64 %r +} + +define i64 @test_pm4add_v4i16_i64(<4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: test_pm4add_v4i16_i64: +; RV32: # %bb.0: +; RV32-NEXT: mv a4, a1 +; RV32-NEXT: pm2wadd.h a0, a0, a2 +; RV32-NEXT: pm2wadda.h a0, a4, a3 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm4add_v4i16_i64: +; RV64: # %bb.0: +; RV64-NEXT: pm4add.h a0, a0, a1 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.pm4add.i64.v4i16(<4 x i16> %a, <4 x i16> %b) + ret i64 %r +} + +define i64 @test_pm4addu_v4i16_i64(<4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: test_pm4addu_v4i16_i64: +; RV32: # %bb.0: +; RV32-NEXT: mv a4, a1 +; RV32-NEXT: pm2waddu.h a0, a0, a2 +; RV32-NEXT: pm2waddau.h a0, a4, a3 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm4addu_v4i16_i64: +; RV64: # %bb.0: +; RV64-NEXT: pm4addu.h a0, a0, a1 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.pm4addu.i64.v4i16(<4 x i16> %a, <4 x i16> %b) + ret i64 %r +} + +define i64 @test_pm4addsu_v4i16_i64(<4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: test_pm4addsu_v4i16_i64: +; RV32: # %bb.0: +; RV32-NEXT: mv a4, a1 +; RV32-NEXT: pm2waddsu.h a0, a0, a2 +; RV32-NEXT: pm2waddasu.h a0, a4, a3 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm4addsu_v4i16_i64: +; RV64: # %bb.0: +; RV64-NEXT: pm4addsu.h a0, a0, a1 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.pm4addsu.i64.v4i16(<4 x i16> %a, <4 x i16> %b) + ret i64 %r +} >From 76e3e5a5294cf87738c562459fa3cb24fb1847c8 Mon Sep 17 00:00:00 2001 From: XChy <[email protected]> Date: Mon, 24 Aug 2026 22:56:04 +0800 Subject: [PATCH 2/4] format --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 8196fbff2ec1a..e2969e965119c 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -12332,8 +12332,7 @@ static unsigned getRVPQFormatAccOpcode(Intrinsic::ID IntNo) { static unsigned getRVPHorizontalMulOpcode(unsigned IntNo) { switch (IntNo) { default: - llvm_unreachable( - "Unexpected RISC-V packed horizontal multiply intrinsic"); + llvm_unreachable("Unexpected RISC-V packed horizontal multiply intrinsic"); case Intrinsic::riscv_pm4add: return RISCVISD::PM4ADD; case Intrinsic::riscv_pm2add: >From 9482d506095661b1f38c65987cc082c0e0a9c535 Mon Sep 17 00:00:00 2001 From: XChy <[email protected]> Date: Tue, 25 Aug 2026 01:22:01 +0800 Subject: [PATCH 3/4] resolve comments --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 68 +++++++++- llvm/lib/Target/RISCV/RISCVInstrInfoP.td | 131 ++++++++------------ llvm/test/CodeGen/RISCV/rvp-simd-64.ll | 30 +++-- 3 files changed, 135 insertions(+), 94 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index e2969e965119c..b64998d98a913 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -12362,6 +12362,70 @@ static unsigned getRVPHorizontalMulOpcode(unsigned IntNo) { } } +static SDValue lowerRV32HorizontalMul64(unsigned IntNo, SDValue Rs1, + SDValue Rs2, const SDLoc &DL, + SelectionDAG &DAG) { + if (Rs1.getSimpleValueType() == MVT::v4i16) { + auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL); + auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL); + unsigned MulOpc, AccOpc; + switch (IntNo) { + default: + llvm_unreachable("Unexpected RV32 horizontal multiply intrinsic"); + case Intrinsic::riscv_pm4add: + MulOpc = RISCVISD::PM2WADD; + AccOpc = RISCVISD::PM2WADDA; + break; + case Intrinsic::riscv_pm4addu: + MulOpc = RISCVISD::PM2WADDU; + AccOpc = RISCVISD::PM2WADDAU; + break; + case Intrinsic::riscv_pm4addsu: + MulOpc = RISCVISD::PM2WADDSU; + AccOpc = RISCVISD::PM2WADDASU; + break; + } + SDValue Acc = DAG.getNode(MulOpc, DL, MVT::v2i32, Rs1Lo, Rs2Lo); + return DAG.getNode(AccOpc, DL, MVT::v2i32, Acc, Rs1Hi, Rs2Hi); + } + + assert(Rs1.getSimpleValueType() == MVT::v2i32 && + "Unexpected RV32 horizontal multiply source type"); + auto Extract = [&](SDValue V, unsigned Idx) { + return DAG.getExtractVectorElt(DL, MVT::i32, V, Idx); + }; + SDValue Rs1Lo = Extract(Rs1, 0); + SDValue Rs1Hi = Extract(Rs1, 1); + SDValue Rs2Lo = Extract(Rs2, 0); + SDValue Rs2Hi = Extract(Rs2, 1); + + if (IntNo == Intrinsic::riscv_pmq2add || IntNo == Intrinsic::riscv_pmqr2add) { + unsigned AccOpc = IntNo == Intrinsic::riscv_pmq2add ? RISCVISD::MQWACC + : RISCVISD::MQRWACC; + SDValue Acc = DAG.getConstant(0, DL, MVT::v2i32); + Acc = DAG.getNode(AccOpc, DL, MVT::v2i32, Acc, Rs1Lo, Rs2Lo); + return DAG.getNode(AccOpc, DL, MVT::v2i32, Acc, Rs1Hi, Rs2Hi); + } + + unsigned MulOpc = ISD::SMUL_LOHI; + if (IntNo == Intrinsic::riscv_pm2addu) + MulOpc = ISD::UMUL_LOHI; + else if (IntNo == Intrinsic::riscv_pm2addsu) + MulOpc = RISCVISD::WMULSU; + bool IsSub = + IntNo == Intrinsic::riscv_pm2sub || IntNo == Intrinsic::riscv_pm2sub_x; + if (IntNo == Intrinsic::riscv_pm2add_x || IntNo == Intrinsic::riscv_pm2sub_x) + std::swap(Rs2Lo, Rs2Hi); + + SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32); + SDValue LoMul = DAG.getNode(MulOpc, DL, VTs, Rs1Lo, Rs2Lo); + SDValue HiMul = DAG.getNode(MulOpc, DL, VTs, Rs1Hi, Rs2Hi); + unsigned Opc = IsSub ? RISCVISD::SUBD : RISCVISD::ADDD; + SDValue Res = DAG.getNode(Opc, DL, VTs, LoMul, LoMul.getValue(1), HiMul, + HiMul.getValue(1)); + return DAG.getBuildVector(MVT::v2i32, DL, {Res, Res.getValue(1)}); +} + SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const { unsigned IntNo = Op.getConstantOperandVal(0); @@ -16952,8 +17016,8 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N, SDValue Rs2 = N->getOperand(2); if (!Subtarget.is64Bit() && VT == MVT::i64) { - SDValue Pair = DAG.getNode(Opc, DL, MVT::v2i32, Rs1, Rs2); - Results.push_back(DAG.getBitcast(MVT::i64, Pair)); + SDValue Res = lowerRV32HorizontalMul64(IntNo, Rs1, Rs2, DL, DAG); + Results.push_back(DAG.getBitcast(MVT::i64, Res)); return; } diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td index 807792928a336..38b0b11397276 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td @@ -1879,6 +1879,28 @@ def riscv_pmulsu_halves_00 def riscv_pmulsu_halves_11 : RVSDNode<"PMULSU_HALVES_11", SDT_RISCVPackedWideningMulByHalves>; +// Packed multiplication with widening horizontal addition. +def SDT_RISCVPackedWideningHorizontalAdd + : SDTypeProfile<1, 2, [SDTCisVT<0, v2i32>, SDTCisVT<1, v2i16>, + SDTCisSameAs<1, 2>]>; +def riscv_pm2wadd + : RVSDNode<"PM2WADD", SDT_RISCVPackedWideningHorizontalAdd>; +def riscv_pm2waddu + : RVSDNode<"PM2WADDU", SDT_RISCVPackedWideningHorizontalAdd>; +def riscv_pm2waddsu + : RVSDNode<"PM2WADDSU", SDT_RISCVPackedWideningHorizontalAdd>; + +// Packed multiplication with widening horizontal addition and accumulate. +def SDT_RISCVPackedWideningHorizontalAddAcc + : SDTypeProfile<1, 3, [SDTCisVT<0, v2i32>, SDTCisSameAs<0, 1>, + SDTCisVT<2, v2i16>, SDTCisSameAs<2, 3>]>; +def riscv_pm2wadda + : RVSDNode<"PM2WADDA", SDT_RISCVPackedWideningHorizontalAddAcc>; +def riscv_pm2waddau + : RVSDNode<"PM2WADDAU", SDT_RISCVPackedWideningHorizontalAddAcc>; +def riscv_pm2waddasu + : RVSDNode<"PM2WADDASU", SDT_RISCVPackedWideningHorizontalAddAcc>; + def SDT_RISCVWideningShiftLeft : SDTypeProfile<2, 2, [SDTCisVT<0, i32>, SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>, @@ -1956,6 +1978,13 @@ def riscv_pm2addsu : RVSDNode<"PM2ADDSU", SDT_RISCVHorizontalMul>; def riscv_mulq : RVSDNode<"MULQ", SDTIntBinOp, [SDNPCommutative]>; def riscv_mulqr : RVSDNode<"MULQR", SDTIntBinOp, [SDNPCommutative]>; +// "Q-format" multiply with widening accumulate. +def SDT_RISCVQWideningMulAcc + : SDTypeProfile<1, 3, [SDTCisVT<0, v2i32>, SDTCisSameAs<0, 1>, + SDTCisVT<2, i32>, SDTCisSameAs<2, 3>]>; +def riscv_mqwacc : RVSDNode<"MQWACC", SDT_RISCVQWideningMulAcc>; +def riscv_mqrwacc : RVSDNode<"MQRWACC", SDT_RISCVQWideningMulAcc>; + def SDT_RISCVPackedShift : SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisSameAs<0, 1>, SDTCisVT<2, XLenVT>]>; @@ -2362,6 +2391,12 @@ let append Predicates = [IsRV32] in { def : PatGprGpr<riscv_mulq, MULQ, i32>; def : PatGprGpr<riscv_mulqr, MULQR, i32>; + // "Q-format" multiply with widening accumulate patterns. + def : Pat<(v2i32 (riscv_mqwacc (v2i32 GPRPair:$rd), GPR:$rs1, GPR:$rs2)), + (MQWACC GPRPair:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(v2i32 (riscv_mqrwacc (v2i32 GPRPair:$rd), GPR:$rs1, GPR:$rs2)), + (MQRWACC GPRPair:$rd, GPR:$rs1, GPR:$rs2)>; + // Packed multiplication with horizontal addition patterns. def : PatHorizontalMul<riscv_pm4add, PM4ADD_B, i32, v4i8>; def : PatHorizontalMul<riscv_pm2add, PM2ADD_H, i32, v2i16>; @@ -2377,82 +2412,6 @@ let append Predicates = [IsRV32] in { def : PatHorizontalMul<riscv_pm4addsu, PM4ADDSU_B, i32, v4i8>; def : PatHorizontalMul<riscv_pm2addsu, PM2ADDSU_H, i32, v2i16>; - // Select the RV32 v2i32 result produced by type legalization into the - // instruction sequences specified for the 64-bit intrinsics. - def : Pat<(v2i32 (riscv_pm2add (v2i32 GPRPair:$rs1), - (v2i32 GPRPair:$rs2))), - (WMACC (WMUL (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even), - (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)), - (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd), - (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>; - def : Pat<(v2i32 (riscv_pm2add_x (v2i32 GPRPair:$rs1), - (v2i32 GPRPair:$rs2))), - (WMACC (WMUL (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even), - (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd)), - (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd), - (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even))>; - def : Pat<(v2i32 (riscv_pm2addu (v2i32 GPRPair:$rs1), - (v2i32 GPRPair:$rs2))), - (WMACCU (WMULU (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even), - (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)), - (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd), - (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>; - def : Pat<(v2i32 (riscv_pmq2add (v2i32 GPRPair:$rs1), - (v2i32 GPRPair:$rs2))), - (MQWACC (v2i32 (MQWACC - (v2i32 (BuildGPRPair (i32 X0), (i32 X0))), - (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even), - (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even))), - (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd), - (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>; - def : Pat<(v2i32 (riscv_pmqr2add (v2i32 GPRPair:$rs1), - (v2i32 GPRPair:$rs2))), - (MQRWACC (v2i32 (MQRWACC - (v2i32 (BuildGPRPair (i32 X0), (i32 X0))), - (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even), - (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even))), - (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd), - (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>; - def : Pat<(v2i32 (riscv_pm2sub (v2i32 GPRPair:$rs1), - (v2i32 GPRPair:$rs2))), - (SUBD (WMUL (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even), - (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)), - (WMUL (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd), - (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd)))>; - def : Pat<(v2i32 (riscv_pm2sub_x (v2i32 GPRPair:$rs1), - (v2i32 GPRPair:$rs2))), - (SUBD (WMUL (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even), - (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd)), - (WMUL (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd), - (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)))>; - def : Pat<(v2i32 (riscv_pm2addsu (v2i32 GPRPair:$rs1), - (v2i32 GPRPair:$rs2))), - (WMACCSU (WMULSU (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even), - (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)), - (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd), - (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>; - def : Pat<(v2i32 (riscv_pm4add (v4i16 GPRPair:$rs1), - (v4i16 GPRPair:$rs2))), - (PM2WADDA_H - (PM2WADD_H (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even), - (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)), - (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd), - (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>; - def : Pat<(v2i32 (riscv_pm4addu (v4i16 GPRPair:$rs1), - (v4i16 GPRPair:$rs2))), - (PM2WADDAU_H - (PM2WADDU_H (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even), - (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)), - (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd), - (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>; - def : Pat<(v2i32 (riscv_pm4addsu (v4i16 GPRPair:$rs1), - (v4i16 GPRPair:$rs2))), - (PM2WADDASU_H - (PM2WADDSU_H (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even), - (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)), - (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd), - (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>; - // Halfword multiply patterns where one operand is a sext.h or zext.h and // the other is a sext.h or zext.h or is known to be sign/zero-extended. We // prefer plain mul when both operands are known to be sign/zero-extended. @@ -2518,6 +2477,26 @@ let append Predicates = [IsRV32] in { def : Pat<(v2i32 (riscv_pwmulsu (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))), (PWMULSU_H GPR:$rs1, GPR:$rs2)>; + // Packed multiplication with widening horizontal addition patterns. + def : Pat<(v2i32 (riscv_pm2wadd (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))), + (PM2WADD_H GPR:$rs1, GPR:$rs2)>; + def : Pat<(v2i32 (riscv_pm2waddu (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))), + (PM2WADDU_H GPR:$rs1, GPR:$rs2)>; + def : Pat<(v2i32 (riscv_pm2waddsu (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))), + (PM2WADDSU_H GPR:$rs1, GPR:$rs2)>; + + // Packed multiplication with widening horizontal addition and accumulate + // patterns. + def : Pat<(v2i32 (riscv_pm2wadda (v2i32 GPRPair:$rd), + (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))), + (PM2WADDA_H GPRPair:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(v2i32 (riscv_pm2waddau (v2i32 GPRPair:$rd), + (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))), + (PM2WADDAU_H GPRPair:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(v2i32 (riscv_pm2waddasu (v2i32 GPRPair:$rd), + (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))), + (PM2WADDASU_H GPRPair:$rd, GPR:$rs1, GPR:$rs2)>; + // 8/16-bit bitreverse patterns // With Zbkb, brev8 reverses the bits within each byte directly; otherwise // reverse all bits then swap the bytes back. diff --git a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll index ad2d4e2dbeb11..4d9e7acd55e0d 100644 --- a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll +++ b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll @@ -7312,9 +7312,9 @@ define <2 x i32> @test_pm2addsu_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) { define i64 @test_pm2add_v2i32_i64(<2 x i32> %a, <2 x i32> %b) { ; RV32-LABEL: test_pm2add_v2i32_i64: ; RV32: # %bb.0: -; RV32-NEXT: mv a4, a1 -; RV32-NEXT: wmul a0, a0, a2 -; RV32-NEXT: wmacc a0, a4, a3 +; RV32-NEXT: mv a4, a0 +; RV32-NEXT: wmul a0, a1, a3 +; RV32-NEXT: wmacc a0, a4, a2 ; RV32-NEXT: ret ; ; RV64-LABEL: test_pm2add_v2i32_i64: @@ -7328,9 +7328,9 @@ define i64 @test_pm2add_v2i32_i64(<2 x i32> %a, <2 x i32> %b) { define i64 @test_pm2add_x_v2i32_i64(<2 x i32> %a, <2 x i32> %b) { ; RV32-LABEL: test_pm2add_x_v2i32_i64: ; RV32: # %bb.0: -; RV32-NEXT: mv a4, a1 -; RV32-NEXT: wmul a0, a0, a3 -; RV32-NEXT: wmacc a0, a4, a2 +; RV32-NEXT: mv a4, a0 +; RV32-NEXT: wmul a0, a1, a2 +; RV32-NEXT: wmacc a0, a4, a3 ; RV32-NEXT: ret ; ; RV64-LABEL: test_pm2add_x_v2i32_i64: @@ -7344,9 +7344,9 @@ define i64 @test_pm2add_x_v2i32_i64(<2 x i32> %a, <2 x i32> %b) { define i64 @test_pm2addu_v2i32_i64(<2 x i32> %a, <2 x i32> %b) { ; RV32-LABEL: test_pm2addu_v2i32_i64: ; RV32: # %bb.0: -; RV32-NEXT: mv a4, a1 -; RV32-NEXT: wmulu a0, a0, a2 -; RV32-NEXT: wmaccu a0, a4, a3 +; RV32-NEXT: mv a4, a0 +; RV32-NEXT: wmulu a0, a1, a3 +; RV32-NEXT: wmaccu a0, a4, a2 ; RV32-NEXT: ret ; ; RV64-LABEL: test_pm2addu_v2i32_i64: @@ -7360,8 +7360,7 @@ define i64 @test_pm2addu_v2i32_i64(<2 x i32> %a, <2 x i32> %b) { define i64 @test_pmq2add_v2i32_i64(<2 x i32> %a, <2 x i32> %b) { ; RV32-LABEL: test_pmq2add_v2i32_i64: ; RV32: # %bb.0: -; RV32-NEXT: li a4, 0 -; RV32-NEXT: li a5, 0 +; RV32-NEXT: mvd a4, zero ; RV32-NEXT: mqwacc a4, a0, a2 ; RV32-NEXT: mqwacc a4, a1, a3 ; RV32-NEXT: mvd a0, a4 @@ -7378,8 +7377,7 @@ define i64 @test_pmq2add_v2i32_i64(<2 x i32> %a, <2 x i32> %b) { define i64 @test_pmqr2add_v2i32_i64(<2 x i32> %a, <2 x i32> %b) { ; RV32-LABEL: test_pmqr2add_v2i32_i64: ; RV32: # %bb.0: -; RV32-NEXT: li a4, 0 -; RV32-NEXT: li a5, 0 +; RV32-NEXT: mvd a4, zero ; RV32-NEXT: mqrwacc a4, a0, a2 ; RV32-NEXT: mqrwacc a4, a1, a3 ; RV32-NEXT: mvd a0, a4 @@ -7428,9 +7426,9 @@ define i64 @test_pm2sub_x_v2i32_i64(<2 x i32> %a, <2 x i32> %b) { define i64 @test_pm2addsu_v2i32_i64(<2 x i32> %a, <2 x i32> %b) { ; RV32-LABEL: test_pm2addsu_v2i32_i64: ; RV32: # %bb.0: -; RV32-NEXT: mv a4, a1 -; RV32-NEXT: wmulsu a0, a0, a2 -; RV32-NEXT: wmaccsu a0, a4, a3 +; RV32-NEXT: mv a4, a0 +; RV32-NEXT: wmulsu a0, a1, a3 +; RV32-NEXT: wmaccsu a0, a4, a2 ; RV32-NEXT: ret ; ; RV64-LABEL: test_pm2addsu_v2i32_i64: >From 7dd47a4f6c0224a692f69d023f0e37790a299401 Mon Sep 17 00:00:00 2001 From: XChy <[email protected]> Date: Thu, 27 Aug 2026 02:09:22 +0800 Subject: [PATCH 4/4] use build_pair --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 22 ++++++++++++--------- llvm/lib/Target/RISCV/RISCVInstrInfoP.td | 13 ------------ llvm/test/CodeGen/RISCV/rvp-simd-64.ll | 6 ++++-- 3 files changed, 17 insertions(+), 24 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index b64998d98a913..622dbce1f3fa3 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -12365,6 +12365,10 @@ static unsigned getRVPHorizontalMulOpcode(unsigned IntNo) { static SDValue lowerRV32HorizontalMul64(unsigned IntNo, SDValue Rs1, SDValue Rs2, const SDLoc &DL, SelectionDAG &DAG) { + auto Extract = [&](SDValue V, unsigned Idx) { + return DAG.getExtractVectorElt(DL, MVT::i32, V, Idx); + }; + if (Rs1.getSimpleValueType() == MVT::v4i16) { auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL); auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL); @@ -12386,14 +12390,12 @@ static SDValue lowerRV32HorizontalMul64(unsigned IntNo, SDValue Rs1, break; } SDValue Acc = DAG.getNode(MulOpc, DL, MVT::v2i32, Rs1Lo, Rs2Lo); - return DAG.getNode(AccOpc, DL, MVT::v2i32, Acc, Rs1Hi, Rs2Hi); + Acc = DAG.getNode(AccOpc, DL, MVT::v2i32, Acc, Rs1Hi, Rs2Hi); + return DAG.getBitcast(MVT::i64, Acc); } assert(Rs1.getSimpleValueType() == MVT::v2i32 && "Unexpected RV32 horizontal multiply source type"); - auto Extract = [&](SDValue V, unsigned Idx) { - return DAG.getExtractVectorElt(DL, MVT::i32, V, Idx); - }; SDValue Rs1Lo = Extract(Rs1, 0); SDValue Rs1Hi = Extract(Rs1, 1); SDValue Rs2Lo = Extract(Rs2, 0); @@ -12402,9 +12404,11 @@ static SDValue lowerRV32HorizontalMul64(unsigned IntNo, SDValue Rs1, if (IntNo == Intrinsic::riscv_pmq2add || IntNo == Intrinsic::riscv_pmqr2add) { unsigned AccOpc = IntNo == Intrinsic::riscv_pmq2add ? RISCVISD::MQWACC : RISCVISD::MQRWACC; - SDValue Acc = DAG.getConstant(0, DL, MVT::v2i32); - Acc = DAG.getNode(AccOpc, DL, MVT::v2i32, Acc, Rs1Lo, Rs2Lo); - return DAG.getNode(AccOpc, DL, MVT::v2i32, Acc, Rs1Hi, Rs2Hi); + SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32); + SDValue Zero = DAG.getConstant(0, DL, MVT::i32); + SDValue Acc = DAG.getNode(AccOpc, DL, VTs, {Zero, Zero, Rs1Lo, Rs2Lo}); + Acc = DAG.getNode(AccOpc, DL, VTs, {Acc, Acc.getValue(1), Rs1Hi, Rs2Hi}); + return DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Acc, Acc.getValue(1)); } unsigned MulOpc = ISD::SMUL_LOHI; @@ -12423,7 +12427,7 @@ static SDValue lowerRV32HorizontalMul64(unsigned IntNo, SDValue Rs1, unsigned Opc = IsSub ? RISCVISD::SUBD : RISCVISD::ADDD; SDValue Res = DAG.getNode(Opc, DL, VTs, LoMul, LoMul.getValue(1), HiMul, HiMul.getValue(1)); - return DAG.getBuildVector(MVT::v2i32, DL, {Res, Res.getValue(1)}); + return DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Res, Res.getValue(1)); } SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, @@ -17017,7 +17021,7 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N, if (!Subtarget.is64Bit() && VT == MVT::i64) { SDValue Res = lowerRV32HorizontalMul64(IntNo, Rs1, Rs2, DL, DAG); - Results.push_back(DAG.getBitcast(MVT::i64, Res)); + Results.push_back(Res); return; } diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td index 38b0b11397276..2a180ceae66e8 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td @@ -1978,13 +1978,6 @@ def riscv_pm2addsu : RVSDNode<"PM2ADDSU", SDT_RISCVHorizontalMul>; def riscv_mulq : RVSDNode<"MULQ", SDTIntBinOp, [SDNPCommutative]>; def riscv_mulqr : RVSDNode<"MULQR", SDTIntBinOp, [SDNPCommutative]>; -// "Q-format" multiply with widening accumulate. -def SDT_RISCVQWideningMulAcc - : SDTypeProfile<1, 3, [SDTCisVT<0, v2i32>, SDTCisSameAs<0, 1>, - SDTCisVT<2, i32>, SDTCisSameAs<2, 3>]>; -def riscv_mqwacc : RVSDNode<"MQWACC", SDT_RISCVQWideningMulAcc>; -def riscv_mqrwacc : RVSDNode<"MQRWACC", SDT_RISCVQWideningMulAcc>; - def SDT_RISCVPackedShift : SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisSameAs<0, 1>, SDTCisVT<2, XLenVT>]>; @@ -2391,12 +2384,6 @@ let append Predicates = [IsRV32] in { def : PatGprGpr<riscv_mulq, MULQ, i32>; def : PatGprGpr<riscv_mulqr, MULQR, i32>; - // "Q-format" multiply with widening accumulate patterns. - def : Pat<(v2i32 (riscv_mqwacc (v2i32 GPRPair:$rd), GPR:$rs1, GPR:$rs2)), - (MQWACC GPRPair:$rd, GPR:$rs1, GPR:$rs2)>; - def : Pat<(v2i32 (riscv_mqrwacc (v2i32 GPRPair:$rd), GPR:$rs1, GPR:$rs2)), - (MQRWACC GPRPair:$rd, GPR:$rs1, GPR:$rs2)>; - // Packed multiplication with horizontal addition patterns. def : PatHorizontalMul<riscv_pm4add, PM4ADD_B, i32, v4i8>; def : PatHorizontalMul<riscv_pm2add, PM2ADD_H, i32, v2i16>; diff --git a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll index 4d9e7acd55e0d..93834337888f1 100644 --- a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll +++ b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll @@ -7360,7 +7360,8 @@ define i64 @test_pm2addu_v2i32_i64(<2 x i32> %a, <2 x i32> %b) { define i64 @test_pmq2add_v2i32_i64(<2 x i32> %a, <2 x i32> %b) { ; RV32-LABEL: test_pmq2add_v2i32_i64: ; RV32: # %bb.0: -; RV32-NEXT: mvd a4, zero +; RV32-NEXT: li a4, 0 +; RV32-NEXT: li a5, 0 ; RV32-NEXT: mqwacc a4, a0, a2 ; RV32-NEXT: mqwacc a4, a1, a3 ; RV32-NEXT: mvd a0, a4 @@ -7377,7 +7378,8 @@ define i64 @test_pmq2add_v2i32_i64(<2 x i32> %a, <2 x i32> %b) { define i64 @test_pmqr2add_v2i32_i64(<2 x i32> %a, <2 x i32> %b) { ; RV32-LABEL: test_pmqr2add_v2i32_i64: ; RV32: # %bb.0: -; RV32-NEXT: mvd a4, zero +; RV32-NEXT: li a4, 0 +; RV32-NEXT: li a5, 0 ; RV32-NEXT: mqrwacc a4, a0, a2 ; RV32-NEXT: mqrwacc a4, a1, a3 ; RV32-NEXT: mvd a0, a4 _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
