https://github.com/sihuan created https://github.com/llvm/llvm-project/pull/222748
Depends on #222571, which adds the accumulate intrinsics this folds into. An add of a multiply-parts product now selects the accumulating instruction, so the natural way to write a dot product ```c int32_t sum = 0; for (int i = 0; i < n; ++i) sum += __riscv_mul_h00_i32(a[i], b[i]); ``` gets `macc.h00` in the loop rather than a separate `mul.h00` and `add`. The add is rewritten before type legalization, so the shapes whose result is illegal are covered by the existing accumulate lowering rather than needing patterns of their own. >From 4003577d908022bcd1d614f5855fbe7cdd30822e Mon Sep 17 00:00:00 2001 From: SiHuaN <[email protected]> Date: Fri, 28 Aug 2026 08:15:57 +0000 Subject: [PATCH 1/2] [RISCV][P-ext] Add packed multiply-parts accumulate intrinsics Add SelectionDAG and intrinsic support for the RISC-V P multiply-parts accumulate operations, which add the selected products into rd. See also https://github.com/riscv/riscv-p-spec/blob/master/P-ext-intrinsics.adoc#packed-multiply-parts-accumulate Also adds the Clang builtins and the `riscv_packed_simd.h` wrappers. --- clang/include/clang/Basic/BuiltinsRISCV.td | 28 ++ clang/lib/CodeGen/TargetBuiltins/RISCV.cpp | 74 ++++++ clang/lib/Headers/riscv_packed_simd.h | 28 ++ .../riscv_packed_simd.c | 172 ++++++++++++ llvm/include/llvm/IR/IntrinsicsRISCV.td | 30 +++ llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp | 26 +- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 139 ++++++++++ llvm/lib/Target/RISCV/RISCVInstrInfoP.td | 63 +++++ llvm/test/CodeGen/RISCV/rvp-simd-32.ll | 122 +++++++++ llvm/test/CodeGen/RISCV/rvp-simd-64.ll | 250 ++++++++++++++++++ 10 files changed, 930 insertions(+), 2 deletions(-) diff --git a/clang/include/clang/Basic/BuiltinsRISCV.td b/clang/include/clang/Basic/BuiltinsRISCV.td index 58184479d3c27..ee840e45a65ba 100644 --- a/clang/include/clang/Basic/BuiltinsRISCV.td +++ b/clang/include/clang/Basic/BuiltinsRISCV.td @@ -426,6 +426,34 @@ def mqracc_w00_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, int>, _Vector<2, def mqracc_w01_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, int>, _Vector<2, int>)">; def mqracc_w11_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, int>, _Vector<2, int>)">; +// Packed Multiply Parts Accumulate (32-bit) +def macc_h00_i32 : RISCVBuiltin<"int(int, _Vector<2, short>, _Vector<2, short>)">; +def macc_h01_i32 : RISCVBuiltin<"int(int, _Vector<2, short>, _Vector<2, short>)">; +def macc_h11_i32 : RISCVBuiltin<"int(int, _Vector<2, short>, _Vector<2, short>)">; +def maccu_h00_u32 : RISCVBuiltin<"unsigned int(unsigned int, _Vector<2, unsigned short>, _Vector<2, unsigned short>)">; +def maccu_h01_u32 : RISCVBuiltin<"unsigned int(unsigned int, _Vector<2, unsigned short>, _Vector<2, unsigned short>)">; +def maccu_h11_u32 : RISCVBuiltin<"unsigned int(unsigned int, _Vector<2, unsigned short>, _Vector<2, unsigned short>)">; +def maccsu_h00_i32 : RISCVBuiltin<"int(int, _Vector<2, short>, _Vector<2, unsigned short>)">; +def maccsu_h11_i32 : RISCVBuiltin<"int(int, _Vector<2, short>, _Vector<2, unsigned short>)">; + +// Packed Multiply Parts Accumulate (64-bit) +def pmacc_h00_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<4, short>, _Vector<4, short>)">; +def pmacc_h01_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<4, short>, _Vector<4, short>)">; +def pmacc_h11_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<4, short>, _Vector<4, short>)">; +def pmaccu_h00_u32x2 : RISCVBuiltin<"_Vector<2, unsigned int>(_Vector<2, unsigned int>, _Vector<4, unsigned short>, _Vector<4, unsigned short>)">; +def pmaccu_h01_u32x2 : RISCVBuiltin<"_Vector<2, unsigned int>(_Vector<2, unsigned int>, _Vector<4, unsigned short>, _Vector<4, unsigned short>)">; +def pmaccu_h11_u32x2 : RISCVBuiltin<"_Vector<2, unsigned int>(_Vector<2, unsigned int>, _Vector<4, unsigned short>, _Vector<4, unsigned short>)">; +def pmaccsu_h00_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<4, short>, _Vector<4, unsigned short>)">; +def pmaccsu_h11_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<4, short>, _Vector<4, unsigned short>)">; +def macc_w00_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, int>, _Vector<2, int>)">; +def macc_w01_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, int>, _Vector<2, int>)">; +def macc_w11_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, int>, _Vector<2, int>)">; +def maccu_w00_u64 : RISCVBuiltin<"uint64_t(uint64_t, _Vector<2, unsigned int>, _Vector<2, unsigned int>)">; +def maccu_w01_u64 : RISCVBuiltin<"uint64_t(uint64_t, _Vector<2, unsigned int>, _Vector<2, unsigned int>)">; +def maccu_w11_u64 : RISCVBuiltin<"uint64_t(uint64_t, _Vector<2, unsigned int>, _Vector<2, unsigned int>)">; +def maccsu_w00_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, int>, _Vector<2, unsigned int>)">; +def maccsu_w11_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, int>, _Vector<2, unsigned int>)">; + // Packed Sign and Zero Extend (32-bit) def psext_b_i16x2 : RISCVBuiltin<"_Vector<2, short>(_Vector<2, short>)">; def pzext_b_u16x2 : RISCVBuiltin<"_Vector<2, unsigned short>(_Vector<2, unsigned short>)">; diff --git a/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp b/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp index 2e9fbf771bed1..f99a05ce673aa 100644 --- a/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp +++ b/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp @@ -1938,6 +1938,80 @@ Value *CodeGenFunction::EmitRISCVBuiltinExpr(unsigned BuiltinID, break; } + // Packed Multiply Parts Accumulate. + case RISCV::BI__builtin_riscv_macc_h00_i32: + case RISCV::BI__builtin_riscv_pmacc_h00_i32x2: + case RISCV::BI__builtin_riscv_macc_w00_i64: + case RISCV::BI__builtin_riscv_macc_h01_i32: + case RISCV::BI__builtin_riscv_pmacc_h01_i32x2: + case RISCV::BI__builtin_riscv_macc_w01_i64: + case RISCV::BI__builtin_riscv_macc_h11_i32: + case RISCV::BI__builtin_riscv_pmacc_h11_i32x2: + case RISCV::BI__builtin_riscv_macc_w11_i64: + case RISCV::BI__builtin_riscv_maccu_h00_u32: + case RISCV::BI__builtin_riscv_pmaccu_h00_u32x2: + case RISCV::BI__builtin_riscv_maccu_w00_u64: + case RISCV::BI__builtin_riscv_maccu_h01_u32: + case RISCV::BI__builtin_riscv_pmaccu_h01_u32x2: + case RISCV::BI__builtin_riscv_maccu_w01_u64: + case RISCV::BI__builtin_riscv_maccu_h11_u32: + case RISCV::BI__builtin_riscv_pmaccu_h11_u32x2: + case RISCV::BI__builtin_riscv_maccu_w11_u64: + case RISCV::BI__builtin_riscv_maccsu_h00_i32: + case RISCV::BI__builtin_riscv_pmaccsu_h00_i32x2: + case RISCV::BI__builtin_riscv_maccsu_w00_i64: + case RISCV::BI__builtin_riscv_maccsu_h11_i32: + case RISCV::BI__builtin_riscv_pmaccsu_h11_i32x2: + case RISCV::BI__builtin_riscv_maccsu_w11_i64: { + switch (BuiltinID) { + default: + llvm_unreachable("unexpected builtin ID"); + case RISCV::BI__builtin_riscv_macc_h00_i32: + case RISCV::BI__builtin_riscv_pmacc_h00_i32x2: + case RISCV::BI__builtin_riscv_macc_w00_i64: + ID = Intrinsic::riscv_macc_00; + break; + case RISCV::BI__builtin_riscv_macc_h01_i32: + case RISCV::BI__builtin_riscv_pmacc_h01_i32x2: + case RISCV::BI__builtin_riscv_macc_w01_i64: + ID = Intrinsic::riscv_macc_01; + break; + case RISCV::BI__builtin_riscv_macc_h11_i32: + case RISCV::BI__builtin_riscv_pmacc_h11_i32x2: + case RISCV::BI__builtin_riscv_macc_w11_i64: + ID = Intrinsic::riscv_macc_11; + break; + case RISCV::BI__builtin_riscv_maccu_h00_u32: + case RISCV::BI__builtin_riscv_pmaccu_h00_u32x2: + case RISCV::BI__builtin_riscv_maccu_w00_u64: + ID = Intrinsic::riscv_maccu_00; + break; + case RISCV::BI__builtin_riscv_maccu_h01_u32: + case RISCV::BI__builtin_riscv_pmaccu_h01_u32x2: + case RISCV::BI__builtin_riscv_maccu_w01_u64: + ID = Intrinsic::riscv_maccu_01; + break; + case RISCV::BI__builtin_riscv_maccu_h11_u32: + case RISCV::BI__builtin_riscv_pmaccu_h11_u32x2: + case RISCV::BI__builtin_riscv_maccu_w11_u64: + ID = Intrinsic::riscv_maccu_11; + break; + case RISCV::BI__builtin_riscv_maccsu_h00_i32: + case RISCV::BI__builtin_riscv_pmaccsu_h00_i32x2: + case RISCV::BI__builtin_riscv_maccsu_w00_i64: + ID = Intrinsic::riscv_maccsu_00; + break; + case RISCV::BI__builtin_riscv_maccsu_h11_i32: + case RISCV::BI__builtin_riscv_pmaccsu_h11_i32x2: + case RISCV::BI__builtin_riscv_maccsu_w11_i64: + ID = Intrinsic::riscv_maccsu_11; + break; + } + + IntrinsicTypes = {ResultType, Ops[1]->getType()}; + break; + } + // Zk builtins // Zknh diff --git a/clang/lib/Headers/riscv_packed_simd.h b/clang/lib/Headers/riscv_packed_simd.h index 37ac140780d02..5c359337d8b7f 100644 --- a/clang/lib/Headers/riscv_packed_simd.h +++ b/clang/lib/Headers/riscv_packed_simd.h @@ -984,6 +984,34 @@ __packed_ternary_builtin_cast(mqracc_w00_i64, int64_t, int32x2_t, __builtin_risc __packed_ternary_builtin_cast(mqracc_w01_i64, int64_t, int32x2_t, __builtin_riscv_mqracc_w01_i64) __packed_ternary_builtin_cast(mqracc_w11_i64, int64_t, int32x2_t, __builtin_riscv_mqracc_w11_i64) +/* Packed Multiply Parts Accumulate (32-bit) */ +__packed_ternary_builtin_mixed(macc_h00_i32, int32_t, int16x2_t, int16x2_t, __builtin_riscv_macc_h00_i32) +__packed_ternary_builtin_mixed(macc_h01_i32, int32_t, int16x2_t, int16x2_t, __builtin_riscv_macc_h01_i32) +__packed_ternary_builtin_mixed(macc_h11_i32, int32_t, int16x2_t, int16x2_t, __builtin_riscv_macc_h11_i32) +__packed_ternary_builtin_mixed(maccu_h00_u32, uint32_t, uint16x2_t, uint16x2_t, __builtin_riscv_maccu_h00_u32) +__packed_ternary_builtin_mixed(maccu_h01_u32, uint32_t, uint16x2_t, uint16x2_t, __builtin_riscv_maccu_h01_u32) +__packed_ternary_builtin_mixed(maccu_h11_u32, uint32_t, uint16x2_t, uint16x2_t, __builtin_riscv_maccu_h11_u32) +__packed_ternary_builtin_mixed(maccsu_h00_i32, int32_t, int16x2_t, uint16x2_t, __builtin_riscv_maccsu_h00_i32) +__packed_ternary_builtin_mixed(maccsu_h11_i32, int32_t, int16x2_t, uint16x2_t, __builtin_riscv_maccsu_h11_i32) + +/* Packed Multiply Parts Accumulate (64-bit) */ +__packed_ternary_builtin_mixed(pmacc_h00_i32x2, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pmacc_h00_i32x2) +__packed_ternary_builtin_mixed(pmacc_h01_i32x2, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pmacc_h01_i32x2) +__packed_ternary_builtin_mixed(pmacc_h11_i32x2, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pmacc_h11_i32x2) +__packed_ternary_builtin_mixed(pmaccu_h00_u32x2, uint32x2_t, uint16x4_t, uint16x4_t, __builtin_riscv_pmaccu_h00_u32x2) +__packed_ternary_builtin_mixed(pmaccu_h01_u32x2, uint32x2_t, uint16x4_t, uint16x4_t, __builtin_riscv_pmaccu_h01_u32x2) +__packed_ternary_builtin_mixed(pmaccu_h11_u32x2, uint32x2_t, uint16x4_t, uint16x4_t, __builtin_riscv_pmaccu_h11_u32x2) +__packed_ternary_builtin_mixed(pmaccsu_h00_i32x2, int32x2_t, int16x4_t, uint16x4_t, __builtin_riscv_pmaccsu_h00_i32x2) +__packed_ternary_builtin_mixed(pmaccsu_h11_i32x2, int32x2_t, int16x4_t, uint16x4_t, __builtin_riscv_pmaccsu_h11_i32x2) +__packed_ternary_builtin_mixed(macc_w00_i64, int64_t, int32x2_t, int32x2_t, __builtin_riscv_macc_w00_i64) +__packed_ternary_builtin_mixed(macc_w01_i64, int64_t, int32x2_t, int32x2_t, __builtin_riscv_macc_w01_i64) +__packed_ternary_builtin_mixed(macc_w11_i64, int64_t, int32x2_t, int32x2_t, __builtin_riscv_macc_w11_i64) +__packed_ternary_builtin_mixed(maccu_w00_u64, uint64_t, uint32x2_t, uint32x2_t, __builtin_riscv_maccu_w00_u64) +__packed_ternary_builtin_mixed(maccu_w01_u64, uint64_t, uint32x2_t, uint32x2_t, __builtin_riscv_maccu_w01_u64) +__packed_ternary_builtin_mixed(maccu_w11_u64, uint64_t, uint32x2_t, uint32x2_t, __builtin_riscv_maccu_w11_u64) +__packed_ternary_builtin_mixed(maccsu_w00_i64, int64_t, int32x2_t, uint32x2_t, __builtin_riscv_maccsu_w00_i64) +__packed_ternary_builtin_mixed(maccsu_w11_i64, int64_t, int32x2_t, uint32x2_t, __builtin_riscv_maccsu_w11_i64) + /* Packed Narrowing Clip Pair (32-bit) */ __packed_binary_builtin_cast(pnclipp_i8x4, int16x2_t, int8x4_t, __builtin_riscv_pnclipp_i8x4) __packed_binary_builtin_cast(pnclipup_u8x4, uint16x2_t, uint8x4_t, __builtin_riscv_pnclipup_u8x4) diff --git a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c index 48fcf036794ff..df9623edfa71a 100644 --- a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c +++ b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c @@ -4248,3 +4248,175 @@ int32_t test_pget_i32x2_i32(int32x2_t v) { uint32_t test_pget_u32x2_u32(uint32x2_t v) { return __riscv_pget_u32x2_u32(v, 1); } + +/* Packed Multiply Parts Accumulate (32-bit) */ + +// CHECK-LABEL: test_macc_h00_i32: +// RV32: macc.h00 +// RV64: pmacc.w.h00 +int32_t test_macc_h00_i32(int32_t rd, int16x2_t a, int16x2_t b) { + return __riscv_macc_h00_i32(rd, a, b); +} + +// CHECK-LABEL: test_macc_h01_i32: +// RV32: macc.h01 +// RV64: pmacc.w.h01 +int32_t test_macc_h01_i32(int32_t rd, int16x2_t a, int16x2_t b) { + return __riscv_macc_h01_i32(rd, a, b); +} + +// CHECK-LABEL: test_macc_h11_i32: +// RV32: macc.h11 +// RV64: pmacc.w.h11 +int32_t test_macc_h11_i32(int32_t rd, int16x2_t a, int16x2_t b) { + return __riscv_macc_h11_i32(rd, a, b); +} + +// CHECK-LABEL: test_maccu_h00_u32: +// RV32: maccu.h00 +// RV64: pmaccu.w.h00 +uint32_t test_maccu_h00_u32(uint32_t rd, uint16x2_t a, uint16x2_t b) { + return __riscv_maccu_h00_u32(rd, a, b); +} + +// CHECK-LABEL: test_maccu_h01_u32: +// RV32: maccu.h01 +// RV64: pmaccu.w.h01 +uint32_t test_maccu_h01_u32(uint32_t rd, uint16x2_t a, uint16x2_t b) { + return __riscv_maccu_h01_u32(rd, a, b); +} + +// CHECK-LABEL: test_maccu_h11_u32: +// RV32: maccu.h11 +// RV64: pmaccu.w.h11 +uint32_t test_maccu_h11_u32(uint32_t rd, uint16x2_t a, uint16x2_t b) { + return __riscv_maccu_h11_u32(rd, a, b); +} + +// CHECK-LABEL: test_maccsu_h00_i32: +// RV32: maccsu.h00 +// RV64: pmaccsu.w.h00 +int32_t test_maccsu_h00_i32(int32_t rd, int16x2_t a, uint16x2_t b) { + return __riscv_maccsu_h00_i32(rd, a, b); +} + +// CHECK-LABEL: test_maccsu_h11_i32: +// RV32: maccsu.h11 +// RV64: pmaccsu.w.h11 +int32_t test_maccsu_h11_i32(int32_t rd, int16x2_t a, uint16x2_t b) { + return __riscv_maccsu_h11_i32(rd, a, b); +} + +/* Packed Multiply Parts Accumulate (64-bit) */ + +// CHECK-LABEL: test_pmacc_h00_i32x2: +// RV32-COUNT-2: macc.h00 +// RV64: pmacc.w.h00 +int32x2_t test_pmacc_h00_i32x2(int32x2_t rd, int16x4_t a, int16x4_t b) { + return __riscv_pmacc_h00_i32x2(rd, a, b); +} + +// CHECK-LABEL: test_pmacc_h01_i32x2: +// RV32-COUNT-2: macc.h01 +// RV64: pmacc.w.h01 +int32x2_t test_pmacc_h01_i32x2(int32x2_t rd, int16x4_t a, int16x4_t b) { + return __riscv_pmacc_h01_i32x2(rd, a, b); +} + +// CHECK-LABEL: test_pmacc_h11_i32x2: +// RV32-COUNT-2: macc.h11 +// RV64: pmacc.w.h11 +int32x2_t test_pmacc_h11_i32x2(int32x2_t rd, int16x4_t a, int16x4_t b) { + return __riscv_pmacc_h11_i32x2(rd, a, b); +} + +// CHECK-LABEL: test_pmaccu_h00_u32x2: +// RV32-COUNT-2: maccu.h00 +// RV64: pmaccu.w.h00 +uint32x2_t test_pmaccu_h00_u32x2(uint32x2_t rd, uint16x4_t a, uint16x4_t b) { + return __riscv_pmaccu_h00_u32x2(rd, a, b); +} + +// CHECK-LABEL: test_pmaccu_h01_u32x2: +// RV32-COUNT-2: maccu.h01 +// RV64: pmaccu.w.h01 +uint32x2_t test_pmaccu_h01_u32x2(uint32x2_t rd, uint16x4_t a, uint16x4_t b) { + return __riscv_pmaccu_h01_u32x2(rd, a, b); +} + +// CHECK-LABEL: test_pmaccu_h11_u32x2: +// RV32-COUNT-2: maccu.h11 +// RV64: pmaccu.w.h11 +uint32x2_t test_pmaccu_h11_u32x2(uint32x2_t rd, uint16x4_t a, uint16x4_t b) { + return __riscv_pmaccu_h11_u32x2(rd, a, b); +} + +// CHECK-LABEL: test_pmaccsu_h00_i32x2: +// RV32-COUNT-2: maccsu.h00 +// RV64: pmaccsu.w.h00 +int32x2_t test_pmaccsu_h00_i32x2(int32x2_t rd, int16x4_t a, uint16x4_t b) { + return __riscv_pmaccsu_h00_i32x2(rd, a, b); +} + +// CHECK-LABEL: test_pmaccsu_h11_i32x2: +// RV32-COUNT-2: maccsu.h11 +// RV64: pmaccsu.w.h11 +int32x2_t test_pmaccsu_h11_i32x2(int32x2_t rd, int16x4_t a, uint16x4_t b) { + return __riscv_pmaccsu_h11_i32x2(rd, a, b); +} + +// CHECK-LABEL: test_macc_w00_i64: +// RV32: wmacc +// RV64: macc.w00 +int64_t test_macc_w00_i64(int64_t rd, int32x2_t a, int32x2_t b) { + return __riscv_macc_w00_i64(rd, a, b); +} + +// CHECK-LABEL: test_macc_w01_i64: +// RV32: wmacc +// RV64: macc.w01 +int64_t test_macc_w01_i64(int64_t rd, int32x2_t a, int32x2_t b) { + return __riscv_macc_w01_i64(rd, a, b); +} + +// CHECK-LABEL: test_macc_w11_i64: +// RV32: wmacc +// RV64: macc.w11 +int64_t test_macc_w11_i64(int64_t rd, int32x2_t a, int32x2_t b) { + return __riscv_macc_w11_i64(rd, a, b); +} + +// CHECK-LABEL: test_maccu_w00_u64: +// RV32: wmaccu +// RV64: maccu.w00 +uint64_t test_maccu_w00_u64(uint64_t rd, uint32x2_t a, uint32x2_t b) { + return __riscv_maccu_w00_u64(rd, a, b); +} + +// CHECK-LABEL: test_maccu_w01_u64: +// RV32: wmaccu +// RV64: maccu.w01 +uint64_t test_maccu_w01_u64(uint64_t rd, uint32x2_t a, uint32x2_t b) { + return __riscv_maccu_w01_u64(rd, a, b); +} + +// CHECK-LABEL: test_maccu_w11_u64: +// RV32: wmaccu +// RV64: maccu.w11 +uint64_t test_maccu_w11_u64(uint64_t rd, uint32x2_t a, uint32x2_t b) { + return __riscv_maccu_w11_u64(rd, a, b); +} + +// CHECK-LABEL: test_maccsu_w00_i64: +// RV32: wmaccsu +// RV64: maccsu.w00 +int64_t test_maccsu_w00_i64(int64_t rd, int32x2_t a, uint32x2_t b) { + return __riscv_maccsu_w00_i64(rd, a, b); +} + +// CHECK-LABEL: test_maccsu_w11_i64: +// RV32: wmaccsu +// RV64: maccsu.w11 +int64_t test_maccsu_w11_i64(int64_t rd, int32x2_t a, uint32x2_t b) { + return __riscv_maccsu_w11_i64(rd, a, b); +} diff --git a/llvm/include/llvm/IR/IntrinsicsRISCV.td b/llvm/include/llvm/IR/IntrinsicsRISCV.td index e4ec9e9beb5ee..e70caa8b09700 100644 --- a/llvm/include/llvm/IR/IntrinsicsRISCV.td +++ b/llvm/include/llvm/IR/IntrinsicsRISCV.td @@ -2186,6 +2186,36 @@ class RVPBinaryIntrinsic def int_riscv_mulsu_00 : RVPScalarMulPartsIntrinsic; def int_riscv_mulsu_11 : RVPScalarMulPartsIntrinsic; + // Packed Multiply Parts Accumulate. + class RVPPackedMulPartsAccIntrinsic + : DefaultAttrsIntrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, + LLVMSubdivide2VectorType<0>, + LLVMSubdivide2VectorType<0>], + [IntrNoMem, IntrSpeculatable]>; + def int_riscv_pmacc_00 : RVPPackedMulPartsAccIntrinsic; + def int_riscv_pmacc_01 : RVPPackedMulPartsAccIntrinsic; + def int_riscv_pmacc_11 : RVPPackedMulPartsAccIntrinsic; + def int_riscv_pmaccu_00 : RVPPackedMulPartsAccIntrinsic; + def int_riscv_pmaccu_01 : RVPPackedMulPartsAccIntrinsic; + def int_riscv_pmaccu_11 : RVPPackedMulPartsAccIntrinsic; + def int_riscv_pmaccsu_00 : RVPPackedMulPartsAccIntrinsic; + def int_riscv_pmaccsu_11 : RVPPackedMulPartsAccIntrinsic; + + class RVPScalarMulPartsAccIntrinsic + : DefaultAttrsIntrinsic<[llvm_anyint_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, + LLVMMatchType<1>], + [IntrNoMem, IntrSpeculatable]>; + def int_riscv_macc_00 : RVPScalarMulPartsAccIntrinsic; + def int_riscv_macc_01 : RVPScalarMulPartsAccIntrinsic; + def int_riscv_macc_11 : RVPScalarMulPartsAccIntrinsic; + def int_riscv_maccu_00 : RVPScalarMulPartsAccIntrinsic; + def int_riscv_maccu_01 : RVPScalarMulPartsAccIntrinsic; + def int_riscv_maccu_11 : RVPScalarMulPartsAccIntrinsic; + def int_riscv_maccsu_00 : RVPScalarMulPartsAccIntrinsic; + def int_riscv_maccsu_11 : RVPScalarMulPartsAccIntrinsic; + // Packed Absolute Difference Sum. def int_riscv_pabdsumu : DefaultAttrsIntrinsic<[llvm_anyint_ty], diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp index d263d0320839b..c1c7031a17e71 100644 --- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp @@ -2073,13 +2073,35 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) { return; } case RISCVISD::MQWACC: - case RISCVISD::MQRWACC: { + case RISCVISD::MQRWACC: + case RISCVISD::WMACC: + case RISCVISD::WMACCU: + case RISCVISD::WMACCSU: { assert(!Subtarget->is64Bit() && Subtarget->hasStdExtP() && "Unexpected opcode"); SDValue Op0 = buildGPRPair(CurDAG, DL, MVT::Untyped, Node->getOperand(0), Node->getOperand(1)); - unsigned Opc = Opcode == RISCVISD::MQRWACC ? RISCV::MQRWACC : RISCV::MQWACC; + unsigned Opc; + switch (Opcode) { + default: + llvm_unreachable("Unexpected opcode"); + case RISCVISD::MQWACC: + Opc = RISCV::MQWACC; + break; + case RISCVISD::MQRWACC: + Opc = RISCV::MQRWACC; + break; + case RISCVISD::WMACC: + Opc = RISCV::WMACC; + break; + case RISCVISD::WMACCU: + Opc = RISCV::WMACCU; + break; + case RISCVISD::WMACCSU: + Opc = RISCV::WMACCSU; + break; + } MachineSDNode *New = CurDAG->getMachineNode( Opc, DL, MVT::Untyped, Op0, Node->getOperand(2), Node->getOperand(3)); auto [Lo, Hi] = extractGPRPair(CurDAG, DL, SDValue(New, 0)); diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 5f8ad5da42da1..e2d91fc4c4cfa 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -12548,6 +12548,38 @@ static Intrinsic::ID getRVPScalarMulPartsIntrinsic(unsigned IntNo) { } } +/// Return the multiply-parts accumulate node for \p IntNo. +static unsigned getRVPMulAccHalvesOpcode(unsigned IntNo) { + switch (IntNo) { + default: + llvm_unreachable("Unexpected RISC-V multiply-parts accumulate intrinsic"); + case Intrinsic::riscv_pmacc_00: + case Intrinsic::riscv_macc_00: + return RISCVISD::PMACC_HALVES_00; + case Intrinsic::riscv_pmacc_01: + case Intrinsic::riscv_macc_01: + return RISCVISD::PMACC_HALVES_01; + case Intrinsic::riscv_pmacc_11: + case Intrinsic::riscv_macc_11: + return RISCVISD::PMACC_HALVES_11; + case Intrinsic::riscv_pmaccu_00: + case Intrinsic::riscv_maccu_00: + return RISCVISD::PMACCU_HALVES_00; + case Intrinsic::riscv_pmaccu_01: + case Intrinsic::riscv_maccu_01: + return RISCVISD::PMACCU_HALVES_01; + case Intrinsic::riscv_pmaccu_11: + case Intrinsic::riscv_maccu_11: + return RISCVISD::PMACCU_HALVES_11; + case Intrinsic::riscv_pmaccsu_00: + case Intrinsic::riscv_maccsu_00: + return RISCVISD::PMACCSU_HALVES_00; + case Intrinsic::riscv_pmaccsu_11: + case Intrinsic::riscv_maccsu_11: + return RISCVISD::PMACCSU_HALVES_11; + } +} + /// Return {opcode, rs1 lane, rs2 lane} for the word form of \p IntNo. static std::tuple<unsigned, unsigned, unsigned> getRVPWordMulPartsOpcodeAndLanes(unsigned IntNo) { @@ -12573,6 +12605,32 @@ getRVPWordMulPartsOpcodeAndLanes(unsigned IntNo) { } } +/// Return {opcode, rs1 lane, rs2 lane} for the word form of accumulate +/// intrinsic \p IntNo. +static std::tuple<unsigned, unsigned, unsigned> +getRVPWordMulPartsAccOpcodeAndLanes(unsigned IntNo) { + switch (IntNo) { + default: + llvm_unreachable("Unexpected RISC-V multiply-parts accumulate intrinsic"); + case Intrinsic::riscv_macc_00: + return {RISCVISD::WMACC, 0, 0}; + case Intrinsic::riscv_macc_01: + return {RISCVISD::WMACC, 0, 1}; + case Intrinsic::riscv_macc_11: + return {RISCVISD::WMACC, 1, 1}; + case Intrinsic::riscv_maccu_00: + return {RISCVISD::WMACCU, 0, 0}; + case Intrinsic::riscv_maccu_01: + return {RISCVISD::WMACCU, 0, 1}; + case Intrinsic::riscv_maccu_11: + return {RISCVISD::WMACCU, 1, 1}; + case Intrinsic::riscv_maccsu_00: + return {RISCVISD::WMACCSU, 0, 0}; + case Intrinsic::riscv_maccsu_11: + return {RISCVISD::WMACCSU, 1, 1}; + } +} + SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const { unsigned IntNo = Op.getConstantOperandVal(0); @@ -12635,6 +12693,42 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SDValue Hi = DAG.getNode(Opc, DL, HalfVT, Rs1Hi, Rs2Hi); return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi); } + case Intrinsic::riscv_pmacc_00: + case Intrinsic::riscv_pmacc_01: + case Intrinsic::riscv_pmacc_11: + case Intrinsic::riscv_pmaccu_00: + case Intrinsic::riscv_pmaccu_01: + case Intrinsic::riscv_pmaccu_11: + case Intrinsic::riscv_pmaccsu_00: + case Intrinsic::riscv_pmaccsu_11: + case Intrinsic::riscv_macc_00: + case Intrinsic::riscv_macc_01: + case Intrinsic::riscv_macc_11: + case Intrinsic::riscv_maccu_00: + case Intrinsic::riscv_maccu_01: + case Intrinsic::riscv_maccu_11: + case Intrinsic::riscv_maccsu_00: + case Intrinsic::riscv_maccsu_11: { + MVT VT = Op.getSimpleValueType(); + SDValue Rd = Op.getOperand(1); + SDValue Rs1 = Op.getOperand(2); + SDValue Rs2 = Op.getOperand(3); + unsigned Opc = getRVPMulAccHalvesOpcode(IntNo); + if (VT != MVT::v2i32 || !Subtarget.isPExtPackedDoubleType(VT)) + return DAG.getNode(Opc, DL, VT, Rd, Rs1, Rs2); + + // On RV32 a 64-bit result lives in a GPR pair; accumulate each half with + // the 32-bit form of the same product. + auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL); + auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL); + SDValue Lo = + DAG.getNode(Opc, DL, MVT::i32, + DAG.getExtractVectorElt(DL, MVT::i32, Rd, 0), Rs1Lo, Rs2Lo); + SDValue Hi = + DAG.getNode(Opc, DL, MVT::i32, + DAG.getExtractVectorElt(DL, MVT::i32, Rd, 1), Rs1Hi, Rs2Hi); + return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Lo, Hi); + } case Intrinsic::riscv_pas: case Intrinsic::riscv_psa: case Intrinsic::riscv_psas: @@ -17263,6 +17357,51 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N, } reportFatalUsageError("unsupported llvm.riscv multiply-parts intrinsic"); } + case Intrinsic::riscv_macc_00: + case Intrinsic::riscv_macc_01: + case Intrinsic::riscv_macc_11: + case Intrinsic::riscv_maccu_00: + case Intrinsic::riscv_maccu_01: + case Intrinsic::riscv_maccu_11: + case Intrinsic::riscv_maccsu_00: + case Intrinsic::riscv_maccsu_11: { + // macc.hXX exists only on RV32 and macc.wXX only on RV64; the other XLEN + // has to build the product here. + MVT VT = N->getSimpleValueType(0); + MVT SrcVT = N->getOperand(2).getSimpleValueType(); + if (Subtarget.hasStdExtP() && Subtarget.is64Bit() && VT == MVT::i32 && + SrcVT == MVT::v2i16) { + // Accumulate into the first element of the packed product. + SDValue Undef = DAG.getUNDEF(SrcVT); + SDValue Rd = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2i32, + N->getOperand(1)); + SDValue Rs1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i16, + N->getOperand(2), Undef); + SDValue Rs2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i16, + N->getOperand(3), Undef); + SDValue Res = DAG.getNode(getRVPMulAccHalvesOpcode(IntNo), DL, + MVT::v2i32, Rd, Rs1, Rs2); + Results.push_back(DAG.getExtractVectorElt(DL, MVT::i32, Res, 0)); + return; + } + if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() && VT == MVT::i64 && + SrcVT == MVT::v2i32) { + auto [Opc, Rs1Lane, Rs2Lane] = + getRVPWordMulPartsAccOpcodeAndLanes(IntNo); + auto [RdLo, RdHi] = + DAG.SplitScalar(N->getOperand(1), DL, MVT::i32, MVT::i32); + SDValue Rs1 = + DAG.getExtractVectorElt(DL, MVT::i32, N->getOperand(2), Rs1Lane); + SDValue Rs2 = + DAG.getExtractVectorElt(DL, MVT::i32, N->getOperand(3), Rs2Lane); + SDValue Res = DAG.getNode(Opc, DL, DAG.getVTList(MVT::i32, MVT::i32), + RdLo, RdHi, Rs1, Rs2); + Results.push_back( + DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Res, Res.getValue(1))); + return; + } + reportFatalUsageError("unsupported llvm.riscv multiply-parts intrinsic"); + } case Intrinsic::riscv_paadd: case Intrinsic::riscv_paaddu: case Intrinsic::riscv_pasub: diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td index 3637ee98df990..dfdfb5435c3c3 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td @@ -1791,6 +1791,11 @@ class PatMulParts<SDPatternOperator OpNode, RVInst Inst, ValueType ResultVT, ValueType SourceVT> : Pat<(ResultVT (OpNode (SourceVT GPR:$rs1), (SourceVT GPR:$rs2))), (Inst GPR:$rs1, GPR:$rs2)>; +class PatMulPartsAcc<SDPatternOperator OpNode, RVInst Inst, ValueType ResultVT, + ValueType SourceVT> + : Pat<(ResultVT (OpNode (ResultVT GPR:$rd), (SourceVT GPR:$rs1), + (SourceVT GPR:$rs2))), + (Inst GPR:$rd, GPR:$rs1, GPR:$rs2)>; class PatGprGprGpr<SDPatternOperator OpNode, RVInst Inst, ValueType VT> : Pat<(VT (OpNode (VT GPR:$rd), (VT GPR:$rs1), (VT GPR:$rs2))), @@ -1853,6 +1858,11 @@ def riscv_wsub : RVSDNode<"WSUB", SDTIntBinHiLoOp>; def riscv_wmulsu : RVSDNode<"WMULSU", SDTIntBinHiLoOp>; +// Widening multiply-accumulate into a GPR pair: rd_p = rd_p + rs1 * rs2. +def riscv_wmacc : RVSDNode<"WMACC", SDT_RISCVWideningAddSubAccumulate>; +def riscv_wmaccu : RVSDNode<"WMACCU", SDT_RISCVWideningAddSubAccumulate>; +def riscv_wmaccsu : RVSDNode<"WMACCSU", SDT_RISCVWideningAddSubAccumulate>; + def SDT_RISCVPackedWideningMul : SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisSameAs<1, 2>, SDTCisOpSmallerThanOp<1, 0>, @@ -1917,6 +1927,29 @@ def riscv_pm2addu_h : RVSDNode<"PM2ADDU_H", SDT_RISCVPM2Halfword, [SDNPCommutative]>; def riscv_pm2sub_h : RVSDNode<"PM2SUB_H", SDT_RISCVPM2Halfword>; +def SDT_RISCVWideningMulAccByHalves + : SDTypeProfile<1, 3, [SDTCisSameAs<0, 1>, + SDTCisSameAs<2, 3>, + SDTCisSameSizeAs<0, 2>]>; +def riscv_pmacc_halves_00 + : RVSDNode<"PMACC_HALVES_00", SDT_RISCVWideningMulAccByHalves>; +def riscv_pmacc_halves_01 + : RVSDNode<"PMACC_HALVES_01", SDT_RISCVWideningMulAccByHalves>; +def riscv_pmacc_halves_11 + : RVSDNode<"PMACC_HALVES_11", SDT_RISCVWideningMulAccByHalves>; + +def riscv_pmaccu_halves_00 + : RVSDNode<"PMACCU_HALVES_00", SDT_RISCVWideningMulAccByHalves>; +def riscv_pmaccu_halves_01 + : RVSDNode<"PMACCU_HALVES_01", SDT_RISCVWideningMulAccByHalves>; +def riscv_pmaccu_halves_11 + : RVSDNode<"PMACCU_HALVES_11", SDT_RISCVWideningMulAccByHalves>; + +def riscv_pmaccsu_halves_00 + : RVSDNode<"PMACCSU_HALVES_00", SDT_RISCVWideningMulAccByHalves>; +def riscv_pmaccsu_halves_11 + : RVSDNode<"PMACCSU_HALVES_11", SDT_RISCVWideningMulAccByHalves>; + def SDT_RISCVWideningShiftLeft : SDTypeProfile<2, 2, [SDTCisVT<0, i32>, SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>, @@ -2382,6 +2415,16 @@ let Predicates = [HasStdExtP] in { (PMULSU_H_B11 GPR:$rs1, GPR:$rs2)>; let append Predicates = [IsRV32] in { + // Scalar halfword multiply-parts accumulate patterns. + def : PatMulPartsAcc<riscv_pmacc_halves_00, MACC_H00, i32, v2i16>; + def : PatMulPartsAcc<riscv_pmacc_halves_01, MACC_H01, i32, v2i16>; + def : PatMulPartsAcc<riscv_pmacc_halves_11, MACC_H11, i32, v2i16>; + def : PatMulPartsAcc<riscv_pmaccu_halves_00, MACCU_H00, i32, v2i16>; + def : PatMulPartsAcc<riscv_pmaccu_halves_01, MACCU_H01, i32, v2i16>; + def : PatMulPartsAcc<riscv_pmaccu_halves_11, MACCU_H11, i32, v2i16>; + def : PatMulPartsAcc<riscv_pmaccsu_halves_00, MACCSU_H00, i32, v2i16>; + def : PatMulPartsAcc<riscv_pmaccsu_halves_11, MACCSU_H11, i32, v2i16>; + // Scalar halfword multiply-parts patterns. def : PatMulParts<int_riscv_mul_00, MUL_H00, i32, v2i16>; def : PatMulParts<int_riscv_mul_01, MUL_H01, i32, v2i16>; @@ -3075,6 +3118,26 @@ let append Predicates = [IsRV64] in { def : PatGprGpr<riscv_asub, PASUB_W, v2i32>; def : PatGprGpr<riscv_asubu, PASUBU_W, v2i32>; + // Packed halfword multiply-parts accumulate patterns. + def : PatMulPartsAcc<riscv_pmacc_halves_00, PMACC_W_H00, v2i32, v4i16>; + def : PatMulPartsAcc<riscv_pmacc_halves_01, PMACC_W_H01, v2i32, v4i16>; + def : PatMulPartsAcc<riscv_pmacc_halves_11, PMACC_W_H11, v2i32, v4i16>; + def : PatMulPartsAcc<riscv_pmaccu_halves_00, PMACCU_W_H00, v2i32, v4i16>; + def : PatMulPartsAcc<riscv_pmaccu_halves_01, PMACCU_W_H01, v2i32, v4i16>; + def : PatMulPartsAcc<riscv_pmaccu_halves_11, PMACCU_W_H11, v2i32, v4i16>; + def : PatMulPartsAcc<riscv_pmaccsu_halves_00, PMACCSU_W_H00, v2i32, v4i16>; + def : PatMulPartsAcc<riscv_pmaccsu_halves_11, PMACCSU_W_H11, v2i32, v4i16>; + + // Scalar word multiply-parts accumulate patterns. + def : PatMulPartsAcc<riscv_pmacc_halves_00, MACC_W00, i64, v2i32>; + def : PatMulPartsAcc<riscv_pmacc_halves_01, MACC_W01, i64, v2i32>; + def : PatMulPartsAcc<riscv_pmacc_halves_11, MACC_W11, i64, v2i32>; + def : PatMulPartsAcc<riscv_pmaccu_halves_00, MACCU_W00, i64, v2i32>; + def : PatMulPartsAcc<riscv_pmaccu_halves_01, MACCU_W01, i64, v2i32>; + def : PatMulPartsAcc<riscv_pmaccu_halves_11, MACCU_W11, i64, v2i32>; + def : PatMulPartsAcc<riscv_pmaccsu_halves_00, MACCSU_W00, i64, v2i32>; + def : PatMulPartsAcc<riscv_pmaccsu_halves_11, MACCSU_W11, i64, v2i32>; + // Scalar word multiply-parts patterns. def : PatMulParts<int_riscv_mul_00, MUL_W00, i64, v2i32>; def : PatMulParts<int_riscv_mul_01, MUL_W01, i64, v2i32>; diff --git a/llvm/test/CodeGen/RISCV/rvp-simd-32.ll b/llvm/test/CodeGen/RISCV/rvp-simd-32.ll index c11d9716ec053..e9e04eb6b5ebe 100644 --- a/llvm/test/CodeGen/RISCV/rvp-simd-32.ll +++ b/llvm/test/CodeGen/RISCV/rvp-simd-32.ll @@ -3705,3 +3705,125 @@ define i32 @test_pm2addsu_v2i16_i32(<2 x i16> %a, <2 x i16> %b) { %r = call i32 @llvm.riscv.pm2addsu.i32.v2i16(<2 x i16> %a, <2 x i16> %b) ret i32 %r } + +; Packed Multiply Parts Accumulate. +declare i32 @llvm.riscv.macc.00.i32.v2i16(i32, <2 x i16>, <2 x i16>) +declare i32 @llvm.riscv.macc.01.i32.v2i16(i32, <2 x i16>, <2 x i16>) +declare i32 @llvm.riscv.macc.11.i32.v2i16(i32, <2 x i16>, <2 x i16>) +declare i32 @llvm.riscv.maccu.00.i32.v2i16(i32, <2 x i16>, <2 x i16>) +declare i32 @llvm.riscv.maccu.01.i32.v2i16(i32, <2 x i16>, <2 x i16>) +declare i32 @llvm.riscv.maccu.11.i32.v2i16(i32, <2 x i16>, <2 x i16>) +declare i32 @llvm.riscv.maccsu.00.i32.v2i16(i32, <2 x i16>, <2 x i16>) +declare i32 @llvm.riscv.maccsu.11.i32.v2i16(i32, <2 x i16>, <2 x i16>) + +define i32 @test_macc_h00_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: test_macc_h00_i32: +; RV32: # %bb.0: +; RV32-NEXT: macc.h00 a0, a1, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: test_macc_h00_i32: +; RV64: # %bb.0: +; RV64-NEXT: pmacc.w.h00 a0, a1, a2 +; RV64-NEXT: ret + %r = call i32 @llvm.riscv.macc.00.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b) + ret i32 %r +} + +define i32 @test_macc_h01_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: test_macc_h01_i32: +; RV32: # %bb.0: +; RV32-NEXT: macc.h01 a0, a1, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: test_macc_h01_i32: +; RV64: # %bb.0: +; RV64-NEXT: pmacc.w.h01 a0, a1, a2 +; RV64-NEXT: ret + %r = call i32 @llvm.riscv.macc.01.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b) + ret i32 %r +} + +define i32 @test_macc_h11_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: test_macc_h11_i32: +; RV32: # %bb.0: +; RV32-NEXT: macc.h11 a0, a1, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: test_macc_h11_i32: +; RV64: # %bb.0: +; RV64-NEXT: pmacc.w.h11 a0, a1, a2 +; RV64-NEXT: ret + %r = call i32 @llvm.riscv.macc.11.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b) + ret i32 %r +} + +define i32 @test_maccu_h00_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: test_maccu_h00_i32: +; RV32: # %bb.0: +; RV32-NEXT: maccu.h00 a0, a1, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: test_maccu_h00_i32: +; RV64: # %bb.0: +; RV64-NEXT: pmaccu.w.h00 a0, a1, a2 +; RV64-NEXT: ret + %r = call i32 @llvm.riscv.maccu.00.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b) + ret i32 %r +} + +define i32 @test_maccu_h01_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: test_maccu_h01_i32: +; RV32: # %bb.0: +; RV32-NEXT: maccu.h01 a0, a1, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: test_maccu_h01_i32: +; RV64: # %bb.0: +; RV64-NEXT: pmaccu.w.h01 a0, a1, a2 +; RV64-NEXT: ret + %r = call i32 @llvm.riscv.maccu.01.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b) + ret i32 %r +} + +define i32 @test_maccu_h11_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: test_maccu_h11_i32: +; RV32: # %bb.0: +; RV32-NEXT: maccu.h11 a0, a1, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: test_maccu_h11_i32: +; RV64: # %bb.0: +; RV64-NEXT: pmaccu.w.h11 a0, a1, a2 +; RV64-NEXT: ret + %r = call i32 @llvm.riscv.maccu.11.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b) + ret i32 %r +} + +define i32 @test_maccsu_h00_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: test_maccsu_h00_i32: +; RV32: # %bb.0: +; RV32-NEXT: maccsu.h00 a0, a1, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: test_maccsu_h00_i32: +; RV64: # %bb.0: +; RV64-NEXT: pmaccsu.w.h00 a0, a1, a2 +; RV64-NEXT: ret + %r = call i32 @llvm.riscv.maccsu.00.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b) + ret i32 %r +} + +define i32 @test_maccsu_h11_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: test_maccsu_h11_i32: +; RV32: # %bb.0: +; RV32-NEXT: maccsu.h11 a0, a1, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: test_maccsu_h11_i32: +; RV64: # %bb.0: +; RV64-NEXT: pmaccsu.w.h11 a0, a1, a2 +; RV64-NEXT: ret + %r = call i32 @llvm.riscv.maccsu.11.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b) + ret i32 %r +} diff --git a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll index fd9cf65c2ef53..43107137fe1fa 100644 --- a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll +++ b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll @@ -7866,3 +7866,253 @@ define i64 @test_pm4addsu_v4i16_i64(<4 x i16> %a, <4 x i16> %b) { %r = call i64 @llvm.riscv.pm4addsu.i64.v4i16(<4 x i16> %a, <4 x i16> %b) ret i64 %r } + +; Packed Multiply Parts Accumulate. +declare <2 x i32> @llvm.riscv.pmacc.00.v2i32(<2 x i32>, <4 x i16>, <4 x i16>) +declare <2 x i32> @llvm.riscv.pmacc.01.v2i32(<2 x i32>, <4 x i16>, <4 x i16>) +declare <2 x i32> @llvm.riscv.pmacc.11.v2i32(<2 x i32>, <4 x i16>, <4 x i16>) +declare <2 x i32> @llvm.riscv.pmaccu.00.v2i32(<2 x i32>, <4 x i16>, <4 x i16>) +declare <2 x i32> @llvm.riscv.pmaccu.01.v2i32(<2 x i32>, <4 x i16>, <4 x i16>) +declare <2 x i32> @llvm.riscv.pmaccu.11.v2i32(<2 x i32>, <4 x i16>, <4 x i16>) +declare <2 x i32> @llvm.riscv.pmaccsu.00.v2i32(<2 x i32>, <4 x i16>, <4 x i16>) +declare <2 x i32> @llvm.riscv.pmaccsu.11.v2i32(<2 x i32>, <4 x i16>, <4 x i16>) +declare i64 @llvm.riscv.macc.00.i64.v2i32(i64, <2 x i32>, <2 x i32>) +declare i64 @llvm.riscv.macc.01.i64.v2i32(i64, <2 x i32>, <2 x i32>) +declare i64 @llvm.riscv.macc.11.i64.v2i32(i64, <2 x i32>, <2 x i32>) +declare i64 @llvm.riscv.maccu.00.i64.v2i32(i64, <2 x i32>, <2 x i32>) +declare i64 @llvm.riscv.maccu.01.i64.v2i32(i64, <2 x i32>, <2 x i32>) +declare i64 @llvm.riscv.maccu.11.i64.v2i32(i64, <2 x i32>, <2 x i32>) +declare i64 @llvm.riscv.maccsu.00.i64.v2i32(i64, <2 x i32>, <2 x i32>) +declare i64 @llvm.riscv.maccsu.11.i64.v2i32(i64, <2 x i32>, <2 x i32>) + +define <2 x i32> @test_pmacc_h00_v2i32(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: test_pmacc_h00_v2i32: +; RV32: # %bb.0: +; RV32-NEXT: macc.h00 a1, a3, a5 +; RV32-NEXT: macc.h00 a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pmacc_h00_v2i32: +; RV64: # %bb.0: +; RV64-NEXT: pmacc.w.h00 a0, a1, a2 +; RV64-NEXT: ret + %r = call <2 x i32> @llvm.riscv.pmacc.00.v2i32(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) + ret <2 x i32> %r +} + +define <2 x i32> @test_pmacc_h01_v2i32(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: test_pmacc_h01_v2i32: +; RV32: # %bb.0: +; RV32-NEXT: macc.h01 a1, a3, a5 +; RV32-NEXT: macc.h01 a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pmacc_h01_v2i32: +; RV64: # %bb.0: +; RV64-NEXT: pmacc.w.h01 a0, a1, a2 +; RV64-NEXT: ret + %r = call <2 x i32> @llvm.riscv.pmacc.01.v2i32(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) + ret <2 x i32> %r +} + +define <2 x i32> @test_pmacc_h11_v2i32(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: test_pmacc_h11_v2i32: +; RV32: # %bb.0: +; RV32-NEXT: macc.h11 a1, a3, a5 +; RV32-NEXT: macc.h11 a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pmacc_h11_v2i32: +; RV64: # %bb.0: +; RV64-NEXT: pmacc.w.h11 a0, a1, a2 +; RV64-NEXT: ret + %r = call <2 x i32> @llvm.riscv.pmacc.11.v2i32(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) + ret <2 x i32> %r +} + +define <2 x i32> @test_pmaccu_h00_v2i32(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: test_pmaccu_h00_v2i32: +; RV32: # %bb.0: +; RV32-NEXT: maccu.h00 a1, a3, a5 +; RV32-NEXT: maccu.h00 a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pmaccu_h00_v2i32: +; RV64: # %bb.0: +; RV64-NEXT: pmaccu.w.h00 a0, a1, a2 +; RV64-NEXT: ret + %r = call <2 x i32> @llvm.riscv.pmaccu.00.v2i32(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) + ret <2 x i32> %r +} + +define <2 x i32> @test_pmaccu_h01_v2i32(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: test_pmaccu_h01_v2i32: +; RV32: # %bb.0: +; RV32-NEXT: maccu.h01 a1, a3, a5 +; RV32-NEXT: maccu.h01 a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pmaccu_h01_v2i32: +; RV64: # %bb.0: +; RV64-NEXT: pmaccu.w.h01 a0, a1, a2 +; RV64-NEXT: ret + %r = call <2 x i32> @llvm.riscv.pmaccu.01.v2i32(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) + ret <2 x i32> %r +} + +define <2 x i32> @test_pmaccu_h11_v2i32(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: test_pmaccu_h11_v2i32: +; RV32: # %bb.0: +; RV32-NEXT: maccu.h11 a1, a3, a5 +; RV32-NEXT: maccu.h11 a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pmaccu_h11_v2i32: +; RV64: # %bb.0: +; RV64-NEXT: pmaccu.w.h11 a0, a1, a2 +; RV64-NEXT: ret + %r = call <2 x i32> @llvm.riscv.pmaccu.11.v2i32(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) + ret <2 x i32> %r +} + +define <2 x i32> @test_pmaccsu_h00_v2i32(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: test_pmaccsu_h00_v2i32: +; RV32: # %bb.0: +; RV32-NEXT: maccsu.h00 a1, a3, a5 +; RV32-NEXT: maccsu.h00 a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pmaccsu_h00_v2i32: +; RV64: # %bb.0: +; RV64-NEXT: pmaccsu.w.h00 a0, a1, a2 +; RV64-NEXT: ret + %r = call <2 x i32> @llvm.riscv.pmaccsu.00.v2i32(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) + ret <2 x i32> %r +} + +define <2 x i32> @test_pmaccsu_h11_v2i32(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: test_pmaccsu_h11_v2i32: +; RV32: # %bb.0: +; RV32-NEXT: maccsu.h11 a1, a3, a5 +; RV32-NEXT: maccsu.h11 a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pmaccsu_h11_v2i32: +; RV64: # %bb.0: +; RV64-NEXT: pmaccsu.w.h11 a0, a1, a2 +; RV64-NEXT: ret + %r = call <2 x i32> @llvm.riscv.pmaccsu.11.v2i32(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) + ret <2 x i32> %r +} + +define i64 @test_macc_w00_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: test_macc_w00_i64: +; RV32: # %bb.0: +; RV32-NEXT: wmacc a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: test_macc_w00_i64: +; RV64: # %bb.0: +; RV64-NEXT: macc.w00 a0, a1, a2 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.macc.00.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b) + ret i64 %r +} + +define i64 @test_macc_w01_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: test_macc_w01_i64: +; RV32: # %bb.0: +; RV32-NEXT: wmacc a0, a2, a5 +; RV32-NEXT: ret +; +; RV64-LABEL: test_macc_w01_i64: +; RV64: # %bb.0: +; RV64-NEXT: macc.w01 a0, a1, a2 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.macc.01.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b) + ret i64 %r +} + +define i64 @test_macc_w11_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: test_macc_w11_i64: +; RV32: # %bb.0: +; RV32-NEXT: wmacc a0, a3, a5 +; RV32-NEXT: ret +; +; RV64-LABEL: test_macc_w11_i64: +; RV64: # %bb.0: +; RV64-NEXT: macc.w11 a0, a1, a2 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.macc.11.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b) + ret i64 %r +} + +define i64 @test_maccu_w00_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: test_maccu_w00_i64: +; RV32: # %bb.0: +; RV32-NEXT: wmaccu a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: test_maccu_w00_i64: +; RV64: # %bb.0: +; RV64-NEXT: maccu.w00 a0, a1, a2 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.maccu.00.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b) + ret i64 %r +} + +define i64 @test_maccu_w01_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: test_maccu_w01_i64: +; RV32: # %bb.0: +; RV32-NEXT: wmaccu a0, a2, a5 +; RV32-NEXT: ret +; +; RV64-LABEL: test_maccu_w01_i64: +; RV64: # %bb.0: +; RV64-NEXT: maccu.w01 a0, a1, a2 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.maccu.01.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b) + ret i64 %r +} + +define i64 @test_maccu_w11_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: test_maccu_w11_i64: +; RV32: # %bb.0: +; RV32-NEXT: wmaccu a0, a3, a5 +; RV32-NEXT: ret +; +; RV64-LABEL: test_maccu_w11_i64: +; RV64: # %bb.0: +; RV64-NEXT: maccu.w11 a0, a1, a2 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.maccu.11.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b) + ret i64 %r +} + +define i64 @test_maccsu_w00_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: test_maccsu_w00_i64: +; RV32: # %bb.0: +; RV32-NEXT: wmaccsu a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: test_maccsu_w00_i64: +; RV64: # %bb.0: +; RV64-NEXT: maccsu.w00 a0, a1, a2 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.maccsu.00.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b) + ret i64 %r +} + +define i64 @test_maccsu_w11_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: test_maccsu_w11_i64: +; RV32: # %bb.0: +; RV32-NEXT: wmaccsu a0, a3, a5 +; RV32-NEXT: ret +; +; RV64-LABEL: test_maccsu_w11_i64: +; RV64: # %bb.0: +; RV64-NEXT: maccsu.w11 a0, a1, a2 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.maccsu.11.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b) + ret i64 %r +} >From 22f533e5b682411a31a87b51d5262cb22d87186b Mon Sep 17 00:00:00 2001 From: SiHuaN <[email protected]> Date: Thu, 10 Sep 2026 10:35:01 +0000 Subject: [PATCH 2/2] [RISCV][P-ext] Fold an add of a multiply-parts product into the accumulate form An add of a multiply-parts product selects the accumulating instruction, so that a loop written with `sum += __riscv_mul_h00_i32(a, b)` gets `macc.h00` rather than a separate multiply and add. Rewriting the add before type legalization keeps the shapes whose result is illegal, so the existing accumulate lowering covers all of them. --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 67 +++ .../CodeGen/RISCV/rvp-mul-parts-acc-fold.ll | 429 ++++++++++++++++++ 2 files changed, 496 insertions(+) create mode 100644 llvm/test/CodeGen/RISCV/rvp-mul-parts-acc-fold.ll diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index e2d91fc4c4cfa..20b45a9b011c1 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -12548,6 +12548,47 @@ static Intrinsic::ID getRVPScalarMulPartsIntrinsic(unsigned IntNo) { } } +/// Return the accumulate form of multiply-parts intrinsic \p IntNo, or +/// Intrinsic::not_intrinsic if there is none. +static Intrinsic::ID getRVPMulPartsAccIntrinsic(unsigned IntNo) { + switch (IntNo) { + default: + return Intrinsic::not_intrinsic; + case Intrinsic::riscv_mul_00: + return Intrinsic::riscv_macc_00; + case Intrinsic::riscv_pmul_00: + return Intrinsic::riscv_pmacc_00; + case Intrinsic::riscv_mul_01: + return Intrinsic::riscv_macc_01; + case Intrinsic::riscv_pmul_01: + return Intrinsic::riscv_pmacc_01; + case Intrinsic::riscv_mul_11: + return Intrinsic::riscv_macc_11; + case Intrinsic::riscv_pmul_11: + return Intrinsic::riscv_pmacc_11; + case Intrinsic::riscv_mulu_00: + return Intrinsic::riscv_maccu_00; + case Intrinsic::riscv_pmulu_00: + return Intrinsic::riscv_pmaccu_00; + case Intrinsic::riscv_mulu_01: + return Intrinsic::riscv_maccu_01; + case Intrinsic::riscv_pmulu_01: + return Intrinsic::riscv_pmaccu_01; + case Intrinsic::riscv_mulu_11: + return Intrinsic::riscv_maccu_11; + case Intrinsic::riscv_pmulu_11: + return Intrinsic::riscv_pmaccu_11; + case Intrinsic::riscv_mulsu_00: + return Intrinsic::riscv_maccsu_00; + case Intrinsic::riscv_pmulsu_00: + return Intrinsic::riscv_pmaccsu_00; + case Intrinsic::riscv_mulsu_11: + return Intrinsic::riscv_maccsu_11; + case Intrinsic::riscv_pmulsu_11: + return Intrinsic::riscv_pmaccsu_11; + } +} + /// Return the multiply-parts accumulate node for \p IntNo. static unsigned getRVPMulAccHalvesOpcode(unsigned IntNo) { switch (IntNo) { @@ -18510,6 +18551,30 @@ static SDValue combineAddMulh(SDNode *N, SelectionDAG &DAG, return DAG.getNode(RISCVISD::MULHSU, DL, VT, X, Mulh.getOperand(1)); } +// Fold an add of a multiply-parts product into the accumulating form. +static SDValue combineAddMulParts(SDNode *N, SelectionDAG &DAG, + const RISCVSubtarget &Subtarget) { + if (!Subtarget.hasStdExtP()) + return SDValue(); + + for (unsigned I = 0; I != 2; ++I) { + SDValue Mul = N->getOperand(I); + if (Mul.getOpcode() != ISD::INTRINSIC_WO_CHAIN || !Mul.hasOneUse()) + continue; + Intrinsic::ID AccId = + getRVPMulPartsAccIntrinsic(Mul.getConstantOperandVal(0)); + if (AccId == Intrinsic::not_intrinsic) + continue; + + SDLoc DL(N); + return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, N->getValueType(0), + DAG.getTargetConstant(AccId, DL, MVT::i32), + N->getOperand(1 - I), Mul.getOperand(1), + Mul.getOperand(2)); + } + return SDValue(); +} + static SDValue combinePExtWideningAddSub(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget) { // Recognize the RV64 decompositions listed for the 32-bit packed widening @@ -18595,6 +18660,8 @@ static SDValue performADDCombine(SDNode *N, return V; if (SDValue V = combineBinOpOfZExt(N, DAG)) return V; + if (SDValue V = combineAddMulParts(N, DAG, Subtarget)) + return V; if (SDValue V = combineAddMulh(N, DAG, Subtarget)) return V; diff --git a/llvm/test/CodeGen/RISCV/rvp-mul-parts-acc-fold.ll b/llvm/test/CodeGen/RISCV/rvp-mul-parts-acc-fold.ll new file mode 100644 index 0000000000000..fa660331631d0 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/rvp-mul-parts-acc-fold.ll @@ -0,0 +1,429 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 +; RUN: llc -mtriple=riscv32 -mattr=+experimental-p,+m,+zbb -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV32 +; RUN: llc -mtriple=riscv64 -mattr=+experimental-p,+m,+zbb -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV64 + +; An add of a multiply-parts product selects the accumulating instruction. + +define i32 @macc_h00(i32 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: macc_h00: +; RV32: # %bb.0: +; RV32-NEXT: macc.h00 a0, a1, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: macc_h00: +; RV64: # %bb.0: +; RV64-NEXT: pmacc.w.h00 a0, a1, a2 +; RV64-NEXT: ret + %m = call i32 @llvm.riscv.mul.00.i32.v2i16(<2 x i16> %a, <2 x i16> %b) + %r = add i32 %rd, %m + ret i32 %r +} + +define i32 @macc_h01(i32 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: macc_h01: +; RV32: # %bb.0: +; RV32-NEXT: macc.h01 a0, a1, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: macc_h01: +; RV64: # %bb.0: +; RV64-NEXT: pmacc.w.h01 a0, a1, a2 +; RV64-NEXT: ret + %m = call i32 @llvm.riscv.mul.01.i32.v2i16(<2 x i16> %a, <2 x i16> %b) + %r = add i32 %rd, %m + ret i32 %r +} + +define i32 @macc_h11(i32 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: macc_h11: +; RV32: # %bb.0: +; RV32-NEXT: macc.h11 a0, a1, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: macc_h11: +; RV64: # %bb.0: +; RV64-NEXT: pmacc.w.h11 a0, a1, a2 +; RV64-NEXT: ret + %m = call i32 @llvm.riscv.mul.11.i32.v2i16(<2 x i16> %a, <2 x i16> %b) + %r = add i32 %rd, %m + ret i32 %r +} + +define i32 @maccu_h00(i32 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: maccu_h00: +; RV32: # %bb.0: +; RV32-NEXT: maccu.h00 a0, a1, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: maccu_h00: +; RV64: # %bb.0: +; RV64-NEXT: pmaccu.w.h00 a0, a1, a2 +; RV64-NEXT: ret + %m = call i32 @llvm.riscv.mulu.00.i32.v2i16(<2 x i16> %a, <2 x i16> %b) + %r = add i32 %rd, %m + ret i32 %r +} + +define i32 @maccu_h01(i32 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: maccu_h01: +; RV32: # %bb.0: +; RV32-NEXT: maccu.h01 a0, a1, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: maccu_h01: +; RV64: # %bb.0: +; RV64-NEXT: pmaccu.w.h01 a0, a1, a2 +; RV64-NEXT: ret + %m = call i32 @llvm.riscv.mulu.01.i32.v2i16(<2 x i16> %a, <2 x i16> %b) + %r = add i32 %rd, %m + ret i32 %r +} + +define i32 @maccu_h11(i32 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: maccu_h11: +; RV32: # %bb.0: +; RV32-NEXT: maccu.h11 a0, a1, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: maccu_h11: +; RV64: # %bb.0: +; RV64-NEXT: pmaccu.w.h11 a0, a1, a2 +; RV64-NEXT: ret + %m = call i32 @llvm.riscv.mulu.11.i32.v2i16(<2 x i16> %a, <2 x i16> %b) + %r = add i32 %rd, %m + ret i32 %r +} + +define i32 @maccsu_h00(i32 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: maccsu_h00: +; RV32: # %bb.0: +; RV32-NEXT: maccsu.h00 a0, a1, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: maccsu_h00: +; RV64: # %bb.0: +; RV64-NEXT: pmaccsu.w.h00 a0, a1, a2 +; RV64-NEXT: ret + %m = call i32 @llvm.riscv.mulsu.00.i32.v2i16(<2 x i16> %a, <2 x i16> %b) + %r = add i32 %rd, %m + ret i32 %r +} + +define i32 @maccsu_h11(i32 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: maccsu_h11: +; RV32: # %bb.0: +; RV32-NEXT: maccsu.h11 a0, a1, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: maccsu_h11: +; RV64: # %bb.0: +; RV64-NEXT: pmaccsu.w.h11 a0, a1, a2 +; RV64-NEXT: ret + %m = call i32 @llvm.riscv.mulsu.11.i32.v2i16(<2 x i16> %a, <2 x i16> %b) + %r = add i32 %rd, %m + ret i32 %r +} + +define i64 @macc_w00(i64 %rd, <2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: macc_w00: +; RV32: # %bb.0: +; RV32-NEXT: wmacc a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: macc_w00: +; RV64: # %bb.0: +; RV64-NEXT: macc.w00 a0, a1, a2 +; RV64-NEXT: ret + %m = call i64 @llvm.riscv.mul.00.i64.v2i32(<2 x i32> %a, <2 x i32> %b) + %r = add i64 %rd, %m + ret i64 %r +} + +define i64 @macc_w01(i64 %rd, <2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: macc_w01: +; RV32: # %bb.0: +; RV32-NEXT: wmacc a0, a2, a5 +; RV32-NEXT: ret +; +; RV64-LABEL: macc_w01: +; RV64: # %bb.0: +; RV64-NEXT: macc.w01 a0, a1, a2 +; RV64-NEXT: ret + %m = call i64 @llvm.riscv.mul.01.i64.v2i32(<2 x i32> %a, <2 x i32> %b) + %r = add i64 %rd, %m + ret i64 %r +} + +define i64 @macc_w11(i64 %rd, <2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: macc_w11: +; RV32: # %bb.0: +; RV32-NEXT: wmacc a0, a3, a5 +; RV32-NEXT: ret +; +; RV64-LABEL: macc_w11: +; RV64: # %bb.0: +; RV64-NEXT: macc.w11 a0, a1, a2 +; RV64-NEXT: ret + %m = call i64 @llvm.riscv.mul.11.i64.v2i32(<2 x i32> %a, <2 x i32> %b) + %r = add i64 %rd, %m + ret i64 %r +} + +define i64 @maccu_w00(i64 %rd, <2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: maccu_w00: +; RV32: # %bb.0: +; RV32-NEXT: wmaccu a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: maccu_w00: +; RV64: # %bb.0: +; RV64-NEXT: maccu.w00 a0, a1, a2 +; RV64-NEXT: ret + %m = call i64 @llvm.riscv.mulu.00.i64.v2i32(<2 x i32> %a, <2 x i32> %b) + %r = add i64 %rd, %m + ret i64 %r +} + +define i64 @maccu_w01(i64 %rd, <2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: maccu_w01: +; RV32: # %bb.0: +; RV32-NEXT: wmaccu a0, a2, a5 +; RV32-NEXT: ret +; +; RV64-LABEL: maccu_w01: +; RV64: # %bb.0: +; RV64-NEXT: maccu.w01 a0, a1, a2 +; RV64-NEXT: ret + %m = call i64 @llvm.riscv.mulu.01.i64.v2i32(<2 x i32> %a, <2 x i32> %b) + %r = add i64 %rd, %m + ret i64 %r +} + +define i64 @maccu_w11(i64 %rd, <2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: maccu_w11: +; RV32: # %bb.0: +; RV32-NEXT: wmaccu a0, a3, a5 +; RV32-NEXT: ret +; +; RV64-LABEL: maccu_w11: +; RV64: # %bb.0: +; RV64-NEXT: maccu.w11 a0, a1, a2 +; RV64-NEXT: ret + %m = call i64 @llvm.riscv.mulu.11.i64.v2i32(<2 x i32> %a, <2 x i32> %b) + %r = add i64 %rd, %m + ret i64 %r +} + +define i64 @maccsu_w00(i64 %rd, <2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: maccsu_w00: +; RV32: # %bb.0: +; RV32-NEXT: wmaccsu a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: maccsu_w00: +; RV64: # %bb.0: +; RV64-NEXT: maccsu.w00 a0, a1, a2 +; RV64-NEXT: ret + %m = call i64 @llvm.riscv.mulsu.00.i64.v2i32(<2 x i32> %a, <2 x i32> %b) + %r = add i64 %rd, %m + ret i64 %r +} + +define i64 @maccsu_w11(i64 %rd, <2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: maccsu_w11: +; RV32: # %bb.0: +; RV32-NEXT: wmaccsu a0, a3, a5 +; RV32-NEXT: ret +; +; RV64-LABEL: maccsu_w11: +; RV64: # %bb.0: +; RV64-NEXT: maccsu.w11 a0, a1, a2 +; RV64-NEXT: ret + %m = call i64 @llvm.riscv.mulsu.11.i64.v2i32(<2 x i32> %a, <2 x i32> %b) + %r = add i64 %rd, %m + ret i64 %r +} + +define <2 x i32> @pmacc_h00(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: pmacc_h00: +; RV32: # %bb.0: +; RV32-NEXT: macc.h00 a1, a3, a5 +; RV32-NEXT: macc.h00 a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: pmacc_h00: +; RV64: # %bb.0: +; RV64-NEXT: pmacc.w.h00 a0, a1, a2 +; RV64-NEXT: ret + %m = call <2 x i32> @llvm.riscv.pmul.00.v2i32(<4 x i16> %a, <4 x i16> %b) + %r = add <2 x i32> %rd, %m + ret <2 x i32> %r +} + +define <2 x i32> @pmacc_h01(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: pmacc_h01: +; RV32: # %bb.0: +; RV32-NEXT: macc.h01 a1, a3, a5 +; RV32-NEXT: macc.h01 a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: pmacc_h01: +; RV64: # %bb.0: +; RV64-NEXT: pmacc.w.h01 a0, a1, a2 +; RV64-NEXT: ret + %m = call <2 x i32> @llvm.riscv.pmul.01.v2i32(<4 x i16> %a, <4 x i16> %b) + %r = add <2 x i32> %rd, %m + ret <2 x i32> %r +} + +define <2 x i32> @pmacc_h11(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: pmacc_h11: +; RV32: # %bb.0: +; RV32-NEXT: macc.h11 a1, a3, a5 +; RV32-NEXT: macc.h11 a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: pmacc_h11: +; RV64: # %bb.0: +; RV64-NEXT: pmacc.w.h11 a0, a1, a2 +; RV64-NEXT: ret + %m = call <2 x i32> @llvm.riscv.pmul.11.v2i32(<4 x i16> %a, <4 x i16> %b) + %r = add <2 x i32> %rd, %m + ret <2 x i32> %r +} + +define <2 x i32> @pmaccu_h00(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: pmaccu_h00: +; RV32: # %bb.0: +; RV32-NEXT: maccu.h00 a1, a3, a5 +; RV32-NEXT: maccu.h00 a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: pmaccu_h00: +; RV64: # %bb.0: +; RV64-NEXT: pmaccu.w.h00 a0, a1, a2 +; RV64-NEXT: ret + %m = call <2 x i32> @llvm.riscv.pmulu.00.v2i32(<4 x i16> %a, <4 x i16> %b) + %r = add <2 x i32> %rd, %m + ret <2 x i32> %r +} + +define <2 x i32> @pmaccu_h01(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: pmaccu_h01: +; RV32: # %bb.0: +; RV32-NEXT: maccu.h01 a1, a3, a5 +; RV32-NEXT: maccu.h01 a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: pmaccu_h01: +; RV64: # %bb.0: +; RV64-NEXT: pmaccu.w.h01 a0, a1, a2 +; RV64-NEXT: ret + %m = call <2 x i32> @llvm.riscv.pmulu.01.v2i32(<4 x i16> %a, <4 x i16> %b) + %r = add <2 x i32> %rd, %m + ret <2 x i32> %r +} + +define <2 x i32> @pmaccu_h11(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: pmaccu_h11: +; RV32: # %bb.0: +; RV32-NEXT: maccu.h11 a1, a3, a5 +; RV32-NEXT: maccu.h11 a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: pmaccu_h11: +; RV64: # %bb.0: +; RV64-NEXT: pmaccu.w.h11 a0, a1, a2 +; RV64-NEXT: ret + %m = call <2 x i32> @llvm.riscv.pmulu.11.v2i32(<4 x i16> %a, <4 x i16> %b) + %r = add <2 x i32> %rd, %m + ret <2 x i32> %r +} + +define <2 x i32> @pmaccsu_h00(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: pmaccsu_h00: +; RV32: # %bb.0: +; RV32-NEXT: maccsu.h00 a1, a3, a5 +; RV32-NEXT: maccsu.h00 a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: pmaccsu_h00: +; RV64: # %bb.0: +; RV64-NEXT: pmaccsu.w.h00 a0, a1, a2 +; RV64-NEXT: ret + %m = call <2 x i32> @llvm.riscv.pmulsu.00.v2i32(<4 x i16> %a, <4 x i16> %b) + %r = add <2 x i32> %rd, %m + ret <2 x i32> %r +} + +define <2 x i32> @pmaccsu_h11(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: pmaccsu_h11: +; RV32: # %bb.0: +; RV32-NEXT: maccsu.h11 a1, a3, a5 +; RV32-NEXT: maccsu.h11 a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: pmaccsu_h11: +; RV64: # %bb.0: +; RV64-NEXT: pmaccsu.w.h11 a0, a1, a2 +; RV64-NEXT: ret + %m = call <2 x i32> @llvm.riscv.pmulsu.11.v2i32(<4 x i16> %a, <4 x i16> %b) + %r = add <2 x i32> %rd, %m + ret <2 x i32> %r +} + +; The product is used twice, so folding would recompute it. +define i32 @multi_use(i32 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: multi_use: +; RV32: # %bb.0: +; RV32-NEXT: mul.h00 a1, a1, a2 +; RV32-NEXT: add a0, a0, a1 +; RV32-NEXT: add a0, a0, a1 +; RV32-NEXT: ret +; +; RV64-LABEL: multi_use: +; RV64: # %bb.0: +; RV64-NEXT: pmul.w.h00 a1, a1, a2 +; RV64-NEXT: add a0, a0, a1 +; RV64-NEXT: addw a0, a0, a1 +; RV64-NEXT: ret + %m = call i32 @llvm.riscv.mul.00.i32.v2i16(<2 x i16> %a, <2 x i16> %b) + %r = add i32 %rd, %m + %s = add i32 %r, %m + ret i32 %s +} + +; add is commutative. +define i32 @swapped_operands(i32 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: swapped_operands: +; RV32: # %bb.0: +; RV32-NEXT: macc.h00 a0, a1, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: swapped_operands: +; RV64: # %bb.0: +; RV64-NEXT: pmacc.w.h00 a0, a1, a2 +; RV64-NEXT: ret + %m = call i32 @llvm.riscv.mul.00.i32.v2i16(<2 x i16> %a, <2 x i16> %b) + %r = add i32 %m, %rd + ret i32 %r +} + +define i32 @accumulate_chain(i32 %rd, <2 x i16> %a, <2 x i16> %b, <2 x i16> %c, <2 x i16> %d) { +; RV32-LABEL: accumulate_chain: +; RV32: # %bb.0: +; RV32-NEXT: macc.h00 a0, a1, a2 +; RV32-NEXT: macc.h00 a0, a3, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: accumulate_chain: +; RV64: # %bb.0: +; RV64-NEXT: pmacc.w.h00 a0, a1, a2 +; RV64-NEXT: pmacc.w.h00 a0, a3, a4 +; RV64-NEXT: ret + %m1 = call i32 @llvm.riscv.mul.00.i32.v2i16(<2 x i16> %a, <2 x i16> %b) + %r1 = add i32 %rd, %m1 + %m2 = call i32 @llvm.riscv.mul.00.i32.v2i16(<2 x i16> %c, <2 x i16> %d) + %r2 = add i32 %r1, %m2 + ret i32 %r2 +} _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
