https://github.com/wangpc-pp updated https://github.com/llvm/llvm-project/pull/215517
>From e252d0cbeb0a1a3ee8c7e03052d1e37035f70d4a Mon Sep 17 00:00:00 2001 From: Pengcheng Wang <[email protected]> Date: Tue, 11 Aug 2026 18:58:52 +0800 Subject: [PATCH 1/2] Format test Created using spr 1.3.6-beta.1 --- .../RISCV/partial-reduce-dot-product-i64.ll | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product-i64.ll b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product-i64.ll index 459c896357809..a1e6d460d32c6 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product-i64.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product-i64.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 4 -; RUN: opt -passes=loop-vectorize -mattr=+v,+experimental-zvdot4a8i -riscv-v-register-bit-width-lmul=8 -tail-folding-policy=dont-fold-tail -S < %s | FileCheck %s --check-prefix=NOTAILFOLD -; RUN: opt -passes=loop-vectorize -mattr=+v,+experimental-zvdot4a8i -riscv-v-register-bit-width-lmul=8 -S < %s | FileCheck %s --check-prefix=TAILFOLD +; RUN: opt -passes=loop-vectorize -mattr=+v,+experimental-zvdot4a8i -riscv-v-register-bit-width-lmul=8 -tail-folding-policy=dont-fold-tail -S < %s \ +; RUN: | FileCheck %s --check-prefix=NOTAILFOLD +; RUN: opt -passes=loop-vectorize -mattr=+v,+experimental-zvdot4a8i -riscv-v-register-bit-width-lmul=8 -S < %s \ +; RUN: | FileCheck %s --check-prefix=TAILFOLD ; Dot product with an i64 accumulator and i8 inputs. This forms a scale-8 ; partial reduction (i8 -> i64), which the RISC-V backend lowers using vdot4a* @@ -11,7 +13,7 @@ target triple = "riscv64-none-unknown-elf" -define i64 @vqdot_i64(ptr %a, ptr %b) #0 { +define i64 @vqdot_i64(ptr %a, ptr %b) { ; NOTAILFOLD-LABEL: define i64 @vqdot_i64( ; NOTAILFOLD-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] { ; NOTAILFOLD-NEXT: entry: @@ -102,5 +104,3 @@ for.body: for.exit: ret i64 %add } - -attributes #0 = { "target-features"="+v,+experimental-zvdot4a8i" } >From b43a10a9a81bad361f1bb7d3a34f6d76cbef73bf Mon Sep 17 00:00:00 2001 From: Pengcheng Wang <[email protected]> Date: Thu, 13 Aug 2026 16:02:14 +0800 Subject: [PATCH 2/2] Address comments: add more tests and do i32 reduction first Created using spr 1.3.6-beta.1 --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 41 +- .../CodeGen/RISCV/rvv/zvdot4a8i-i64-sdnode.ll | 705 +++++++++++++++++- 2 files changed, 698 insertions(+), 48 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 48b4baca61994..152e04ffb26ca 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -9902,34 +9902,35 @@ SDValue RISCVTargetLowering::lowerPARTIAL_REDUCE_MLA(SDValue Op, // vdot4a* only produces an i32 result. For an i64 accumulator, perform the // dot product into a fresh i32 accumulator (each result is the sum of four - // i8 products, which cannot overflow i32), then widen the i32 partial sums - // to i64 and accumulate. This mirrors the AArch64 sdot+sadalp idiom. + // i8 products, which cannot overflow i32), reduce those i32 partial sums + // down to the accumulator's element count while still in i32 (a sum of eight + // i8 products still cannot overflow i32), and only then extend to i64 and add + // to the accumulator. if (VT.getVectorElementType() == MVT::i64) { assert(Accum.getSimpleValueType() == VT); // vdot4a* reduces each group of four i8 lanes into one i32 lane, so the // intermediate i32 result has 1/4 the element count of the i8 inputs. - MVT I32VT = MVT::getVectorVT( + MVT DotVT = MVT::getVectorVT( MVT::i32, ArgVT.getVectorElementCount().divideCoefficientBy(4)); - SDValue Dot = DAG.getNode(Op.getOpcode(), DL, I32VT, - {DAG.getConstant(0, DL, I32VT), A, B}); - // Widen the i32 partial sums to i64. They are signed for SMLA/SUMLA and - // unsigned for UMLA. + SDValue Dot = DAG.getNode(Op.getOpcode(), DL, DotVT, + {DAG.getConstant(0, DL, DotVT), A, B}); + // Add the i32 partial sums down to the accumulator's element count by + // extracting and summing the subvectors (still in i32 to avoid a wider + // extend). + MVT NarrowVT = VT.changeVectorElementType(MVT::i32); + unsigned Stride = NarrowVT.getVectorMinNumElements(); + SDValue Sum = DAG.getExtractSubvector(DL, NarrowVT, Dot, 0); + for (unsigned I = 1, E = DotVT.getVectorMinNumElements() / Stride; I != E; + ++I) + Sum = DAG.getNode(ISD::ADD, DL, NarrowVT, Sum, + DAG.getExtractSubvector(DL, NarrowVT, Dot, I * Stride)); + // Extend the reduced i32 sums to i64 and accumulate. They are signed for + // SMLA/SUMLA and unsigned for UMLA. unsigned ExtOpc = Op.getOpcode() == ISD::PARTIAL_REDUCE_UMLA ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND; - MVT WideVT = I32VT.changeVectorElementType(MVT::i64); - SDValue Wide = DAG.getNode(ExtOpc, DL, WideVT, Dot); - // The widened dot result has twice the elements of the i64 accumulator. - // Reduce it in by splitting into subvectors matching the accumulator and - // adding them together (the same lowering the generic expander would use - // for a multiplier-free partial reduction, but without a redundant mul). - unsigned Stride = VT.getVectorMinNumElements(); - SDValue Res = Accum; - for (unsigned I = 0, E = WideVT.getVectorMinNumElements() / Stride; I != E; - ++I) - Res = DAG.getNode(ISD::ADD, DL, VT, Res, - DAG.getExtractSubvector(DL, VT, Wide, I * Stride)); - return Res; + return DAG.getNode(ISD::ADD, DL, VT, Accum, + DAG.getNode(ExtOpc, DL, VT, Sum)); } assert(Accum.getSimpleValueType() == VT && diff --git a/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-i64-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-i64-sdnode.ll index 740da5c6584da..10b0fa0723128 100644 --- a/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-i64-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-i64-sdnode.ll @@ -49,13 +49,14 @@ define <vscale x 1 x i64> @vdot4a_i64(<vscale x 1 x i64> %acc, <vscale x 8 x i8> ; DOT-LABEL: vdot4a_i64: ; DOT: # %bb.0: # %entry ; DOT-NEXT: vsetvli a0, zero, e32, m1, ta, ma -; DOT-NEXT: vmv.v.i v12, 0 -; DOT-NEXT: vdot4a.vv v12, v9, v10 -; DOT-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; DOT-NEXT: vsext.vf2 v10, v12 -; DOT-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; DOT-NEXT: vadd.vv v8, v8, v10 -; DOT-NEXT: vadd.vv v8, v8, v11 +; DOT-NEXT: vmv.v.i v11, 0 +; DOT-NEXT: vdot4a.vv v11, v9, v10 +; DOT-NEXT: csrr a0, vlenb +; DOT-NEXT: srli a0, a0, 3 +; DOT-NEXT: vslidedown.vx v9, v11, a0 +; DOT-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; DOT-NEXT: vadd.vv v9, v11, v9 +; DOT-NEXT: vwadd.wv v8, v8, v9 ; DOT-NEXT: ret entry: %a.sext = sext <vscale x 8 x i8> %a to <vscale x 8 x i64> @@ -105,13 +106,14 @@ define <vscale x 1 x i64> @vdot4au_i64(<vscale x 1 x i64> %acc, <vscale x 8 x i8 ; DOT-LABEL: vdot4au_i64: ; DOT: # %bb.0: # %entry ; DOT-NEXT: vsetvli a0, zero, e32, m1, ta, ma -; DOT-NEXT: vmv.v.i v12, 0 -; DOT-NEXT: vdot4au.vv v12, v9, v10 -; DOT-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; DOT-NEXT: vzext.vf2 v10, v12 -; DOT-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; DOT-NEXT: vadd.vv v8, v8, v10 -; DOT-NEXT: vadd.vv v8, v8, v11 +; DOT-NEXT: vmv.v.i v11, 0 +; DOT-NEXT: vdot4au.vv v11, v9, v10 +; DOT-NEXT: csrr a0, vlenb +; DOT-NEXT: srli a0, a0, 3 +; DOT-NEXT: vslidedown.vx v9, v11, a0 +; DOT-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; DOT-NEXT: vadd.vv v9, v11, v9 +; DOT-NEXT: vwaddu.wv v8, v8, v9 ; DOT-NEXT: ret entry: %a.zext = zext <vscale x 8 x i8> %a to <vscale x 8 x i64> @@ -162,13 +164,14 @@ define <vscale x 1 x i64> @vdot4asu_i64(<vscale x 1 x i64> %acc, <vscale x 8 x i ; DOT-LABEL: vdot4asu_i64: ; DOT: # %bb.0: # %entry ; DOT-NEXT: vsetvli a0, zero, e32, m1, ta, ma -; DOT-NEXT: vmv.v.i v12, 0 -; DOT-NEXT: vdot4asu.vv v12, v9, v10 -; DOT-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; DOT-NEXT: vsext.vf2 v10, v12 -; DOT-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; DOT-NEXT: vadd.vv v8, v8, v10 -; DOT-NEXT: vadd.vv v8, v8, v11 +; DOT-NEXT: vmv.v.i v11, 0 +; DOT-NEXT: vdot4asu.vv v11, v9, v10 +; DOT-NEXT: csrr a0, vlenb +; DOT-NEXT: srli a0, a0, 3 +; DOT-NEXT: vslidedown.vx v9, v11, a0 +; DOT-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; DOT-NEXT: vadd.vv v9, v11, v9 +; DOT-NEXT: vwadd.wv v8, v8, v9 ; DOT-NEXT: ret entry: %a.sext = sext <vscale x 8 x i8> %a to <vscale x 8 x i64> @@ -228,13 +231,11 @@ define <vscale x 2 x i64> @vdot4a_i64_m2(<vscale x 2 x i64> %acc, <vscale x 16 x ; DOT-LABEL: vdot4a_i64_m2: ; DOT: # %bb.0: # %entry ; DOT-NEXT: vsetvli a0, zero, e32, m2, ta, ma -; DOT-NEXT: vmv.v.i v16, 0 -; DOT-NEXT: vdot4a.vv v16, v10, v12 -; DOT-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; DOT-NEXT: vsext.vf2 v12, v16 -; DOT-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; DOT-NEXT: vadd.vv v8, v8, v12 -; DOT-NEXT: vadd.vv v8, v8, v14 +; DOT-NEXT: vmv.v.i v14, 0 +; DOT-NEXT: vdot4a.vv v14, v10, v12 +; DOT-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; DOT-NEXT: vadd.vv v10, v14, v15 +; DOT-NEXT: vwadd.wv v8, v8, v10 ; DOT-NEXT: ret entry: %a.sext = sext <vscale x 16 x i8> %a to <vscale x 16 x i64> @@ -243,6 +244,654 @@ entry: %res = call <vscale x 2 x i64> @llvm.experimental.vector.partial.reduce.add.nxv2i64.nxv16i64(<vscale x 2 x i64> %acc, <vscale x 16 x i64> %mul) ret <vscale x 2 x i64> %res } + +; The <vscale x 8 x i64> accumulator has an illegal result type, so the +; sign/zero extend produced by the lowering must itself be legalized. +define <vscale x 8 x i64> @vdot4a_i64_m8(<vscale x 8 x i64> %acc, <vscale x 64 x i8> %a, <vscale x 64 x i8> %b) { +; NODOT32-LABEL: vdot4a_i64_m8: +; NODOT32: # %bb.0: # %entry +; NODOT32-NEXT: addi sp, sp, -16 +; NODOT32-NEXT: .cfi_def_cfa_offset 16 +; NODOT32-NEXT: csrr a1, vlenb +; NODOT32-NEXT: slli a1, a1, 4 +; NODOT32-NEXT: sub sp, sp, a1 +; NODOT32-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb +; NODOT32-NEXT: csrr a1, vlenb +; NODOT32-NEXT: slli a1, a1, 3 +; NODOT32-NEXT: add a1, sp, a1 +; NODOT32-NEXT: addi a1, a1, 16 +; NODOT32-NEXT: vs8r.v v8, (a1) # vscale x 64-byte Folded Spill +; NODOT32-NEXT: vl8r.v v8, (a0) +; NODOT32-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; NODOT32-NEXT: vsext.vf4 v4, v16 +; NODOT32-NEXT: vsext.vf4 v0, v8 +; NODOT32-NEXT: vwmul.vv v24, v4, v0 +; NODOT32-NEXT: addi a0, sp, 16 +; NODOT32-NEXT: vs8r.v v24, (a0) # vscale x 64-byte Folded Spill +; NODOT32-NEXT: li a0, 1 +; NODOT32-NEXT: csrr a1, vlenb +; NODOT32-NEXT: slli a1, a1, 3 +; NODOT32-NEXT: add a1, sp, a1 +; NODOT32-NEXT: addi a1, a1, 16 +; NODOT32-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload +; NODOT32-NEXT: addi a1, sp, 16 +; NODOT32-NEXT: vl8r.v v0, (a1) # vscale x 64-byte Folded Reload +; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; NODOT32-NEXT: vmacc.vx v24, a0, v0 +; NODOT32-NEXT: csrr a1, vlenb +; NODOT32-NEXT: slli a1, a1, 3 +; NODOT32-NEXT: add a1, sp, a1 +; NODOT32-NEXT: addi a1, a1, 16 +; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill +; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; NODOT32-NEXT: vsext.vf4 v28, v17 +; NODOT32-NEXT: vmv4r.v v4, v12 +; NODOT32-NEXT: vmv2r.v v2, v10 +; NODOT32-NEXT: vmv1r.v v1, v9 +; NODOT32-NEXT: vsext.vf4 v24, v9 +; NODOT32-NEXT: vwmul.vv v0, v28, v24 +; NODOT32-NEXT: csrr a1, vlenb +; NODOT32-NEXT: slli a1, a1, 3 +; NODOT32-NEXT: add a1, sp, a1 +; NODOT32-NEXT: addi a1, a1, 16 +; NODOT32-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload +; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; NODOT32-NEXT: vmacc.vx v24, a0, v0 +; NODOT32-NEXT: csrr a1, vlenb +; NODOT32-NEXT: slli a1, a1, 3 +; NODOT32-NEXT: add a1, sp, a1 +; NODOT32-NEXT: addi a1, a1, 16 +; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill +; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; NODOT32-NEXT: vsext.vf4 v24, v18 +; NODOT32-NEXT: vmv4r.v v4, v12 +; NODOT32-NEXT: vmv2r.v v2, v10 +; NODOT32-NEXT: vsext.vf4 v28, v2 +; NODOT32-NEXT: vwmul.vv v0, v24, v28 +; NODOT32-NEXT: csrr a1, vlenb +; NODOT32-NEXT: slli a1, a1, 3 +; NODOT32-NEXT: add a1, sp, a1 +; NODOT32-NEXT: addi a1, a1, 16 +; NODOT32-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload +; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; NODOT32-NEXT: vmacc.vx v24, a0, v0 +; NODOT32-NEXT: csrr a1, vlenb +; NODOT32-NEXT: slli a1, a1, 3 +; NODOT32-NEXT: add a1, sp, a1 +; NODOT32-NEXT: addi a1, a1, 16 +; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill +; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; NODOT32-NEXT: vsext.vf4 v24, v19 +; NODOT32-NEXT: vmv4r.v v4, v12 +; NODOT32-NEXT: vmv1r.v v3, v11 +; NODOT32-NEXT: vsext.vf4 v16, v11 +; NODOT32-NEXT: vwmul.vv v0, v24, v16 +; NODOT32-NEXT: csrr a1, vlenb +; NODOT32-NEXT: slli a1, a1, 3 +; NODOT32-NEXT: add a1, sp, a1 +; NODOT32-NEXT: addi a1, a1, 16 +; NODOT32-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload +; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; NODOT32-NEXT: vmacc.vx v24, a0, v0 +; NODOT32-NEXT: csrr a1, vlenb +; NODOT32-NEXT: slli a1, a1, 3 +; NODOT32-NEXT: add a1, sp, a1 +; NODOT32-NEXT: addi a1, a1, 16 +; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill +; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; NODOT32-NEXT: vsext.vf4 v16, v20 +; NODOT32-NEXT: vmv4r.v v28, v12 +; NODOT32-NEXT: vsext.vf4 v24, v28 +; NODOT32-NEXT: vwmul.vv v0, v16, v24 +; NODOT32-NEXT: csrr a1, vlenb +; NODOT32-NEXT: slli a1, a1, 3 +; NODOT32-NEXT: add a1, sp, a1 +; NODOT32-NEXT: addi a1, a1, 16 +; NODOT32-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload +; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; NODOT32-NEXT: vmacc.vx v24, a0, v0 +; NODOT32-NEXT: csrr a1, vlenb +; NODOT32-NEXT: slli a1, a1, 3 +; NODOT32-NEXT: add a1, sp, a1 +; NODOT32-NEXT: addi a1, a1, 16 +; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill +; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; NODOT32-NEXT: vsext.vf4 v16, v21 +; NODOT32-NEXT: vsext.vf4 v24, v13 +; NODOT32-NEXT: vwmul.vv v0, v16, v24 +; NODOT32-NEXT: csrr a1, vlenb +; NODOT32-NEXT: slli a1, a1, 3 +; NODOT32-NEXT: add a1, sp, a1 +; NODOT32-NEXT: addi a1, a1, 16 +; NODOT32-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload +; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; NODOT32-NEXT: vmacc.vx v24, a0, v0 +; NODOT32-NEXT: csrr a1, vlenb +; NODOT32-NEXT: slli a1, a1, 3 +; NODOT32-NEXT: add a1, sp, a1 +; NODOT32-NEXT: addi a1, a1, 16 +; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill +; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; NODOT32-NEXT: vsext.vf4 v16, v22 +; NODOT32-NEXT: vmv2r.v v30, v14 +; NODOT32-NEXT: vsext.vf4 v24, v30 +; NODOT32-NEXT: vwmul.vv v0, v16, v24 +; NODOT32-NEXT: csrr a1, vlenb +; NODOT32-NEXT: slli a1, a1, 3 +; NODOT32-NEXT: add a1, sp, a1 +; NODOT32-NEXT: addi a1, a1, 16 +; NODOT32-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload +; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; NODOT32-NEXT: vmacc.vx v24, a0, v0 +; NODOT32-NEXT: csrr a1, vlenb +; NODOT32-NEXT: slli a1, a1, 3 +; NODOT32-NEXT: add a1, sp, a1 +; NODOT32-NEXT: addi a1, a1, 16 +; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill +; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; NODOT32-NEXT: vsext.vf4 v24, v23 +; NODOT32-NEXT: vsext.vf4 v28, v15 +; NODOT32-NEXT: vwmul.vv v16, v24, v28 +; NODOT32-NEXT: csrr a1, vlenb +; NODOT32-NEXT: slli a1, a1, 3 +; NODOT32-NEXT: add a1, sp, a1 +; NODOT32-NEXT: addi a1, a1, 16 +; NODOT32-NEXT: vl8r.v v8, (a1) # vscale x 64-byte Folded Reload +; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; NODOT32-NEXT: vmacc.vx v8, a0, v16 +; NODOT32-NEXT: csrr a0, vlenb +; NODOT32-NEXT: slli a0, a0, 4 +; NODOT32-NEXT: add sp, sp, a0 +; NODOT32-NEXT: .cfi_def_cfa sp, 16 +; NODOT32-NEXT: addi sp, sp, 16 +; NODOT32-NEXT: .cfi_def_cfa_offset 0 +; NODOT32-NEXT: ret +; +; NODOT64-LABEL: vdot4a_i64_m8: +; NODOT64: # %bb.0: # %entry +; NODOT64-NEXT: vl8r.v v24, (a0) +; NODOT64-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; NODOT64-NEXT: vsext.vf4 v4, v16 +; NODOT64-NEXT: vsext.vf4 v0, v24 +; NODOT64-NEXT: vwmacc.vv v8, v4, v0 +; NODOT64-NEXT: vsext.vf4 v4, v17 +; NODOT64-NEXT: vsext.vf4 v0, v25 +; NODOT64-NEXT: vwmacc.vv v8, v4, v0 +; NODOT64-NEXT: vsext.vf4 v4, v18 +; NODOT64-NEXT: vsext.vf4 v0, v26 +; NODOT64-NEXT: vwmacc.vv v8, v4, v0 +; NODOT64-NEXT: vsext.vf4 v4, v19 +; NODOT64-NEXT: vsext.vf4 v16, v27 +; NODOT64-NEXT: vwmacc.vv v8, v4, v16 +; NODOT64-NEXT: vsext.vf4 v16, v20 +; NODOT64-NEXT: vsext.vf4 v24, v28 +; NODOT64-NEXT: vwmacc.vv v8, v16, v24 +; NODOT64-NEXT: vsext.vf4 v16, v21 +; NODOT64-NEXT: vsext.vf4 v24, v29 +; NODOT64-NEXT: vwmacc.vv v8, v16, v24 +; NODOT64-NEXT: vsext.vf4 v16, v22 +; NODOT64-NEXT: vsext.vf4 v24, v30 +; NODOT64-NEXT: vwmacc.vv v8, v16, v24 +; NODOT64-NEXT: vsext.vf4 v16, v23 +; NODOT64-NEXT: vsext.vf4 v20, v31 +; NODOT64-NEXT: vwmacc.vv v8, v16, v20 +; NODOT64-NEXT: ret +; +; DOT-LABEL: vdot4a_i64_m8: +; DOT: # %bb.0: # %entry +; DOT-NEXT: vl8r.v v24, (a0) +; DOT-NEXT: vsetvli a0, zero, e32, m8, ta, ma +; DOT-NEXT: vmv.v.i v0, 0 +; DOT-NEXT: vdot4a.vv v0, v16, v24 +; DOT-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; DOT-NEXT: vadd.vv v16, v0, v4 +; DOT-NEXT: vwadd.wv v8, v8, v16 +; DOT-NEXT: ret +entry: + %a.sext = sext <vscale x 64 x i8> %a to <vscale x 64 x i64> + %b.sext = sext <vscale x 64 x i8> %b to <vscale x 64 x i64> + %mul = mul <vscale x 64 x i64> %a.sext, %b.sext + %res = call <vscale x 8 x i64> @llvm.experimental.vector.partial.reduce.add.nxv8i64.nxv64i64(<vscale x 8 x i64> %acc, <vscale x 64 x i64> %mul) + ret <vscale x 8 x i64> %res +} + +define <2 x i64> @vdot4a_v2i64(<2 x i64> %acc, <16 x i8> %a, <16 x i8> %b) { +; NODOT32-LABEL: vdot4a_v2i64: +; NODOT32: # %bb.0: # %entry +; NODOT32-NEXT: addi sp, sp, -16 +; NODOT32-NEXT: .cfi_def_cfa_offset 16 +; NODOT32-NEXT: csrr a0, vlenb +; NODOT32-NEXT: slli a1, a0, 3 +; NODOT32-NEXT: add a0, a1, a0 +; NODOT32-NEXT: sub sp, sp, a0 +; NODOT32-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x09, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 9 * vlenb +; NODOT32-NEXT: csrr a0, vlenb +; NODOT32-NEXT: slli a0, a0, 3 +; NODOT32-NEXT: add a0, sp, a0 +; NODOT32-NEXT: addi a0, a0, 16 +; NODOT32-NEXT: vs1r.v v8, (a0) # vscale x 8-byte Folded Spill +; NODOT32-NEXT: li a0, 1 +; NODOT32-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; NODOT32-NEXT: vsext.vf4 v16, v9 +; NODOT32-NEXT: vsext.vf4 v20, v10 +; NODOT32-NEXT: vwmul.vv v8, v16, v20 +; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; NODOT32-NEXT: vmul.vx v16, v8, a0 +; NODOT32-NEXT: vsetivli zero, 2, e64, m8, ta, ma +; NODOT32-NEXT: vslidedown.vi v8, v16, 14 +; NODOT32-NEXT: addi a0, sp, 16 +; NODOT32-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill +; NODOT32-NEXT: vslidedown.vi v0, v16, 12 +; NODOT32-NEXT: vslidedown.vi v24, v16, 10 +; NODOT32-NEXT: vslidedown.vi v8, v16, 8 +; NODOT32-NEXT: vsetivli zero, 2, e64, m4, ta, ma +; NODOT32-NEXT: vslidedown.vi v12, v16, 6 +; NODOT32-NEXT: vslidedown.vi v20, v16, 4 +; NODOT32-NEXT: vsetivli zero, 2, e64, m2, ta, ma +; NODOT32-NEXT: vslidedown.vi v10, v16, 2 +; NODOT32-NEXT: csrr a0, vlenb +; NODOT32-NEXT: slli a0, a0, 3 +; NODOT32-NEXT: add a0, sp, a0 +; NODOT32-NEXT: addi a0, a0, 16 +; NODOT32-NEXT: vl1r.v v9, (a0) # vscale x 8-byte Folded Reload +; NODOT32-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; NODOT32-NEXT: vadd.vv v9, v9, v16 +; NODOT32-NEXT: vadd.vv v11, v24, v0 +; NODOT32-NEXT: addi a0, sp, 16 +; NODOT32-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload +; NODOT32-NEXT: vadd.vv v9, v24, v9 +; NODOT32-NEXT: vadd.vv v8, v12, v8 +; NODOT32-NEXT: vadd.vv v10, v10, v20 +; NODOT32-NEXT: vadd.vv v9, v11, v9 +; NODOT32-NEXT: vadd.vv v8, v10, v8 +; NODOT32-NEXT: vadd.vv v8, v8, v9 +; NODOT32-NEXT: csrr a0, vlenb +; NODOT32-NEXT: slli a1, a0, 3 +; NODOT32-NEXT: add a0, a1, a0 +; NODOT32-NEXT: add sp, sp, a0 +; NODOT32-NEXT: .cfi_def_cfa sp, 16 +; NODOT32-NEXT: addi sp, sp, 16 +; NODOT32-NEXT: .cfi_def_cfa_offset 0 +; NODOT32-NEXT: ret +; +; NODOT64-LABEL: vdot4a_v2i64: +; NODOT64: # %bb.0: # %entry +; NODOT64-NEXT: addi sp, sp, -16 +; NODOT64-NEXT: .cfi_def_cfa_offset 16 +; NODOT64-NEXT: csrr a0, vlenb +; NODOT64-NEXT: slli a1, a0, 3 +; NODOT64-NEXT: add a0, a1, a0 +; NODOT64-NEXT: sub sp, sp, a0 +; NODOT64-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x09, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 9 * vlenb +; NODOT64-NEXT: csrr a0, vlenb +; NODOT64-NEXT: slli a0, a0, 3 +; NODOT64-NEXT: add a0, sp, a0 +; NODOT64-NEXT: addi a0, a0, 16 +; NODOT64-NEXT: vs1r.v v8, (a0) # vscale x 8-byte Folded Spill +; NODOT64-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; NODOT64-NEXT: vsext.vf4 v12, v9 +; NODOT64-NEXT: vsext.vf4 v24, v10 +; NODOT64-NEXT: vwmul.vv v16, v12, v24 +; NODOT64-NEXT: vsetivli zero, 2, e64, m8, ta, ma +; NODOT64-NEXT: vslidedown.vi v8, v16, 14 +; NODOT64-NEXT: addi a0, sp, 16 +; NODOT64-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill +; NODOT64-NEXT: vslidedown.vi v0, v16, 12 +; NODOT64-NEXT: vslidedown.vi v24, v16, 10 +; NODOT64-NEXT: vslidedown.vi v8, v16, 8 +; NODOT64-NEXT: vsetivli zero, 2, e64, m4, ta, ma +; NODOT64-NEXT: vslidedown.vi v12, v16, 6 +; NODOT64-NEXT: vslidedown.vi v20, v16, 4 +; NODOT64-NEXT: vsetivli zero, 2, e64, m2, ta, ma +; NODOT64-NEXT: vslidedown.vi v10, v16, 2 +; NODOT64-NEXT: csrr a0, vlenb +; NODOT64-NEXT: slli a0, a0, 3 +; NODOT64-NEXT: add a0, sp, a0 +; NODOT64-NEXT: addi a0, a0, 16 +; NODOT64-NEXT: vl1r.v v9, (a0) # vscale x 8-byte Folded Reload +; NODOT64-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; NODOT64-NEXT: vadd.vv v9, v9, v16 +; NODOT64-NEXT: vadd.vv v11, v24, v0 +; NODOT64-NEXT: addi a0, sp, 16 +; NODOT64-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload +; NODOT64-NEXT: vadd.vv v9, v24, v9 +; NODOT64-NEXT: vadd.vv v8, v12, v8 +; NODOT64-NEXT: vadd.vv v10, v10, v20 +; NODOT64-NEXT: vadd.vv v9, v11, v9 +; NODOT64-NEXT: vadd.vv v8, v10, v8 +; NODOT64-NEXT: vadd.vv v8, v8, v9 +; NODOT64-NEXT: csrr a0, vlenb +; NODOT64-NEXT: slli a1, a0, 3 +; NODOT64-NEXT: add a0, a1, a0 +; NODOT64-NEXT: add sp, sp, a0 +; NODOT64-NEXT: .cfi_def_cfa sp, 16 +; NODOT64-NEXT: addi sp, sp, 16 +; NODOT64-NEXT: .cfi_def_cfa_offset 0 +; NODOT64-NEXT: ret +; +; DOT-LABEL: vdot4a_v2i64: +; DOT: # %bb.0: # %entry +; DOT-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; DOT-NEXT: vmv.v.i v11, 0 +; DOT-NEXT: vdot4a.vv v11, v9, v10 +; DOT-NEXT: vsetivli zero, 2, e32, m1, ta, ma +; DOT-NEXT: vslidedown.vi v9, v11, 2 +; DOT-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; DOT-NEXT: vadd.vv v9, v11, v9 +; DOT-NEXT: vwadd.wv v8, v8, v9 +; DOT-NEXT: ret +entry: + %a.sext = sext <16 x i8> %a to <16 x i64> + %b.sext = sext <16 x i8> %b to <16 x i64> + %mul = mul <16 x i64> %a.sext, %b.sext + %res = call <2 x i64> @llvm.experimental.vector.partial.reduce.add.v2i64.v16i64(<2 x i64> %acc, <16 x i64> %mul) + ret <2 x i64> %res +} + +define <4 x i64> @vdot4au_v4i64(<4 x i64> %acc, <32 x i8> %a, <32 x i8> %b) { +; NODOT32-LABEL: vdot4au_v4i64: +; NODOT32: # %bb.0: # %entry +; NODOT32-NEXT: addi sp, sp, -16 +; NODOT32-NEXT: .cfi_def_cfa_offset 16 +; NODOT32-NEXT: csrr a0, vlenb +; NODOT32-NEXT: slli a0, a0, 1 +; NODOT32-NEXT: mv a1, a0 +; NODOT32-NEXT: slli a0, a0, 3 +; NODOT32-NEXT: add a0, a0, a1 +; NODOT32-NEXT: sub sp, sp, a0 +; NODOT32-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x12, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 18 * vlenb +; NODOT32-NEXT: csrr a0, vlenb +; NODOT32-NEXT: slli a0, a0, 4 +; NODOT32-NEXT: add a0, sp, a0 +; NODOT32-NEXT: addi a0, a0, 16 +; NODOT32-NEXT: vs2r.v v8, (a0) # vscale x 16-byte Folded Spill +; NODOT32-NEXT: li a0, 1 +; NODOT32-NEXT: vsetivli zero, 16, e8, m1, ta, ma +; NODOT32-NEXT: vwmulu.vv v8, v10, v12 +; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; NODOT32-NEXT: vzext.vf2 v24, v8 +; NODOT32-NEXT: vwmulu.vx v16, v24, a0 +; NODOT32-NEXT: vsetivli zero, 4, e64, m8, ta, ma +; NODOT32-NEXT: vslidedown.vi v24, v16, 12 +; NODOT32-NEXT: csrr a1, vlenb +; NODOT32-NEXT: slli a1, a1, 3 +; NODOT32-NEXT: add a1, sp, a1 +; NODOT32-NEXT: addi a1, a1, 16 +; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill +; NODOT32-NEXT: vslidedown.vi v24, v16, 8 +; NODOT32-NEXT: addi a1, sp, 16 +; NODOT32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill +; NODOT32-NEXT: vsetivli zero, 16, e8, m2, ta, ma +; NODOT32-NEXT: vslidedown.vi v8, v10, 16 +; NODOT32-NEXT: vslidedown.vi v10, v12, 16 +; NODOT32-NEXT: vsetivli zero, 16, e8, m1, ta, ma +; NODOT32-NEXT: vwmulu.vv v12, v8, v10 +; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; NODOT32-NEXT: vzext.vf2 v20, v12 +; NODOT32-NEXT: vwmulu.vx v8, v20, a0 +; NODOT32-NEXT: vsetivli zero, 4, e64, m8, ta, ma +; NODOT32-NEXT: vslidedown.vi v24, v8, 12 +; NODOT32-NEXT: vslidedown.vi v0, v8, 8 +; NODOT32-NEXT: vsetivli zero, 4, e64, m4, ta, ma +; NODOT32-NEXT: vslidedown.vi v12, v16, 4 +; NODOT32-NEXT: vslidedown.vi v20, v8, 4 +; NODOT32-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; NODOT32-NEXT: vadd.vv v8, v24, v8 +; NODOT32-NEXT: addi a0, sp, 16 +; NODOT32-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload +; NODOT32-NEXT: vadd.vv v10, v12, v24 +; NODOT32-NEXT: csrr a0, vlenb +; NODOT32-NEXT: slli a0, a0, 4 +; NODOT32-NEXT: add a0, sp, a0 +; NODOT32-NEXT: addi a0, a0, 16 +; NODOT32-NEXT: vl2r.v v12, (a0) # vscale x 16-byte Folded Reload +; NODOT32-NEXT: vadd.vv v12, v12, v16 +; NODOT32-NEXT: csrr a0, vlenb +; NODOT32-NEXT: slli a0, a0, 3 +; NODOT32-NEXT: add a0, sp, a0 +; NODOT32-NEXT: addi a0, a0, 16 +; NODOT32-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload +; NODOT32-NEXT: vadd.vv v12, v24, v12 +; NODOT32-NEXT: vadd.vv v10, v10, v12 +; NODOT32-NEXT: vadd.vv v8, v8, v10 +; NODOT32-NEXT: vadd.vv v10, v20, v0 +; NODOT32-NEXT: vadd.vv v8, v10, v8 +; NODOT32-NEXT: csrr a0, vlenb +; NODOT32-NEXT: slli a0, a0, 1 +; NODOT32-NEXT: mv a1, a0 +; NODOT32-NEXT: slli a0, a0, 3 +; NODOT32-NEXT: add a0, a0, a1 +; NODOT32-NEXT: add sp, sp, a0 +; NODOT32-NEXT: .cfi_def_cfa sp, 16 +; NODOT32-NEXT: addi sp, sp, 16 +; NODOT32-NEXT: .cfi_def_cfa_offset 0 +; NODOT32-NEXT: ret +; +; NODOT64-LABEL: vdot4au_v4i64: +; NODOT64: # %bb.0: # %entry +; NODOT64-NEXT: addi sp, sp, -16 +; NODOT64-NEXT: .cfi_def_cfa_offset 16 +; NODOT64-NEXT: csrr a0, vlenb +; NODOT64-NEXT: slli a0, a0, 1 +; NODOT64-NEXT: mv a1, a0 +; NODOT64-NEXT: slli a0, a0, 3 +; NODOT64-NEXT: add a0, a0, a1 +; NODOT64-NEXT: sub sp, sp, a0 +; NODOT64-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x12, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 18 * vlenb +; NODOT64-NEXT: csrr a0, vlenb +; NODOT64-NEXT: slli a0, a0, 4 +; NODOT64-NEXT: add a0, sp, a0 +; NODOT64-NEXT: addi a0, a0, 16 +; NODOT64-NEXT: vs2r.v v8, (a0) # vscale x 16-byte Folded Spill +; NODOT64-NEXT: vsetivli zero, 16, e8, m1, ta, ma +; NODOT64-NEXT: vwmulu.vv v8, v10, v12 +; NODOT64-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; NODOT64-NEXT: vzext.vf4 v16, v8 +; NODOT64-NEXT: vsetivli zero, 4, e64, m8, ta, ma +; NODOT64-NEXT: vslidedown.vi v24, v16, 12 +; NODOT64-NEXT: csrr a0, vlenb +; NODOT64-NEXT: slli a0, a0, 3 +; NODOT64-NEXT: add a0, sp, a0 +; NODOT64-NEXT: addi a0, a0, 16 +; NODOT64-NEXT: vs8r.v v24, (a0) # vscale x 64-byte Folded Spill +; NODOT64-NEXT: vslidedown.vi v24, v16, 8 +; NODOT64-NEXT: addi a0, sp, 16 +; NODOT64-NEXT: vs8r.v v24, (a0) # vscale x 64-byte Folded Spill +; NODOT64-NEXT: vsetivli zero, 16, e8, m2, ta, ma +; NODOT64-NEXT: vslidedown.vi v8, v10, 16 +; NODOT64-NEXT: vslidedown.vi v10, v12, 16 +; NODOT64-NEXT: vsetivli zero, 16, e8, m1, ta, ma +; NODOT64-NEXT: vwmulu.vv v20, v8, v10 +; NODOT64-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; NODOT64-NEXT: vzext.vf4 v8, v20 +; NODOT64-NEXT: vsetivli zero, 4, e64, m8, ta, ma +; NODOT64-NEXT: vslidedown.vi v24, v8, 12 +; NODOT64-NEXT: vslidedown.vi v0, v8, 8 +; NODOT64-NEXT: vsetivli zero, 4, e64, m4, ta, ma +; NODOT64-NEXT: vslidedown.vi v12, v16, 4 +; NODOT64-NEXT: vslidedown.vi v20, v8, 4 +; NODOT64-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; NODOT64-NEXT: vadd.vv v8, v24, v8 +; NODOT64-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload +; NODOT64-NEXT: vadd.vv v10, v12, v24 +; NODOT64-NEXT: csrr a0, vlenb +; NODOT64-NEXT: slli a0, a0, 4 +; NODOT64-NEXT: add a0, sp, a0 +; NODOT64-NEXT: addi a0, a0, 16 +; NODOT64-NEXT: vl2r.v v12, (a0) # vscale x 16-byte Folded Reload +; NODOT64-NEXT: vadd.vv v12, v12, v16 +; NODOT64-NEXT: csrr a0, vlenb +; NODOT64-NEXT: slli a0, a0, 3 +; NODOT64-NEXT: add a0, sp, a0 +; NODOT64-NEXT: addi a0, a0, 16 +; NODOT64-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload +; NODOT64-NEXT: vadd.vv v12, v24, v12 +; NODOT64-NEXT: vadd.vv v10, v10, v12 +; NODOT64-NEXT: vadd.vv v8, v8, v10 +; NODOT64-NEXT: vadd.vv v10, v20, v0 +; NODOT64-NEXT: vadd.vv v8, v10, v8 +; NODOT64-NEXT: csrr a0, vlenb +; NODOT64-NEXT: slli a0, a0, 1 +; NODOT64-NEXT: mv a1, a0 +; NODOT64-NEXT: slli a0, a0, 3 +; NODOT64-NEXT: add a0, a0, a1 +; NODOT64-NEXT: add sp, sp, a0 +; NODOT64-NEXT: .cfi_def_cfa sp, 16 +; NODOT64-NEXT: addi sp, sp, 16 +; NODOT64-NEXT: .cfi_def_cfa_offset 0 +; NODOT64-NEXT: ret +; +; DOT-LABEL: vdot4au_v4i64: +; DOT: # %bb.0: # %entry +; DOT-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; DOT-NEXT: vmv.v.i v14, 0 +; DOT-NEXT: vdot4au.vv v14, v10, v12 +; DOT-NEXT: vsetivli zero, 4, e32, m2, ta, ma +; DOT-NEXT: vslidedown.vi v10, v14, 4 +; DOT-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; DOT-NEXT: vadd.vv v10, v14, v10 +; DOT-NEXT: vwaddu.wv v8, v8, v10 +; DOT-NEXT: ret +entry: + %a.zext = zext <32 x i8> %a to <32 x i64> + %b.zext = zext <32 x i8> %b to <32 x i64> + %mul = mul <32 x i64> %a.zext, %b.zext + %res = call <4 x i64> @llvm.experimental.vector.partial.reduce.add.v4i64.v32i64(<4 x i64> %acc, <32 x i64> %mul) + ret <4 x i64> %res +} + +define <2 x i64> @vdot4asu_v2i64(<2 x i64> %acc, <16 x i8> %a, <16 x i8> %b) { +; NODOT32-LABEL: vdot4asu_v2i64: +; NODOT32: # %bb.0: # %entry +; NODOT32-NEXT: addi sp, sp, -16 +; NODOT32-NEXT: .cfi_def_cfa_offset 16 +; NODOT32-NEXT: csrr a0, vlenb +; NODOT32-NEXT: slli a1, a0, 3 +; NODOT32-NEXT: add a0, a1, a0 +; NODOT32-NEXT: sub sp, sp, a0 +; NODOT32-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x09, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 9 * vlenb +; NODOT32-NEXT: csrr a0, vlenb +; NODOT32-NEXT: slli a0, a0, 3 +; NODOT32-NEXT: add a0, sp, a0 +; NODOT32-NEXT: addi a0, a0, 16 +; NODOT32-NEXT: vs1r.v v8, (a0) # vscale x 8-byte Folded Spill +; NODOT32-NEXT: li a0, 1 +; NODOT32-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; NODOT32-NEXT: vsext.vf4 v16, v9 +; NODOT32-NEXT: vzext.vf4 v20, v10 +; NODOT32-NEXT: vwmulsu.vv v8, v16, v20 +; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; NODOT32-NEXT: vmul.vx v16, v8, a0 +; NODOT32-NEXT: vsetivli zero, 2, e64, m8, ta, ma +; NODOT32-NEXT: vslidedown.vi v8, v16, 14 +; NODOT32-NEXT: addi a0, sp, 16 +; NODOT32-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill +; NODOT32-NEXT: vslidedown.vi v0, v16, 12 +; NODOT32-NEXT: vslidedown.vi v24, v16, 10 +; NODOT32-NEXT: vslidedown.vi v8, v16, 8 +; NODOT32-NEXT: vsetivli zero, 2, e64, m4, ta, ma +; NODOT32-NEXT: vslidedown.vi v12, v16, 6 +; NODOT32-NEXT: vslidedown.vi v20, v16, 4 +; NODOT32-NEXT: vsetivli zero, 2, e64, m2, ta, ma +; NODOT32-NEXT: vslidedown.vi v10, v16, 2 +; NODOT32-NEXT: csrr a0, vlenb +; NODOT32-NEXT: slli a0, a0, 3 +; NODOT32-NEXT: add a0, sp, a0 +; NODOT32-NEXT: addi a0, a0, 16 +; NODOT32-NEXT: vl1r.v v9, (a0) # vscale x 8-byte Folded Reload +; NODOT32-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; NODOT32-NEXT: vadd.vv v9, v9, v16 +; NODOT32-NEXT: vadd.vv v11, v24, v0 +; NODOT32-NEXT: addi a0, sp, 16 +; NODOT32-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload +; NODOT32-NEXT: vadd.vv v9, v24, v9 +; NODOT32-NEXT: vadd.vv v8, v12, v8 +; NODOT32-NEXT: vadd.vv v10, v10, v20 +; NODOT32-NEXT: vadd.vv v9, v11, v9 +; NODOT32-NEXT: vadd.vv v8, v10, v8 +; NODOT32-NEXT: vadd.vv v8, v8, v9 +; NODOT32-NEXT: csrr a0, vlenb +; NODOT32-NEXT: slli a1, a0, 3 +; NODOT32-NEXT: add a0, a1, a0 +; NODOT32-NEXT: add sp, sp, a0 +; NODOT32-NEXT: .cfi_def_cfa sp, 16 +; NODOT32-NEXT: addi sp, sp, 16 +; NODOT32-NEXT: .cfi_def_cfa_offset 0 +; NODOT32-NEXT: ret +; +; NODOT64-LABEL: vdot4asu_v2i64: +; NODOT64: # %bb.0: # %entry +; NODOT64-NEXT: addi sp, sp, -16 +; NODOT64-NEXT: .cfi_def_cfa_offset 16 +; NODOT64-NEXT: csrr a0, vlenb +; NODOT64-NEXT: slli a1, a0, 3 +; NODOT64-NEXT: add a0, a1, a0 +; NODOT64-NEXT: sub sp, sp, a0 +; NODOT64-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x09, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 9 * vlenb +; NODOT64-NEXT: csrr a0, vlenb +; NODOT64-NEXT: slli a0, a0, 3 +; NODOT64-NEXT: add a0, sp, a0 +; NODOT64-NEXT: addi a0, a0, 16 +; NODOT64-NEXT: vs1r.v v8, (a0) # vscale x 8-byte Folded Spill +; NODOT64-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; NODOT64-NEXT: vsext.vf4 v12, v9 +; NODOT64-NEXT: vzext.vf4 v24, v10 +; NODOT64-NEXT: vwmulsu.vv v16, v12, v24 +; NODOT64-NEXT: vsetivli zero, 2, e64, m8, ta, ma +; NODOT64-NEXT: vslidedown.vi v8, v16, 14 +; NODOT64-NEXT: addi a0, sp, 16 +; NODOT64-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill +; NODOT64-NEXT: vslidedown.vi v0, v16, 12 +; NODOT64-NEXT: vslidedown.vi v24, v16, 10 +; NODOT64-NEXT: vslidedown.vi v8, v16, 8 +; NODOT64-NEXT: vsetivli zero, 2, e64, m4, ta, ma +; NODOT64-NEXT: vslidedown.vi v12, v16, 6 +; NODOT64-NEXT: vslidedown.vi v20, v16, 4 +; NODOT64-NEXT: vsetivli zero, 2, e64, m2, ta, ma +; NODOT64-NEXT: vslidedown.vi v10, v16, 2 +; NODOT64-NEXT: csrr a0, vlenb +; NODOT64-NEXT: slli a0, a0, 3 +; NODOT64-NEXT: add a0, sp, a0 +; NODOT64-NEXT: addi a0, a0, 16 +; NODOT64-NEXT: vl1r.v v9, (a0) # vscale x 8-byte Folded Reload +; NODOT64-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; NODOT64-NEXT: vadd.vv v9, v9, v16 +; NODOT64-NEXT: vadd.vv v11, v24, v0 +; NODOT64-NEXT: addi a0, sp, 16 +; NODOT64-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload +; NODOT64-NEXT: vadd.vv v9, v24, v9 +; NODOT64-NEXT: vadd.vv v8, v12, v8 +; NODOT64-NEXT: vadd.vv v10, v10, v20 +; NODOT64-NEXT: vadd.vv v9, v11, v9 +; NODOT64-NEXT: vadd.vv v8, v10, v8 +; NODOT64-NEXT: vadd.vv v8, v8, v9 +; NODOT64-NEXT: csrr a0, vlenb +; NODOT64-NEXT: slli a1, a0, 3 +; NODOT64-NEXT: add a0, a1, a0 +; NODOT64-NEXT: add sp, sp, a0 +; NODOT64-NEXT: .cfi_def_cfa sp, 16 +; NODOT64-NEXT: addi sp, sp, 16 +; NODOT64-NEXT: .cfi_def_cfa_offset 0 +; NODOT64-NEXT: ret +; +; DOT-LABEL: vdot4asu_v2i64: +; DOT: # %bb.0: # %entry +; DOT-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; DOT-NEXT: vmv.v.i v11, 0 +; DOT-NEXT: vdot4asu.vv v11, v9, v10 +; DOT-NEXT: vsetivli zero, 2, e32, m1, ta, ma +; DOT-NEXT: vslidedown.vi v9, v11, 2 +; DOT-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; DOT-NEXT: vadd.vv v9, v11, v9 +; DOT-NEXT: vwadd.wv v8, v8, v9 +; DOT-NEXT: ret +entry: + %a.sext = sext <16 x i8> %a to <16 x i64> + %b.zext = zext <16 x i8> %b to <16 x i64> + %mul = mul <16 x i64> %a.sext, %b.zext + %res = call <2 x i64> @llvm.experimental.vector.partial.reduce.add.v2i64.v16i64(<2 x i64> %acc, <16 x i64> %mul) + ret <2 x i64> %res +} ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: ; CHECK: {{.*}} ; DOT32: {{.*}} _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
