llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-backend-risc-v Author: Pengcheng Wang (wangpc-pp) <details> <summary>Changes</summary> In `Zvdot4a8i` extension, `vdot4a*` only produces an i32 result. This PR extends partial-reduction handling so an i64 accumulator with i8 inputs (a scale-8 partial reduction) is also lowered using the dot-product instructions: perform the dot product into a fresh i32 accumulator (each lane is the sum of four i8 products and cannot overflow i32), then sign/zero-extend the i32 partial sums to i64 and accumulate. This mirrors the AArch64 sdot+sadalp idiom. - `getPartialReductionCost` accepts an i64 accumulator (reduction factor 8), costing the `vdot4a*` plus the extra i32->i64 widen and accumulate. - `setPartialReduceMLAAction` marks the i8->i64 (scalable and fixed-length) partial reductions Custom. - `lowerPARTIAL_REDUCE_MLA` lowers the i64 case via an i32 `vdot4a*` followed by a widening `partial.reduce.add`, which the generic expander turns into extract_subvector + add. The i64 case requires a wide enough VF (LMUL) to reach the scale-8 factor, matching how AArch64 only forms it under SVE. Assisted-by: TRAE CLI (Opus 4.8) --- Patch is 24.50 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/215517.diff 4 Files Affected: - (modified) llvm/lib/Target/RISCV/RISCVISelLowering.cpp (+46-4) - (modified) llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp (+26-5) - (added) llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-i64-sdnode.ll (+250) - (added) llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product-i64.ll (+106) ``````````diff diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index ea4803e59ebc1..48b4baca61994 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -1970,13 +1970,22 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, setPartialReduceMLAAction(MLAOps, MVT::nxv8i32, MVT::nxv32i8, Custom); setPartialReduceMLAAction(MLAOps, MVT::nxv16i32, MVT::nxv64i8, Custom); + // An i64 accumulator is handled by performing an i32 vdot4a* and widening + // the result to i64 (see lowerPARTIAL_REDUCE_MLA). + setPartialReduceMLAAction(MLAOps, MVT::nxv1i64, MVT::nxv8i8, Custom); + setPartialReduceMLAAction(MLAOps, MVT::nxv2i64, MVT::nxv16i8, Custom); + setPartialReduceMLAAction(MLAOps, MVT::nxv4i64, MVT::nxv32i8, Custom); + setPartialReduceMLAAction(MLAOps, MVT::nxv8i64, MVT::nxv64i8, Custom); + if (Subtarget.useRVVForFixedLengthVectors()) { for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) { - if (VT.getVectorElementType() != MVT::i32 || + if ((VT.getVectorElementType() != MVT::i32 && + VT.getVectorElementType() != MVT::i64) || !useRVVForFixedLengthVectorVT(VT)) continue; ElementCount EC = VT.getVectorElementCount(); - MVT ArgVT = MVT::getVectorVT(MVT::i8, EC.multiplyCoefficientBy(4)); + unsigned Scale = VT.getVectorElementType() == MVT::i64 ? 8 : 4; + MVT ArgVT = MVT::getVectorVT(MVT::i8, EC.multiplyCoefficientBy(Scale)); setPartialReduceMLAAction(MLAOps, VT, ArgVT, Custom); } } @@ -9884,8 +9893,6 @@ SDValue RISCVTargetLowering::lowerPARTIAL_REDUCE_MLA(SDValue Op, SDLoc DL(Op); MVT VT = Op.getSimpleValueType(); SDValue Accum = Op.getOperand(0); - assert(Accum.getSimpleValueType() == VT && - VT.getVectorElementType() == MVT::i32); SDValue A = Op.getOperand(1); SDValue B = Op.getOperand(2); MVT ArgVT = A.getSimpleValueType(); @@ -9893,6 +9900,41 @@ SDValue RISCVTargetLowering::lowerPARTIAL_REDUCE_MLA(SDValue Op, ArgVT.getVectorElementType() == MVT::i8); (void)ArgVT; + // vdot4a* only produces an i32 result. For an i64 accumulator, perform the + // dot product into a fresh i32 accumulator (each result is the sum of four + // i8 products, which cannot overflow i32), then widen the i32 partial sums + // to i64 and accumulate. This mirrors the AArch64 sdot+sadalp idiom. + if (VT.getVectorElementType() == MVT::i64) { + assert(Accum.getSimpleValueType() == VT); + // vdot4a* reduces each group of four i8 lanes into one i32 lane, so the + // intermediate i32 result has 1/4 the element count of the i8 inputs. + MVT I32VT = MVT::getVectorVT( + MVT::i32, ArgVT.getVectorElementCount().divideCoefficientBy(4)); + SDValue Dot = DAG.getNode(Op.getOpcode(), DL, I32VT, + {DAG.getConstant(0, DL, I32VT), A, B}); + // Widen the i32 partial sums to i64. They are signed for SMLA/SUMLA and + // unsigned for UMLA. + unsigned ExtOpc = Op.getOpcode() == ISD::PARTIAL_REDUCE_UMLA + ? ISD::ZERO_EXTEND + : ISD::SIGN_EXTEND; + MVT WideVT = I32VT.changeVectorElementType(MVT::i64); + SDValue Wide = DAG.getNode(ExtOpc, DL, WideVT, Dot); + // The widened dot result has twice the elements of the i64 accumulator. + // Reduce it in by splitting into subvectors matching the accumulator and + // adding them together (the same lowering the generic expander would use + // for a multiplier-free partial reduction, but without a redundant mul). + unsigned Stride = VT.getVectorMinNumElements(); + SDValue Res = Accum; + for (unsigned I = 0, E = WideVT.getVectorMinNumElements() / Stride; I != E; + ++I) + Res = DAG.getNode(ISD::ADD, DL, VT, Res, + DAG.getExtractSubvector(DL, VT, Wide, I * Stride)); + return Res; + } + + assert(Accum.getSimpleValueType() == VT && + VT.getVectorElementType() == MVT::i32); + // The zvdot4a8i pseudos are defined with sources and destination both // being i32. This cast is needed for correctness to avoid incorrect // .vx matching of i8 splats. diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index 9868e4d02f905..62f95797bbf49 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -353,17 +353,38 @@ InstructionCost RISCVTTIImpl::getPartialReductionCost( // zve32x is broken for partial_reduce_umla, but let's make sure we // don't generate them. + // vdot4a* reduces four i8 products into an i32 result; an i64 accumulator is + // additionally supported by widening the i32 partial sums to i64 (see + // lowerPARTIAL_REDUCE_MLA). \p VF is the number of i8 input elements, so the + // reduction factor is AccumBits / 8 (4 for i32, 8 for i64). if (!ST->hasStdExtZvdot4a8i() || ST->getELen() < 64 || Opcode != Instruction::Add || !BinOp || *BinOp != Instruction::Mul || InputTypeA != InputTypeB || !InputTypeA->isIntegerTy(8) || - !AccumType->isIntegerTy(32) || !VF.isKnownMultipleOf(4)) + (!AccumType->isIntegerTy(32) && !AccumType->isIntegerTy(64))) return InstructionCost::getInvalid(); - Type *Tp = VectorType::get(AccumType, VF.divideCoefficientBy(4)); - std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Tp); + unsigned Ratio = AccumType->getScalarSizeInBits() / 8; + if (!VF.isKnownMultipleOf(Ratio)) + return InstructionCost::getInvalid(); + + // Cost of the vdot4a* itself, which operates on the i32 intermediate type + // holding VF/4 elements. + Type *DotTp = VectorType::get(Type::getInt32Ty(AccumType->getContext()), + VF.divideCoefficientBy(4)); + std::pair<InstructionCost, MVT> DotLT = getTypeLegalizationCost(DotTp); // Note: Asuming all vdot4a* variants are equal cost - return LT.first * - getRISCVInstructionCost(RISCV::VDOT4A_VV, LT.second, CostKind); + InstructionCost Cost = + DotLT.first * + getRISCVInstructionCost(RISCV::VDOT4A_VV, DotLT.second, CostKind); + + // Account for the widening extend + accumulate needed for an i64 result. + if (AccumType->isIntegerTy(64)) { + Type *AccTp = VectorType::get(AccumType, VF.divideCoefficientBy(Ratio)); + std::pair<InstructionCost, MVT> AccLT = getTypeLegalizationCost(AccTp); + Cost += AccLT.first * 2; + } + + return Cost; } bool RISCVTTIImpl::shouldExpandReduction(const IntrinsicInst *II) const { diff --git a/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-i64-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-i64-sdnode.ll new file mode 100644 index 0000000000000..740da5c6584da --- /dev/null +++ b/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-i64-sdnode.ll @@ -0,0 +1,250 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NODOT,NODOT32 +; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NODOT,NODOT64 +; RUN: llc -mtriple=riscv32 -mattr=+v,+experimental-zvdot4a8i -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,DOT,DOT32 +; RUN: llc -mtriple=riscv64 -mattr=+v,+experimental-zvdot4a8i -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,DOT,DOT64 + +; Tests for partial.reduce.add into an i64 accumulator with i8 inputs. When +; Zvdot4a8i is available these are lowered to a vdot4a* (i8 -> i32) followed by +; a widening (i32 -> i64) accumulate, mirroring the AArch64 sdot+sadalp idiom. + +define <vscale x 1 x i64> @vdot4a_i64(<vscale x 1 x i64> %acc, <vscale x 8 x i8> %a, <vscale x 8 x i8> %b) { +; NODOT32-LABEL: vdot4a_i64: +; NODOT32: # %bb.0: # %entry +; NODOT32-NEXT: li a0, 1 +; NODOT32-NEXT: vsetvli a1, zero, e32, m4, ta, ma +; NODOT32-NEXT: vsext.vf4 v12, v9 +; NODOT32-NEXT: vsext.vf4 v24, v10 +; NODOT32-NEXT: vwmul.vv v16, v12, v24 +; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; NODOT32-NEXT: vmul.vx v16, v16, a0 +; NODOT32-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; NODOT32-NEXT: vadd.vv v9, v19, v20 +; NODOT32-NEXT: vadd.vv v8, v8, v16 +; NODOT32-NEXT: vadd.vv v10, v17, v18 +; NODOT32-NEXT: vadd.vv v8, v23, v8 +; NODOT32-NEXT: vadd.vv v11, v21, v22 +; NODOT32-NEXT: vadd.vv v9, v10, v9 +; NODOT32-NEXT: vadd.vv v8, v11, v8 +; NODOT32-NEXT: vadd.vv v8, v9, v8 +; NODOT32-NEXT: ret +; +; NODOT64-LABEL: vdot4a_i64: +; NODOT64: # %bb.0: # %entry +; NODOT64-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; NODOT64-NEXT: vsext.vf4 v12, v9 +; NODOT64-NEXT: vsext.vf4 v24, v10 +; NODOT64-NEXT: vwmul.vv v16, v12, v24 +; NODOT64-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; NODOT64-NEXT: vadd.vv v9, v19, v20 +; NODOT64-NEXT: vadd.vv v8, v8, v16 +; NODOT64-NEXT: vadd.vv v10, v17, v18 +; NODOT64-NEXT: vadd.vv v8, v23, v8 +; NODOT64-NEXT: vadd.vv v11, v21, v22 +; NODOT64-NEXT: vadd.vv v9, v10, v9 +; NODOT64-NEXT: vadd.vv v8, v11, v8 +; NODOT64-NEXT: vadd.vv v8, v9, v8 +; NODOT64-NEXT: ret +; +; DOT-LABEL: vdot4a_i64: +; DOT: # %bb.0: # %entry +; DOT-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; DOT-NEXT: vmv.v.i v12, 0 +; DOT-NEXT: vdot4a.vv v12, v9, v10 +; DOT-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; DOT-NEXT: vsext.vf2 v10, v12 +; DOT-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; DOT-NEXT: vadd.vv v8, v8, v10 +; DOT-NEXT: vadd.vv v8, v8, v11 +; DOT-NEXT: ret +entry: + %a.sext = sext <vscale x 8 x i8> %a to <vscale x 8 x i64> + %b.sext = sext <vscale x 8 x i8> %b to <vscale x 8 x i64> + %mul = mul <vscale x 8 x i64> %a.sext, %b.sext + %res = call <vscale x 1 x i64> @llvm.experimental.vector.partial.reduce.add.nxv1i64.nxv8i64(<vscale x 1 x i64> %acc, <vscale x 8 x i64> %mul) + ret <vscale x 1 x i64> %res +} + +define <vscale x 1 x i64> @vdot4au_i64(<vscale x 1 x i64> %acc, <vscale x 8 x i8> %a, <vscale x 8 x i8> %b) { +; NODOT32-LABEL: vdot4au_i64: +; NODOT32: # %bb.0: # %entry +; NODOT32-NEXT: li a0, 1 +; NODOT32-NEXT: vsetvli a1, zero, e8, m1, ta, ma +; NODOT32-NEXT: vwmulu.vv v16, v9, v10 +; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; NODOT32-NEXT: vzext.vf2 v12, v16 +; NODOT32-NEXT: vwmulu.vx v16, v12, a0 +; NODOT32-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; NODOT32-NEXT: vadd.vv v9, v19, v20 +; NODOT32-NEXT: vadd.vv v8, v8, v16 +; NODOT32-NEXT: vadd.vv v10, v17, v18 +; NODOT32-NEXT: vadd.vv v8, v23, v8 +; NODOT32-NEXT: vadd.vv v11, v21, v22 +; NODOT32-NEXT: vadd.vv v9, v10, v9 +; NODOT32-NEXT: vadd.vv v8, v11, v8 +; NODOT32-NEXT: vadd.vv v8, v9, v8 +; NODOT32-NEXT: ret +; +; NODOT64-LABEL: vdot4au_i64: +; NODOT64: # %bb.0: # %entry +; NODOT64-NEXT: vsetvli a0, zero, e8, m1, ta, ma +; NODOT64-NEXT: vwmulu.vv v12, v9, v10 +; NODOT64-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; NODOT64-NEXT: vzext.vf4 v16, v12 +; NODOT64-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; NODOT64-NEXT: vadd.vv v9, v19, v20 +; NODOT64-NEXT: vadd.vv v8, v8, v16 +; NODOT64-NEXT: vadd.vv v10, v17, v18 +; NODOT64-NEXT: vadd.vv v8, v23, v8 +; NODOT64-NEXT: vadd.vv v11, v21, v22 +; NODOT64-NEXT: vadd.vv v9, v10, v9 +; NODOT64-NEXT: vadd.vv v8, v11, v8 +; NODOT64-NEXT: vadd.vv v8, v9, v8 +; NODOT64-NEXT: ret +; +; DOT-LABEL: vdot4au_i64: +; DOT: # %bb.0: # %entry +; DOT-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; DOT-NEXT: vmv.v.i v12, 0 +; DOT-NEXT: vdot4au.vv v12, v9, v10 +; DOT-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; DOT-NEXT: vzext.vf2 v10, v12 +; DOT-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; DOT-NEXT: vadd.vv v8, v8, v10 +; DOT-NEXT: vadd.vv v8, v8, v11 +; DOT-NEXT: ret +entry: + %a.zext = zext <vscale x 8 x i8> %a to <vscale x 8 x i64> + %b.zext = zext <vscale x 8 x i8> %b to <vscale x 8 x i64> + %mul = mul <vscale x 8 x i64> %a.zext, %b.zext + %res = call <vscale x 1 x i64> @llvm.experimental.vector.partial.reduce.add.nxv1i64.nxv8i64(<vscale x 1 x i64> %acc, <vscale x 8 x i64> %mul) + ret <vscale x 1 x i64> %res +} + +define <vscale x 1 x i64> @vdot4asu_i64(<vscale x 1 x i64> %acc, <vscale x 8 x i8> %a, <vscale x 8 x i8> %b) { +; NODOT32-LABEL: vdot4asu_i64: +; NODOT32: # %bb.0: # %entry +; NODOT32-NEXT: li a0, 1 +; NODOT32-NEXT: vsetvli a1, zero, e32, m4, ta, ma +; NODOT32-NEXT: vsext.vf4 v12, v9 +; NODOT32-NEXT: vzext.vf4 v24, v10 +; NODOT32-NEXT: vwmulsu.vv v16, v12, v24 +; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; NODOT32-NEXT: vmul.vx v16, v16, a0 +; NODOT32-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; NODOT32-NEXT: vadd.vv v9, v19, v20 +; NODOT32-NEXT: vadd.vv v8, v8, v16 +; NODOT32-NEXT: vadd.vv v10, v17, v18 +; NODOT32-NEXT: vadd.vv v8, v23, v8 +; NODOT32-NEXT: vadd.vv v11, v21, v22 +; NODOT32-NEXT: vadd.vv v9, v10, v9 +; NODOT32-NEXT: vadd.vv v8, v11, v8 +; NODOT32-NEXT: vadd.vv v8, v9, v8 +; NODOT32-NEXT: ret +; +; NODOT64-LABEL: vdot4asu_i64: +; NODOT64: # %bb.0: # %entry +; NODOT64-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; NODOT64-NEXT: vsext.vf4 v12, v9 +; NODOT64-NEXT: vzext.vf4 v24, v10 +; NODOT64-NEXT: vwmulsu.vv v16, v12, v24 +; NODOT64-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; NODOT64-NEXT: vadd.vv v9, v19, v20 +; NODOT64-NEXT: vadd.vv v8, v8, v16 +; NODOT64-NEXT: vadd.vv v10, v17, v18 +; NODOT64-NEXT: vadd.vv v8, v23, v8 +; NODOT64-NEXT: vadd.vv v11, v21, v22 +; NODOT64-NEXT: vadd.vv v9, v10, v9 +; NODOT64-NEXT: vadd.vv v8, v11, v8 +; NODOT64-NEXT: vadd.vv v8, v9, v8 +; NODOT64-NEXT: ret +; +; DOT-LABEL: vdot4asu_i64: +; DOT: # %bb.0: # %entry +; DOT-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; DOT-NEXT: vmv.v.i v12, 0 +; DOT-NEXT: vdot4asu.vv v12, v9, v10 +; DOT-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; DOT-NEXT: vsext.vf2 v10, v12 +; DOT-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; DOT-NEXT: vadd.vv v8, v8, v10 +; DOT-NEXT: vadd.vv v8, v8, v11 +; DOT-NEXT: ret +entry: + %a.sext = sext <vscale x 8 x i8> %a to <vscale x 8 x i64> + %b.zext = zext <vscale x 8 x i8> %b to <vscale x 8 x i64> + %mul = mul <vscale x 8 x i64> %a.sext, %b.zext + %res = call <vscale x 1 x i64> @llvm.experimental.vector.partial.reduce.add.nxv1i64.nxv8i64(<vscale x 1 x i64> %acc, <vscale x 8 x i64> %mul) + ret <vscale x 1 x i64> %res +} + +define <vscale x 2 x i64> @vdot4a_i64_m2(<vscale x 2 x i64> %acc, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) { +; NODOT32-LABEL: vdot4a_i64_m2: +; NODOT32: # %bb.0: # %entry +; NODOT32-NEXT: li a0, 1 +; NODOT32-NEXT: vsetvli a1, zero, e32, m4, ta, ma +; NODOT32-NEXT: vsext.vf4 v24, v10 +; NODOT32-NEXT: vsext.vf4 v4, v11 +; NODOT32-NEXT: vsext.vf4 v28, v12 +; NODOT32-NEXT: vsext.vf4 v0, v13 +; NODOT32-NEXT: vwmul.vv v16, v24, v28 +; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; NODOT32-NEXT: vmul.vx v16, v16, a0 +; NODOT32-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; NODOT32-NEXT: vwmul.vv v24, v4, v0 +; NODOT32-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; NODOT32-NEXT: vmul.vx v24, v24, a0 +; NODOT32-NEXT: vsetvli a0, zero, e64, m2, ta, ma +; NODOT32-NEXT: vadd.vv v10, v18, v20 +; NODOT32-NEXT: vadd.vv v8, v8, v16 +; NODOT32-NEXT: vadd.vv v8, v22, v8 +; NODOT32-NEXT: vadd.vv v10, v10, v24 +; NODOT32-NEXT: vadd.vv v8, v10, v8 +; NODOT32-NEXT: vadd.vv v8, v30, v8 +; NODOT32-NEXT: vadd.vv v10, v26, v28 +; NODOT32-NEXT: vadd.vv v8, v10, v8 +; NODOT32-NEXT: ret +; +; NODOT64-LABEL: vdot4a_i64_m2: +; NODOT64: # %bb.0: # %entry +; NODOT64-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; NODOT64-NEXT: vsext.vf4 v4, v10 +; NODOT64-NEXT: vsext.vf4 v24, v11 +; NODOT64-NEXT: vsext.vf4 v0, v12 +; NODOT64-NEXT: vsext.vf4 v28, v13 +; NODOT64-NEXT: vwmul.vv v16, v24, v28 +; NODOT64-NEXT: vwmul.vv v24, v4, v0 +; NODOT64-NEXT: vsetvli a0, zero, e64, m2, ta, ma +; NODOT64-NEXT: vadd.vv v10, v26, v28 +; NODOT64-NEXT: vadd.vv v8, v8, v24 +; NODOT64-NEXT: vadd.vv v8, v30, v8 +; NODOT64-NEXT: vadd.vv v10, v10, v16 +; NODOT64-NEXT: vadd.vv v8, v10, v8 +; NODOT64-NEXT: vadd.vv v8, v22, v8 +; NODOT64-NEXT: vadd.vv v10, v18, v20 +; NODOT64-NEXT: vadd.vv v8, v10, v8 +; NODOT64-NEXT: ret +; +; DOT-LABEL: vdot4a_i64_m2: +; DOT: # %bb.0: # %entry +; DOT-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; DOT-NEXT: vmv.v.i v16, 0 +; DOT-NEXT: vdot4a.vv v16, v10, v12 +; DOT-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; DOT-NEXT: vsext.vf2 v12, v16 +; DOT-NEXT: vsetvli a0, zero, e64, m2, ta, ma +; DOT-NEXT: vadd.vv v8, v8, v12 +; DOT-NEXT: vadd.vv v8, v8, v14 +; DOT-NEXT: ret +entry: + %a.sext = sext <vscale x 16 x i8> %a to <vscale x 16 x i64> + %b.sext = sext <vscale x 16 x i8> %b to <vscale x 16 x i64> + %mul = mul <vscale x 16 x i64> %a.sext, %b.sext + %res = call <vscale x 2 x i64> @llvm.experimental.vector.partial.reduce.add.nxv2i64.nxv16i64(<vscale x 2 x i64> %acc, <vscale x 16 x i64> %mul) + ret <vscale x 2 x i64> %res +} +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; CHECK: {{.*}} +; DOT32: {{.*}} +; DOT64: {{.*}} +; NODOT: {{.*}} diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product-i64.ll b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product-i64.ll new file mode 100644 index 0000000000000..459c896357809 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product-i64.ll @@ -0,0 +1,106 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 4 +; RUN: opt -passes=loop-vectorize -mattr=+v,+experimental-zvdot4a8i -riscv-v-register-bit-width-lmul=8 -tail-folding-policy=dont-fold-tail -S < %s | FileCheck %s --check-prefix=NOTAILFOLD +; RUN: opt -passes=loop-vectorize -mattr=+v,+experimental-zvdot4a8i -riscv-v-register-bit-width-lmul=8 -S < %s | FileCheck %s --check-prefix=TAILFOLD + +; Dot product with an i64 accumulator and i8 inputs. This forms a scale-8 +; partial reduction (i8 -> i64), which the RISC-V backend lowers using vdot4a* +; plus a widening accumulate. The i64 accumulator is 8x wider than the i8 +; input, so the scale-8 reduction needs a VF of at least (vscale x) 8; the +; -riscv-v-register-bit-width-lmul=8 override raises the VF range high enough +; to reach it, both with and without EVL tail folding. + +target triple = "riscv64-none-unknown-elf" + +define i64 @vqdot_i64(ptr %a, ptr %b) #0 { +; NOTAILFOLD-LABEL: define i64 @vqdot_i64( +; NOTAILFOLD-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] { +; NOTAILFOLD-NEXT: entry: +; NOTAILFOLD-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; NOTAILFOLD-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP4]], 3 +; NOTAILFOLD-NEXT: [[TMP5:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 16) +; NOTAILFOLD-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP5]] +; NOTAILFOLD-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; NOTAILFOLD: vector.ph: +; NOTAILFOLD-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP4]], 3 +; NOTAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] +; NOTAILFOLD-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; NOTAILFOLD-NEXT: br label [[VECTOR_BODY:%.*]] +; NOTAILFOLD: vector.body: +; NOTAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] +; NOTAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 1 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ] +; NOTAILFOLD-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]] +; NOTAILFOLD-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP0]], align 1 +; NOTAILFOLD-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]] +; NOTAILFOLD-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i8>, ptr [[TMP2]], align 1 +; NOTAILFOLD-NEXT: [[TMP6:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i64> +; NOTAILFOLD-NEXT: [[TMP7:%.*]] = sext <vscal... [truncated] `````````` </details> https://github.com/llvm/llvm-project/pull/215517 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
