https://github.com/Stylie777 updated https://github.com/llvm/llvm-project/pull/217609
>From 785b0aa4f2e22c887458160186ad84bcae3befa0 Mon Sep 17 00:00:00 2001 From: Jack Styles <[email protected]> Date: Thu, 20 Aug 2026 13:52:13 +0100 Subject: [PATCH 1/4] [AArch64][SDAG] Enable +sme2p2/+sve2p2 lowering for MSTORE Following on from #215219 it will now be possible to lower llvm.masked.compressstore for sve2p2/sme2p2 using compact for i8/i16/f16/bf16 types. --- .../Target/AArch64/AArch64ISelLowering.cpp | 6 +- .../AArch64/AArch64TargetTransformInfo.h | 5 ++ .../CostModel/AArch64/masked_compress_load.ll | 74 +++++++++---------- .../sve-masked-compressstore-sve2p2.ll | 48 ++++++++++-- 4 files changed, 88 insertions(+), 45 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index c1ca0f789e534..6a08bf6bc0e38 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -2283,6 +2283,11 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, // lowering for non-compressing masked stores. setOperationAction(ISD::MSTORE, VT, Custom); } + if (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2()) { + // With +sve2p2/+sme2p2 the full range of vector types are supported. + for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv8f16, MVT::nxv8bf16}) + setOperationAction(ISD::MSTORE, VT, Custom); + } // Histcnt is SVE2 only if (Subtarget->hasSVE2()) { @@ -33927,7 +33932,6 @@ SDValue AArch64TargetLowering::LowerMSTORE(SDValue Op, ISD::INTRINSIC_WO_CHAIN, DL, MVT::i64, DAG.getTargetConstant(Intrinsic::aarch64_sve_cntp, DL, MVT::i64), Store->getMask(), Store->getMask()); - SDValue CompressedValue = DAG.getNode(ISD::VECTOR_COMPRESS, DL, VT, Store->getValue(), Store->getMask(), DAG.getPOISON(VT)); diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h index 19de90427cb74..64337ff4d5db7 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h @@ -334,6 +334,11 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> { } bool isElementTypeLegalForCompressStore(Type *Ty) const { + if ((ST->hasSVE2p2() || ST->hasSME2p2()) && + ((Ty->isIntegerTy(8) || Ty->isIntegerTy(16)) || + ((Ty->isHalfTy() || Ty->isBFloatTy()) && + Ty->getScalarSizeInBits() == 16))) + return true; return Ty->isFloatTy() || Ty->isDoubleTy() || Ty->isIntegerTy(32) || Ty->isIntegerTy(64); } diff --git a/llvm/test/Analysis/CostModel/AArch64/masked_compress_load.ll b/llvm/test/Analysis/CostModel/AArch64/masked_compress_load.ll index cb59f1016a4d1..7cc880ce5b66d 100644 --- a/llvm/test/Analysis/CostModel/AArch64/masked_compress_load.ll +++ b/llvm/test/Analysis/CostModel/AArch64/masked_compress_load.ll @@ -95,21 +95,21 @@ define void @fixed() { ; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2i8.p0(<2 x i8> poison, ptr poison, <2 x i1> poison) ; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4i8.p0(<4 x i8> poison, ptr poison, <4 x i1> poison) ; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v8i8.p0(<8 x i8> poison, ptr poison, <8 x i1> poison) -; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v16i8.p0(<16 x i8> poison, ptr poison, <16 x i1> poison) +; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v16i8.p0(<16 x i8> poison, ptr poison, <16 x i1> poison) ; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2i16.p0(<2 x i16> poison, ptr poison, <2 x i1> poison) ; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4i16.p0(<4 x i16> poison, ptr poison, <4 x i1> poison) -; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v8i16.p0(<8 x i16> poison, ptr poison, <8 x i1> poison) +; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v8i16.p0(<8 x i16> poison, ptr poison, <8 x i1> poison) ; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2i32.p0(<2 x i32> poison, ptr poison, <2 x i1> poison) ; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> poison, ptr poison, <4 x i1> poison) ; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v2i64.p0(<2 x i64> poison, ptr poison, <2 x i1> poison) ; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2f16.p0(<2 x half> poison, ptr poison, <2 x i1> poison) ; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4f16.p0(<4 x half> poison, ptr poison, <4 x i1> poison) -; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v8f16.p0(<8 x half> poison, ptr poison, <8 x i1> poison) +; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v8f16.p0(<8 x half> poison, ptr poison, <8 x i1> poison) ; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2f32.p0(<2 x float> poison, ptr poison, <2 x i1> poison) ; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v4f32.p0(<4 x float> poison, ptr poison, <4 x i1> poison) ; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v2f64.p0(<2 x double> poison, ptr poison, <2 x i1> poison) ; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v4i64.p0(<4 x i64> poison, ptr poison, <4 x i1> poison) -; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v32f16.p0(<32 x half> poison, ptr poison, <32 x i1> poison) +; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 8 for: call void @llvm.masked.compressstore.v32f16.p0(<32 x half> poison, ptr poison, <32 x i1> poison) ; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; entry: @@ -142,25 +142,25 @@ entry: define void @scalable() { ; SVE2p2-SME2p2-NON-STREAMING-LABEL: 'scalable' -; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison) -; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison) -; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison) -; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison) -; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison) -; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison) -; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison) +; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison) +; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison) +; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison) +; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison) +; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison) +; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison) +; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison) ; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison) ; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison) ; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison) -; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison) -; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison) -; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison) +; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison) +; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison) +; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison) ; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison) ; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison) ; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison) ; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv1i64.p0(<vscale x 1 x i64> poison, ptr poison, <vscale x 1 x i1> poison) ; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 8 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison) -; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison) +; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison) ; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; ; SVE2p2-SME2p2-STREAMING-LABEL: 'scalable' @@ -186,25 +186,25 @@ define void @scalable() { ; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; ; SVE2p2-SME2p2-LABEL: 'scalable' -; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison) -; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison) -; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison) -; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison) -; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison) -; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison) -; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison) +; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison) +; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison) +; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison) +; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison) +; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison) +; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison) +; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison) ; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison) ; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison) ; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison) -; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison) -; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison) -; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison) +; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison) +; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison) +; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison) ; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison) ; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison) ; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison) ; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv1i64.p0(<vscale x 1 x i64> poison, ptr poison, <vscale x 1 x i1> poison) ; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 8 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison) -; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison) +; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison) ; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; ; SVE-ONLY-LABEL: 'scalable' @@ -230,25 +230,25 @@ define void @scalable() { ; SVE-ONLY-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; ; SVE2p2-SME2p2-SVE256-LABEL: 'scalable' -; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison) -; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison) -; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison) -; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison) -; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison) -; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison) -; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison) +; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison) +; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison) +; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison) +; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison) +; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison) +; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison) +; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison) ; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison) ; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison) ; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison) -; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison) -; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison) -; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison) +; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison) +; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison) +; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison) ; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison) ; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison) ; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison) ; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv1i64.p0(<vscale x 1 x i64> poison, ptr poison, <vscale x 1 x i1> poison) ; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 8 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison) -; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison) +; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison) ; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; entry: diff --git a/llvm/test/CodeGen/AArch64/sve-masked-compressstore-sve2p2.ll b/llvm/test/CodeGen/AArch64/sve-masked-compressstore-sve2p2.ll index 92ecc3c83e2c5..5655d53b5ddaa 100644 --- a/llvm/test/CodeGen/AArch64/sve-masked-compressstore-sve2p2.ll +++ b/llvm/test/CodeGen/AArch64/sve-masked-compressstore-sve2p2.ll @@ -1,17 +1,51 @@ -; RUN: llc -mtriple=aarch64 -mattr=+sve2p2 < %s - -;; These masked.compressstore operations could be natively supported with +sve2p2 -;; (or by promoting to 32/64 bit elements + a truncstore), but currently are not -;; supported. - -; XFAIL: * +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 +; RUN: llc -mtriple=aarch64 -mattr=+sve2p2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-BASE --allow-unused-prefixes +; RUN: llc -mtriple=aarch64 -aarch64-sve-vector-bits-min=256 -mattr=+sve2p2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-VL256 --allow-unused-prefixes define void @test_compressstore_nxv8i16(ptr %p, <vscale x 8 x i16> %vec, <vscale x 8 x i1> %mask) { +; CHECK-LABEL: test_compressstore_nxv8i16: +; CHECK: // %bb.0: +; CHECK-NEXT: cntp x8, p0, p0.h +; CHECK-NEXT: compact z0.h, p0, z0.h +; CHECK-NEXT: whilelo p1.h, xzr, x8 +; CHECK-NEXT: st1h { z0.h }, p1, [x0] +; CHECK-NEXT: ret tail call void @llvm.masked.compressstore.nxv8i16(<vscale x 8 x i16> %vec, ptr align 2 %p, <vscale x 8 x i1> %mask) ret void } define void @test_compressstore_nxv16i8(ptr %p, <vscale x 16 x i8> %vec, <vscale x 16 x i1> %mask) { +; CHECK-LABEL: test_compressstore_nxv16i8: +; CHECK: // %bb.0: +; CHECK-NEXT: cntp x8, p0, p0.b +; CHECK-NEXT: compact z0.b, p0, z0.b +; CHECK-NEXT: whilelo p1.b, xzr, x8 +; CHECK-NEXT: st1b { z0.b }, p1, [x0] +; CHECK-NEXT: ret tail call void @llvm.masked.compressstore.nxv16i8(<vscale x 16 x i8> %vec, ptr align 1 %p, <vscale x 16 x i1> %mask) ret void } + +define void @test_compressstore_nxv8f16(ptr %p, <vscale x 8 x half> %vec, <vscale x 8 x i1> %mask) { +; CHECK-LABEL: test_compressstore_nxv8f16: +; CHECK: // %bb.0: +; CHECK-NEXT: cntp x8, p0, p0.h +; CHECK-NEXT: compact z0.h, p0, z0.h +; CHECK-NEXT: whilelo p1.h, xzr, x8 +; CHECK-NEXT: st1h { z0.h }, p1, [x0] +; CHECK-NEXT: ret + tail call void @llvm.masked.compressstore.nxv8f16(<vscale x 8 x half> %vec, ptr align 1 %p, <vscale x 8 x i1> %mask) + ret void +} + +define void @test_compressstore_nxv8bf16(ptr %p, <vscale x 8 x bfloat> %vec, <vscale x 8 x i1> %mask) { +; CHECK-LABEL: test_compressstore_nxv8bf16: +; CHECK: // %bb.0: +; CHECK-NEXT: cntp x8, p0, p0.h +; CHECK-NEXT: compact z0.h, p0, z0.h +; CHECK-NEXT: whilelo p1.h, xzr, x8 +; CHECK-NEXT: st1h { z0.h }, p1, [x0] +; CHECK-NEXT: ret + tail call void @llvm.masked.compressstore.nxv8bf16(<vscale x 8 x bfloat> %vec, ptr align 1 %p, <vscale x 8 x i1> %mask) + ret void +} >From f8a722e61b99837d18fda72e3460662e8d5f8292 Mon Sep 17 00:00:00 2001 From: Jack Styles <[email protected]> Date: Mon, 24 Aug 2026 10:03:30 +0100 Subject: [PATCH 2/4] Remove whitespace change --- llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 6a08bf6bc0e38..07b28395c0dfe 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -33932,6 +33932,7 @@ SDValue AArch64TargetLowering::LowerMSTORE(SDValue Op, ISD::INTRINSIC_WO_CHAIN, DL, MVT::i64, DAG.getTargetConstant(Intrinsic::aarch64_sve_cntp, DL, MVT::i64), Store->getMask(), Store->getMask()); + SDValue CompressedValue = DAG.getNode(ISD::VECTOR_COMPRESS, DL, VT, Store->getValue(), Store->getMask(), DAG.getPOISON(VT)); >From 8e09f73faf8315ede14485f2797935482a9a3452 Mon Sep 17 00:00:00 2001 From: Jack Styles <[email protected]> Date: Mon, 24 Aug 2026 10:48:06 +0100 Subject: [PATCH 3/4] Enable Streaming SME2p2 --- llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 11 ++++++----- .../AArch64/sve-masked-compressstore-sve2p2.ll | 3 +++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 07b28395c0dfe..639d5b079f734 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -2283,11 +2283,6 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, // lowering for non-compressing masked stores. setOperationAction(ISD::MSTORE, VT, Custom); } - if (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2()) { - // With +sve2p2/+sme2p2 the full range of vector types are supported. - for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv8f16, MVT::nxv8bf16}) - setOperationAction(ISD::MSTORE, VT, Custom); - } // Histcnt is SVE2 only if (Subtarget->hasSVE2()) { @@ -2308,6 +2303,12 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, } } + if ((Subtarget->isSVEAvailable() && Subtarget->hasSVE2p2()) || (Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSME2p2())) { + // With +sve2p2/+sme2p2 the full range of vector types are supported. + for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv8f16, MVT::nxv8bf16}) + setOperationAction(ISD::MSTORE, VT, Custom); + } + if (Subtarget->hasMOPS() && Subtarget->hasMTE()) { // Only required for llvm.aarch64.mops.memset.tag setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i8, Custom); diff --git a/llvm/test/CodeGen/AArch64/sve-masked-compressstore-sve2p2.ll b/llvm/test/CodeGen/AArch64/sve-masked-compressstore-sve2p2.ll index 5655d53b5ddaa..4fca173a712ba 100644 --- a/llvm/test/CodeGen/AArch64/sve-masked-compressstore-sve2p2.ll +++ b/llvm/test/CodeGen/AArch64/sve-masked-compressstore-sve2p2.ll @@ -1,6 +1,9 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 ; RUN: llc -mtriple=aarch64 -mattr=+sve2p2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-BASE --allow-unused-prefixes ; RUN: llc -mtriple=aarch64 -aarch64-sve-vector-bits-min=256 -mattr=+sve2p2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-VL256 --allow-unused-prefixes +; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme2p2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SVE2p2 --allow-unused-prefixes +; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2p2 -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SME2p2-STREAMING --allow-unused-prefixes +; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2p2,+sve2p2 -force-streaming-compatible < %s | FileCheck %s --check-prefixes=CHECK,CHECK-STREAMING-COMPAT --allow-unused-prefixes define void @test_compressstore_nxv8i16(ptr %p, <vscale x 8 x i16> %vec, <vscale x 8 x i1> %mask) { ; CHECK-LABEL: test_compressstore_nxv8i16: >From 3865d30d79af0986c28e23b2f713e22d70ebd905 Mon Sep 17 00:00:00 2001 From: Jack Styles <[email protected]> Date: Mon, 24 Aug 2026 10:50:40 +0100 Subject: [PATCH 4/4] format --- llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 639d5b079f734..a0fb5365cd843 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -2303,7 +2303,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, } } - if ((Subtarget->isSVEAvailable() && Subtarget->hasSVE2p2()) || (Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSME2p2())) { + if ((Subtarget->isSVEAvailable() && Subtarget->hasSVE2p2()) || + (Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSME2p2())) { // With +sve2p2/+sme2p2 the full range of vector types are supported. for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv8f16, MVT::nxv8bf16}) setOperationAction(ISD::MSTORE, VT, Custom); _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
