https://github.com/sdesmalen-arm created https://github.com/llvm/llvm-project/pull/231899
This removes the need for the `StreamingIncompatibility` enum and makes the code a little simpler. This PR also moves `hasPossibleIncompatibleOps` to AArch64TTIimpl, so that we can query the costmodel in a follow-up PR. >From 02627b3155c8c92bf1427c024cd44e7fa85f98a8 Mon Sep 17 00:00:00 2001 From: Sander de Smalen <[email protected]> Date: Tue, 6 Oct 2026 14:52:23 +0000 Subject: [PATCH] [AArch64][SME] NFC: Inline getCompatibilityForChangeToStreamingMode. This removes the need for the `StreamingIncompatibility` enum and makes the code a little simpler. This PR also moves hasPossibleIncompatibleOps to AArch64TTIimpl, so that we can query the costmodel in a follow-up PR. --- .../AArch64/AArch64TargetTransformInfo.cpp | 105 +++++++----------- .../AArch64/AArch64TargetTransformInfo.h | 3 + 2 files changed, 46 insertions(+), 62 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index 79a0472b917a9b..f94555a4e3ddb1 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -172,63 +172,13 @@ static cl::opt<TailFoldingOption, true, cl::parser<std::string>> SVETailFolding( "\nnoreverse Inverse of above"), cl::location(TailFoldingOptionLoc)); -// Enum to record why an operation is incompatible in a different streaming -// mode. -enum class StreamingIncompatibility : uint8_t { - None = 0, - VScaleDependent = 1 << 0, - HasIncompatibleInstruction = 1 << 1, - FullyIncompatible = VScaleDependent | HasIncompatibleInstruction, - LLVM_MARK_AS_BITMASK_ENUM(/*LargestFlag=*/HasIncompatibleInstruction) -}; - -/// Returns an enum describing whether \p I is an instruction that is -/// incompatible in the mode of the caller. -static StreamingIncompatibility -getCompatibilityForChangeToStreamingMode(const Instruction *I) { - if (auto *II = dyn_cast<IntrinsicInst>(I)) { - unsigned IID = II->getIntrinsicID(); - switch (IID) { - default: - // Conservatively disallow target intrinsics in other modes. - if (Intrinsic::isTargetIntrinsic(IID)) - return StreamingIncompatibility::FullyIncompatible; - break; - case Intrinsic::vscale: - return StreamingIncompatibility::VScaleDependent; - case Intrinsic::masked_gather: - case Intrinsic::masked_scatter: - case Intrinsic::masked_compressstore: - case Intrinsic::experimental_vector_histogram_add: - // Instructions that are not available in streaming mode. - // Fixed-length operations can still be code-generated by scalarizing. - // FIXME: Consider FEAT_FA64. - if (I->getOperand(0)->getType()->isScalableTy()) - return StreamingIncompatibility::FullyIncompatible; - break; - } - } - - // Inlining operations on scalable vectors is rejected when - // vscale-dependent operations cannot be safely inlined. - if (I->getType()->isScalableTy() || - any_of(I->operand_values(), - [](const Value *V) { return V->getType()->isScalableTy(); }) || - (isa<GetElementPtrInst>(I) && - cast<GetElementPtrInst>(I)->getSourceElementType()->isScalableTy()) || - (isa<AllocaInst>(I) && cast<AllocaInst>(I)->isScalable())) - return StreamingIncompatibility::VScaleDependent; - - return StreamingIncompatibility::None; -} - /// Returns true if the function has explicit operations that can only be /// lowered using incompatible instructions for the selected mode. This also /// returns true if the function F may use or modify ZA state. -static bool hasPossibleIncompatibleOps(const Function *F, - const AArch64TargetLowering &TLI, - bool ConsiderZA, bool ConsiderZT, - bool ConsiderSM) { +bool AArch64TTIImpl::hasPossibleIncompatibleOps(const Function *F, + bool ConsiderZA, + bool ConsiderZT, + bool ConsiderSM) const { if (!ConsiderZA && !ConsiderZT && !ConsiderSM) return false; @@ -247,7 +197,7 @@ static bool hasPossibleIncompatibleOps(const Function *F, for (const BasicBlock &BB : *F) { for (const Instruction &I : BB) { if (auto *CB = dyn_cast<CallBase>(&I)) { - SMECallAttrs CallAttrs(*CB, &TLI.getRuntimeLibcallsInfo()); + SMECallAttrs CallAttrs(*CB, &TLI->getRuntimeLibcallsInfo()); // Inline asm must be rejected as it could use SME state. if (CB->isInlineAsm()) @@ -272,17 +222,49 @@ static bool hasPossibleIncompatibleOps(const Function *F, // Inlining operations on fixed-length vectors when the streaming // mode does not match is rejected because performance may be impacted. // This decision should eventually be moved to the cost-model. + // When IsVScaleDependent is true, then we allow fixed-length vectors as + // the operation may be something like llvm.vector.match, which takes both + // a scalable and a fixed-length vector. if (!IsAlwaysInline && (isa<FixedVectorType>(I.getType()) || any_of(I.operand_values(), [](const Value *V) { return isa<FixedVectorType>(V->getType()); }))) return true; - StreamingIncompatibility C = getCompatibilityForChangeToStreamingMode(&I); - if (any(C & StreamingIncompatibility::HasIncompatibleInstruction) || - (!AssumeVScaleIsEquivalent && - any(C & StreamingIncompatibility::VScaleDependent))) - return true; + if (auto *II = dyn_cast<IntrinsicInst>(&I)) { + unsigned IID = II->getIntrinsicID(); + switch (IID) { + default: + if (!Intrinsic::isTargetIntrinsic(IID)) + break; + // Conservatively assume target intrinsics are vscale dependent + // or invalid in the current mode. + return true; + case Intrinsic::masked_gather: + case Intrinsic::masked_scatter: + case Intrinsic::masked_compressstore: + case Intrinsic::experimental_vector_histogram_add: + // Instructions that are not available in streaming mode. + // Fixed-length operations can still be code-generated by scalarizing. + // FIXME: Consider FEAT_FA64. + if (I.getOperand(0)->getType()->isScalableTy()) + return true; + break; + case Intrinsic::vscale: + return !AssumeVScaleIsEquivalent; + } + } + + // Determine if the operation is vscale dependent by looking at it's + // operand/result types. + bool IsVScaleDependent = false; + if (I.getType()->isScalableTy() || + any_of(I.operand_values(), + [](const Value *V) { return V->getType()->isScalableTy(); }) || + (isa<GetElementPtrInst>(I) && + cast<GetElementPtrInst>(I).getSourceElementType()->isScalableTy()) || + (isa<AllocaInst>(I) && cast<AllocaInst>(I).isScalable())) + return !AssumeVScaleIsEquivalent; } } @@ -387,8 +369,7 @@ bool AArch64TTIImpl::areInlineCompatible(const Function *Caller, if (ConsiderSM && (Caller->isStrictFP() || Callee->isStrictFP())) return false; - if (hasPossibleIncompatibleOps(Callee, *getTLI(), ConsiderZA, ConsiderZT, - ConsiderSM)) + if (hasPossibleIncompatibleOps(Callee, ConsiderZA, ConsiderZT, ConsiderSM)) return false; return BaseT::areInlineCompatible(Caller, Callee); diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h index 65c6364cffc35c..2563fb50df3a43 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h @@ -576,6 +576,9 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> { return ST->enableAggressiveInterleaving(); } /// @} + + bool hasPossibleIncompatibleOps(const Function *F, bool ConsiderZA, + bool ConsiderZT, bool ConsiderSM) const; }; } // end namespace llvm _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
