https://github.com/adamant-pwn updated https://github.com/llvm/llvm-project/pull/230278
>From 58d124aa03c4478553dd3f68c3064af4e65deb44 Mon Sep 17 00:00:00 2001 From: Oleksandr Kulkov <[email protected]> Date: Thu, 8 Oct 2026 22:38:33 +0200 Subject: [PATCH 1/2] [Clang][X86] Resolve target_clones calls through GlobalOpt Emit runtime feature predicates for x86 target_clones and provide the TTI hooks used by GlobalOpt. Share the existing resolver priority calculation through TargetParser so static and runtime selection use the same ordering. Leave CPU-model predicates and non-IFUNC dispatch unchanged. Cover matching and differing version families, local function pointers, global initialization, ISA levels and interposition. Closes #94949 --- clang/lib/Basic/Targets/X86.cpp | 31 +---- clang/lib/CodeGen/CodeGenModule.cpp | 14 +- .../test/CodeGen/attr-target-clones-inline.c | 85 ++++++++++++ .../llvm/TargetParser/X86TargetParser.h | 3 + .../lib/Target/X86/X86TargetTransformInfo.cpp | 26 ++++ llvm/lib/Target/X86/X86TargetTransformInfo.h | 4 + llvm/lib/TargetParser/X86TargetParser.cpp | 33 +++++ .../GlobalOpt/X86/resolve-fmv-ifunc.ll | 126 ++++++++++++++++++ 8 files changed, 291 insertions(+), 31 deletions(-) create mode 100644 clang/test/CodeGen/attr-target-clones-inline.c create mode 100644 llvm/test/Transforms/GlobalOpt/X86/resolve-fmv-ifunc.ll diff --git a/clang/lib/Basic/Targets/X86.cpp b/clang/lib/Basic/Targets/X86.cpp index 670c85a769d71b..718ac419f6c817 100644 --- a/clang/lib/Basic/Targets/X86.cpp +++ b/clang/lib/Basic/Targets/X86.cpp @@ -1349,37 +1349,8 @@ bool X86TargetInfo::validateCpuSupports(StringRef FeatureStr) const { .Default(false); } -static llvm::X86::ProcessorFeatures getFeature(StringRef Name) { - return llvm::StringSwitch<llvm::X86::ProcessorFeatures>(Name) -#define X86_FEATURE_COMPAT(ENUM, STR, PRIORITY, ABI_VALUE) \ - .Case(STR, llvm::X86::FEATURE_##ENUM) - -#include "llvm/TargetParser/X86TargetParser.def" - ; - // Note, this function should only be used after ensuring the value is - // correct, so it asserts if the value is out of range. -} - llvm::APInt X86TargetInfo::getFMVPriority(ArrayRef<StringRef> Features) const { - auto getPriority = [](StringRef Feature) -> unsigned { - // Valid CPUs have a 'key feature' that compares just better than its key - // feature. - using namespace llvm::X86; - CPUKind Kind = parseArchX86(Feature); - if (Kind != CK_None) { - ProcessorFeatures KeyFeature = getKeyFeature(Kind); - return (getFeaturePriority(KeyFeature) << 1) + 1; - } - // Now we know we have a feature, so get its priority and shift it a few so - // that we have sufficient room for the CPUs (above). - return getFeaturePriority(getFeature(Feature)) << 1; - }; - - unsigned Priority = 0; - for (StringRef Feature : Features) - if (!Feature.empty()) - Priority = std::max(Priority, getPriority(Feature)); - return llvm::APInt(32, Priority); + return llvm::APInt(32, llvm::X86::getFMVPriority(Features)); } bool X86TargetInfo::validateCPUSpecificCPUDispatch(StringRef Name) const { diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index 62ef11a697b1c2..32fe69dc6d9db8 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -3533,7 +3533,7 @@ bool CodeGenModule::GetCPUAndFeaturesAttributes(GlobalDecl GD, AddedAttr = true; } } - // Add metadata for AArch64 Function Multi Versioning. + // Add metadata for Function Multi Versioning. if (getTarget().getTriple().isAArch64()) { llvm::SmallVector<StringRef, 8> Feats; bool IsDefault = false; @@ -3556,6 +3556,18 @@ bool CodeGenModule::GetCPUAndFeaturesAttributes(GlobalDecl GD, Attrs.addAttribute("fmv-features", FMVFeatures.substr(1)); AddedAttr = true; } + } else if (getTarget().getTriple().isX86() && getTarget().supportsIFunc() && + TC) { + auto Arch = TC->getX86Architecture(GD.getMultiVersionIndex()); + // CPU-model checks cannot be represented by feature masks. + if (!Arch || Arch->starts_with("x86-64")) { + llvm::SmallVector<StringRef, 8> Feats; + TC->getX86Feature(Feats, GD.getMultiVersionIndex()); + if (Arch) + Feats.push_back(*Arch); + Attrs.addAttribute("fmv-features", llvm::join(Feats, ",")); + AddedAttr = true; + } } return AddedAttr; } diff --git a/clang/test/CodeGen/attr-target-clones-inline.c b/clang/test/CodeGen/attr-target-clones-inline.c new file mode 100644 index 00000000000000..0e4a930a408be1 --- /dev/null +++ b/clang/test/CodeGen/attr-target-clones-inline.c @@ -0,0 +1,85 @@ +// REQUIRES: x86-registered-target +// RUN: %clang_cc1 -triple x86_64-linux-gnu -O1 -disable-llvm-passes -emit-llvm %s -o - | FileCheck %s --check-prefix=ATTR +// RUN: %clang_cc1 -triple x86_64-linux-gnu -O2 -emit-llvm %s -o - | FileCheck %s --check-prefix=OPT +// RUN: %clang_cc1 -triple x86_64-linux-gnu -x c++ -O1 -disable-llvm-passes -emit-llvm %s -o - | FileCheck %s --check-prefix=ATTR +// RUN: %clang_cc1 -triple x86_64-linux-gnu -x c++ -O2 -emit-llvm %s -o - | FileCheck %s --check-prefixes=OPT,CXX +// RUN: %clang_cc1 -triple x86_64-windows-pc -O1 -disable-llvm-passes -emit-llvm %s -o - | FileCheck %s --check-prefix=WINDOWS + +// WINDOWS-NOT: "fmv-features" + +#ifdef __cplusplus +extern "C" { +#endif + +__attribute__((target_clones("default,sse4.2,avx2"))) +int callee(int x) { return x + 1; } + +// Reordering versions and calling through a local pointer still allow inlining. +__attribute__((target_clones("avx2,default,sse4.2"))) +int caller(int x) { + int (*p)(int) = callee; + return p(x); +} + +// OPT-LABEL: define{{.*}} i32 @caller.avx2. +// OPT-NOT: call +// OPT: add nsw i32 +// OPT: ret i32 +// OPT-LABEL: define{{.*}} i32 @caller.default. +// OPT-NOT: call +// OPT: add nsw i32 +// OPT: ret i32 +// OPT-LABEL: define{{.*}} i32 @caller.sse4.2. +// OPT-NOT: call +// OPT: add nsw i32 +// OPT: ret i32 + +// The runtime resolver can still choose callee.avx2 from either caller here. +__attribute__((target_clones("default,sse4.2"))) +int limited(int x) { return callee(x); } + +// OPT-LABEL: define{{.*}} i32 @limited.default. +// OPT: call i32 @callee( +// OPT-LABEL: define{{.*}} i32 @limited.sse4.2. +// OPT: call i32 @callee( + +__attribute__((target_clones("default,arch=x86-64-v3"))) +double widen(_Float16 x) { return (double)x; } + +__attribute__((target_clones("default,arch=x86-64-v3"))) +double wide_caller(_Float16 x) { return widen(x); } + +// OPT-LABEL: define{{.*}} double @wide_caller.default. +// OPT-NOT: call +// OPT: fpext half +// OPT: ret double +// OPT-LABEL: define{{.*}} double @wide_caller.arch_x86-64-v3. +// OPT-NOT: call +// OPT: fpext half +// OPT: ret double + +// CPU-model predicates are deliberately left to the runtime resolver. +__attribute__((target_clones("default,arch=haswell"))) +int cpu_callee(int x) { return x + 2; } + +__attribute__((target_clones("default,arch=haswell"))) +int cpu_caller(int x) { return cpu_callee(x); } + +// OPT-LABEL: define{{.*}} i32 @cpu_caller.default. +// OPT: call i32 @cpu_callee( +// OPT-LABEL: define{{.*}} i32 @cpu_caller.arch_haswell. +// OPT: call i32 @cpu_callee( + +#ifdef __cplusplus +} +// There is no enclosing FunctionDecl while emitting a global initializer. +int dynamic_init = callee(0); +// CXX-LABEL: define internal void @_GLOBAL__sub_I_ +// CXX: call i32 @callee( +#endif + +// Dispatch predicates, rather than the expanded ISA features, describe clones. +// ATTR-DAG: attributes #{{[0-9]+}} = { {{.*}}"fmv-features" {{.*}} } +// ATTR-DAG: attributes #{{[0-9]+}} = { {{.*}}"fmv-features"="sse4.2" {{.*}} } +// ATTR-DAG: attributes #{{[0-9]+}} = { {{.*}}"fmv-features"="avx2" {{.*}} } +// ATTR-DAG: attributes #{{[0-9]+}} = { {{.*}}"fmv-features"="x86-64-v3" {{.*}} } diff --git a/llvm/include/llvm/TargetParser/X86TargetParser.h b/llvm/include/llvm/TargetParser/X86TargetParser.h index 4552287507fd3c..7ef2f2c26a2023 100644 --- a/llvm/include/llvm/TargetParser/X86TargetParser.h +++ b/llvm/include/llvm/TargetParser/X86TargetParser.h @@ -177,6 +177,9 @@ LLVM_ABI std::array<uint32_t, 4> getCpuSupportsMask(ArrayRef<StringRef> FeatureStrs); LLVM_ABI unsigned getFeaturePriority(ProcessorFeatures Feat); +/// Return the runtime function-multiversioning priority of features or CPUs. +LLVM_ABI unsigned getFMVPriority(ArrayRef<StringRef> Features); + } // namespace X86 } // namespace llvm diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp index b1bac7fe4b25a2..cf1aac637a1537 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -57,6 +57,7 @@ #include "llvm/CodeGen/TargetLowering.h" #include "llvm/IR/InstIterator.h" #include "llvm/IR/IntrinsicInst.h" +#include "llvm/TargetParser/X86TargetParser.h" #include <optional> using namespace llvm; @@ -69,6 +70,31 @@ using namespace llvm; // //===----------------------------------------------------------------------===// +static SmallVector<StringRef, 8> getFMVFeatures(const Function &F) { + SmallVector<StringRef, 8> Features; + if (F.hasFnAttribute("fmv-features")) + F.getFnAttribute("fmv-features") + .getValueAsString() + .split(Features, ",", -1, false); + return Features; +} + +APInt X86TTIImpl::getFeatureMask(const Function &F) const { + auto Words = X86::getCpuSupportsMask(getFMVFeatures(F)); + APInt Mask(32 * Words.size(), 0); + for (auto [I, Word] : enumerate(Words)) + Mask |= APInt(Mask.getBitWidth(), Word) << (32 * I); + return Mask; +} + +APInt X86TTIImpl::getPriorityMask(const Function &F) const { + return APInt(32, X86::getFMVPriority(getFMVFeatures(F))); +} + +bool X86TTIImpl::isMultiversionedFunction(const Function &F) const { + return F.hasFnAttribute("fmv-features"); +} + // Helper struct to store/access costs for each cost kind. // TODO: Move this to allow other targets to use it? struct CostKindCosts { diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h index 160a71e1affbb4..5df3098703f0a0 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.h +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h @@ -41,6 +41,10 @@ class X86TTIImpl final : public BasicTTIImplBase<X86TTIImpl> { : BaseT(TM, F.getDataLayout()), ST(TM->getSubtargetImpl(F)), TLI(ST->getTargetLowering()) {} + APInt getFeatureMask(const Function &F) const override; + APInt getPriorityMask(const Function &F) const override; + bool isMultiversionedFunction(const Function &F) const override; + /// \name Scalar TTI Implementations /// @{ TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth) const override; diff --git a/llvm/lib/TargetParser/X86TargetParser.cpp b/llvm/lib/TargetParser/X86TargetParser.cpp index 384687702c7c25..0b2aef130432f0 100644 --- a/llvm/lib/TargetParser/X86TargetParser.cpp +++ b/llvm/lib/TargetParser/X86TargetParser.cpp @@ -829,3 +829,36 @@ unsigned llvm::X86::getFeaturePriority(ProcessorFeatures Feat) { llvm_unreachable("No Feature Priority for non-CPUSupports Features"); } } + +static llvm::X86::ProcessorFeatures getFeature(StringRef Name) { + return llvm::StringSwitch<llvm::X86::ProcessorFeatures>(Name) +#define X86_FEATURE_COMPAT(ENUM, STR, PRIORITY, ABI_VALUE) \ + .Case(STR, llvm::X86::FEATURE_##ENUM) + +#include "llvm/TargetParser/X86TargetParser.def" + ; + // Note, this function should only be used after ensuring the value is + // correct, so it asserts if the value is out of range. +} + +unsigned llvm::X86::getFMVPriority(ArrayRef<StringRef> Features) { + auto getPriority = [](StringRef Feature) -> unsigned { + // Valid CPUs have a 'key feature' that compares just better than its key + // feature. + using namespace llvm::X86; + CPUKind Kind = parseArchX86(Feature); + if (Kind != CK_None) { + ProcessorFeatures KeyFeature = getKeyFeature(Kind); + return (getFeaturePriority(KeyFeature) << 1) + 1; + } + // Now we know we have a feature, so get its priority and shift it a few so + // that we have sufficient room for the CPUs (above). + return getFeaturePriority(getFeature(Feature)) << 1; + }; + + unsigned Priority = 0; + for (StringRef Feature : Features) + if (!Feature.empty()) + Priority = std::max(Priority, getPriority(Feature)); + return Priority; +} diff --git a/llvm/test/Transforms/GlobalOpt/X86/resolve-fmv-ifunc.ll b/llvm/test/Transforms/GlobalOpt/X86/resolve-fmv-ifunc.ll new file mode 100644 index 00000000000000..b78b3644105442 --- /dev/null +++ b/llvm/test/Transforms/GlobalOpt/X86/resolve-fmv-ifunc.ll @@ -0,0 +1,126 @@ +; NOTE: This test uses separate bits for the runtime predicates, including the +; x86-64-v3 level. They are not expanded sets of code-generation features. +; REQUIRES: x86-registered-target +; RUN: opt -passes=globalopt -S %s | FileCheck %s + +target triple = "x86_64-unknown-linux-gnu" + +@features = external global i32 +@callee = weak_odr ifunc i32 (i32), ptr @callee.resolver +@caller = weak_odr ifunc i32 (i32), ptr @caller.resolver +@limited = weak_odr ifunc i32 (i32), ptr @limited.resolver +@preemptible = weak ifunc i32 (i32), ptr @callee.resolver +@cpu_callee = weak_odr ifunc i32 (i32), ptr @cpu_callee.resolver + +define ptr @callee.resolver() { + %features = load i32, ptr @features + %avx2 = icmp ne i32 %features, 0 + %result = select i1 %avx2, ptr @callee.avx2, ptr @callee.default + ret ptr %result +} + +define ptr @caller.resolver() { + %features = load i32, ptr @features + %avx2 = icmp ne i32 %features, 0 + %result = select i1 %avx2, ptr @caller.avx2, ptr @caller.default + ret ptr %result +} + +define ptr @limited.resolver() { + ret ptr @limited.default +} + +define ptr @cpu_callee.resolver() { + %features = load i32, ptr @features + %cpu = icmp ne i32 %features, 0 + %result = select i1 %cpu, ptr @callee.cpu, ptr @callee.default + ret ptr %result +} + +declare i32 @callee.avx2(i32) #1 +declare i32 @callee.default(i32) #0 +declare i32 @callee.cpu(i32) #2 + +define i32 @caller.avx2(i32 %x) #1 { +; CHECK-LABEL: define i32 @caller.avx2( +; CHECK: call i32 @callee.avx2( +; CHECK: call i32 @preemptible( +; CHECK: call i32 @cpu_callee( + %a = call i32 @callee(i32 %x) + %b = call i32 @preemptible(i32 %a) + %c = call i32 @cpu_callee(i32 %b) + ret i32 %c +} + +define i32 @caller.default(i32 %x) #0 { +; CHECK-LABEL: define i32 @caller.default( +; CHECK: call i32 @callee.default( + %a = call i32 @callee(i32 %x) + ret i32 %a +} + +define i32 @limited.default(i32 %x) #0 { +; CHECK-LABEL: define i32 @limited.default( +; CHECK: call i32 @callee( + %a = call i32 @callee(i32 %x) + ret i32 %a +} + +define i32 @ordinary(i32 %x) #3 { +; CHECK-LABEL: define i32 @ordinary( +; CHECK: call i32 @callee( + %a = call i32 @callee(i32 %x) + ret i32 %a +} + +@levels = weak_odr ifunc i32 (i32), ptr @levels.resolver +@level_caller = weak_odr ifunc i32 (i32), ptr @level_caller.resolver + +define ptr @levels.resolver() { + %features = load i32, ptr @features + %v3 = icmp eq i32 %features, 2 + %avx2 = icmp ne i32 %features, 0 + %low = select i1 %avx2, ptr @level.avx2, ptr @level.default + %result = select i1 %v3, ptr @level.v3, ptr %low + ret ptr %result +} + +define ptr @level_caller.resolver() { + %features = load i32, ptr @features + %v3 = icmp eq i32 %features, 2 + %avx2 = icmp ne i32 %features, 0 + %low = select i1 %avx2, ptr @level_caller.avx2, ptr @level_caller.default + %result = select i1 %v3, ptr @level_caller.v3, ptr %low + ret ptr %result +} + +declare i32 @level.default(i32) #0 +declare i32 @level.avx2(i32) #1 +declare i32 @level.v3(i32) #4 + +define i32 @level_caller.default(i32 %x) #0 { +; CHECK-LABEL: define i32 @level_caller.default( +; CHECK: call i32 @level.default( + %a = call i32 @levels(i32 %x) + ret i32 %a +} + +define i32 @level_caller.avx2(i32 %x) #1 { +; CHECK-LABEL: define i32 @level_caller.avx2( +; CHECK: call i32 @level.avx2( + %a = call i32 @levels(i32 %x) + ret i32 %a +} + +define i32 @level_caller.v3(i32 %x) #4 { +; CHECK-LABEL: define i32 @level_caller.v3( +; CHECK: call i32 @level.v3( + %a = call i32 @levels(i32 %x) + ret i32 %a +} + +attributes #0 = { "fmv-features" } +attributes #1 = { "fmv-features"="avx2" } +attributes #2 = { "target-cpu"="haswell" } +attributes #3 = { "target-features"="+avx2" } +attributes #4 = { "fmv-features"="x86-64-v3" } >From 1968eadff4b80a48ce0fa500c353cdf3dfc4240e Mon Sep 17 00:00:00 2001 From: Oleksandr Kulkov <[email protected]> Date: Thu, 8 Oct 2026 23:19:11 +0200 Subject: [PATCH 2/2] [Clang][X86] Simplify FMV attribute handling Use the single x86 clone predicate directly instead of constructing and joining a temporary vector. Read absent fmv-features attributes as empty strings, as supported by the existing Attribute API. --- clang/lib/CodeGen/CodeGenModule.cpp | 8 +++----- llvm/lib/Target/X86/X86TargetTransformInfo.cpp | 7 +++---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index 32fe69dc6d9db8..3e71525027b1c0 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -3561,11 +3561,9 @@ bool CodeGenModule::GetCPUAndFeaturesAttributes(GlobalDecl GD, auto Arch = TC->getX86Architecture(GD.getMultiVersionIndex()); // CPU-model checks cannot be represented by feature masks. if (!Arch || Arch->starts_with("x86-64")) { - llvm::SmallVector<StringRef, 8> Feats; - TC->getX86Feature(Feats, GD.getMultiVersionIndex()); - if (Arch) - Feats.push_back(*Arch); - Attrs.addAttribute("fmv-features", llvm::join(Feats, ",")); + StringRef Feature = + Arch.value_or(TC->getFeatureStr(GD.getMultiVersionIndex())); + Attrs.addAttribute("fmv-features", Feature == "default" ? "" : Feature); AddedAttr = true; } } diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp index cf1aac637a1537..1122c1ebe823f5 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -72,10 +72,9 @@ using namespace llvm; static SmallVector<StringRef, 8> getFMVFeatures(const Function &F) { SmallVector<StringRef, 8> Features; - if (F.hasFnAttribute("fmv-features")) - F.getFnAttribute("fmv-features") - .getValueAsString() - .split(Features, ",", -1, false); + F.getFnAttribute("fmv-features") + .getValueAsString() + .split(Features, ",", -1, false); return Features; } _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
