llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-openmp Author: Matt Arsenault (arsenm) <details> <summary>Changes</summary> When the unrolled loop's CanonicalLoopInfo is needed by an enclosing loop directive and no factor was given, unrollLoopPartial computed one by constructing its own TargetMachine with default TargetOptions and a standalone FunctionAnalysisManager, then running LoopUnrollPass's cost model on the function while it was still being built. This is a layering violation. Frontends should not be using codegen optimization information to construct the original program. This will not have a properly configured TargetTransformInfo from a pass manager. The cost model is also not meaningful on incomplete, unoptimized IR. Use a factor of 2, matching what Sema's OpenMP loop transformation uses for a partial clause without a factor. This removes the hidden -openmp-ir-builder-unroll-threshold-factor option. Co-authored-by: Claude Opus 5.5 <noreply@<!-- -->anthropic.com> --- Full diff: https://github.com/llvm/llvm-project/pull/240030.diff 5 Files Affected: - (modified) clang/test/OpenMP/irbuilder_unroll_partial_heuristic_constant_for.c (+4-4) - (modified) clang/test/OpenMP/irbuilder_unroll_partial_heuristic_runtime_for.c (+4-4) - (modified) clang/test/OpenMP/irbuilder_unroll_unroll_partial_heuristic.c (+4-4) - (modified) llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h (+4-8) - (modified) llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp (+2-189) ``````````diff diff --git a/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_constant_for.c b/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_constant_for.c index acf0e5164411c1..dd2bbe63b1e485 100644 --- a/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_constant_for.c +++ b/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_constant_for.c @@ -59,8 +59,8 @@ void unroll_partial_heuristic_constant_for(float *a, float *b, float *c, float * // CHECK-NEXT: [[DOTCOUNT:%.*]] = load i32, ptr [[DOTCOUNT_ADDR]], align 4 // CHECK-NEXT: br label [[OMP_LOOP_PREHEADER:%.*]] // CHECK: omp_loop.preheader: -// CHECK-NEXT: [[TMP3:%.*]] = udiv i32 [[DOTCOUNT]], 4 -// CHECK-NEXT: [[TMP4:%.*]] = urem i32 [[DOTCOUNT]], 4 +// CHECK-NEXT: [[TMP3:%.*]] = udiv i32 [[DOTCOUNT]], 2 +// CHECK-NEXT: [[TMP4:%.*]] = urem i32 [[DOTCOUNT]], 2 // CHECK-NEXT: [[TMP5:%.*]] = icmp ne i32 [[TMP4]], 0 // CHECK-NEXT: [[TMP6:%.*]] = zext i1 [[TMP5]] to i32 // CHECK-NEXT: [[OMP_FLOOR0_TRIPCOUNT:%.*]] = add nuw i32 [[TMP3]], [[TMP6]] @@ -86,7 +86,7 @@ void unroll_partial_heuristic_constant_for(float *a, float *b, float *c, float * // CHECK: omp_floor0.body: // CHECK-NEXT: [[TMP12:%.*]] = add i32 [[OMP_FLOOR0_IV]], [[TMP8]] // CHECK-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP12]], [[TMP3]] -// CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP4]], i32 4 +// CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP4]], i32 2 // CHECK-NEXT: br label [[OMP_TILE0_PREHEADER:%.*]] // CHECK: omp_tile0.preheader: // CHECK-NEXT: br label [[OMP_TILE0_HEADER:%.*]] @@ -97,7 +97,7 @@ void unroll_partial_heuristic_constant_for(float *a, float *b, float *c, float * // CHECK-NEXT: [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], [[TMP14]] // CHECK-NEXT: br i1 [[OMP_TILE0_CMP]], label [[OMP_TILE0_BODY:%.*]], label [[OMP_TILE0_EXIT:%.*]] // CHECK: omp_tile0.body: -// CHECK-NEXT: [[TMP15:%.*]] = mul nuw i32 4, [[TMP12]] +// CHECK-NEXT: [[TMP15:%.*]] = mul nuw i32 2, [[TMP12]] // CHECK-NEXT: [[TMP16:%.*]] = add nuw i32 [[TMP15]], [[OMP_TILE0_IV]] // CHECK-NEXT: br label [[OMP_LOOP_BODY:%.*]] // CHECK: omp_loop.body: diff --git a/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_runtime_for.c b/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_runtime_for.c index 8fee5773e53b46..0ccd0c2825fcd8 100644 --- a/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_runtime_for.c +++ b/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_runtime_for.c @@ -61,8 +61,8 @@ void unroll_partial_heuristic_runtime_for(int n, float *a, float *b, float *c, f // CHECK-NEXT: [[DOTCOUNT:%.*]] = load i32, ptr [[DOTCOUNT_ADDR]], align 4 // CHECK-NEXT: br label [[OMP_LOOP_PREHEADER:%.*]] // CHECK: omp_loop.preheader: -// CHECK-NEXT: [[TMP4:%.*]] = udiv i32 [[DOTCOUNT]], 4 -// CHECK-NEXT: [[TMP5:%.*]] = urem i32 [[DOTCOUNT]], 4 +// CHECK-NEXT: [[TMP4:%.*]] = udiv i32 [[DOTCOUNT]], 2 +// CHECK-NEXT: [[TMP5:%.*]] = urem i32 [[DOTCOUNT]], 2 // CHECK-NEXT: [[TMP6:%.*]] = icmp ne i32 [[TMP5]], 0 // CHECK-NEXT: [[TMP7:%.*]] = zext i1 [[TMP6]] to i32 // CHECK-NEXT: [[OMP_FLOOR0_TRIPCOUNT:%.*]] = add nuw i32 [[TMP4]], [[TMP7]] @@ -88,7 +88,7 @@ void unroll_partial_heuristic_runtime_for(int n, float *a, float *b, float *c, f // CHECK: omp_floor0.body: // CHECK-NEXT: [[TMP13:%.*]] = add i32 [[OMP_FLOOR0_IV]], [[TMP9]] // CHECK-NEXT: [[TMP14:%.*]] = icmp eq i32 [[TMP13]], [[TMP4]] -// CHECK-NEXT: [[TMP15:%.*]] = select i1 [[TMP14]], i32 [[TMP5]], i32 4 +// CHECK-NEXT: [[TMP15:%.*]] = select i1 [[TMP14]], i32 [[TMP5]], i32 2 // CHECK-NEXT: br label [[OMP_TILE0_PREHEADER:%.*]] // CHECK: omp_tile0.preheader: // CHECK-NEXT: br label [[OMP_TILE0_HEADER:%.*]] @@ -99,7 +99,7 @@ void unroll_partial_heuristic_runtime_for(int n, float *a, float *b, float *c, f // CHECK-NEXT: [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], [[TMP15]] // CHECK-NEXT: br i1 [[OMP_TILE0_CMP]], label [[OMP_TILE0_BODY:%.*]], label [[OMP_TILE0_EXIT:%.*]] // CHECK: omp_tile0.body: -// CHECK-NEXT: [[TMP16:%.*]] = mul nuw i32 4, [[TMP13]] +// CHECK-NEXT: [[TMP16:%.*]] = mul nuw i32 2, [[TMP13]] // CHECK-NEXT: [[TMP17:%.*]] = add nuw i32 [[TMP16]], [[OMP_TILE0_IV]] // CHECK-NEXT: br label [[OMP_LOOP_BODY:%.*]] // CHECK: omp_loop.body: diff --git a/clang/test/OpenMP/irbuilder_unroll_unroll_partial_heuristic.c b/clang/test/OpenMP/irbuilder_unroll_unroll_partial_heuristic.c index 24d42d265d6a6e..20226e21c605a9 100644 --- a/clang/test/OpenMP/irbuilder_unroll_unroll_partial_heuristic.c +++ b/clang/test/OpenMP/irbuilder_unroll_unroll_partial_heuristic.c @@ -44,8 +44,8 @@ void unroll_unroll_partial_heuristic(float *a, float *b, float *c, float *d) { // CHECK-NEXT: [[DOTCOUNT:%.*]] = load i32, ptr [[DOTCOUNT_ADDR]], align 4 // CHECK-NEXT: br label [[OMP_LOOP_PREHEADER:%.*]] // CHECK: omp_loop.preheader: -// CHECK-NEXT: [[TMP3:%.*]] = udiv i32 [[DOTCOUNT]], 8 -// CHECK-NEXT: [[TMP4:%.*]] = urem i32 [[DOTCOUNT]], 8 +// CHECK-NEXT: [[TMP3:%.*]] = udiv i32 [[DOTCOUNT]], 2 +// CHECK-NEXT: [[TMP4:%.*]] = urem i32 [[DOTCOUNT]], 2 // CHECK-NEXT: [[TMP5:%.*]] = icmp ne i32 [[TMP4]], 0 // CHECK-NEXT: [[TMP6:%.*]] = zext i1 [[TMP5]] to i32 // CHECK-NEXT: [[OMP_FLOOR0_TRIPCOUNT:%.*]] = add nuw i32 [[TMP3]], [[TMP6]] @@ -60,7 +60,7 @@ void unroll_unroll_partial_heuristic(float *a, float *b, float *c, float *d) { // CHECK-NEXT: br i1 [[OMP_FLOOR0_CMP]], label [[OMP_FLOOR0_BODY:%.*]], label [[OMP_FLOOR0_EXIT:%.*]] // CHECK: omp_floor0.body: // CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[OMP_FLOOR0_IV]], [[TMP3]] -// CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP4]], i32 8 +// CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP4]], i32 2 // CHECK-NEXT: br label [[OMP_TILE0_PREHEADER:%.*]] // CHECK: omp_tile0.preheader: // CHECK-NEXT: br label [[OMP_TILE0_HEADER:%.*]] @@ -71,7 +71,7 @@ void unroll_unroll_partial_heuristic(float *a, float *b, float *c, float *d) { // CHECK-NEXT: [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], [[TMP8]] // CHECK-NEXT: br i1 [[OMP_TILE0_CMP]], label [[OMP_TILE0_BODY:%.*]], label [[OMP_TILE0_EXIT:%.*]] // CHECK: omp_tile0.body: -// CHECK-NEXT: [[TMP9:%.*]] = mul nuw i32 8, [[OMP_FLOOR0_IV]] +// CHECK-NEXT: [[TMP9:%.*]] = mul nuw i32 2, [[OMP_FLOOR0_IV]] // CHECK-NEXT: [[TMP10:%.*]] = add nuw i32 [[TMP9]], [[OMP_TILE0_IV]] // CHECK-NEXT: br label [[OMP_LOOP_BODY:%.*]] // CHECK: omp_loop.body: diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h index 4bb38028b24da2..732215a666cdb4 100644 --- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h +++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h @@ -1553,18 +1553,14 @@ class OpenMPIRBuilder { /// needing the CanonicalLoopInfo allows more efficient code generation by /// deferring the actual unrolling to the LoopUnrollPass using loop metadata. /// A loop-associated directive applied to the unrolled loop needs to know the - /// new trip count which means that if using a heuristically determined unroll - /// factor (\p Factor == 0), that factor must be computed immediately. We are - /// using the same logic as the LoopUnrollPass to derived the unroll factor, - /// but which assumes that some canonicalization has taken place (e.g. - /// Mem2Reg, LICM, GVN, Inlining, etc.). That is, the heuristic will perform - /// better when the unrolled loop's CanonicalLoopInfo is not needed. + /// new trip count, so if the CanonicalLoopInfo is requested with \p Factor == + /// 0, a fixed factor of 2 is used. /// /// \param DL Debug location for instructions added by unrolling. /// \param Loop The loop to unroll. The loop will be invalidated. /// \param Factor The factor to unroll the loop by. A factor of 0 - /// indicates that a heuristic should be used to determine - /// the unroll-factor. + /// leaves the choice of unroll-factor to the + /// implementation. /// \param UnrolledCLI If non-null, receives the CanonicalLoopInfo of the /// partially unrolled loop. Otherwise, uses loop metadata /// to defer unrolling to the LoopUnrollPass. diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp index 2a5e3caba2a5f6..0018fd71db36bf 100644 --- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp +++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp @@ -19,11 +19,8 @@ #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/Analysis/AssumptionCache.h" -#include "llvm/Analysis/CodeMetrics.h" #include "llvm/Analysis/LoopInfo.h" -#include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/Analysis/PostDominators.h" -#include "llvm/Analysis/ScalarEvolution.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Bitcode/BitcodeReader.h" #include "llvm/Frontend/Offloading/Utility.h" @@ -48,19 +45,14 @@ #include "llvm/IR/PassManager.h" #include "llvm/IR/ReplaceConstant.h" #include "llvm/IR/Value.h" -#include "llvm/MC/TargetRegistry.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/NVVMAttributes.h" #include "llvm/Support/VirtualFileSystem.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetOptions.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/Cloning.h" #include "llvm/Transforms/Utils/CodeExtractor.h" -#include "llvm/Transforms/Utils/LoopPeel.h" -#include "llvm/Transforms/Utils/UnrollLoop.h" #include <cstdint> #include <optional> @@ -76,12 +68,6 @@ static cl::opt<bool> "'as-if' properties of runtime calls."), cl::init(false)); -static cl::opt<double> UnrollThresholdFactor( - "openmp-ir-builder-unroll-threshold-factor", cl::Hidden, - cl::desc("Factor for the unroll threshold to account for code " - "simplifications still taking place"), - cl::init(1.5)); - static cl::opt<bool> UseDefaultMaxThreads( "openmp-ir-builder-use-default-max-threads", cl::Hidden, cl::desc("Use a default max threads if none is provided."), cl::init(true)); @@ -7984,179 +7970,6 @@ void OpenMPIRBuilder::applySimd(CanonicalLoopInfo *CanonicalLoop, addLoopMetadata(CanonicalLoop, LoopMDList); } -/// Create the TargetMachine object to query the backend for optimization -/// preferences. -/// -/// Ideally, this would be passed from the front-end to the OpenMPBuilder, but -/// e.g. Clang does not pass it to its CodeGen layer and creates it only when -/// needed for the LLVM pass pipline. We use some default options to avoid -/// having to pass too many settings from the frontend that probably do not -/// matter. -/// -/// Currently, TargetMachine is only used sometimes by the unrollLoopPartial -/// method. If we are going to use TargetMachine for more purposes, especially -/// those that are sensitive to TargetOptions, RelocModel and CodeModel, it -/// might become be worth requiring front-ends to pass on their TargetMachine, -/// or at least cache it between methods. Note that while fontends such as Clang -/// have just a single main TargetMachine per translation unit, "target-cpu" and -/// "target-features" that determine the TargetMachine are per-function and can -/// be overrided using __attribute__((target("OPTIONS"))). -static std::unique_ptr<TargetMachine> -createTargetMachine(Function *F, CodeGenOptLevel OptLevel) { - Module *M = F->getParent(); - - StringRef CPU = F->getFnAttribute("target-cpu").getValueAsString(); - StringRef Features = F->getFnAttribute("target-features").getValueAsString(); - const llvm::Triple &Triple = M->getTargetTriple(); - - std::string Error; - const llvm::Target *TheTarget = TargetRegistry::lookupTarget(Triple, Error); - if (!TheTarget) - return {}; - - llvm::TargetOptions Options; - return std::unique_ptr<TargetMachine>(TheTarget->createTargetMachine( - Triple, CPU, Features, Options, /*RelocModel=*/std::nullopt, - /*CodeModel=*/std::nullopt, OptLevel)); -} - -/// Heuristically determine the best-performant unroll factor for \p CLI. This -/// depends on the target processor. We are re-using the same heuristics as the -/// LoopUnrollPass. -static int32_t computeHeuristicUnrollFactor(CanonicalLoopInfo *CLI) { - Function *F = CLI->getFunction(); - - // Assume the user requests the most aggressive unrolling, even if the rest of - // the code is optimized using a lower setting. - CodeGenOptLevel OptLevel = CodeGenOptLevel::Aggressive; - std::unique_ptr<TargetMachine> TM = createTargetMachine(F, OptLevel); - - // Blocks must have terminators. - // FIXME: Don't run analyses on incomplete/invalid IR. - SmallVector<Instruction *> UIs; - for (BasicBlock &BB : *F) - if (!BB.hasTerminator()) - UIs.push_back(new UnreachableInst(F->getContext(), &BB)); - - FunctionAnalysisManager FAM; - FAM.registerPass([]() { return TargetLibraryAnalysis(); }); - FAM.registerPass([]() { return AssumptionAnalysis(); }); - FAM.registerPass([]() { return DominatorTreeAnalysis(); }); - FAM.registerPass([]() { return LoopAnalysis(); }); - FAM.registerPass([]() { return ScalarEvolutionAnalysis(); }); - FAM.registerPass([]() { return PassInstrumentationAnalysis(); }); - TargetIRAnalysis TIRA; - if (TM) - TIRA = TM->getTargetIRAnalysis(); - FAM.registerPass([&]() { return TIRA; }); - - TargetIRAnalysis::Result &&TTI = TIRA.run(*F, FAM); - ScalarEvolutionAnalysis SEA; - ScalarEvolution &&SE = SEA.run(*F, FAM); - DominatorTreeAnalysis DTA; - DominatorTree &&DT = DTA.run(*F, FAM); - LoopAnalysis LIA; - LoopInfo &&LI = LIA.run(*F, FAM); - AssumptionAnalysis ACT; - AssumptionCache &&AC = ACT.run(*F, FAM); - OptimizationRemarkEmitter ORE{F}; - - for (Instruction *I : UIs) - I->eraseFromParent(); - - Loop *L = LI.getLoopFor(CLI->getHeader()); - assert(L && "Expecting CanonicalLoopInfo to be recognized as a loop"); - - TargetTransformInfo::UnrollingPreferences UP = gatherUnrollingPreferences( - L, SE, TTI, - /*BlockFrequencyInfo=*/nullptr, - /*ProfileSummaryInfo=*/nullptr, ORE, static_cast<int>(OptLevel), - /*UserThreshold=*/std::nullopt, - /*UserAllowPartial=*/true, - /*UserAllowRuntime=*/true, - /*UserUpperBound=*/std::nullopt, - /*UserFullUnrollMaxCount=*/std::nullopt); - - UP.Force = true; - - // Account for additional optimizations taking place before the LoopUnrollPass - // would unroll the loop. - UP.Threshold *= UnrollThresholdFactor; - UP.PartialThreshold *= UnrollThresholdFactor; - - // Use normal unroll factors even if the rest of the code is optimized for - // size. - UP.OptSizeThreshold = UP.Threshold; - UP.PartialOptSizeThreshold = UP.PartialThreshold; - - LLVM_DEBUG(dbgs() << "Unroll heuristic thresholds:\n" - << " Threshold=" << UP.Threshold << "\n" - << " PartialThreshold=" << UP.PartialThreshold << "\n" - << " OptSizeThreshold=" << UP.OptSizeThreshold << "\n" - << " PartialOptSizeThreshold=" - << UP.PartialOptSizeThreshold << "\n"); - - // Disable peeling. - TargetTransformInfo::PeelingPreferences PP = - gatherPeelingPreferences(L, SE, TTI, - /*UserAllowPeeling=*/false, - /*UserAllowProfileBasedPeeling=*/false, - /*UnrollingSpecficValues=*/false); - - SmallPtrSet<const Value *, 32> EphValues; - CodeMetrics::collectEphemeralValues(L, &AC, EphValues); - - // Assume that reads and writes to stack variables can be eliminated by - // Mem2Reg, SROA or LICM. That is, don't count them towards the loop body's - // size. - for (BasicBlock *BB : L->blocks()) { - for (Instruction &I : *BB) { - Value *Ptr; - if (auto *Load = dyn_cast<LoadInst>(&I)) { - Ptr = Load->getPointerOperand(); - } else if (auto *Store = dyn_cast<StoreInst>(&I)) { - Ptr = Store->getPointerOperand(); - } else - continue; - - Ptr = Ptr->stripPointerCasts(); - - if (auto *Alloca = dyn_cast<AllocaInst>(Ptr)) { - if (Alloca->getParent() == &F->getEntryBlock()) - EphValues.insert(&I); - } - } - } - - UnrollCostEstimator UCE(L, TTI, EphValues, UP.BEInsns); - - // Loop is not unrollable if the loop contains certain instructions. - if (!UCE.canUnroll()) { - LLVM_DEBUG(dbgs() << "Loop not considered unrollable\n"); - return 1; - } - - LLVM_DEBUG(dbgs() << "Estimated loop size is " << UCE.getRolledLoopSize() - << "\n"); - - // TODO: Determine trip count of \p CLI if constant, computeUnrollCount might - // be able to use it. - int TripCount = 0; - int MaxTripCount = 0; - bool MaxOrZero = false; - unsigned TripMultiple = 0; - - unsigned Factor = - computeUnrollCount(L, TTI, DT, &LI, &AC, SE, EphValues, &ORE, TripCount, - MaxTripCount, MaxOrZero, TripMultiple, UCE, UP, PP); - LLVM_DEBUG(dbgs() << "Suggesting unroll factor of " << Factor << "\n"); - - // This function returns 1 to signal to not unroll a loop. - if (Factor == 0) - return 1; - return Factor; -} - void OpenMPIRBuilder::unrollLoopPartial(DebugLoc DL, CanonicalLoopInfo *Loop, int32_t Factor, CanonicalLoopInfo **UnrolledCLI) { @@ -8183,11 +7996,11 @@ void OpenMPIRBuilder::unrollLoopPartial(DebugLoc DL, CanonicalLoopInfo *Loop, return; } - // Heuristically determine the unroll factor. + // TODO: Use a better profitability model. if (Factor == 0) { if (Loop->VirtualCollapse) Loop->reportUnsupportedVirtualCollapse(); - Factor = computeHeuristicUnrollFactor(Loop); + Factor = 2; } // No change required with unroll factor 1. `````````` </details> https://github.com/llvm/llvm-project/pull/240030 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
