https://github.com/yxsamliu updated https://github.com/llvm/llvm-project/pull/233184
>From 1feef69c8ee948d8b7a449ae99d0f3df0b887cce Mon Sep 17 00:00:00 2001 From: "Yaxun (Sam) Liu" <[email protected]> Date: Sat, 17 Oct 2026 11:30:35 -0400 Subject: [PATCH 1/5] [clang][Driver] Honor `--offload-jobs` in device-only compilation `--offload-jobs=N` runs independent device compilations in parallel, but it had no effect when only device code is compiled; multi-target device-only builds still compiled each target one at a time. Two things got in the way. The device-only compile produces an object directly, and that step was not treated as one that can run in parallel. Each target's compile is also followed by its own device link, so the compiles that could run together are not next to each other, and only neighboring steps were grouped for parallel execution. Treat the object-producing device compile as parallelizable, and group same-kind compiles even when other independent steps sit between them. The grouped compiles do not depend on those in-between steps, so the output is unchanged and only the build time drops. --- clang/lib/Driver/Compilation.cpp | 73 ++++++++++--------- clang/lib/Driver/Driver.cpp | 3 +- .../Driver/offload-parallel-device-cc1.cu | 7 ++ 3 files changed, 46 insertions(+), 37 deletions(-) diff --git a/clang/lib/Driver/Compilation.cpp b/clang/lib/Driver/Compilation.cpp index c81c4445a29f95..e9de5ab4d8de73 100644 --- a/clang/lib/Driver/Compilation.cpp +++ b/clang/lib/Driver/Compilation.cpp @@ -16,6 +16,7 @@ #include "clang/Driver/Util.h" #include "clang/Options/Options.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallPtrSet.h" #include "llvm/Option/ArgList.h" #include "llvm/Option/OptSpecifier.h" #include "llvm/Option/Option.h" @@ -291,40 +292,40 @@ struct ParallelJobResult { std::string Error; }; -struct ParallelOffloadJobGroupResult { - size_t NumJobs = 0; -}; - -static std::optional<ParallelOffloadJobGroupResult> +// Collect the device jobs that share StartIndex's parallel group and can run +// concurrently, run them on a thread pool, and record them in \p Done. Group +// members need not be adjacent in the job list: in device-only compilation each +// target's compile is followed by its own device link, so the compiles that +// share a group are separated by those links. Returns true if a parallel group +// (including the start job) was executed. +static bool tryExecuteParallelOffloadJobGroup(const Driver &D, const ArgList &Args, ArrayRef<std::optional<StringRef>> Redirects, const JobList::list_type &JobStorage, size_t StartIndex, + llvm::SmallPtrSetImpl<const Command *> &Done, FailingCommandList &FailingCommands) { const Command &Job = *JobStorage[StartIndex]; if (!CanRunInParallelOffloadJobGroup(Job)) - return std::nullopt; + return false; - SmallVector<const Command *, 4> ParallelJobs; - for (size_t I = StartIndex; I < JobStorage.size(); ++I) { + // The start job's dependencies precede it in the topologically ordered job + // list and have already run, so it is always safe to run here. + SmallVector<const Command *, 4> ParallelJobs{&Job}; + for (size_t I = StartIndex + 1; I < JobStorage.size(); ++I) { const Command &Candidate = *JobStorage[I]; - if (ActionFailed(&Candidate.getSource(), FailingCommands)) - break; - - if (!CanRunInParallelOffloadJobGroup(Candidate)) - break; - - if (!SameParallelOffloadJobGroup(Job, Candidate)) - break; - - if (!HasDistinctBoundArch(Candidate, ParallelJobs)) - break; + if (Done.contains(&Candidate) || + ActionFailed(&Candidate.getSource(), FailingCommands) || + !CanRunInParallelOffloadJobGroup(Candidate) || + !SameParallelOffloadJobGroup(Job, Candidate) || + !HasDistinctBoundArch(Candidate, ParallelJobs)) + continue; if (!llvm::all_of(ParallelJobs, [&](const Command *Other) { return ActionsAreIndependent(&Candidate.getSource(), &Other->getSource()); })) - break; + continue; ParallelJobs.push_back(&Candidate); } @@ -332,7 +333,7 @@ tryExecuteParallelOffloadJobGroup(const Driver &D, const ArgList &Args, std::optional<llvm::ThreadPoolStrategy> Strategy = getParallelOffloadJobsStrategy(Args, ParallelJobs.size()); if (!Strategy) - return std::nullopt; + return false; SmallVector<ParallelJobResult, 4> Results(ParallelJobs.size()); llvm::DefaultThreadPool Pool(*Strategy); @@ -359,7 +360,9 @@ tryExecuteParallelOffloadJobGroup(const Driver &D, const ArgList &Args, } } - return ParallelOffloadJobGroupResult{ParallelJobs.size()}; + for (const Command *ParallelJob : ParallelJobs) + Done.insert(ParallelJob); + return true; } void Compilation::ExecuteJobs(const JobList &Jobs, @@ -375,23 +378,22 @@ void Compilation::ExecuteJobs(const JobList &Jobs, !getDriver().IsCLMode() && !getArgs().hasArg(options::OPT_v) && Redirects.empty() && !PostCallback; + // Jobs run as part of a parallel offload group. Group members are not + // necessarily adjacent, so track them and skip them when the loop reaches + // them. + llvm::SmallPtrSet<const Command *, 16> Done; const auto &JobStorage = Jobs.getJobs(); - for (size_t I = 0; I < JobStorage.size();) { + for (size_t I = 0; I < JobStorage.size(); ++I) { const auto &Job = *JobStorage[I]; - if (ActionFailed(&Job.getSource(), FailingCommands)) { - ++I; + if (Done.contains(&Job)) + continue; + if (ActionFailed(&Job.getSource(), FailingCommands)) continue; - } - if (CanRunJobsInParallel) { - if (std::optional<ParallelOffloadJobGroupResult> Result = - tryExecuteParallelOffloadJobGroup(getDriver(), getArgs(), - Redirects, JobStorage, I, - FailingCommands)) { - I += Result->NumJobs; - continue; - } - } + if (CanRunJobsInParallel && + tryExecuteParallelOffloadJobGroup(getDriver(), getArgs(), Redirects, + JobStorage, I, Done, FailingCommands)) + continue; const Command *FailingCommand = nullptr; if (int Res = ExecuteCommand(Job, FailingCommand, LogOnly)) { @@ -400,7 +402,6 @@ void Compilation::ExecuteJobs(const JobList &Jobs, if (TheDriver.IsCLMode()) return; } - ++I; } } diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp index b2dbc32f1b89fd..4a064219fbfb34 100644 --- a/clang/lib/Driver/Driver.cpp +++ b/clang/lib/Driver/Driver.cpp @@ -4530,7 +4530,8 @@ Action *Driver::ConstructPhaseAction( static bool isOffloadDeviceCC1JobCandidate(Command &Job) { const Action &Source = Job.getSource(); - if (!isa<CompileJobAction>(Source) && !isa<BackendJobAction>(Source)) + if (!isa<CompileJobAction>(Source) && !isa<BackendJobAction>(Source) && + !isa<AssembleJobAction>(Source)) return false; if (Job.getBoundArch().empty() && !Source.getOffloadingArch().empty()) diff --git a/clang/test/Driver/offload-parallel-device-cc1.cu b/clang/test/Driver/offload-parallel-device-cc1.cu index ff68770926aa17..769e8b822e81a1 100644 --- a/clang/test/Driver/offload-parallel-device-cc1.cu +++ b/clang/test/Driver/offload-parallel-device-cc1.cu @@ -17,6 +17,13 @@ // RUN: --cuda-gpu-arch=sm_70 --cuda-gpu-arch=sm_80 --offload-jobs=2 \ // RUN: --cuda-device-only -S -Werror %s +// Device-only compilation to an object parallelizes the per-target compiles +// even though each is followed by its own device link. +// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \ +// RUN: -nostdinc -nogpuinc -nohipwrapperinc -nogpulib \ +// RUN: --offload-arch=gfx900 --offload-arch=gfx906 --offload-jobs=2 \ +// RUN: --offload-device-only -O3 -c -Werror %s -o %t/hip-dev.o + // RUN: not %clang -x cuda --target=x86_64-unknown-linux-gnu \ // RUN: -nocudainc -nocudalib \ // RUN: --cuda-gpu-arch=sm_70 --cuda-gpu-arch=sm_80 --offload-jobs=0x4 \ >From 6e02e90fb3a1f7884abd243499d49006500bf191 Mon Sep 17 00:00:00 2001 From: "Yaxun (Sam) Liu" <[email protected]> Date: Sun, 18 Oct 2026 14:20:12 -0400 Subject: [PATCH 2/5] [clang][Driver] Drop unused HIP wrapper flag from device-only test --- clang/test/Driver/offload-parallel-device-cc1.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/test/Driver/offload-parallel-device-cc1.cu b/clang/test/Driver/offload-parallel-device-cc1.cu index 769e8b822e81a1..b44d9dfe4dd97a 100644 --- a/clang/test/Driver/offload-parallel-device-cc1.cu +++ b/clang/test/Driver/offload-parallel-device-cc1.cu @@ -20,7 +20,7 @@ // Device-only compilation to an object parallelizes the per-target compiles // even though each is followed by its own device link. // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \ -// RUN: -nostdinc -nogpuinc -nohipwrapperinc -nogpulib \ +// RUN: -nostdinc -nogpuinc -nogpulib \ // RUN: --offload-arch=gfx900 --offload-arch=gfx906 --offload-jobs=2 \ // RUN: --offload-device-only -O3 -c -Werror %s -o %t/hip-dev.o >From 9206d1ddc7eca04b3fbbd4e7759033f69690e17a Mon Sep 17 00:00:00 2001 From: "Yaxun (Sam) Liu" <[email protected]> Date: Tue, 20 Oct 2026 07:39:18 -0400 Subject: [PATCH 3/5] [clang][Driver] Respect unfinished prerequisites in parallel offload groups Nonadjacent device compiles can be independent of each other but depend on intervening jobs that have not run. With -save-temps, a later compile could start before its preprocessing output existed. Check all unfinished earlier jobs before adding a candidate to a parallel group. Continue to allow independent compiles separated by device links. Add execution coverage for HIP and CUDA saved intermediates, device-only syntax checks, and target-specific HIP PCH consumption. --- clang/lib/Driver/Compilation.cpp | 14 ++++---- .../Driver/offload-parallel-device-cc1.cu | 32 +++++++++++++++++++ .../Driver/offload-parallel-device-pch.cu | 32 +++++++++++++++++++ 3 files changed, 71 insertions(+), 7 deletions(-) create mode 100644 clang/test/Driver/offload-parallel-device-pch.cu diff --git a/clang/lib/Driver/Compilation.cpp b/clang/lib/Driver/Compilation.cpp index e9de5ab4d8de73..0ecfc24cbdb496 100644 --- a/clang/lib/Driver/Compilation.cpp +++ b/clang/lib/Driver/Compilation.cpp @@ -245,10 +245,6 @@ static bool ActionDependsOn(const Action *A, const Action *Other) { }); } -static bool ActionsAreIndependent(const Action *A, const Action *B) { - return !ActionDependsOn(A, B) && !ActionDependsOn(B, A); -} - static bool CanRunInParallelOffloadJobGroup(const Command &Job) { return !Job.InProcess && !Job.PrintInputFilenames && !Job.getBoundArch().empty() && @@ -321,9 +317,13 @@ tryExecuteParallelOffloadJobGroup(const Driver &D, const ArgList &Args, !HasDistinctBoundArch(Candidate, ParallelJobs)) continue; - if (!llvm::all_of(ParallelJobs, [&](const Command *Other) { - return ActionsAreIndependent(&Candidate.getSource(), - &Other->getSource()); + // In this topologically ordered list, only the candidate can depend on + // an earlier job, such as preprocessing with -save-temps. + auto EarlierJobs = llvm::make_range(JobStorage.begin() + StartIndex, + JobStorage.begin() + I); + if (!llvm::all_of(EarlierJobs, [&](const auto &Other) { + return Done.contains(Other.get()) || + !ActionDependsOn(&Candidate.getSource(), &Other->getSource()); })) continue; diff --git a/clang/test/Driver/offload-parallel-device-cc1.cu b/clang/test/Driver/offload-parallel-device-cc1.cu index b44d9dfe4dd97a..125e787d58e965 100644 --- a/clang/test/Driver/offload-parallel-device-cc1.cu +++ b/clang/test/Driver/offload-parallel-device-cc1.cu @@ -1,3 +1,5 @@ +// NOTE: Do not autogenerate. These checks cover job execution and selective +// driver diagnostics. // REQUIRES: x86-registered-target, amdgpu-registered-target // REQUIRES: nvptx-registered-target, lld @@ -24,6 +26,36 @@ // RUN: --offload-arch=gfx900 --offload-arch=gfx906 --offload-jobs=2 \ // RUN: --offload-device-only -O3 -c -Werror %s -o %t/hip-dev.o +// Saved intermediates introduce unfinished preprocessing jobs between device +// compiles. A later compile must wait for its own preprocessing job. +// RUN: mkdir -p %t/save-temps +// RUN: cd %t/save-temps && %clang -x hip --target=x86_64-unknown-linux-gnu \ +// RUN: -nostdinc -nogpuinc -nogpulib \ +// RUN: --offload-arch=gfx900 --offload-arch=gfx906 --offload-jobs=2 \ +// RUN: --offload-device-only -O3 -c -save-temps -Werror %s -o hip-dev.o +// RUN: mkdir -p %t/save-temps-obj +// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \ +// RUN: -nostdinc -nogpuinc -nogpulib \ +// RUN: --offload-arch=gfx900 --offload-arch=gfx906 --offload-jobs=2 \ +// RUN: --offload-device-only -O3 -c -save-temps=obj -Werror %s \ +// RUN: -o %t/save-temps-obj/hip-dev.o + +// RUN: mkdir -p %t/cuda-save-temps +// RUN: cd %t/cuda-save-temps && %clang -x cuda --target=x86_64-unknown-linux-gnu \ +// RUN: -nocudainc -nocudalib \ +// RUN: --cuda-gpu-arch=sm_70 --cuda-gpu-arch=sm_80 --offload-jobs=2 \ +// RUN: --cuda-device-only -S -save-temps -Werror %s + +// Syntax-only device compiles have no output artifacts. +// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \ +// RUN: -nostdinc -nogpuinc -nogpulib \ +// RUN: --offload-arch=gfx900 --offload-arch=gfx906 --offload-jobs=2 \ +// RUN: --offload-device-only -fsyntax-only -Werror %s +// RUN: %clang -x cuda --target=x86_64-unknown-linux-gnu \ +// RUN: -nocudainc -nocudalib \ +// RUN: --cuda-gpu-arch=sm_70 --cuda-gpu-arch=sm_80 --offload-jobs=2 \ +// RUN: --cuda-device-only -fsyntax-only -Werror %s + // RUN: not %clang -x cuda --target=x86_64-unknown-linux-gnu \ // RUN: -nocudainc -nocudalib \ // RUN: --cuda-gpu-arch=sm_70 --cuda-gpu-arch=sm_80 --offload-jobs=0x4 \ diff --git a/clang/test/Driver/offload-parallel-device-pch.cu b/clang/test/Driver/offload-parallel-device-pch.cu new file mode 100644 index 00000000000000..cc710e5115eb36 --- /dev/null +++ b/clang/test/Driver/offload-parallel-device-pch.cu @@ -0,0 +1,32 @@ +// NOTE: Do not autogenerate. Successful compilation checks PCH consumption +// by each device target, including syntax-only jobs without output artifacts. +// REQUIRES: x86-registered-target, amdgpu-registered-target, lld +// RUN: split-file %s %t + +// Build a PCH for each device target with the same frontend options as its user. +// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \ +// RUN: -nostdinc -nogpuinc -nogpulib --offload-device-only \ +// RUN: --offload-arch=gfx900 -O3 -Werror -fsyntax-only -Xclang -emit-pch \ +// RUN: %t/header.h > %t/gfx900.h.pch +// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \ +// RUN: -nostdinc -nogpuinc -nogpulib --offload-device-only \ +// RUN: --offload-arch=gfx906 -O3 -Werror -fsyntax-only -Xclang -emit-pch \ +// RUN: %t/header.h > %t/gfx906.h.pch + +// Each device compile reads its own PCH while running in parallel. +// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \ +// RUN: -nostdinc -nogpuinc -nogpulib --offload-device-only \ +// RUN: --offload-arch=gfx900 --offload-arch=gfx906 --offload-jobs=2 \ +// RUN: -Xarch_gfx900 -include%t/gfx900.h -Xarch_gfx906 -include%t/gfx906.h \ +// RUN: -O3 -Werror -c %t/input.cu -o %t/hip-dev.o +// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \ +// RUN: -nostdinc -nogpuinc -nogpulib --offload-device-only \ +// RUN: --offload-arch=gfx900 --offload-arch=gfx906 --offload-jobs=2 \ +// RUN: -Xarch_gfx900 -include%t/gfx900.h -Xarch_gfx906 -include%t/gfx906.h \ +// RUN: -O3 -Werror -fsyntax-only %t/input.cu + +//--- header.h +__attribute__((device)) inline int get_value() { return 42; } + +//--- input.cu +__attribute__((global)) void kernel(int *p) { *p = get_value(); } >From 4db5e8895aa9682eb2b1c1736fcf0ff00347b1f9 Mon Sep 17 00:00:00 2001 From: "Yaxun (Sam) Liu" <[email protected]> Date: Wed, 21 Oct 2026 09:55:41 -0400 Subject: [PATCH 4/5] [clang][Driver] Trim parallel offload job comment --- clang/lib/Driver/Compilation.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/clang/lib/Driver/Compilation.cpp b/clang/lib/Driver/Compilation.cpp index 0ecfc24cbdb496..4e24ab94b01da3 100644 --- a/clang/lib/Driver/Compilation.cpp +++ b/clang/lib/Driver/Compilation.cpp @@ -288,12 +288,7 @@ struct ParallelJobResult { std::string Error; }; -// Collect the device jobs that share StartIndex's parallel group and can run -// concurrently, run them on a thread pool, and record them in \p Done. Group -// members need not be adjacent in the job list: in device-only compilation each -// target's compile is followed by its own device link, so the compiles that -// share a group are separated by those links. Returns true if a parallel group -// (including the start job) was executed. +// Device-only compiles in one group can be separated by device links. static bool tryExecuteParallelOffloadJobGroup(const Driver &D, const ArgList &Args, ArrayRef<std::optional<StringRef>> Redirects, >From c31d78397f73703da110197bc041e59216eefd0a Mon Sep 17 00:00:00 2001 From: "Yaxun (Sam) Liu" <[email protected]> Date: Wed, 21 Oct 2026 17:57:07 -0400 Subject: [PATCH 5/5] [clang][CIR] Build llvm-strings for check-clang With CIR enabled, check-clang discovers the bytecode tests. They invoke llvm-strings, but only check-clang-cir declares it as a dependency. Add it to the common Clang test dependencies for non-standalone CIR builds so a fresh check-clang build has the tool. --- clang/test/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/clang/test/CMakeLists.txt b/clang/test/CMakeLists.txt index 4063f58ad0e080..f58222b979bb6d 100644 --- a/clang/test/CMakeLists.txt +++ b/clang/test/CMakeLists.txt @@ -118,6 +118,9 @@ if(CLANG_ENABLE_CIR) cir-opt cir-translate ) + if(NOT CLANG_BUILT_STANDALONE) + list(APPEND CLANG_TEST_DEPS llvm-strings) + endif() add_subdirectory(CIR/lib) endif() _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
