[clang] [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use thread pool in P1689 per file mode (PR #84285)
https://github.com/ChuanqiXu9 closed https://github.com/llvm/llvm-project/pull/84285 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use thread pool in P1689 per file mode (PR #84285)
https://github.com/benlangmuir approved this pull request. https://github.com/llvm/llvm-project/pull/84285 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use thread pool in P1689 per file mode (PR #84285)
https://github.com/jansvoboda11 approved this pull request. LGTM, thanks! https://github.com/llvm/llvm-project/pull/84285 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use thread pool in P1689 per file mode (PR #84285)
@@ -893,102 +889,118 @@ int clang_scan_deps_main(int argc, char **argv, const
llvm::ToolContext &) {
if (Format == ScanningOutputFormat::Full)
FD.emplace(ModuleName.empty() ? Inputs.size() : 0);
- if (Verbose) {
-llvm::outs() << "Running clang-scan-deps on " << Inputs.size()
- << " files using " << Pool.getMaxConcurrency() << "
workers\n";
- }
-
- llvm::Timer T;
- T.startTimer();
-
- for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I) {
-Pool.async([&, I]() {
- llvm::DenseSet AlreadySeenModules;
- while (auto MaybeInputIndex = GetNextInputIndex()) {
-size_t LocalIndex = *MaybeInputIndex;
-const tooling::CompileCommand *Input = [LocalIndex];
-std::string Filename = std::move(Input->Filename);
-std::string CWD = std::move(Input->Directory);
-
-std::optional MaybeModuleName;
-if (!ModuleName.empty())
- MaybeModuleName = ModuleName;
-
-std::string OutputDir(ModuleFilesDir);
-if (OutputDir.empty())
- OutputDir = getModuleCachePath(Input->CommandLine);
-auto LookupOutput = [&](const ModuleID , ModuleOutputKind MOK) {
- return ::lookupModuleOutput(MID, MOK, OutputDir);
-};
-
-// Run the tool on it.
-if (Format == ScanningOutputFormat::Make) {
- auto MaybeFile =
- WorkerTools[I]->getDependencyFile(Input->CommandLine, CWD);
- if (handleMakeDependencyToolResult(Filename, MaybeFile, DependencyOS,
- Errs))
-HadErrors = true;
-} else if (Format == ScanningOutputFormat::P1689) {
- // It is useful to generate the make-format dependency output during
- // the scanning for P1689. Otherwise the users need to scan again for
- // it. We will generate the make-format dependency output if we find
- // `-MF` in the command lines.
- std::string MakeformatOutputPath;
- std::string MakeformatOutput;
-
- auto MaybeRule = WorkerTools[I]->getP1689ModuleDependencyFile(
- *Input, CWD, MakeformatOutput, MakeformatOutputPath);
-
- if (handleP1689DependencyToolResult(Filename, MaybeRule, PD, Errs))
-HadErrors = true;
+ std::vector> WorkerTools;
ChuanqiXu9 wrote:
Nice catch. Done.
https://github.com/llvm/llvm-project/pull/84285
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use thread pool in P1689 per file mode (PR #84285)
https://github.com/ChuanqiXu9 updated
https://github.com/llvm/llvm-project/pull/84285
>From 48b3261e1d217b7ce78180314a222dca4d6aba18 Mon Sep 17 00:00:00 2001
From: Chuanqi Xu
Date: Thu, 7 Mar 2024 15:19:28 +0800
Subject: [PATCH 1/5] [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use
thread pool in P1689 per file mode
I suddenly found that the clang scan deps may use all concurrent threads
to scan the files. It makes sense in the batch mode. But in P1689
per file mode, it simply wastes times.
---
clang/tools/clang-scan-deps/ClangScanDeps.cpp | 204 +-
1 file changed, 108 insertions(+), 96 deletions(-)
diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
index d042fecc3dbe63..843816a8ed6515 100644
--- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -744,6 +744,9 @@ getCompilationDataBase(int argc, char **argv, std::string
) {
return nullptr;
}
+ // Only 1 threads is required if P1689 per file mode.
+ NumThreads = 1;
+
// There might be multiple jobs for a compilation. Extract the specified
// output filename from the last job.
auto LastCmd = C->getJobs().end();
@@ -867,13 +870,6 @@ int clang_scan_deps_main(int argc, char **argv, const
llvm::ToolContext &) {
// Print out the dependency results to STDOUT by default.
SharedStream DependencyOS(llvm::outs());
- DependencyScanningService Service(ScanMode, Format, OptimizeArgs,
-EagerLoadModules);
- llvm::DefaultThreadPool Pool(llvm::hardware_concurrency(NumThreads));
- std::vector> WorkerTools;
- for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I)
-WorkerTools.push_back(std::make_unique(Service));
-
std::vector Inputs =
AdjustingCompilations->getAllCompileCommands();
@@ -893,102 +889,118 @@ int clang_scan_deps_main(int argc, char **argv, const
llvm::ToolContext &) {
if (Format == ScanningOutputFormat::Full)
FD.emplace(ModuleName.empty() ? Inputs.size() : 0);
- if (Verbose) {
-llvm::outs() << "Running clang-scan-deps on " << Inputs.size()
- << " files using " << Pool.getMaxConcurrency() << "
workers\n";
- }
-
- llvm::Timer T;
- T.startTimer();
-
- for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I) {
-Pool.async([&, I]() {
- llvm::DenseSet AlreadySeenModules;
- while (auto MaybeInputIndex = GetNextInputIndex()) {
-size_t LocalIndex = *MaybeInputIndex;
-const tooling::CompileCommand *Input = [LocalIndex];
-std::string Filename = std::move(Input->Filename);
-std::string CWD = std::move(Input->Directory);
-
-std::optional MaybeModuleName;
-if (!ModuleName.empty())
- MaybeModuleName = ModuleName;
-
-std::string OutputDir(ModuleFilesDir);
-if (OutputDir.empty())
- OutputDir = getModuleCachePath(Input->CommandLine);
-auto LookupOutput = [&](const ModuleID , ModuleOutputKind MOK) {
- return ::lookupModuleOutput(MID, MOK, OutputDir);
-};
-
-// Run the tool on it.
-if (Format == ScanningOutputFormat::Make) {
- auto MaybeFile =
- WorkerTools[I]->getDependencyFile(Input->CommandLine, CWD);
- if (handleMakeDependencyToolResult(Filename, MaybeFile, DependencyOS,
- Errs))
-HadErrors = true;
-} else if (Format == ScanningOutputFormat::P1689) {
- // It is useful to generate the make-format dependency output during
- // the scanning for P1689. Otherwise the users need to scan again for
- // it. We will generate the make-format dependency output if we find
- // `-MF` in the command lines.
- std::string MakeformatOutputPath;
- std::string MakeformatOutput;
-
- auto MaybeRule = WorkerTools[I]->getP1689ModuleDependencyFile(
- *Input, CWD, MakeformatOutput, MakeformatOutputPath);
-
- if (handleP1689DependencyToolResult(Filename, MaybeRule, PD, Errs))
-HadErrors = true;
+ std::vector> WorkerTools;
- if (!MakeformatOutputPath.empty() && !MakeformatOutput.empty() &&
- !HadErrors) {
-static std::mutex Lock;
-// With compilation database, we may open different files
-// concurrently or we may write the same file concurrently. So we
-// use a map here to allow multiple compile commands to write to
the
-// same file. Also we need a lock here to avoid data race.
-static llvm::StringMap OSs;
-std::unique_lock LockGuard(Lock);
-
-auto OSIter = OSs.find(MakeformatOutputPath);
-if (OSIter == OSs.end()) {
- std::error_code EC;
- OSIter = OSs.try_emplace(MakeformatOutputPath,
- MakeformatOutputPath, EC)
-
[clang] [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use thread pool in P1689 per file mode (PR #84285)
https://github.com/ChuanqiXu9 updated
https://github.com/llvm/llvm-project/pull/84285
>From 48b3261e1d217b7ce78180314a222dca4d6aba18 Mon Sep 17 00:00:00 2001
From: Chuanqi Xu
Date: Thu, 7 Mar 2024 15:19:28 +0800
Subject: [PATCH 1/4] [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use
thread pool in P1689 per file mode
I suddenly found that the clang scan deps may use all concurrent threads
to scan the files. It makes sense in the batch mode. But in P1689
per file mode, it simply wastes times.
---
clang/tools/clang-scan-deps/ClangScanDeps.cpp | 204 +-
1 file changed, 108 insertions(+), 96 deletions(-)
diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
index d042fecc3dbe63..843816a8ed6515 100644
--- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -744,6 +744,9 @@ getCompilationDataBase(int argc, char **argv, std::string
) {
return nullptr;
}
+ // Only 1 threads is required if P1689 per file mode.
+ NumThreads = 1;
+
// There might be multiple jobs for a compilation. Extract the specified
// output filename from the last job.
auto LastCmd = C->getJobs().end();
@@ -867,13 +870,6 @@ int clang_scan_deps_main(int argc, char **argv, const
llvm::ToolContext &) {
// Print out the dependency results to STDOUT by default.
SharedStream DependencyOS(llvm::outs());
- DependencyScanningService Service(ScanMode, Format, OptimizeArgs,
-EagerLoadModules);
- llvm::DefaultThreadPool Pool(llvm::hardware_concurrency(NumThreads));
- std::vector> WorkerTools;
- for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I)
-WorkerTools.push_back(std::make_unique(Service));
-
std::vector Inputs =
AdjustingCompilations->getAllCompileCommands();
@@ -893,102 +889,118 @@ int clang_scan_deps_main(int argc, char **argv, const
llvm::ToolContext &) {
if (Format == ScanningOutputFormat::Full)
FD.emplace(ModuleName.empty() ? Inputs.size() : 0);
- if (Verbose) {
-llvm::outs() << "Running clang-scan-deps on " << Inputs.size()
- << " files using " << Pool.getMaxConcurrency() << "
workers\n";
- }
-
- llvm::Timer T;
- T.startTimer();
-
- for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I) {
-Pool.async([&, I]() {
- llvm::DenseSet AlreadySeenModules;
- while (auto MaybeInputIndex = GetNextInputIndex()) {
-size_t LocalIndex = *MaybeInputIndex;
-const tooling::CompileCommand *Input = [LocalIndex];
-std::string Filename = std::move(Input->Filename);
-std::string CWD = std::move(Input->Directory);
-
-std::optional MaybeModuleName;
-if (!ModuleName.empty())
- MaybeModuleName = ModuleName;
-
-std::string OutputDir(ModuleFilesDir);
-if (OutputDir.empty())
- OutputDir = getModuleCachePath(Input->CommandLine);
-auto LookupOutput = [&](const ModuleID , ModuleOutputKind MOK) {
- return ::lookupModuleOutput(MID, MOK, OutputDir);
-};
-
-// Run the tool on it.
-if (Format == ScanningOutputFormat::Make) {
- auto MaybeFile =
- WorkerTools[I]->getDependencyFile(Input->CommandLine, CWD);
- if (handleMakeDependencyToolResult(Filename, MaybeFile, DependencyOS,
- Errs))
-HadErrors = true;
-} else if (Format == ScanningOutputFormat::P1689) {
- // It is useful to generate the make-format dependency output during
- // the scanning for P1689. Otherwise the users need to scan again for
- // it. We will generate the make-format dependency output if we find
- // `-MF` in the command lines.
- std::string MakeformatOutputPath;
- std::string MakeformatOutput;
-
- auto MaybeRule = WorkerTools[I]->getP1689ModuleDependencyFile(
- *Input, CWD, MakeformatOutput, MakeformatOutputPath);
-
- if (handleP1689DependencyToolResult(Filename, MaybeRule, PD, Errs))
-HadErrors = true;
+ std::vector> WorkerTools;
- if (!MakeformatOutputPath.empty() && !MakeformatOutput.empty() &&
- !HadErrors) {
-static std::mutex Lock;
-// With compilation database, we may open different files
-// concurrently or we may write the same file concurrently. So we
-// use a map here to allow multiple compile commands to write to
the
-// same file. Also we need a lock here to avoid data race.
-static llvm::StringMap OSs;
-std::unique_lock LockGuard(Lock);
-
-auto OSIter = OSs.find(MakeformatOutputPath);
-if (OSIter == OSs.end()) {
- std::error_code EC;
- OSIter = OSs.try_emplace(MakeformatOutputPath,
- MakeformatOutputPath, EC)
-
[clang] [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use thread pool in P1689 per file mode (PR #84285)
@@ -744,6 +744,9 @@ getCompilationDataBase(int argc, char **argv, std::string
) {
return nullptr;
}
+ // Only 1 threads is required if P1689 per file mode.
+ NumThreads = 1;
ChuanqiXu9 wrote:
Oh, sorry. I don't know why I missed this somehow. This should be done now.
https://github.com/llvm/llvm-project/pull/84285
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use thread pool in P1689 per file mode (PR #84285)
@@ -893,102 +889,118 @@ int clang_scan_deps_main(int argc, char **argv, const
llvm::ToolContext &) {
if (Format == ScanningOutputFormat::Full)
FD.emplace(ModuleName.empty() ? Inputs.size() : 0);
- if (Verbose) {
-llvm::outs() << "Running clang-scan-deps on " << Inputs.size()
- << " files using " << Pool.getMaxConcurrency() << "
workers\n";
- }
-
- llvm::Timer T;
- T.startTimer();
-
- for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I) {
-Pool.async([&, I]() {
- llvm::DenseSet AlreadySeenModules;
- while (auto MaybeInputIndex = GetNextInputIndex()) {
-size_t LocalIndex = *MaybeInputIndex;
-const tooling::CompileCommand *Input = [LocalIndex];
-std::string Filename = std::move(Input->Filename);
-std::string CWD = std::move(Input->Directory);
-
-std::optional MaybeModuleName;
-if (!ModuleName.empty())
- MaybeModuleName = ModuleName;
-
-std::string OutputDir(ModuleFilesDir);
-if (OutputDir.empty())
- OutputDir = getModuleCachePath(Input->CommandLine);
-auto LookupOutput = [&](const ModuleID , ModuleOutputKind MOK) {
- return ::lookupModuleOutput(MID, MOK, OutputDir);
-};
-
-// Run the tool on it.
-if (Format == ScanningOutputFormat::Make) {
- auto MaybeFile =
- WorkerTools[I]->getDependencyFile(Input->CommandLine, CWD);
- if (handleMakeDependencyToolResult(Filename, MaybeFile, DependencyOS,
- Errs))
-HadErrors = true;
-} else if (Format == ScanningOutputFormat::P1689) {
- // It is useful to generate the make-format dependency output during
- // the scanning for P1689. Otherwise the users need to scan again for
- // it. We will generate the make-format dependency output if we find
- // `-MF` in the command lines.
- std::string MakeformatOutputPath;
- std::string MakeformatOutput;
-
- auto MaybeRule = WorkerTools[I]->getP1689ModuleDependencyFile(
- *Input, CWD, MakeformatOutput, MakeformatOutputPath);
-
- if (handleP1689DependencyToolResult(Filename, MaybeRule, PD, Errs))
-HadErrors = true;
+ std::vector> WorkerTools;
jansvoboda11 wrote:
Right. Can't we create `DependencyScanningTool` that's local to `ScanningTask`?
https://github.com/llvm/llvm-project/pull/84285
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use thread pool in P1689 per file mode (PR #84285)
@@ -744,6 +744,9 @@ getCompilationDataBase(int argc, char **argv, std::string
) {
return nullptr;
}
+ // Only 1 threads is required if P1689 per file mode.
+ NumThreads = 1;
jansvoboda11 wrote:
I still see `NumThreads = 1` here.
https://github.com/llvm/llvm-project/pull/84285
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use thread pool in P1689 per file mode (PR #84285)
https://github.com/ChuanqiXu9 updated
https://github.com/llvm/llvm-project/pull/84285
>From dd9711be2368a299b408d0ff06ec9c1c0540083b Mon Sep 17 00:00:00 2001
From: Chuanqi Xu
Date: Thu, 7 Mar 2024 15:19:28 +0800
Subject: [PATCH 1/3] [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use
thread pool in P1689 per file mode
I suddenly found that the clang scan deps may use all concurrent threads
to scan the files. It makes sense in the batch mode. But in P1689
per file mode, it simply wastes times.
---
clang/tools/clang-scan-deps/ClangScanDeps.cpp | 204 +-
1 file changed, 108 insertions(+), 96 deletions(-)
diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
index d042fecc3dbe63..843816a8ed6515 100644
--- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -744,6 +744,9 @@ getCompilationDataBase(int argc, char **argv, std::string
) {
return nullptr;
}
+ // Only 1 threads is required if P1689 per file mode.
+ NumThreads = 1;
+
// There might be multiple jobs for a compilation. Extract the specified
// output filename from the last job.
auto LastCmd = C->getJobs().end();
@@ -867,13 +870,6 @@ int clang_scan_deps_main(int argc, char **argv, const
llvm::ToolContext &) {
// Print out the dependency results to STDOUT by default.
SharedStream DependencyOS(llvm::outs());
- DependencyScanningService Service(ScanMode, Format, OptimizeArgs,
-EagerLoadModules);
- llvm::DefaultThreadPool Pool(llvm::hardware_concurrency(NumThreads));
- std::vector> WorkerTools;
- for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I)
-WorkerTools.push_back(std::make_unique(Service));
-
std::vector Inputs =
AdjustingCompilations->getAllCompileCommands();
@@ -893,102 +889,118 @@ int clang_scan_deps_main(int argc, char **argv, const
llvm::ToolContext &) {
if (Format == ScanningOutputFormat::Full)
FD.emplace(ModuleName.empty() ? Inputs.size() : 0);
- if (Verbose) {
-llvm::outs() << "Running clang-scan-deps on " << Inputs.size()
- << " files using " << Pool.getMaxConcurrency() << "
workers\n";
- }
-
- llvm::Timer T;
- T.startTimer();
-
- for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I) {
-Pool.async([&, I]() {
- llvm::DenseSet AlreadySeenModules;
- while (auto MaybeInputIndex = GetNextInputIndex()) {
-size_t LocalIndex = *MaybeInputIndex;
-const tooling::CompileCommand *Input = [LocalIndex];
-std::string Filename = std::move(Input->Filename);
-std::string CWD = std::move(Input->Directory);
-
-std::optional MaybeModuleName;
-if (!ModuleName.empty())
- MaybeModuleName = ModuleName;
-
-std::string OutputDir(ModuleFilesDir);
-if (OutputDir.empty())
- OutputDir = getModuleCachePath(Input->CommandLine);
-auto LookupOutput = [&](const ModuleID , ModuleOutputKind MOK) {
- return ::lookupModuleOutput(MID, MOK, OutputDir);
-};
-
-// Run the tool on it.
-if (Format == ScanningOutputFormat::Make) {
- auto MaybeFile =
- WorkerTools[I]->getDependencyFile(Input->CommandLine, CWD);
- if (handleMakeDependencyToolResult(Filename, MaybeFile, DependencyOS,
- Errs))
-HadErrors = true;
-} else if (Format == ScanningOutputFormat::P1689) {
- // It is useful to generate the make-format dependency output during
- // the scanning for P1689. Otherwise the users need to scan again for
- // it. We will generate the make-format dependency output if we find
- // `-MF` in the command lines.
- std::string MakeformatOutputPath;
- std::string MakeformatOutput;
-
- auto MaybeRule = WorkerTools[I]->getP1689ModuleDependencyFile(
- *Input, CWD, MakeformatOutput, MakeformatOutputPath);
-
- if (handleP1689DependencyToolResult(Filename, MaybeRule, PD, Errs))
-HadErrors = true;
+ std::vector> WorkerTools;
- if (!MakeformatOutputPath.empty() && !MakeformatOutput.empty() &&
- !HadErrors) {
-static std::mutex Lock;
-// With compilation database, we may open different files
-// concurrently or we may write the same file concurrently. So we
-// use a map here to allow multiple compile commands to write to
the
-// same file. Also we need a lock here to avoid data race.
-static llvm::StringMap OSs;
-std::unique_lock LockGuard(Lock);
-
-auto OSIter = OSs.find(MakeformatOutputPath);
-if (OSIter == OSs.end()) {
- std::error_code EC;
- OSIter = OSs.try_emplace(MakeformatOutputPath,
- MakeformatOutputPath, EC)
-
[clang] [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use thread pool in P1689 per file mode (PR #84285)
@@ -893,102 +889,118 @@ int clang_scan_deps_main(int argc, char **argv, const
llvm::ToolContext &) {
if (Format == ScanningOutputFormat::Full)
FD.emplace(ModuleName.empty() ? Inputs.size() : 0);
- if (Verbose) {
-llvm::outs() << "Running clang-scan-deps on " << Inputs.size()
- << " files using " << Pool.getMaxConcurrency() << "
workers\n";
- }
-
- llvm::Timer T;
- T.startTimer();
-
- for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I) {
-Pool.async([&, I]() {
- llvm::DenseSet AlreadySeenModules;
- while (auto MaybeInputIndex = GetNextInputIndex()) {
-size_t LocalIndex = *MaybeInputIndex;
-const tooling::CompileCommand *Input = [LocalIndex];
-std::string Filename = std::move(Input->Filename);
-std::string CWD = std::move(Input->Directory);
-
-std::optional MaybeModuleName;
-if (!ModuleName.empty())
- MaybeModuleName = ModuleName;
-
-std::string OutputDir(ModuleFilesDir);
-if (OutputDir.empty())
- OutputDir = getModuleCachePath(Input->CommandLine);
-auto LookupOutput = [&](const ModuleID , ModuleOutputKind MOK) {
- return ::lookupModuleOutput(MID, MOK, OutputDir);
-};
-
-// Run the tool on it.
-if (Format == ScanningOutputFormat::Make) {
- auto MaybeFile =
- WorkerTools[I]->getDependencyFile(Input->CommandLine, CWD);
- if (handleMakeDependencyToolResult(Filename, MaybeFile, DependencyOS,
- Errs))
-HadErrors = true;
-} else if (Format == ScanningOutputFormat::P1689) {
- // It is useful to generate the make-format dependency output during
- // the scanning for P1689. Otherwise the users need to scan again for
- // it. We will generate the make-format dependency output if we find
- // `-MF` in the command lines.
- std::string MakeformatOutputPath;
- std::string MakeformatOutput;
-
- auto MaybeRule = WorkerTools[I]->getP1689ModuleDependencyFile(
- *Input, CWD, MakeformatOutput, MakeformatOutputPath);
-
- if (handleP1689DependencyToolResult(Filename, MaybeRule, PD, Errs))
-HadErrors = true;
+ std::vector> WorkerTools;
ChuanqiXu9 wrote:
Done by moving this into the `ThreadPool` only branch and get rid of
`std::unique_ptr`. It looks like we can't remove the vector entirely. Since the
parameter type of `ScanningTask` is `DependencyScanningTool &` instead of
`const DependencyScanningTool &`, so we may face lifetime issues if we
construct DependencyScanningTool in place.
https://github.com/llvm/llvm-project/pull/84285
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use thread pool in P1689 per file mode (PR #84285)
github-actions[bot] wrote:
:warning: C/C++ code formatter, clang-format found issues in your code.
:warning:
You can test this locally with the following command:
``bash
git-clang-format --diff da4957be2365831c94eab0b52612367c29f1d299
e875b07ad16c7c66d8d9dd70efca30276981f639 --
clang/tools/clang-scan-deps/ClangScanDeps.cpp
``
View the diff from clang-format here.
``diff
diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
index 2fd85c8e65..832be0df64 100644
--- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -910,8 +910,7 @@ int clang_scan_deps_main(int argc, char **argv, const
llvm::ToolContext &) {
// Run the tool on it.
if (Format == ScanningOutputFormat::Make) {
-auto MaybeFile =
-WorkerTool.getDependencyFile(Input->CommandLine, CWD);
+auto MaybeFile = WorkerTool.getDependencyFile(Input->CommandLine, CWD);
if (handleMakeDependencyToolResult(Filename, MaybeFile, DependencyOS,
Errs))
HadErrors = true;
@@ -996,9 +995,8 @@ int clang_scan_deps_main(int argc, char **argv, const
llvm::ToolContext &) {
}
for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I) {
- Pool.async([ScanningTask, , I]() {
-ScanningTask(WorkerTools[I]);
- });
+ Pool.async(
+ [ScanningTask, , I]() { ScanningTask(WorkerTools[I]); });
}
Pool.wait();
}
``
https://github.com/llvm/llvm-project/pull/84285
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use thread pool in P1689 per file mode (PR #84285)
@@ -893,102 +889,118 @@ int clang_scan_deps_main(int argc, char **argv, const
llvm::ToolContext &) {
if (Format == ScanningOutputFormat::Full)
FD.emplace(ModuleName.empty() ? Inputs.size() : 0);
- if (Verbose) {
-llvm::outs() << "Running clang-scan-deps on " << Inputs.size()
- << " files using " << Pool.getMaxConcurrency() << "
workers\n";
- }
-
- llvm::Timer T;
- T.startTimer();
-
- for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I) {
-Pool.async([&, I]() {
- llvm::DenseSet AlreadySeenModules;
- while (auto MaybeInputIndex = GetNextInputIndex()) {
-size_t LocalIndex = *MaybeInputIndex;
-const tooling::CompileCommand *Input = [LocalIndex];
-std::string Filename = std::move(Input->Filename);
-std::string CWD = std::move(Input->Directory);
-
-std::optional MaybeModuleName;
-if (!ModuleName.empty())
- MaybeModuleName = ModuleName;
-
-std::string OutputDir(ModuleFilesDir);
-if (OutputDir.empty())
- OutputDir = getModuleCachePath(Input->CommandLine);
-auto LookupOutput = [&](const ModuleID , ModuleOutputKind MOK) {
- return ::lookupModuleOutput(MID, MOK, OutputDir);
-};
-
-// Run the tool on it.
-if (Format == ScanningOutputFormat::Make) {
- auto MaybeFile =
- WorkerTools[I]->getDependencyFile(Input->CommandLine, CWD);
- if (handleMakeDependencyToolResult(Filename, MaybeFile, DependencyOS,
- Errs))
-HadErrors = true;
-} else if (Format == ScanningOutputFormat::P1689) {
- // It is useful to generate the make-format dependency output during
- // the scanning for P1689. Otherwise the users need to scan again for
- // it. We will generate the make-format dependency output if we find
- // `-MF` in the command lines.
- std::string MakeformatOutputPath;
- std::string MakeformatOutput;
-
- auto MaybeRule = WorkerTools[I]->getP1689ModuleDependencyFile(
- *Input, CWD, MakeformatOutput, MakeformatOutputPath);
-
- if (handleP1689DependencyToolResult(Filename, MaybeRule, PD, Errs))
-HadErrors = true;
+ std::vector> WorkerTools;
- if (!MakeformatOutputPath.empty() && !MakeformatOutput.empty() &&
- !HadErrors) {
-static std::mutex Lock;
-// With compilation database, we may open different files
-// concurrently or we may write the same file concurrently. So we
-// use a map here to allow multiple compile commands to write to
the
-// same file. Also we need a lock here to avoid data race.
-static llvm::StringMap OSs;
-std::unique_lock LockGuard(Lock);
-
-auto OSIter = OSs.find(MakeformatOutputPath);
-if (OSIter == OSs.end()) {
- std::error_code EC;
- OSIter = OSs.try_emplace(MakeformatOutputPath,
- MakeformatOutputPath, EC)
- .first;
- if (EC)
-llvm::errs()
-<< "Failed to open P1689 make format output file \""
-<< MakeformatOutputPath << "\" for " << EC.message()
-<< "\n";
-}
+ auto ScanningTask = [&](unsigned I) {
ChuanqiXu9 wrote:
Done. It looks indeed better.
https://github.com/llvm/llvm-project/pull/84285
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use thread pool in P1689 per file mode (PR #84285)
@@ -744,6 +744,9 @@ getCompilationDataBase(int argc, char **argv, std::string
) {
return nullptr;
}
+ // Only 1 threads is required if P1689 per file mode.
+ NumThreads = 1;
ChuanqiXu9 wrote:
Makes sense. Done.
https://github.com/llvm/llvm-project/pull/84285
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use thread pool in P1689 per file mode (PR #84285)
https://github.com/ChuanqiXu9 updated
https://github.com/llvm/llvm-project/pull/84285
>From dd9711be2368a299b408d0ff06ec9c1c0540083b Mon Sep 17 00:00:00 2001
From: Chuanqi Xu
Date: Thu, 7 Mar 2024 15:19:28 +0800
Subject: [PATCH 1/2] [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use
thread pool in P1689 per file mode
I suddenly found that the clang scan deps may use all concurrent threads
to scan the files. It makes sense in the batch mode. But in P1689
per file mode, it simply wastes times.
---
clang/tools/clang-scan-deps/ClangScanDeps.cpp | 204 +-
1 file changed, 108 insertions(+), 96 deletions(-)
diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
index d042fecc3dbe63..843816a8ed6515 100644
--- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -744,6 +744,9 @@ getCompilationDataBase(int argc, char **argv, std::string
) {
return nullptr;
}
+ // Only 1 threads is required if P1689 per file mode.
+ NumThreads = 1;
+
// There might be multiple jobs for a compilation. Extract the specified
// output filename from the last job.
auto LastCmd = C->getJobs().end();
@@ -867,13 +870,6 @@ int clang_scan_deps_main(int argc, char **argv, const
llvm::ToolContext &) {
// Print out the dependency results to STDOUT by default.
SharedStream DependencyOS(llvm::outs());
- DependencyScanningService Service(ScanMode, Format, OptimizeArgs,
-EagerLoadModules);
- llvm::DefaultThreadPool Pool(llvm::hardware_concurrency(NumThreads));
- std::vector> WorkerTools;
- for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I)
-WorkerTools.push_back(std::make_unique(Service));
-
std::vector Inputs =
AdjustingCompilations->getAllCompileCommands();
@@ -893,102 +889,118 @@ int clang_scan_deps_main(int argc, char **argv, const
llvm::ToolContext &) {
if (Format == ScanningOutputFormat::Full)
FD.emplace(ModuleName.empty() ? Inputs.size() : 0);
- if (Verbose) {
-llvm::outs() << "Running clang-scan-deps on " << Inputs.size()
- << " files using " << Pool.getMaxConcurrency() << "
workers\n";
- }
-
- llvm::Timer T;
- T.startTimer();
-
- for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I) {
-Pool.async([&, I]() {
- llvm::DenseSet AlreadySeenModules;
- while (auto MaybeInputIndex = GetNextInputIndex()) {
-size_t LocalIndex = *MaybeInputIndex;
-const tooling::CompileCommand *Input = [LocalIndex];
-std::string Filename = std::move(Input->Filename);
-std::string CWD = std::move(Input->Directory);
-
-std::optional MaybeModuleName;
-if (!ModuleName.empty())
- MaybeModuleName = ModuleName;
-
-std::string OutputDir(ModuleFilesDir);
-if (OutputDir.empty())
- OutputDir = getModuleCachePath(Input->CommandLine);
-auto LookupOutput = [&](const ModuleID , ModuleOutputKind MOK) {
- return ::lookupModuleOutput(MID, MOK, OutputDir);
-};
-
-// Run the tool on it.
-if (Format == ScanningOutputFormat::Make) {
- auto MaybeFile =
- WorkerTools[I]->getDependencyFile(Input->CommandLine, CWD);
- if (handleMakeDependencyToolResult(Filename, MaybeFile, DependencyOS,
- Errs))
-HadErrors = true;
-} else if (Format == ScanningOutputFormat::P1689) {
- // It is useful to generate the make-format dependency output during
- // the scanning for P1689. Otherwise the users need to scan again for
- // it. We will generate the make-format dependency output if we find
- // `-MF` in the command lines.
- std::string MakeformatOutputPath;
- std::string MakeformatOutput;
-
- auto MaybeRule = WorkerTools[I]->getP1689ModuleDependencyFile(
- *Input, CWD, MakeformatOutput, MakeformatOutputPath);
-
- if (handleP1689DependencyToolResult(Filename, MaybeRule, PD, Errs))
-HadErrors = true;
+ std::vector> WorkerTools;
- if (!MakeformatOutputPath.empty() && !MakeformatOutput.empty() &&
- !HadErrors) {
-static std::mutex Lock;
-// With compilation database, we may open different files
-// concurrently or we may write the same file concurrently. So we
-// use a map here to allow multiple compile commands to write to
the
-// same file. Also we need a lock here to avoid data race.
-static llvm::StringMap OSs;
-std::unique_lock LockGuard(Lock);
-
-auto OSIter = OSs.find(MakeformatOutputPath);
-if (OSIter == OSs.end()) {
- std::error_code EC;
- OSIter = OSs.try_emplace(MakeformatOutputPath,
- MakeformatOutputPath, EC)
-
[clang] [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use thread pool in P1689 per file mode (PR #84285)
https://github.com/jansvoboda11 edited https://github.com/llvm/llvm-project/pull/84285 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use thread pool in P1689 per file mode (PR #84285)
@@ -893,102 +889,118 @@ int clang_scan_deps_main(int argc, char **argv, const
llvm::ToolContext &) {
if (Format == ScanningOutputFormat::Full)
FD.emplace(ModuleName.empty() ? Inputs.size() : 0);
- if (Verbose) {
-llvm::outs() << "Running clang-scan-deps on " << Inputs.size()
- << " files using " << Pool.getMaxConcurrency() << "
workers\n";
- }
-
- llvm::Timer T;
- T.startTimer();
-
- for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I) {
-Pool.async([&, I]() {
- llvm::DenseSet AlreadySeenModules;
- while (auto MaybeInputIndex = GetNextInputIndex()) {
-size_t LocalIndex = *MaybeInputIndex;
-const tooling::CompileCommand *Input = [LocalIndex];
-std::string Filename = std::move(Input->Filename);
-std::string CWD = std::move(Input->Directory);
-
-std::optional MaybeModuleName;
-if (!ModuleName.empty())
- MaybeModuleName = ModuleName;
-
-std::string OutputDir(ModuleFilesDir);
-if (OutputDir.empty())
- OutputDir = getModuleCachePath(Input->CommandLine);
-auto LookupOutput = [&](const ModuleID , ModuleOutputKind MOK) {
- return ::lookupModuleOutput(MID, MOK, OutputDir);
-};
-
-// Run the tool on it.
-if (Format == ScanningOutputFormat::Make) {
- auto MaybeFile =
- WorkerTools[I]->getDependencyFile(Input->CommandLine, CWD);
- if (handleMakeDependencyToolResult(Filename, MaybeFile, DependencyOS,
- Errs))
-HadErrors = true;
-} else if (Format == ScanningOutputFormat::P1689) {
- // It is useful to generate the make-format dependency output during
- // the scanning for P1689. Otherwise the users need to scan again for
- // it. We will generate the make-format dependency output if we find
- // `-MF` in the command lines.
- std::string MakeformatOutputPath;
- std::string MakeformatOutput;
-
- auto MaybeRule = WorkerTools[I]->getP1689ModuleDependencyFile(
- *Input, CWD, MakeformatOutput, MakeformatOutputPath);
-
- if (handleP1689DependencyToolResult(Filename, MaybeRule, PD, Errs))
-HadErrors = true;
+ std::vector> WorkerTools;
jansvoboda11 wrote:
If `ScanningTask` takes `DependencyScanningTool &`, we could move this into the
`ThreadPool`-only branch.
https://github.com/llvm/llvm-project/pull/84285
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use thread pool in P1689 per file mode (PR #84285)
https://github.com/jansvoboda11 edited https://github.com/llvm/llvm-project/pull/84285 ___ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use thread pool in P1689 per file mode (PR #84285)
@@ -893,102 +889,118 @@ int clang_scan_deps_main(int argc, char **argv, const
llvm::ToolContext &) {
if (Format == ScanningOutputFormat::Full)
FD.emplace(ModuleName.empty() ? Inputs.size() : 0);
- if (Verbose) {
-llvm::outs() << "Running clang-scan-deps on " << Inputs.size()
- << " files using " << Pool.getMaxConcurrency() << "
workers\n";
- }
-
- llvm::Timer T;
- T.startTimer();
-
- for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I) {
-Pool.async([&, I]() {
- llvm::DenseSet AlreadySeenModules;
- while (auto MaybeInputIndex = GetNextInputIndex()) {
-size_t LocalIndex = *MaybeInputIndex;
-const tooling::CompileCommand *Input = [LocalIndex];
-std::string Filename = std::move(Input->Filename);
-std::string CWD = std::move(Input->Directory);
-
-std::optional MaybeModuleName;
-if (!ModuleName.empty())
- MaybeModuleName = ModuleName;
-
-std::string OutputDir(ModuleFilesDir);
-if (OutputDir.empty())
- OutputDir = getModuleCachePath(Input->CommandLine);
-auto LookupOutput = [&](const ModuleID , ModuleOutputKind MOK) {
- return ::lookupModuleOutput(MID, MOK, OutputDir);
-};
-
-// Run the tool on it.
-if (Format == ScanningOutputFormat::Make) {
- auto MaybeFile =
- WorkerTools[I]->getDependencyFile(Input->CommandLine, CWD);
- if (handleMakeDependencyToolResult(Filename, MaybeFile, DependencyOS,
- Errs))
-HadErrors = true;
-} else if (Format == ScanningOutputFormat::P1689) {
- // It is useful to generate the make-format dependency output during
- // the scanning for P1689. Otherwise the users need to scan again for
- // it. We will generate the make-format dependency output if we find
- // `-MF` in the command lines.
- std::string MakeformatOutputPath;
- std::string MakeformatOutput;
-
- auto MaybeRule = WorkerTools[I]->getP1689ModuleDependencyFile(
- *Input, CWD, MakeformatOutput, MakeformatOutputPath);
-
- if (handleP1689DependencyToolResult(Filename, MaybeRule, PD, Errs))
-HadErrors = true;
+ std::vector> WorkerTools;
- if (!MakeformatOutputPath.empty() && !MakeformatOutput.empty() &&
- !HadErrors) {
-static std::mutex Lock;
-// With compilation database, we may open different files
-// concurrently or we may write the same file concurrently. So we
-// use a map here to allow multiple compile commands to write to
the
-// same file. Also we need a lock here to avoid data race.
-static llvm::StringMap OSs;
-std::unique_lock LockGuard(Lock);
-
-auto OSIter = OSs.find(MakeformatOutputPath);
-if (OSIter == OSs.end()) {
- std::error_code EC;
- OSIter = OSs.try_emplace(MakeformatOutputPath,
- MakeformatOutputPath, EC)
- .first;
- if (EC)
-llvm::errs()
-<< "Failed to open P1689 make format output file \""
-<< MakeformatOutputPath << "\" for " << EC.message()
-<< "\n";
-}
+ auto ScanningTask = [&](unsigned I) {
jansvoboda11 wrote:
IIUC `I` is only used to index into `WorkerTools`. Could we just directly take
`WorkerTools &`? Makes this a bit clearer IMO - for a second I thought `I` is
an input index.
https://github.com/llvm/llvm-project/pull/84285
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use thread pool in P1689 per file mode (PR #84285)
@@ -744,6 +744,9 @@ getCompilationDataBase(int argc, char **argv, std::string
) {
return nullptr;
}
+ // Only 1 threads is required if P1689 per file mode.
+ NumThreads = 1;
jansvoboda11 wrote:
Instead of setting `NumThreads` for the P1689 format, can we check if size of
`Inputs` below and avoid creating the thread pool for any single-compilation
database?
https://github.com/llvm/llvm-project/pull/84285
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
[clang] [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use thread pool in P1689 per file mode (PR #84285)
llvmbot wrote:
@llvm/pr-subscribers-clang-modules
Author: Chuanqi Xu (ChuanqiXu9)
Changes
I suddenly found that the clang scan deps may use all concurrent threads to
scan the files. It makes sense in the batch mode. But in P1689 per file mode,
it simply wastes times and resources.
This patch itself should be a NFC patch. It simply moves codes.
---
Full diff: https://github.com/llvm/llvm-project/pull/84285.diff
1 Files Affected:
- (modified) clang/tools/clang-scan-deps/ClangScanDeps.cpp (+108-96)
``diff
diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
index d042fecc3dbe63..843816a8ed6515 100644
--- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -744,6 +744,9 @@ getCompilationDataBase(int argc, char **argv, std::string
) {
return nullptr;
}
+ // Only 1 threads is required if P1689 per file mode.
+ NumThreads = 1;
+
// There might be multiple jobs for a compilation. Extract the specified
// output filename from the last job.
auto LastCmd = C->getJobs().end();
@@ -867,13 +870,6 @@ int clang_scan_deps_main(int argc, char **argv, const
llvm::ToolContext &) {
// Print out the dependency results to STDOUT by default.
SharedStream DependencyOS(llvm::outs());
- DependencyScanningService Service(ScanMode, Format, OptimizeArgs,
-EagerLoadModules);
- llvm::DefaultThreadPool Pool(llvm::hardware_concurrency(NumThreads));
- std::vector> WorkerTools;
- for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I)
-WorkerTools.push_back(std::make_unique(Service));
-
std::vector Inputs =
AdjustingCompilations->getAllCompileCommands();
@@ -893,102 +889,118 @@ int clang_scan_deps_main(int argc, char **argv, const
llvm::ToolContext &) {
if (Format == ScanningOutputFormat::Full)
FD.emplace(ModuleName.empty() ? Inputs.size() : 0);
- if (Verbose) {
-llvm::outs() << "Running clang-scan-deps on " << Inputs.size()
- << " files using " << Pool.getMaxConcurrency() << "
workers\n";
- }
-
- llvm::Timer T;
- T.startTimer();
-
- for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I) {
-Pool.async([&, I]() {
- llvm::DenseSet AlreadySeenModules;
- while (auto MaybeInputIndex = GetNextInputIndex()) {
-size_t LocalIndex = *MaybeInputIndex;
-const tooling::CompileCommand *Input = [LocalIndex];
-std::string Filename = std::move(Input->Filename);
-std::string CWD = std::move(Input->Directory);
-
-std::optional MaybeModuleName;
-if (!ModuleName.empty())
- MaybeModuleName = ModuleName;
-
-std::string OutputDir(ModuleFilesDir);
-if (OutputDir.empty())
- OutputDir = getModuleCachePath(Input->CommandLine);
-auto LookupOutput = [&](const ModuleID , ModuleOutputKind MOK) {
- return ::lookupModuleOutput(MID, MOK, OutputDir);
-};
-
-// Run the tool on it.
-if (Format == ScanningOutputFormat::Make) {
- auto MaybeFile =
- WorkerTools[I]->getDependencyFile(Input->CommandLine, CWD);
- if (handleMakeDependencyToolResult(Filename, MaybeFile, DependencyOS,
- Errs))
-HadErrors = true;
-} else if (Format == ScanningOutputFormat::P1689) {
- // It is useful to generate the make-format dependency output during
- // the scanning for P1689. Otherwise the users need to scan again for
- // it. We will generate the make-format dependency output if we find
- // `-MF` in the command lines.
- std::string MakeformatOutputPath;
- std::string MakeformatOutput;
-
- auto MaybeRule = WorkerTools[I]->getP1689ModuleDependencyFile(
- *Input, CWD, MakeformatOutput, MakeformatOutputPath);
-
- if (handleP1689DependencyToolResult(Filename, MaybeRule, PD, Errs))
-HadErrors = true;
+ std::vector> WorkerTools;
- if (!MakeformatOutputPath.empty() && !MakeformatOutput.empty() &&
- !HadErrors) {
-static std::mutex Lock;
-// With compilation database, we may open different files
-// concurrently or we may write the same file concurrently. So we
-// use a map here to allow multiple compile commands to write to
the
-// same file. Also we need a lock here to avoid data race.
-static llvm::StringMap OSs;
-std::unique_lock LockGuard(Lock);
-
-auto OSIter = OSs.find(MakeformatOutputPath);
-if (OSIter == OSs.end()) {
- std::error_code EC;
- OSIter = OSs.try_emplace(MakeformatOutputPath,
- MakeformatOutputPath, EC)
- .first;
- if (EC)
-llvm::errs()
-<<
[clang] [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use thread pool in P1689 per file mode (PR #84285)
https://github.com/ChuanqiXu9 created
https://github.com/llvm/llvm-project/pull/84285
I suddenly found that the clang scan deps may use all concurrent threads to
scan the files. It makes sense in the batch mode. But in P1689 per file mode,
it simply wastes times and resources.
This patch itself should be a NFC patch. It simply moves codes.
>From 8f3060da8059dc300152a1a1bc297cffcba51a61 Mon Sep 17 00:00:00 2001
From: Chuanqi Xu
Date: Thu, 7 Mar 2024 15:19:28 +0800
Subject: [PATCH] [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use thread
pool in P1689 per file mode
I suddenly found that the clang scan deps may use all concurrent threads
to scan the files. It makes sense in the batch mode. But in P1689
per file mode, it simply wastes times.
---
clang/tools/clang-scan-deps/ClangScanDeps.cpp | 204 +-
1 file changed, 108 insertions(+), 96 deletions(-)
diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
index d042fecc3dbe63..843816a8ed6515 100644
--- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -744,6 +744,9 @@ getCompilationDataBase(int argc, char **argv, std::string
) {
return nullptr;
}
+ // Only 1 threads is required if P1689 per file mode.
+ NumThreads = 1;
+
// There might be multiple jobs for a compilation. Extract the specified
// output filename from the last job.
auto LastCmd = C->getJobs().end();
@@ -867,13 +870,6 @@ int clang_scan_deps_main(int argc, char **argv, const
llvm::ToolContext &) {
// Print out the dependency results to STDOUT by default.
SharedStream DependencyOS(llvm::outs());
- DependencyScanningService Service(ScanMode, Format, OptimizeArgs,
-EagerLoadModules);
- llvm::DefaultThreadPool Pool(llvm::hardware_concurrency(NumThreads));
- std::vector> WorkerTools;
- for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I)
-WorkerTools.push_back(std::make_unique(Service));
-
std::vector Inputs =
AdjustingCompilations->getAllCompileCommands();
@@ -893,102 +889,118 @@ int clang_scan_deps_main(int argc, char **argv, const
llvm::ToolContext &) {
if (Format == ScanningOutputFormat::Full)
FD.emplace(ModuleName.empty() ? Inputs.size() : 0);
- if (Verbose) {
-llvm::outs() << "Running clang-scan-deps on " << Inputs.size()
- << " files using " << Pool.getMaxConcurrency() << "
workers\n";
- }
-
- llvm::Timer T;
- T.startTimer();
-
- for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I) {
-Pool.async([&, I]() {
- llvm::DenseSet AlreadySeenModules;
- while (auto MaybeInputIndex = GetNextInputIndex()) {
-size_t LocalIndex = *MaybeInputIndex;
-const tooling::CompileCommand *Input = [LocalIndex];
-std::string Filename = std::move(Input->Filename);
-std::string CWD = std::move(Input->Directory);
-
-std::optional MaybeModuleName;
-if (!ModuleName.empty())
- MaybeModuleName = ModuleName;
-
-std::string OutputDir(ModuleFilesDir);
-if (OutputDir.empty())
- OutputDir = getModuleCachePath(Input->CommandLine);
-auto LookupOutput = [&](const ModuleID , ModuleOutputKind MOK) {
- return ::lookupModuleOutput(MID, MOK, OutputDir);
-};
-
-// Run the tool on it.
-if (Format == ScanningOutputFormat::Make) {
- auto MaybeFile =
- WorkerTools[I]->getDependencyFile(Input->CommandLine, CWD);
- if (handleMakeDependencyToolResult(Filename, MaybeFile, DependencyOS,
- Errs))
-HadErrors = true;
-} else if (Format == ScanningOutputFormat::P1689) {
- // It is useful to generate the make-format dependency output during
- // the scanning for P1689. Otherwise the users need to scan again for
- // it. We will generate the make-format dependency output if we find
- // `-MF` in the command lines.
- std::string MakeformatOutputPath;
- std::string MakeformatOutput;
-
- auto MaybeRule = WorkerTools[I]->getP1689ModuleDependencyFile(
- *Input, CWD, MakeformatOutput, MakeformatOutputPath);
-
- if (handleP1689DependencyToolResult(Filename, MaybeRule, PD, Errs))
-HadErrors = true;
+ std::vector> WorkerTools;
- if (!MakeformatOutputPath.empty() && !MakeformatOutput.empty() &&
- !HadErrors) {
-static std::mutex Lock;
-// With compilation database, we may open different files
-// concurrently or we may write the same file concurrently. So we
-// use a map here to allow multiple compile commands to write to
the
-// same file. Also we need a lock here to avoid data race.
-static llvm::StringMap OSs;
-std::unique_lock LockGuard(Lock);
-
-auto
