qchateau updated this revision to Diff 429478.
qchateau added a comment.
Small correction
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D112661/new/
https://reviews.llvm.org/D112661
Files:
clang-tools-extra/clangd/ClangdServer.cpp
clang-tools-extra/clangd/ClangdServer.h
clang-tools-extra/clangd/ParsedAST.cpp
clang-tools-extra/clangd/TUScheduler.cpp
clang-tools-extra/clangd/TUScheduler.h
clang-tools-extra/clangd/tool/ClangdMain.cpp
Index: clang-tools-extra/clangd/tool/ClangdMain.cpp
===================================================================
--- clang-tools-extra/clangd/tool/ClangdMain.cpp
+++ clang-tools-extra/clangd/tool/ClangdMain.cpp
@@ -334,6 +334,13 @@
init(getDefaultAsyncThreadsCount()),
};
+opt<unsigned> ClosedPreambleCacheSize{
+ "closed-preamble-cache-size",
+ cat(Misc),
+ desc("Number of preambles of closed files to keep in the cache."),
+ init(1),
+};
+
opt<Path> IndexFile{
"index-file",
cat(Misc),
@@ -879,6 +886,7 @@
Opts.StaticIndex = PAI.get();
}
Opts.AsyncThreadsCount = WorkerThreadsCount;
+ Opts.ClosedPreambleCacheSize = ClosedPreambleCacheSize;
Opts.FoldingRanges = FoldingRanges;
Opts.MemoryCleanup = getMemoryCleanupFunction();
Index: clang-tools-extra/clangd/TUScheduler.h
===================================================================
--- clang-tools-extra/clangd/TUScheduler.h
+++ clang-tools-extra/clangd/TUScheduler.h
@@ -190,6 +190,9 @@
/// If 0, executes actions synchronously on the calling thread.
unsigned AsyncThreadsCount = getDefaultAsyncThreadsCount();
+ // Number of preambles of closed files to keep in the cache.
+ unsigned ClosedPreambleCacheSize = 1;
+
/// Cache (large) preamble data in RAM rather than temporary files on disk.
bool StorePreamblesInMemory = false;
@@ -313,6 +316,8 @@
class ASTCache;
/// Tracks headers included by open files, to get known-good compile commands.
class HeaderIncluderCache;
+ /// Store all known preambles
+ class PreambleStore;
// The file being built/processed in the current thread. This is a hack in
// order to get the file name into the index implementations. Do not depend on
@@ -336,6 +341,7 @@
llvm::StringMap<std::unique_ptr<FileData>> Files;
std::unique_ptr<ASTCache> IdleASTs;
std::unique_ptr<HeaderIncluderCache> HeaderIncluders;
+ std::unique_ptr<PreambleStore> Preambles;
// None when running tasks synchronously and non-None when running tasks
// asynchronously.
llvm::Optional<AsyncTaskRunner> PreambleTasks;
Index: clang-tools-extra/clangd/TUScheduler.cpp
===================================================================
--- clang-tools-extra/clangd/TUScheduler.cpp
+++ clang-tools-extra/clangd/TUScheduler.cpp
@@ -342,6 +342,65 @@
}
};
+class TUScheduler::PreambleStore {
+public:
+ struct Entry {
+ std::shared_ptr<const PreambleData> Preamble;
+ size_t Score;
+ Path FileName;
+
+ bool operator>(const Entry &Other) const { return Score > Other.Score; }
+ };
+
+ explicit PreambleStore(size_t ClosedPreamblesToKeep)
+ : ClosedPreamblesToKeep(ClosedPreamblesToKeep) {}
+
+ auto getAll() {
+ std::unique_lock<std::mutex> Lock(Mut);
+ return Store;
+ }
+
+ void push(PathRef FileName, std::shared_ptr<const PreambleData> Preamble) {
+ std::unique_lock<std::mutex> Lock(Mut);
+ auto It = llvm::find_if(
+ Store, [&](const auto &Item) { return Item.FileName == FileName; });
+ if (It == Store.end()) {
+ Store.push_back(Entry{std::move(Preamble), 0, FileName.str()});
+ popWorstPreambles();
+ } else {
+ It->Preamble = std::move(Preamble);
+ }
+ vlog("Store contains {0} preambles", Store.size());
+ }
+
+ void hit(PathRef FileName) {
+ std::unique_lock<std::mutex> Lock(Mut);
+ auto It = llvm::find_if(
+ Store, [&](const auto &Item) { return Item.FileName == FileName; });
+ if (It == Store.end()) {
+ return;
+ }
+ It->Score++;
+ }
+
+private:
+ void popWorstPreambles() {
+ auto Begin = llvm::partition(
+ Store, [](const auto &Item) { return Item.Preamble.use_count() > 1; });
+ if (static_cast<size_t>(std::distance(Begin, Store.end())) <=
+ ClosedPreamblesToKeep) {
+ return;
+ }
+ auto Nth = Begin + ClosedPreamblesToKeep;
+ std::nth_element(Begin, Nth, Store.end(), std::greater<Entry>());
+ Store.erase(Nth, Store.end());
+ }
+
+ std::mutex Mut;
+ std::vector<Entry> Store;
+ size_t ClosedPreamblesToKeep;
+};
+
namespace {
bool isReliable(const tooling::CompileCommand &Cmd) {
@@ -541,7 +600,8 @@
ASTWorker(PathRef FileName, const GlobalCompilationDatabase &CDB,
TUScheduler::ASTCache &LRUCache,
TUScheduler::HeaderIncluderCache &HeaderIncluders,
- Semaphore &Barrier, bool RunSync, const TUScheduler::Options &Opts,
+ TUScheduler::PreambleStore &Preambles, Semaphore &Barrier,
+ bool RunSync, const TUScheduler::Options &Opts,
ParsingCallbacks &Callbacks);
public:
@@ -554,8 +614,9 @@
create(PathRef FileName, const GlobalCompilationDatabase &CDB,
TUScheduler::ASTCache &IdleASTs,
TUScheduler::HeaderIncluderCache &HeaderIncluders,
- AsyncTaskRunner *Tasks, Semaphore &Barrier,
- const TUScheduler::Options &Opts, ParsingCallbacks &Callbacks);
+ TUScheduler::PreambleStore &Preambles, AsyncTaskRunner *Tasks,
+ Semaphore &Barrier, const TUScheduler::Options &Opts,
+ ParsingCallbacks &Callbacks);
~ASTWorker();
void update(ParseInputs Inputs, WantDiagnostics, bool ContentChanged);
@@ -567,6 +628,9 @@
std::shared_ptr<const PreambleData> getPossiblyStalePreamble(
std::shared_ptr<const ASTSignals> *ASTSignals = nullptr) const;
+ std::tuple<std::shared_ptr<const PreambleData>, Optional<PreamblePatch>>
+ getBestPreamble(
+ std::shared_ptr<const ASTSignals> *ASTSignals = nullptr) const;
/// Used to inform ASTWorker about a new preamble build by PreambleThread.
/// Diagnostics are only published through this callback. This ensures they
@@ -635,6 +699,9 @@
/// Should the first task in the queue be skipped instead of run?
bool shouldSkipHeadLocked() const;
+ /// Get all preambles that may be used to accelerate the AST build
+ std::vector<TUScheduler::PreambleStore::Entry> getCompatiblePreambles() const;
+
struct Request {
llvm::unique_function<void()> Action;
std::string Name;
@@ -649,6 +716,7 @@
/// Handles retention of ASTs.
TUScheduler::ASTCache &IdleASTs;
TUScheduler::HeaderIncluderCache &HeaderIncluders;
+ TUScheduler::PreambleStore &Preambles;
const bool RunSync;
/// Time to wait after an update to see whether another update obsoletes it.
const DebouncePolicy UpdateDebounce;
@@ -751,12 +819,12 @@
ASTWorker::create(PathRef FileName, const GlobalCompilationDatabase &CDB,
TUScheduler::ASTCache &IdleASTs,
TUScheduler::HeaderIncluderCache &HeaderIncluders,
- AsyncTaskRunner *Tasks, Semaphore &Barrier,
- const TUScheduler::Options &Opts,
+ TUScheduler::PreambleStore &Preambles, AsyncTaskRunner *Tasks,
+ Semaphore &Barrier, const TUScheduler::Options &Opts,
ParsingCallbacks &Callbacks) {
- std::shared_ptr<ASTWorker> Worker(
- new ASTWorker(FileName, CDB, IdleASTs, HeaderIncluders, Barrier,
- /*RunSync=*/!Tasks, Opts, Callbacks));
+ std::shared_ptr<ASTWorker> Worker(new ASTWorker(
+ FileName, CDB, IdleASTs, HeaderIncluders, Preambles, Barrier,
+ /*RunSync=*/!Tasks, Opts, Callbacks));
if (Tasks) {
Tasks->runAsync("ASTWorker:" + llvm::sys::path::filename(FileName),
[Worker]() { Worker->run(); });
@@ -770,10 +838,11 @@
ASTWorker::ASTWorker(PathRef FileName, const GlobalCompilationDatabase &CDB,
TUScheduler::ASTCache &LRUCache,
TUScheduler::HeaderIncluderCache &HeaderIncluders,
- Semaphore &Barrier, bool RunSync,
- const TUScheduler::Options &Opts,
+ TUScheduler::PreambleStore &Preambles, Semaphore &Barrier,
+ bool RunSync, const TUScheduler::Options &Opts,
ParsingCallbacks &Callbacks)
- : IdleASTs(LRUCache), HeaderIncluders(HeaderIncluders), RunSync(RunSync),
+ : IdleASTs(LRUCache), HeaderIncluders(HeaderIncluders),
+ Preambles(Preambles), RunSync(RunSync),
UpdateDebounce(Opts.UpdateDebounce), FileName(FileName),
ContextProvider(Opts.ContextProvider), CDB(CDB), Callbacks(Callbacks),
Barrier(Barrier), Done(false), Status(FileName, Callbacks),
@@ -882,14 +951,18 @@
PreamblePeer.update(std::move(Invocation), std::move(Inputs),
std::move(CompilerInvocationDiags), WantDiags);
+
std::unique_lock<std::mutex> Lock(Mutex);
PreambleCV.wait(Lock, [this] {
// Block until we reiceve a preamble request, unless a preamble already
// exists, as patching an empty preamble would imply rebuilding it from
// scratch.
+ // It is also acceptable to unblock is we have compatible preambles
+ // that may be used to accelerate the AST build.
// We block here instead of the consumer to prevent any deadlocks. Since
// LatestPreamble is only populated by ASTWorker thread.
- return LatestPreamble || !PreambleRequests.empty() || Done;
+ return LatestPreamble || !PreambleRequests.empty() || Done ||
+ !getCompatiblePreambles().empty();
});
};
startTask(TaskName, std::move(Task), UpdateType{WantDiags, ContentChanged},
@@ -916,13 +989,17 @@
vlog("ASTWorker rebuilding evicted AST to run {0}: {1} version {2}", Name,
FileName, FileInputs.Version);
// FIXME: We might need to build a patched ast once preamble thread starts
- // running async. Currently getPossiblyStalePreamble below will always
+ // running async. Currently getBestPreamble below will always
// return a compatible preamble as ASTWorker::update blocks.
+ auto BestPreamble = getBestPreamble();
+ if (const auto &Patch = std::get<Optional<PreamblePatch>>(BestPreamble))
+ Patch->apply(*Invocation);
llvm::Optional<ParsedAST> NewAST;
if (Invocation) {
- NewAST = ParsedAST::build(FileName, FileInputs, std::move(Invocation),
- CompilerInvocationDiagConsumer.take(),
- getPossiblyStalePreamble());
+ NewAST = ParsedAST::build(
+ FileName, FileInputs, std::move(Invocation),
+ CompilerInvocationDiagConsumer.take(),
+ std::get<std::shared_ptr<const PreambleData>>(BestPreamble));
++ASTBuildCount;
}
AST = NewAST ? std::make_unique<ParsedAST>(std::move(*NewAST)) : nullptr;
@@ -1051,6 +1128,7 @@
else
LatestPreamble = std::move(Preamble);
}
+ Preambles.push(FileName, *LatestPreamble);
// Notify anyone waiting for a preamble.
PreambleCV.notify_all();
// Give up our ownership to old preamble before starting expensive AST
@@ -1193,6 +1271,51 @@
return LatestPreamble ? *LatestPreamble : nullptr;
}
+std::vector<TUScheduler::PreambleStore::Entry>
+ASTWorker::getCompatiblePreambles() const {
+ auto AllPreamblesEntries = Preambles.getAll();
+ auto End = llvm::remove_if(AllPreamblesEntries, [&](const auto &Item) {
+ auto TransferedCmd = tooling::transferCompileCommand(
+ Item.Preamble->CompileCommand, FileName);
+ return TransferedCmd != FileInputs.CompileCommand;
+ });
+ AllPreamblesEntries.erase(End, AllPreamblesEntries.end());
+ return AllPreamblesEntries;
+}
+
+std::tuple<std::shared_ptr<const PreambleData>, Optional<PreamblePatch>>
+ASTWorker::getBestPreamble(
+ std::shared_ptr<const ASTSignals> *ASTSignals) const {
+ if (auto Preamble = getPossiblyStalePreamble(ASTSignals)) {
+ vlog("Best premable is the real one");
+ return {Preamble, None};
+ }
+
+ auto CompatiblePreambles = getCompatiblePreambles();
+ vlog("Looking for the best of {0} preambles ...", CompatiblePreambles.size());
+ if (CompatiblePreambles.empty()) {
+ vlog("No compatible preamble");
+ return {nullptr, None};
+ }
+
+ auto Best = CompatiblePreambles.begin();
+ auto BestPatch =
+ PreamblePatch::createFullPatch(FileName, FileInputs, *Best->Preamble);
+
+ for (auto It = Best + 1; It != CompatiblePreambles.end(); ++It) {
+ auto Patch =
+ PreamblePatch::createFullPatch(FileName, FileInputs, *It->Preamble);
+ if (Patch.preambleIncludes().size() > BestPatch.preambleIncludes().size()) {
+ BestPatch = std::move(Patch);
+ Best = It;
+ }
+ }
+
+ vlog("Using preamble from: {0}", Best->FileName);
+ Preambles.hit(Best->FileName);
+ return {Best->Preamble, BestPatch};
+}
+
void ASTWorker::waitForFirstPreamble() const {
std::unique_lock<std::mutex> Lock(Mutex);
PreambleCV.wait(Lock, [this] { return LatestPreamble.hasValue() || Done; });
@@ -1549,7 +1672,8 @@
Barrier(Opts.AsyncThreadsCount), QuickRunBarrier(Opts.AsyncThreadsCount),
IdleASTs(
std::make_unique<ASTCache>(Opts.RetentionPolicy.MaxRetainedASTs)),
- HeaderIncluders(std::make_unique<HeaderIncluderCache>()) {
+ HeaderIncluders(std::make_unique<HeaderIncluderCache>()),
+ Preambles(std::make_unique<PreambleStore>(Opts.ClosedPreambleCacheSize)) {
// Avoid null checks everywhere.
if (!Opts.ContextProvider) {
this->Opts.ContextProvider = [](llvm::StringRef) {
@@ -1591,7 +1715,7 @@
if (!FD) {
// Create a new worker to process the AST-related tasks.
ASTWorkerHandle Worker =
- ASTWorker::create(File, CDB, *IdleASTs, *HeaderIncluders,
+ ASTWorker::create(File, CDB, *IdleASTs, *HeaderIncluders, *Preambles,
WorkerThreads ? WorkerThreads.getPointer() : nullptr,
Barrier, Opts, *Callbacks);
FD = std::unique_ptr<FileData>(
@@ -1684,11 +1808,11 @@
trace::Span Tracer(Name);
SPAN_ATTACH(Tracer, "file", File);
std::shared_ptr<const ASTSignals> Signals;
- std::shared_ptr<const PreambleData> Preamble =
- It->second->Worker->getPossiblyStalePreamble(&Signals);
+ auto Preamble = std::get<std::shared_ptr<const PreambleData>>(
+ It->second->Worker->getBestPreamble(&Signals));
+ auto CompileCommand = It->second->Worker->getCurrentCompileCommand();
WithContext WithProvidedContext(Opts.ContextProvider(File));
- Action(InputsAndPreamble{It->second->Contents,
- It->second->Worker->getCurrentCompileCommand(),
+ Action(InputsAndPreamble{It->second->Contents, CompileCommand,
Preamble.get(), Signals.get()});
return;
}
@@ -1705,7 +1829,6 @@
crashDumpCompileCommand(llvm::errs(), Command);
crashDumpFileContents(llvm::errs(), Contents);
});
- std::shared_ptr<const PreambleData> Preamble;
if (Consistency == PreambleConsistency::Stale) {
// Wait until the preamble is built for the first time, if preamble
// is required. This avoids extra work of processing the preamble
@@ -1713,7 +1836,8 @@
Worker->waitForFirstPreamble();
}
std::shared_ptr<const ASTSignals> Signals;
- Preamble = Worker->getPossiblyStalePreamble(&Signals);
+ auto Preamble = std::get<std::shared_ptr<const PreambleData>>(
+ Worker->getBestPreamble(&Signals));
std::lock_guard<Semaphore> BarrierLock(Barrier);
WithContext Guard(std::move(Ctx));
Index: clang-tools-extra/clangd/ParsedAST.cpp
===================================================================
--- clang-tools-extra/clangd/ParsedAST.cpp
+++ clang-tools-extra/clangd/ParsedAST.cpp
@@ -41,6 +41,7 @@
#include "clang/Lex/Lexer.h"
#include "clang/Lex/PPCallbacks.h"
#include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/PreprocessorOptions.h"
#include "clang/Serialization/ASTWriter.h"
#include "clang/Tooling/CompilationDatabase.h"
#include "clang/Tooling/Syntax/Tokens.h"
@@ -587,8 +588,9 @@
// Copy over the macros in the preamble region of the main file, and combine
// with non-preamble macros below.
MainFileMacros Macros;
- if (Preamble)
+ if (Preamble && Preamble->CompileCommand.Filename == Filename) {
Macros = Preamble->Macros;
+ }
Clang->getPreprocessor().addPPCallbacks(
std::make_unique<CollectMainFileMacros>(Clang->getSourceManager(),
Macros));
Index: clang-tools-extra/clangd/ClangdServer.h
===================================================================
--- clang-tools-extra/clangd/ClangdServer.h
+++ clang-tools-extra/clangd/ClangdServer.h
@@ -98,6 +98,9 @@
/// thread, and callbacks are invoked before "async" functions return.
unsigned AsyncThreadsCount = getDefaultAsyncThreadsCount();
+ // Number of preambles of closed files to keep in the cache.
+ unsigned ClosedPreambleCacheSize = 1;
+
/// AST caching policy. The default is to keep up to 3 ASTs in memory.
ASTRetentionPolicy RetentionPolicy;
Index: clang-tools-extra/clangd/ClangdServer.cpp
===================================================================
--- clang-tools-extra/clangd/ClangdServer.cpp
+++ clang-tools-extra/clangd/ClangdServer.cpp
@@ -129,12 +129,14 @@
Opts.UpdateDebounce = DebouncePolicy::fixed(/*zero*/ {});
Opts.StorePreamblesInMemory = true;
Opts.AsyncThreadsCount = 4; // Consistent!
+ Opts.ClosedPreambleCacheSize = 0;
return Opts;
}
ClangdServer::Options::operator TUScheduler::Options() const {
TUScheduler::Options Opts;
Opts.AsyncThreadsCount = AsyncThreadsCount;
+ Opts.ClosedPreambleCacheSize = ClosedPreambleCacheSize;
Opts.RetentionPolicy = RetentionPolicy;
Opts.StorePreamblesInMemory = StorePreamblesInMemory;
Opts.UpdateDebounce = UpdateDebounce;
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
