https://github.com/jasonwilliams updated https://github.com/llvm/llvm-project/pull/175209
>From 86f4d8beccc66b57fe30f86b1d8b7a6bc2fbea7f Mon Sep 17 00:00:00 2001 From: jasonwilliams <[email protected]> Date: Fri, 9 Jan 2026 17:23:51 +0000 Subject: [PATCH 1/8] [clangd] Add background index format support to clangd-indexer Add support for generating background index shards directly from clangd-indexer, enabling offline pre-indexing of projects for use with clangd's background index. --- .../clangd/index/Serialization.h | 5 +- .../clangd/indexer/IndexerMain.cpp | 205 ++++++++++++++++-- 2 files changed, 191 insertions(+), 19 deletions(-) diff --git a/clang-tools-extra/clangd/index/Serialization.h b/clang-tools-extra/clangd/index/Serialization.h index bf8e036afcb6c..1553e702a5881 100644 --- a/clang-tools-extra/clangd/index/Serialization.h +++ b/clang-tools-extra/clangd/index/Serialization.h @@ -35,8 +35,9 @@ namespace clang { namespace clangd { enum class IndexFileFormat { - RIFF, // Versioned binary format, suitable for production use. - YAML, // Human-readable format, suitable for experiments and debugging. + RIFF, // Versioned binary format, suitable for production use. + YAML, // Human-readable format, suitable for experiments and debugging. + BACKGROUND // Background index format, suitable for language server use. }; // Holds the contents of an index file that was read. diff --git a/clang-tools-extra/clangd/indexer/IndexerMain.cpp b/clang-tools-extra/clangd/indexer/IndexerMain.cpp index 94db860f0b9b5..bf5eecaaaa6b4 100644 --- a/clang-tools-extra/clangd/indexer/IndexerMain.cpp +++ b/clang-tools-extra/clangd/indexer/IndexerMain.cpp @@ -12,6 +12,8 @@ #include "CompileCommands.h" #include "Compiler.h" +#include "GlobalCompilationDatabase.h" +#include "index/Background.h" #include "index/IndexAction.h" #include "index/Merge.h" #include "index/Ref.h" @@ -30,13 +32,14 @@ namespace clang { namespace clangd { namespace { -static llvm::cl::opt<IndexFileFormat> - Format("format", llvm::cl::desc("Format of the index to be written"), - llvm::cl::values(clEnumValN(IndexFileFormat::YAML, "yaml", - "human-readable YAML format"), - clEnumValN(IndexFileFormat::RIFF, "binary", - "binary RIFF format")), - llvm::cl::init(IndexFileFormat::RIFF)); +static llvm::cl::opt<IndexFileFormat> Format( + "format", llvm::cl::desc("Format of the index to be written"), + llvm::cl::values( + clEnumValN(IndexFileFormat::YAML, "yaml", "human-readable YAML format"), + clEnumValN(IndexFileFormat::RIFF, "binary", "binary RIFF format"), + clEnumValN(IndexFileFormat::BACKGROUND, "background", + "background index format for language servers")), + llvm::cl::init(IndexFileFormat::RIFF)); static llvm::cl::list<std::string> QueryDriverGlobs{ "query-driver", @@ -48,6 +51,16 @@ static llvm::cl::list<std::string> QueryDriverGlobs{ llvm::cl::CommaSeparated, }; +static llvm::cl::opt<std::string> ProjectRoot{ + "project-root", + llvm::cl::desc( + "Path to the project root for --format=background. " + "Determines where to store index shards. Shards are stored in " + "<project-root>/.cache/clangd/index/. " + "Defaults to current directory if not specified."), +}; + +// Action factory that merges all symbols into a single index (for YAML/RIFF). class IndexActionFactory : public tooling::FrontendActionFactory { public: IndexActionFactory(IndexFileIn &Result) : Result(Result) {} @@ -123,6 +136,117 @@ class IndexActionFactory : public tooling::FrontendActionFactory { RelationSlab::Builder Relations; }; +// Action factory that writes per-file shards (for background index format). +class BackgroundIndexActionFactory : public tooling::FrontendActionFactory { +public: + BackgroundIndexActionFactory(BackgroundIndexStorage &Storage) + : Storage(Storage), Symbols(std::make_unique<SymbolSlab::Builder>()), + Refs(std::make_unique<RefSlab::Builder>()), + Relations(std::make_unique<RelationSlab::Builder>()) {} + + std::unique_ptr<FrontendAction> create() override { + SymbolCollector::Options Opts; + Opts.CountReferences = true; + Opts.FileFilter = [&](const SourceManager &SM, FileID FID) { + const auto F = SM.getFileEntryRefForID(FID); + if (!F) + return false; + auto AbsPath = getCanonicalPath(*F, SM.getFileManager()); + if (!AbsPath) + return false; + std::lock_guard<std::mutex> Lock(FilesMu); + return Files.insert(*AbsPath).second; + }; + return createStaticIndexingAction( + Opts, + [&](SymbolSlab S) { + std::lock_guard<std::mutex> Lock(SymbolsMu); + for (const auto &Sym : S) { + if (const auto *Existing = Symbols->find(Sym.ID)) + Symbols->insert(mergeSymbol(*Existing, Sym)); + else + Symbols->insert(Sym); + } + }, + [&](RefSlab S) { + std::lock_guard<std::mutex> Lock(RefsMu); + for (const auto &Sym : S) { + for (const auto &Ref : Sym.second) + Refs->insert(Sym.first, Ref); + } + }, + [&](RelationSlab S) { + std::lock_guard<std::mutex> Lock(RelsMu); + for (const auto &R : S) + Relations->insert(R); + }, + /*IncludeGraphCallback=*/nullptr); + } + + bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation, + FileManager *Files, + std::shared_ptr<PCHContainerOperations> PCHContainerOps, + DiagnosticConsumer *DiagConsumer) override { + disableUnsupportedOptions(*Invocation); + + // Get the main file path before running. + std::string MainFile; + if (!Invocation->getFrontendOpts().Inputs.empty()) + MainFile = Invocation->getFrontendOpts().Inputs[0].getFile().str(); + + bool Success = tooling::FrontendActionFactory::runInvocation( + std::move(Invocation), Files, std::move(PCHContainerOps), DiagConsumer); + + // After processing, write a shard for this file. + if (Success && !MainFile.empty()) + writeShardForFile(MainFile); + + return Success; + } + +private: + void writeShardForFile(llvm::StringRef MainFile) { + IndexFileIn Data; + { + std::lock_guard<std::mutex> Lock(SymbolsMu); + Data.Symbols = std::move(*Symbols).build(); + Symbols = std::make_unique<SymbolSlab::Builder>(); + } + { + std::lock_guard<std::mutex> Lock(RefsMu); + Data.Refs = std::move(*Refs).build(); + Refs = std::make_unique<RefSlab::Builder>(); + } + { + std::lock_guard<std::mutex> Lock(RelsMu); + Data.Relations = std::move(*Relations).build(); + Relations = std::make_unique<RelationSlab::Builder>(); + } + + IndexFileOut Out(Data); + Out.Format = IndexFileFormat::RIFF; // Shards use RIFF format. + + if (auto Err = Storage.storeShard(MainFile, Out)) { + elog("Failed to write shard for {0}: {1}", MainFile, std::move(Err)); + } else { + std::lock_guard<std::mutex> Lock(FilesMu); + ++ShardsWritten; + log("Wrote shard for {0} ({1} total)", MainFile, ShardsWritten); + } + } + + BackgroundIndexStorage &Storage; + std::mutex FilesMu; + llvm::StringSet<> Files; + unsigned ShardsWritten = 0; + std::mutex SymbolsMu; + std::unique_ptr<SymbolSlab::Builder> Symbols; + std::mutex RefsMu; + std::unique_ptr<RefSlab::Builder> Refs; + std::mutex RelsMu; + std::unique_ptr<RelationSlab::Builder> Relations; +}; + } // namespace } // namespace clangd } // namespace clang @@ -141,6 +265,13 @@ int main(int argc, const char **argv) { $ clangd-indexer File1.cpp File2.cpp ... FileN.cpp > clangd.dex + Example usage for background index format (writes shards to disk): + + $ clangd-indexer --format=background --executor=all-TUs build/ + + This writes index shards to .cache/clangd/index/ in the current directory. + Use --project-root to specify a different location for the shards. + Note: only symbols from header files will be indexed. )"; @@ -152,23 +283,63 @@ int main(int argc, const char **argv) { return 1; } - // Collect symbols found in each translation unit, merging as we go. - clang::clangd::IndexFileIn Data; auto Mangler = std::make_shared<clang::clangd::CommandMangler>( clang::clangd::CommandMangler::detect()); Mangler->SystemIncludeExtractor = clang::clangd::getSystemIncludeExtractor( static_cast<llvm::ArrayRef<std::string>>( clang::clangd::QueryDriverGlobs)); + + auto Adjuster = clang::tooling::ArgumentsAdjuster( + [Mangler = std::move(Mangler)](const std::vector<std::string> &Args, + llvm::StringRef File) { + clang::tooling::CompileCommand Cmd; + Cmd.CommandLine = Args; + Mangler->operator()(Cmd, File); + return Cmd.CommandLine; + }); + + // Handle background index format separately - writes per-file shards. + if (clang::clangd::Format == clang::clangd::IndexFileFormat::BACKGROUND) { + // Default to current directory if --project-root not specified. + std::string Root = clang::clangd::ProjectRoot; + if (Root.empty()) { + llvm::SmallString<256> CurrentDir; + if (auto EC = llvm::sys::fs::current_path(CurrentDir)) { + llvm::errs() << "Error: Failed to get current directory: " + << EC.message() << "\n"; + return 1; + } + Root = std::string(CurrentDir); + } + + // Create storage factory for disk-backed index shards. + auto IndexStorageFactory = + clang::clangd::BackgroundIndexStorage::createDiskBackedStorageFactory( + [Root](clang::clangd::PathRef) { + return clang::clangd::ProjectInfo{Root}; + }); + + // Get storage for the project root. + clang::clangd::BackgroundIndexStorage *Storage = IndexStorageFactory(Root); + + auto Err = Executor->get()->execute( + std::make_unique<clang::clangd::BackgroundIndexActionFactory>(*Storage), + std::move(Adjuster)); + if (Err) { + clang::clangd::elog("{0}", std::move(Err)); + return 1; + } + + llvm::errs() << "Background index shards written to " << Root + << "/.cache/clangd/index/\n"; + return 0; + } + + // Standard mode: collect and merge symbols, then emit to stdout. + clang::clangd::IndexFileIn Data; auto Err = Executor->get()->execute( std::make_unique<clang::clangd::IndexActionFactory>(Data), - clang::tooling::ArgumentsAdjuster( - [Mangler = std::move(Mangler)](const std::vector<std::string> &Args, - llvm::StringRef File) { - clang::tooling::CompileCommand Cmd; - Cmd.CommandLine = Args; - Mangler->operator()(Cmd, File); - return Cmd.CommandLine; - })); + std::move(Adjuster)); if (Err) { clang::clangd::elog("{0}", std::move(Err)); } >From c6988120561d54a988f3ee268c771dacbad8053b Mon Sep 17 00:00:00 2001 From: jasonwilliams <[email protected]> Date: Sun, 11 Jan 2026 15:01:40 +0000 Subject: [PATCH 2/8] handle background case for serialization --- clang-tools-extra/clangd/index/Serialization.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/clang-tools-extra/clangd/index/Serialization.cpp b/clang-tools-extra/clangd/index/Serialization.cpp index f03839599612c..10388b1948f43 100644 --- a/clang-tools-extra/clangd/index/Serialization.cpp +++ b/clang-tools-extra/clangd/index/Serialization.cpp @@ -686,6 +686,8 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const IndexFileOut &O) { case IndexFileFormat::YAML: writeYAML(O, OS); break; + case IndexFileFormat::BACKGROUND: + llvm_unreachable("BACKGROUND format not supported for serialization"); } return OS; } >From c4d335016379a89162785593d94a2de868b3f052 Mon Sep 17 00:00:00 2001 From: jasonwilliams <[email protected]> Date: Fri, 23 Jan 2026 10:43:13 +0000 Subject: [PATCH 3/8] change name to sharded instead of background --- .../clangd/index/Serialization.cpp | 4 ++-- .../clangd/index/Serialization.h | 6 ++--- .../clangd/indexer/IndexerMain.cpp | 22 +++++++++---------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/clang-tools-extra/clangd/index/Serialization.cpp b/clang-tools-extra/clangd/index/Serialization.cpp index 10388b1948f43..fbb672f184dd2 100644 --- a/clang-tools-extra/clangd/index/Serialization.cpp +++ b/clang-tools-extra/clangd/index/Serialization.cpp @@ -686,8 +686,8 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const IndexFileOut &O) { case IndexFileFormat::YAML: writeYAML(O, OS); break; - case IndexFileFormat::BACKGROUND: - llvm_unreachable("BACKGROUND format not supported for serialization"); + case IndexFileFormat::SHARDED: + llvm_unreachable("SHARDED format not supported for serialization"); } return OS; } diff --git a/clang-tools-extra/clangd/index/Serialization.h b/clang-tools-extra/clangd/index/Serialization.h index 1553e702a5881..d0939e051ecaf 100644 --- a/clang-tools-extra/clangd/index/Serialization.h +++ b/clang-tools-extra/clangd/index/Serialization.h @@ -35,9 +35,9 @@ namespace clang { namespace clangd { enum class IndexFileFormat { - RIFF, // Versioned binary format, suitable for production use. - YAML, // Human-readable format, suitable for experiments and debugging. - BACKGROUND // Background index format, suitable for language server use. + RIFF, // Versioned binary format, suitable for production use. + YAML, // Human-readable format, suitable for experiments and debugging. + SHARDED // Sharded index format, suitable for language server use. }; // Holds the contents of an index file that was read. diff --git a/clang-tools-extra/clangd/indexer/IndexerMain.cpp b/clang-tools-extra/clangd/indexer/IndexerMain.cpp index bf5eecaaaa6b4..64f610590d246 100644 --- a/clang-tools-extra/clangd/indexer/IndexerMain.cpp +++ b/clang-tools-extra/clangd/indexer/IndexerMain.cpp @@ -37,8 +37,8 @@ static llvm::cl::opt<IndexFileFormat> Format( llvm::cl::values( clEnumValN(IndexFileFormat::YAML, "yaml", "human-readable YAML format"), clEnumValN(IndexFileFormat::RIFF, "binary", "binary RIFF format"), - clEnumValN(IndexFileFormat::BACKGROUND, "background", - "background index format for language servers")), + clEnumValN(IndexFileFormat::SHARDED, "sharded", + "Sharded index format for language servers")), llvm::cl::init(IndexFileFormat::RIFF)); static llvm::cl::list<std::string> QueryDriverGlobs{ @@ -54,7 +54,7 @@ static llvm::cl::list<std::string> QueryDriverGlobs{ static llvm::cl::opt<std::string> ProjectRoot{ "project-root", llvm::cl::desc( - "Path to the project root for --format=background. " + "Path to the project root for --format=sharded. " "Determines where to store index shards. Shards are stored in " "<project-root>/.cache/clangd/index/. " "Defaults to current directory if not specified."), @@ -136,10 +136,10 @@ class IndexActionFactory : public tooling::FrontendActionFactory { RelationSlab::Builder Relations; }; -// Action factory that writes per-file shards (for background index format). -class BackgroundIndexActionFactory : public tooling::FrontendActionFactory { +// Action factory that writes per-file shards (for sharded index format). +class ShardedIndexActionFactory : public tooling::FrontendActionFactory { public: - BackgroundIndexActionFactory(BackgroundIndexStorage &Storage) + ShardedIndexActionFactory(BackgroundIndexStorage &Storage) : Storage(Storage), Symbols(std::make_unique<SymbolSlab::Builder>()), Refs(std::make_unique<RefSlab::Builder>()), Relations(std::make_unique<RelationSlab::Builder>()) {} @@ -265,9 +265,9 @@ int main(int argc, const char **argv) { $ clangd-indexer File1.cpp File2.cpp ... FileN.cpp > clangd.dex - Example usage for background index format (writes shards to disk): + Example usage for sharded index format (writes shards to disk): - $ clangd-indexer --format=background --executor=all-TUs build/ + $ clangd-indexer --format=sharded --executor=all-TUs build/ This writes index shards to .cache/clangd/index/ in the current directory. Use --project-root to specify a different location for the shards. @@ -298,8 +298,8 @@ int main(int argc, const char **argv) { return Cmd.CommandLine; }); - // Handle background index format separately - writes per-file shards. - if (clang::clangd::Format == clang::clangd::IndexFileFormat::BACKGROUND) { + // Handle sharded index format separately - writes per-file shards. + if (clang::clangd::Format == clang::clangd::IndexFileFormat::SHARDED) { // Default to current directory if --project-root not specified. std::string Root = clang::clangd::ProjectRoot; if (Root.empty()) { @@ -323,7 +323,7 @@ int main(int argc, const char **argv) { clang::clangd::BackgroundIndexStorage *Storage = IndexStorageFactory(Root); auto Err = Executor->get()->execute( - std::make_unique<clang::clangd::BackgroundIndexActionFactory>(*Storage), + std::make_unique<clang::clangd::ShardedIndexActionFactory>(*Storage), std::move(Adjuster)); if (Err) { clang::clangd::elog("{0}", std::move(Err)); >From 15da5a92673205da23fa6a6823b7017a9663f15d Mon Sep 17 00:00:00 2001 From: jasonwilliams <[email protected]> Date: Fri, 23 Jan 2026 10:50:59 +0000 Subject: [PATCH 4/8] create base class for both indexActionFactory and ShardedIndexActionFactory --- .../clangd/indexer/IndexerMain.cpp | 92 ++++++------------- 1 file changed, 27 insertions(+), 65 deletions(-) diff --git a/clang-tools-extra/clangd/indexer/IndexerMain.cpp b/clang-tools-extra/clangd/indexer/IndexerMain.cpp index 64f610590d246..c592551012336 100644 --- a/clang-tools-extra/clangd/indexer/IndexerMain.cpp +++ b/clang-tools-extra/clangd/indexer/IndexerMain.cpp @@ -60,10 +60,13 @@ static llvm::cl::opt<std::string> ProjectRoot{ "Defaults to current directory if not specified."), }; -// Action factory that merges all symbols into a single index (for YAML/RIFF). -class IndexActionFactory : public tooling::FrontendActionFactory { +// Base class for index action factories that provides common symbol collection. +class IndexActionFactoryBase : public tooling::FrontendActionFactory { public: - IndexActionFactory(IndexFileIn &Result) : Result(Result) {} + IndexActionFactoryBase() + : Symbols(std::make_unique<SymbolSlab::Builder>()), + Refs(std::make_unique<RefSlab::Builder>()), + Relations(std::make_unique<RelationSlab::Builder>()) {} std::unique_ptr<FrontendAction> create() override { SymbolCollector::Options Opts; @@ -116,72 +119,39 @@ class IndexActionFactory : public tooling::FrontendActionFactory { std::move(Invocation), Files, std::move(PCHContainerOps), DiagConsumer); } +protected: + std::mutex FilesMu; + llvm::StringSet<> Files; + std::mutex SymbolsMu; + std::unique_ptr<SymbolSlab::Builder> Symbols; + std::mutex RefsMu; + std::unique_ptr<RefSlab::Builder> Refs; + std::mutex RelsMu; + std::unique_ptr<RelationSlab::Builder> Relations; +}; + +// Action factory that merges all symbols into a single index (for YAML/RIFF). +class IndexActionFactory : public IndexActionFactoryBase { +public: + IndexActionFactory(IndexFileIn &Result) : Result(Result) {} + // Awkward: we write the result in the destructor, because the executor // takes ownership so it's the easiest way to get our data back out. ~IndexActionFactory() { - Result.Symbols = std::move(Symbols).build(); - Result.Refs = std::move(Refs).build(); - Result.Relations = std::move(Relations).build(); + Result.Symbols = std::move(*Symbols).build(); + Result.Refs = std::move(*Refs).build(); + Result.Relations = std::move(*Relations).build(); } private: IndexFileIn &Result; - std::mutex FilesMu; - llvm::StringSet<> Files; - std::mutex SymbolsMu; - SymbolSlab::Builder Symbols; - std::mutex RefsMu; - RefSlab::Builder Refs; - std::mutex RelsMu; - RelationSlab::Builder Relations; }; // Action factory that writes per-file shards (for sharded index format). -class ShardedIndexActionFactory : public tooling::FrontendActionFactory { +class ShardedIndexActionFactory : public IndexActionFactoryBase { public: ShardedIndexActionFactory(BackgroundIndexStorage &Storage) - : Storage(Storage), Symbols(std::make_unique<SymbolSlab::Builder>()), - Refs(std::make_unique<RefSlab::Builder>()), - Relations(std::make_unique<RelationSlab::Builder>()) {} - - std::unique_ptr<FrontendAction> create() override { - SymbolCollector::Options Opts; - Opts.CountReferences = true; - Opts.FileFilter = [&](const SourceManager &SM, FileID FID) { - const auto F = SM.getFileEntryRefForID(FID); - if (!F) - return false; - auto AbsPath = getCanonicalPath(*F, SM.getFileManager()); - if (!AbsPath) - return false; - std::lock_guard<std::mutex> Lock(FilesMu); - return Files.insert(*AbsPath).second; - }; - return createStaticIndexingAction( - Opts, - [&](SymbolSlab S) { - std::lock_guard<std::mutex> Lock(SymbolsMu); - for (const auto &Sym : S) { - if (const auto *Existing = Symbols->find(Sym.ID)) - Symbols->insert(mergeSymbol(*Existing, Sym)); - else - Symbols->insert(Sym); - } - }, - [&](RefSlab S) { - std::lock_guard<std::mutex> Lock(RefsMu); - for (const auto &Sym : S) { - for (const auto &Ref : Sym.second) - Refs->insert(Sym.first, Ref); - } - }, - [&](RelationSlab S) { - std::lock_guard<std::mutex> Lock(RelsMu); - for (const auto &R : S) - Relations->insert(R); - }, - /*IncludeGraphCallback=*/nullptr); - } + : Storage(Storage) {} bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation, FileManager *Files, @@ -236,15 +206,7 @@ class ShardedIndexActionFactory : public tooling::FrontendActionFactory { } BackgroundIndexStorage &Storage; - std::mutex FilesMu; - llvm::StringSet<> Files; unsigned ShardsWritten = 0; - std::mutex SymbolsMu; - std::unique_ptr<SymbolSlab::Builder> Symbols; - std::mutex RefsMu; - std::unique_ptr<RefSlab::Builder> Refs; - std::mutex RelsMu; - std::unique_ptr<RelationSlab::Builder> Relations; }; } // namespace >From e88c938833277ef44ceb416e3db63e3f35a7eb31 Mon Sep 17 00:00:00 2001 From: jasonwilliams <[email protected]> Date: Mon, 26 Jan 2026 11:32:44 +0000 Subject: [PATCH 5/8] Change sharded from format to index-type instead --- .../clangd/index/Serialization.cpp | 2 - .../clangd/index/Serialization.h | 5 +-- .../clangd/indexer/IndexerMain.cpp | 37 ++++++++++++------- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/clang-tools-extra/clangd/index/Serialization.cpp b/clang-tools-extra/clangd/index/Serialization.cpp index fbb672f184dd2..f03839599612c 100644 --- a/clang-tools-extra/clangd/index/Serialization.cpp +++ b/clang-tools-extra/clangd/index/Serialization.cpp @@ -686,8 +686,6 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const IndexFileOut &O) { case IndexFileFormat::YAML: writeYAML(O, OS); break; - case IndexFileFormat::SHARDED: - llvm_unreachable("SHARDED format not supported for serialization"); } return OS; } diff --git a/clang-tools-extra/clangd/index/Serialization.h b/clang-tools-extra/clangd/index/Serialization.h index d0939e051ecaf..bf8e036afcb6c 100644 --- a/clang-tools-extra/clangd/index/Serialization.h +++ b/clang-tools-extra/clangd/index/Serialization.h @@ -35,9 +35,8 @@ namespace clang { namespace clangd { enum class IndexFileFormat { - RIFF, // Versioned binary format, suitable for production use. - YAML, // Human-readable format, suitable for experiments and debugging. - SHARDED // Sharded index format, suitable for language server use. + RIFF, // Versioned binary format, suitable for production use. + YAML, // Human-readable format, suitable for experiments and debugging. }; // Holds the contents of an index file that was read. diff --git a/clang-tools-extra/clangd/indexer/IndexerMain.cpp b/clang-tools-extra/clangd/indexer/IndexerMain.cpp index c592551012336..36c66dc3356ef 100644 --- a/clang-tools-extra/clangd/indexer/IndexerMain.cpp +++ b/clang-tools-extra/clangd/indexer/IndexerMain.cpp @@ -30,16 +30,27 @@ namespace clang { namespace clangd { + +enum class IndexOutputMode { Monolithic, Sharded }; + namespace { -static llvm::cl::opt<IndexFileFormat> Format( - "format", llvm::cl::desc("Format of the index to be written"), +static llvm::cl::opt<IndexFileFormat> + Format("format", llvm::cl::desc("Format of the index to be written"), + llvm::cl::values(clEnumValN(IndexFileFormat::YAML, "yaml", + "human-readable YAML format"), + clEnumValN(IndexFileFormat::RIFF, "binary", + "binary RIFF format")), + llvm::cl::init(IndexFileFormat::RIFF)); + +static llvm::cl::opt<IndexOutputMode> OutputMode( + "index-type", llvm::cl::desc("Type of index output"), llvm::cl::values( - clEnumValN(IndexFileFormat::YAML, "yaml", "human-readable YAML format"), - clEnumValN(IndexFileFormat::RIFF, "binary", "binary RIFF format"), - clEnumValN(IndexFileFormat::SHARDED, "sharded", - "Sharded index format for language servers")), - llvm::cl::init(IndexFileFormat::RIFF)); + clEnumValN(IndexOutputMode::Monolithic, "monolithic", + "Single merged index file written to stdout (default)"), + clEnumValN(IndexOutputMode::Sharded, "sharded", + "Per-file shards written to disk")), + llvm::cl::init(IndexOutputMode::Monolithic)); static llvm::cl::list<std::string> QueryDriverGlobs{ "query-driver", @@ -54,7 +65,7 @@ static llvm::cl::list<std::string> QueryDriverGlobs{ static llvm::cl::opt<std::string> ProjectRoot{ "project-root", llvm::cl::desc( - "Path to the project root for --format=sharded. " + "Path to the project root for --index-type=sharded. " "Determines where to store index shards. Shards are stored in " "<project-root>/.cache/clangd/index/. " "Defaults to current directory if not specified."), @@ -227,9 +238,9 @@ int main(int argc, const char **argv) { $ clangd-indexer File1.cpp File2.cpp ... FileN.cpp > clangd.dex - Example usage for sharded index format (writes shards to disk): + Example usage for sharded index (writes shards to disk): - $ clangd-indexer --format=sharded --executor=all-TUs build/ + $ clangd-indexer --index-type=sharded --executor=all-TUs build/ This writes index shards to .cache/clangd/index/ in the current directory. Use --project-root to specify a different location for the shards. @@ -260,8 +271,8 @@ int main(int argc, const char **argv) { return Cmd.CommandLine; }); - // Handle sharded index format separately - writes per-file shards. - if (clang::clangd::Format == clang::clangd::IndexFileFormat::SHARDED) { + // Handle sharded index type separately - writes per-file shards. + if (clang::clangd::OutputMode == clang::clangd::IndexOutputMode::Sharded) { // Default to current directory if --project-root not specified. std::string Root = clang::clangd::ProjectRoot; if (Root.empty()) { @@ -292,7 +303,7 @@ int main(int argc, const char **argv) { return 1; } - llvm::errs() << "Background index shards written to " << Root + llvm::errs() << "Index shards written to " << Root << "/.cache/clangd/index/\n"; return 0; } >From efea3c23424e5dc9f1e9c787467583b1fad15a5e Mon Sep 17 00:00:00 2001 From: jasonwilliams <[email protected]> Date: Mon, 26 Jan 2026 17:00:25 +0000 Subject: [PATCH 6/8] Match how clangd's background index works --- .../clangd/indexer/IndexerMain.cpp | 107 ++++++++++++++++-- 1 file changed, 96 insertions(+), 11 deletions(-) diff --git a/clang-tools-extra/clangd/indexer/IndexerMain.cpp b/clang-tools-extra/clangd/indexer/IndexerMain.cpp index 36c66dc3356ef..7bdd2f68e609d 100644 --- a/clang-tools-extra/clangd/indexer/IndexerMain.cpp +++ b/clang-tools-extra/clangd/indexer/IndexerMain.cpp @@ -14,6 +14,7 @@ #include "Compiler.h" #include "GlobalCompilationDatabase.h" #include "index/Background.h" +#include "index/FileIndex.h" #include "index/IndexAction.h" #include "index/Merge.h" #include "index/Ref.h" @@ -21,6 +22,7 @@ #include "index/Symbol.h" #include "index/SymbolCollector.h" #include "support/Logger.h" +#include "URI.h" #include "clang/Tooling/ArgumentsAdjusters.h" #include "clang/Tooling/Execution.h" #include "clang/Tooling/Tooling.h" @@ -164,6 +166,54 @@ class ShardedIndexActionFactory : public IndexActionFactoryBase { ShardedIndexActionFactory(BackgroundIndexStorage &Storage) : Storage(Storage) {} + std::unique_ptr<FrontendAction> create() override { + SymbolCollector::Options Opts; + Opts.CountReferences = true; + Opts.FileFilter = [&](const SourceManager &SM, FileID FID) { + const auto F = SM.getFileEntryRefForID(FID); + if (!F) + return false; // Skip invalid files. + auto AbsPath = getCanonicalPath(*F, SM.getFileManager()); + if (!AbsPath) + return false; // Skip files without absolute path. + std::lock_guard<std::mutex> Lock(FilesMu); + return Files.insert(*AbsPath).second; // Skip already processed files. + }; + return createStaticIndexingAction( + Opts, + [&](SymbolSlab S) { + // Merge as we go. + std::lock_guard<std::mutex> Lock(SymbolsMu); + for (const auto &Sym : S) { + if (const auto *Existing = Symbols->find(Sym.ID)) + Symbols->insert(mergeSymbol(*Existing, Sym)); + else + Symbols->insert(Sym); + } + }, + [&](RefSlab S) { + std::lock_guard<std::mutex> Lock(RefsMu); + for (const auto &Sym : S) { + // Deduplication happens during insertion. + for (const auto &Ref : Sym.second) + Refs->insert(Sym.first, Ref); + } + }, + [&](RelationSlab S) { + std::lock_guard<std::mutex> Lock(RelsMu); + for (const auto &R : S) { + Relations->insert(R); + } + }, + [&](IncludeGraph IG) { + std::lock_guard<std::mutex> Lock(SourcesMu); + for (auto &Entry : IG) { + // Merge include graphs from different TUs. + Sources.try_emplace(Entry.first(), Entry.second); + } + }); + } + bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation, FileManager *Files, std::shared_ptr<PCHContainerOperations> PCHContainerOps, @@ -178,15 +228,16 @@ class ShardedIndexActionFactory : public IndexActionFactoryBase { bool Success = tooling::FrontendActionFactory::runInvocation( std::move(Invocation), Files, std::move(PCHContainerOps), DiagConsumer); - // After processing, write a shard for this file. + // After processing, write shards for all files in this TU. if (Success && !MainFile.empty()) - writeShardForFile(MainFile); + writeShardsForTU(MainFile); return Success; } private: - void writeShardForFile(llvm::StringRef MainFile) { + void writeShardsForTU(llvm::StringRef MainFile) { + // Build the complete index data for this TU. IndexFileIn Data; { std::lock_guard<std::mutex> Lock(SymbolsMu); @@ -203,20 +254,54 @@ class ShardedIndexActionFactory : public IndexActionFactoryBase { Data.Relations = std::move(*Relations).build(); Relations = std::make_unique<RelationSlab::Builder>(); } + { + std::lock_guard<std::mutex> Lock(SourcesMu); + Data.Sources = std::move(Sources); + Sources.clear(); + } - IndexFileOut Out(Data); - Out.Format = IndexFileFormat::RIFF; // Shards use RIFF format. + // Shard the index data per-file. + FileShardedIndex ShardedIndex(std::move(Data)); - if (auto Err = Storage.storeShard(MainFile, Out)) { - elog("Failed to write shard for {0}: {1}", MainFile, std::move(Err)); - } else { - std::lock_guard<std::mutex> Lock(FilesMu); - ++ShardsWritten; - log("Wrote shard for {0} ({1} total)", MainFile, ShardsWritten); + // Write a shard for each file. + unsigned TUShardsWritten = 0; + for (llvm::StringRef Uri : ShardedIndex.getAllSources()) { + auto Shard = ShardedIndex.getShard(Uri); + if (!Shard) { + elog("Failed to get shard for {0}", Uri); + continue; + } + + // Resolve URI to absolute path. + auto AbsPath = URI::resolve(Uri, MainFile); + if (!AbsPath) { + elog("Failed to resolve URI {0}: {1}", Uri, AbsPath.takeError()); + continue; + } + + // Only store command line for the main file. + if (*AbsPath != MainFile) + Shard->Cmd.reset(); + + IndexFileOut Out(*Shard); + Out.Format = IndexFileFormat::RIFF; // Shards use RIFF format. + + if (auto Err = Storage.storeShard(*AbsPath, Out)) { + elog("Failed to write shard for {0}: {1}", *AbsPath, std::move(Err)); + } else { + ++TUShardsWritten; + } } + + std::lock_guard<std::mutex> Lock(FilesMu); + ShardsWritten += TUShardsWritten; + log("Wrote {0} shards for TU {1} ({2} total)", TUShardsWritten, MainFile, + ShardsWritten); } BackgroundIndexStorage &Storage; + std::mutex SourcesMu; + IncludeGraph Sources; unsigned ShardsWritten = 0; }; >From f65767524a762ca1bc225ba7e8fede5c88cce5bf Mon Sep 17 00:00:00 2001 From: jasonwilliams <[email protected]> Date: Mon, 26 Jan 2026 18:41:16 +0000 Subject: [PATCH 7/8] fixup formatting --- clang-tools-extra/clangd/indexer/IndexerMain.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang-tools-extra/clangd/indexer/IndexerMain.cpp b/clang-tools-extra/clangd/indexer/IndexerMain.cpp index 7bdd2f68e609d..98902af73ebdb 100644 --- a/clang-tools-extra/clangd/indexer/IndexerMain.cpp +++ b/clang-tools-extra/clangd/indexer/IndexerMain.cpp @@ -13,6 +13,7 @@ #include "CompileCommands.h" #include "Compiler.h" #include "GlobalCompilationDatabase.h" +#include "URI.h" #include "index/Background.h" #include "index/FileIndex.h" #include "index/IndexAction.h" @@ -22,7 +23,6 @@ #include "index/Symbol.h" #include "index/SymbolCollector.h" #include "support/Logger.h" -#include "URI.h" #include "clang/Tooling/ArgumentsAdjusters.h" #include "clang/Tooling/Execution.h" #include "clang/Tooling/Tooling.h" >From 9d6c7f78ee7eb439eec78d66eff1765909701846 Mon Sep 17 00:00:00 2001 From: jasonwilliams <[email protected]> Date: Mon, 13 Jul 2026 19:22:45 +0100 Subject: [PATCH 8/8] =?UTF-8?q?-=20Removed=20IndexActionFactoryBase=20?= =?UTF-8?q?=E2=80=94=20the=20shared=20base=20class=20is=20gone.=20=20=20In?= =?UTF-8?q?dexActionFactory=20is=20now=20a=20standalone=20class=20(restore?= =?UTF-8?q?d=20to=20its=20pre-PR=20=20=20state)=20with=20direct=20member?= =?UTF-8?q?=20builders=20(SymbolSlab::Builder=20Symbols,=20etc.)=20=20=20r?= =?UTF-8?q?ather=20than=20unique=5Fptrs.=20-=20Completely=20rewrote=20Shar?= =?UTF-8?q?dedIndexActionFactory:=20=20=20-=20No=20more=20merging=20betwee?= =?UTF-8?q?n=20TUs,=20each=20IndexFileIn=20from=20the=20=20=20=20=20create?= =?UTF-8?q?StaticIndexingAction=20callback=20is=20passed=20directly=20to?= =?UTF-8?q?=20=20=20=20=20FileShardedIndex=20=20=20-=20No=20SymbolSlab::Bu?= =?UTF-8?q?ilder=20/=20RefSlab::Builder=20/=20RelationSlab::Builder=20=20?= =?UTF-8?q?=20=20=20objects,=20slabs=20go=20straight=20from=20the=20callba?= =?UTF-8?q?ck=20into=20FileShardedIndex=20=20=20-=20Shards=20are=20written?= =?UTF-8?q?=20directly=20from=20the=20createStaticIndexingAction=20=20=20?= =?UTF-8?q?=20=20callback=20(no=20more=20runInvocation=20override=20needed?= =?UTF-8?q?=20for=20the=20=20=20=20=20shard-writing=20logic)=20=20=20-=20N?= =?UTF-8?q?o=20longer=20overrides=20create()=20with=20the=20(non-existent)?= =?UTF-8?q?=204-callback=20form=20-=20Added=20header=20shard=20deduplicati?= =?UTF-8?q?on=20=E2=80=94=20ShardVersions=20(a=20=20=20StringMap<ShardVers?= =?UTF-8?q?ion>)=20tracks=20the=20content=20digest=20of=20each=20shard=20w?= =?UTF-8?q?e've=20=20=20written.=20Before=20each=20TU=20is=20indexed,=20cr?= =?UTF-8?q?eate()=20snapshots=20this=20map.=20=20=20writeShards()=20then?= =?UTF-8?q?=20only=20writes=20shards=20for=20files=20whose=20digest=20diff?= =?UTF-8?q?ers=20=20=20from=20the=20snapshot,=20mirroring=20exactly=20how?= =?UTF-8?q?=20BackgroundIndex::update()=20=20=20avoids=20re-writing=20unch?= =?UTF-8?q?anged=20header=20shards.=20-=20Moved=20ShardVersion=20to=20name?= =?UTF-8?q?space=20scope=20as=20requested=20by=20the=20reviewer.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../clangd/indexer/IndexerMain.cpp | 216 +++++++----------- 1 file changed, 84 insertions(+), 132 deletions(-) diff --git a/clang-tools-extra/clangd/indexer/IndexerMain.cpp b/clang-tools-extra/clangd/indexer/IndexerMain.cpp index 98902af73ebdb..d41b68a427c88 100644 --- a/clang-tools-extra/clangd/indexer/IndexerMain.cpp +++ b/clang-tools-extra/clangd/indexer/IndexerMain.cpp @@ -73,13 +73,17 @@ static llvm::cl::opt<std::string> ProjectRoot{ "Defaults to current directory if not specified."), }; -// Base class for index action factories that provides common symbol collection. -class IndexActionFactoryBase : public tooling::FrontendActionFactory { +// Tracks the content digest of the last shard written for a given file. +// Used to avoid writing duplicate shards when multiple TUs include the same +// header. +struct ShardVersion { + FileDigest Digest{{0}}; +}; + +// Action factory that merges all symbols into a single index (for YAML/RIFF). +class IndexActionFactory : public tooling::FrontendActionFactory { public: - IndexActionFactoryBase() - : Symbols(std::make_unique<SymbolSlab::Builder>()), - Refs(std::make_unique<RefSlab::Builder>()), - Relations(std::make_unique<RelationSlab::Builder>()) {} + IndexActionFactory(IndexFileIn &Result) : Result(Result) {} std::unique_ptr<FrontendAction> create() override { SymbolCollector::Options Opts; @@ -94,11 +98,11 @@ class IndexActionFactoryBase : public tooling::FrontendActionFactory { std::lock_guard<std::mutex> Lock(FilesMu); return Files.insert(*AbsPath).second; // Skip already processed files. }; - return createStaticIndexingAction(Opts, [&](IndexFileIn Result) { + return createStaticIndexingAction(Opts, [&](IndexFileIn TUResult) { { // Merge as we go. std::lock_guard<std::mutex> Lock(SymbolsMu); - for (const auto &Sym : *Result.Symbols) { + for (const auto &Sym : *TUResult.Symbols) { if (const auto *Existing = Symbols.find(Sym.ID)) Symbols.insert(mergeSymbol(*Existing, Sym)); else @@ -107,7 +111,7 @@ class IndexActionFactoryBase : public tooling::FrontendActionFactory { } { std::lock_guard<std::mutex> Lock(RefsMu); - for (const auto &Sym : *Result.Refs) { + for (const auto &Sym : *TUResult.Refs) { // Deduplication happens during insertion. for (const auto &Ref : Sym.second) Refs.insert(Sym.first, Ref); @@ -115,11 +119,11 @@ class IndexActionFactoryBase : public tooling::FrontendActionFactory { } { std::lock_guard<std::mutex> Lock(RelsMu); - for (const auto &R : *Result.Relations) { + for (const auto &R : *TUResult.Relations) { Relations.insert(R); } } - // FIXME: Handle Result.Sources? + // FIXME: Handle TUResult.Sources? }); } @@ -132,85 +136,50 @@ class IndexActionFactoryBase : public tooling::FrontendActionFactory { std::move(Invocation), Files, std::move(PCHContainerOps), DiagConsumer); } -protected: - std::mutex FilesMu; - llvm::StringSet<> Files; - std::mutex SymbolsMu; - std::unique_ptr<SymbolSlab::Builder> Symbols; - std::mutex RefsMu; - std::unique_ptr<RefSlab::Builder> Refs; - std::mutex RelsMu; - std::unique_ptr<RelationSlab::Builder> Relations; -}; - -// Action factory that merges all symbols into a single index (for YAML/RIFF). -class IndexActionFactory : public IndexActionFactoryBase { -public: - IndexActionFactory(IndexFileIn &Result) : Result(Result) {} - // Awkward: we write the result in the destructor, because the executor // takes ownership so it's the easiest way to get our data back out. ~IndexActionFactory() { - Result.Symbols = std::move(*Symbols).build(); - Result.Refs = std::move(*Refs).build(); - Result.Relations = std::move(*Relations).build(); + Result.Symbols = std::move(Symbols).build(); + Result.Refs = std::move(Refs).build(); + Result.Relations = std::move(Relations).build(); } private: IndexFileIn &Result; + std::mutex FilesMu; + llvm::StringSet<> Files; + std::mutex SymbolsMu; + SymbolSlab::Builder Symbols; + std::mutex RefsMu; + RefSlab::Builder Refs; + std::mutex RelsMu; + RelationSlab::Builder Relations; }; // Action factory that writes per-file shards (for sharded index format). -class ShardedIndexActionFactory : public IndexActionFactoryBase { +// Each TU's index data is sharded independently — no merging across TUs. +// Header shards are deduplicated: if a header's content hasn't changed since +// the last time we wrote its shard, we skip writing it again. +class ShardedIndexActionFactory : public tooling::FrontendActionFactory { public: ShardedIndexActionFactory(BackgroundIndexStorage &Storage) : Storage(Storage) {} std::unique_ptr<FrontendAction> create() override { + // Snapshot the current shard versions so the callback can check staleness + // without holding the lock during indexing. + llvm::StringMap<ShardVersion> Snapshot; + { + std::lock_guard<std::mutex> Lock(ShardVersionsMu); + Snapshot = ShardVersions; + } + SymbolCollector::Options Opts; Opts.CountReferences = true; - Opts.FileFilter = [&](const SourceManager &SM, FileID FID) { - const auto F = SM.getFileEntryRefForID(FID); - if (!F) - return false; // Skip invalid files. - auto AbsPath = getCanonicalPath(*F, SM.getFileManager()); - if (!AbsPath) - return false; // Skip files without absolute path. - std::lock_guard<std::mutex> Lock(FilesMu); - return Files.insert(*AbsPath).second; // Skip already processed files. - }; + return createStaticIndexingAction( - Opts, - [&](SymbolSlab S) { - // Merge as we go. - std::lock_guard<std::mutex> Lock(SymbolsMu); - for (const auto &Sym : S) { - if (const auto *Existing = Symbols->find(Sym.ID)) - Symbols->insert(mergeSymbol(*Existing, Sym)); - else - Symbols->insert(Sym); - } - }, - [&](RefSlab S) { - std::lock_guard<std::mutex> Lock(RefsMu); - for (const auto &Sym : S) { - // Deduplication happens during insertion. - for (const auto &Ref : Sym.second) - Refs->insert(Sym.first, Ref); - } - }, - [&](RelationSlab S) { - std::lock_guard<std::mutex> Lock(RelsMu); - for (const auto &R : S) { - Relations->insert(R); - } - }, - [&](IncludeGraph IG) { - std::lock_guard<std::mutex> Lock(SourcesMu); - for (auto &Entry : IG) { - // Merge include graphs from different TUs. - Sources.try_emplace(Entry.first(), Entry.second); - } + Opts, [this, Snapshot](IndexFileIn Index) { + writeShards(std::move(Index), Snapshot); }); } @@ -219,90 +188,73 @@ class ShardedIndexActionFactory : public IndexActionFactoryBase { std::shared_ptr<PCHContainerOperations> PCHContainerOps, DiagnosticConsumer *DiagConsumer) override { disableUnsupportedOptions(*Invocation); - - // Get the main file path before running. - std::string MainFile; - if (!Invocation->getFrontendOpts().Inputs.empty()) - MainFile = Invocation->getFrontendOpts().Inputs[0].getFile().str(); - - bool Success = tooling::FrontendActionFactory::runInvocation( + return tooling::FrontendActionFactory::runInvocation( std::move(Invocation), Files, std::move(PCHContainerOps), DiagConsumer); - - // After processing, write shards for all files in this TU. - if (Success && !MainFile.empty()) - writeShardsForTU(MainFile); - - return Success; } private: - void writeShardsForTU(llvm::StringRef MainFile) { - // Build the complete index data for this TU. - IndexFileIn Data; - { - std::lock_guard<std::mutex> Lock(SymbolsMu); - Data.Symbols = std::move(*Symbols).build(); - Symbols = std::make_unique<SymbolSlab::Builder>(); - } - { - std::lock_guard<std::mutex> Lock(RefsMu); - Data.Refs = std::move(*Refs).build(); - Refs = std::make_unique<RefSlab::Builder>(); - } - { - std::lock_guard<std::mutex> Lock(RelsMu); - Data.Relations = std::move(*Relations).build(); - Relations = std::make_unique<RelationSlab::Builder>(); + void writeShards(IndexFileIn Index, + const llvm::StringMap<ShardVersion> &Snapshot) { + if (!Index.Sources) + return; + + // Find the URI of the main file (the TU root). Store as owned string + // because we move Index below, which would invalidate any StringRef into it. + std::string MainUri; + for (const auto &[Uri, Node] : *Index.Sources) { + if (Node.Flags & IncludeGraphNode::SourceFlag::IsTU) { + MainUri = Uri.str(); + break; + } } - { - std::lock_guard<std::mutex> Lock(SourcesMu); - Data.Sources = std::move(Sources); - Sources.clear(); + + // Collect files that need updated shards, based on content digest. + // Files whose shard was already written with the same digest are skipped. + llvm::StringMap<std::pair<Path, FileDigest>> FilesToUpdate; + for (const auto &[Uri, Node] : *Index.Sources) { + auto AbsPath = URI::resolve(Uri, MainUri); + if (!AbsPath) { + elog("Failed to resolve URI {0}: {1}", Uri, AbsPath.takeError()); + continue; + } + auto DigestIt = Snapshot.find(*AbsPath); + if (DigestIt == Snapshot.end() || DigestIt->second.Digest != Node.Digest) + FilesToUpdate[Uri] = {std::move(*AbsPath), Node.Digest}; } - // Shard the index data per-file. - FileShardedIndex ShardedIndex(std::move(Data)); + // Shard the index data by file. + FileShardedIndex ShardedIndex(std::move(Index)); - // Write a shard for each file. - unsigned TUShardsWritten = 0; - for (llvm::StringRef Uri : ShardedIndex.getAllSources()) { + unsigned Written = 0; + for (const auto &[Uri, PathAndDigest] : FilesToUpdate) { auto Shard = ShardedIndex.getShard(Uri); if (!Shard) { elog("Failed to get shard for {0}", Uri); continue; } + PathRef Path = PathAndDigest.first; - // Resolve URI to absolute path. - auto AbsPath = URI::resolve(Uri, MainFile); - if (!AbsPath) { - elog("Failed to resolve URI {0}: {1}", Uri, AbsPath.takeError()); - continue; - } - - // Only store command line for the main file. - if (*AbsPath != MainFile) + // Command line is only meaningful for the TU's main file. + if (Uri != MainUri) Shard->Cmd.reset(); IndexFileOut Out(*Shard); - Out.Format = IndexFileFormat::RIFF; // Shards use RIFF format. - - if (auto Err = Storage.storeShard(*AbsPath, Out)) { - elog("Failed to write shard for {0}: {1}", *AbsPath, std::move(Err)); + Out.Format = IndexFileFormat::RIFF; + if (auto Err = Storage.storeShard(Path, Out)) { + elog("Failed to write shard for {0}: {1}", Path, std::move(Err)); } else { - ++TUShardsWritten; + std::lock_guard<std::mutex> Lock(ShardVersionsMu); + ShardVersions[Path].Digest = PathAndDigest.second; + ++Written; } } - std::lock_guard<std::mutex> Lock(FilesMu); - ShardsWritten += TUShardsWritten; - log("Wrote {0} shards for TU {1} ({2} total)", TUShardsWritten, MainFile, - ShardsWritten); + log("Wrote {0} shards for TU {1}", Written, MainUri); } BackgroundIndexStorage &Storage; - std::mutex SourcesMu; - IncludeGraph Sources; - unsigned ShardsWritten = 0; + std::mutex ShardVersionsMu; + llvm::StringMap<ShardVersion> ShardVersions; }; } // namespace _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
