Author: Kadir Cetinkaya Date: 2020-10-12T15:25:29+02:00 New Revision: a74d594948611164f88a79ca0544721183a0b19c
URL: https://github.com/llvm/llvm-project/commit/a74d594948611164f88a79ca0544721183a0b19c DIFF: https://github.com/llvm/llvm-project/commit/a74d594948611164f88a79ca0544721183a0b19c.diff LOG: [clangd] Introduce memory dumping to FileIndex, FileSymbols and BackgroundIndex File-granular information is considered details. Depends on D88411 Differential Revision: https://reviews.llvm.org/D88414 Added: Modified: clang-tools-extra/clangd/index/Background.cpp clang-tools-extra/clangd/index/Background.h clang-tools-extra/clangd/index/FileIndex.cpp clang-tools-extra/clangd/index/FileIndex.h clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp clang-tools-extra/clangd/unittests/FileIndexTests.cpp Removed: ################################################################################ diff --git a/clang-tools-extra/clangd/index/Background.cpp b/clang-tools-extra/clangd/index/Background.cpp index a1aafeaf31a9..4779cb8d4c23 100644 --- a/clang-tools-extra/clangd/index/Background.cpp +++ b/clang-tools-extra/clangd/index/Background.cpp @@ -16,6 +16,7 @@ #include "URI.h" #include "index/BackgroundIndexLoader.h" #include "index/FileIndex.h" +#include "index/Index.h" #include "index/IndexAction.h" #include "index/MemIndex.h" #include "index/Ref.h" @@ -414,5 +415,10 @@ BackgroundIndex::loadProject(std::vector<std::string> MainFiles) { return {TUsToIndex.begin(), TUsToIndex.end()}; } +void BackgroundIndex::profile(MemoryTree &MT) const { + IndexedSymbols.profile(MT.child("symbols")); + // We don't want to mix memory used by index and symbols, so call base class. + MT.child("index").addUsage(SwapIndex::estimateMemoryUsage()); +} } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/index/Background.h b/clang-tools-extra/clangd/index/Background.h index 472603013a53..e8f9468889f2 100644 --- a/clang-tools-extra/clangd/index/Background.h +++ b/clang-tools-extra/clangd/index/Background.h @@ -16,9 +16,11 @@ #include "index/Index.h" #include "index/Serialization.h" #include "support/Context.h" +#include "support/MemoryTree.h" #include "support/Path.h" #include "support/Threading.h" #include "support/ThreadsafeFS.h" +#include "support/Trace.h" #include "clang/Tooling/CompilationDatabase.h" #include "llvm/ADT/StringMap.h" #include "llvm/Support/Threading.h" @@ -172,6 +174,8 @@ class BackgroundIndex : public SwapIndex { return Queue.blockUntilIdleForTest(TimeoutSeconds); } + void profile(MemoryTree &MT) const; + private: /// Represents the state of a single file when indexing was performed. struct ShardVersion { diff --git a/clang-tools-extra/clangd/index/FileIndex.cpp b/clang-tools-extra/clangd/index/FileIndex.cpp index ad55b6ad7f5d..587c7eb78170 100644 --- a/clang-tools-extra/clangd/index/FileIndex.cpp +++ b/clang-tools-extra/clangd/index/FileIndex.cpp @@ -22,6 +22,7 @@ #include "index/SymbolOrigin.h" #include "index/dex/Dex.h" #include "support/Logger.h" +#include "support/MemoryTree.h" #include "support/Path.h" #include "clang/AST/ASTContext.h" #include "clang/Index/IndexingAction.h" @@ -388,6 +389,25 @@ FileSymbols::buildIndex(IndexType Type, DuplicateHandling DuplicateHandle, llvm_unreachable("Unknown clangd::IndexType"); } +void FileSymbols::profile(MemoryTree &MT) const { + std::lock_guard<std::mutex> Lock(Mutex); + for (const auto &SymSlab : SymbolsSnapshot) { + MT.detail(SymSlab.first()) + .child("symbols") + .addUsage(SymSlab.second->bytes()); + } + for (const auto &RefSlab : RefsSnapshot) { + MT.detail(RefSlab.first()) + .child("references") + .addUsage(RefSlab.second.Slab->bytes()); + } + for (const auto &RelSlab : RelationsSnapshot) { + MT.detail(RelSlab.first()) + .child("relations") + .addUsage(RelSlab.second->bytes()); + } +} + FileIndex::FileIndex(bool UseDex, bool CollectMainFileRefs) : MergedIndex(&MainFileIndex, &PreambleIndex), UseDex(UseDex), CollectMainFileRefs(CollectMainFileRefs), @@ -457,5 +477,15 @@ void FileIndex::updateMain(PathRef Path, ParsedAST &AST) { } } +void FileIndex::profile(MemoryTree &MT) const { + PreambleSymbols.profile(MT.child("preamble").child("symbols")); + MT.child("preamble") + .child("index") + .addUsage(PreambleIndex.estimateMemoryUsage()); + MainFileSymbols.profile(MT.child("main_file").child("symbols")); + MT.child("main_file") + .child("index") + .addUsage(MainFileIndex.estimateMemoryUsage()); +} } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/index/FileIndex.h b/clang-tools-extra/clangd/index/FileIndex.h index 127203c84c48..8ecae66373a5 100644 --- a/clang-tools-extra/clangd/index/FileIndex.h +++ b/clang-tools-extra/clangd/index/FileIndex.h @@ -24,6 +24,7 @@ #include "index/Relation.h" #include "index/Serialization.h" #include "index/Symbol.h" +#include "support/MemoryTree.h" #include "support/Path.h" #include "clang/Lex/Preprocessor.h" #include "clang/Tooling/CompilationDatabase.h" @@ -87,6 +88,8 @@ class FileSymbols { DuplicateHandling DuplicateHandle = DuplicateHandling::PickOne, size_t *Version = nullptr); + void profile(MemoryTree &MT) const; + private: struct RefSlabAndCountReferences { std::shared_ptr<RefSlab> Slab; @@ -116,6 +119,8 @@ class FileIndex : public MergedIndex { /// `indexMainDecls`. void updateMain(PathRef Path, ParsedAST &AST); + void profile(MemoryTree &MT) const; + private: bool UseDex; // FIXME: this should be always on. bool CollectMainFileRefs; diff --git a/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp b/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp index adf39a915c1a..4f089519530a 100644 --- a/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp +++ b/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp @@ -21,6 +21,7 @@ using ::testing::AllOf; using ::testing::Contains; using ::testing::ElementsAre; using ::testing::Not; +using ::testing::Pair; using ::testing::UnorderedElementsAre; namespace clang { @@ -916,5 +917,18 @@ TEST(BackgroundQueueTest, Progress) { EXPECT_EQ(S.LastIdle, 2000u); } +TEST(BackgroundIndex, Profile) { + MockFS FS; + MockCompilationDatabase CDB; + BackgroundIndex Idx(FS, CDB, [](llvm::StringRef) { return nullptr; }, + /*Opts=*/{}); + + llvm::BumpPtrAllocator Alloc; + MemoryTree MT(&Alloc); + Idx.profile(MT); + ASSERT_THAT(MT.children(), + UnorderedElementsAre(Pair("symbols", _), Pair("index", _))); +} + } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/unittests/FileIndexTests.cpp b/clang-tools-extra/clangd/unittests/FileIndexTests.cpp index c5bfbe132d37..2b20b7e7fef0 100644 --- a/clang-tools-extra/clangd/unittests/FileIndexTests.cpp +++ b/clang-tools-extra/clangd/unittests/FileIndexTests.cpp @@ -22,20 +22,25 @@ #include "index/Relation.h" #include "index/Serialization.h" #include "index/Symbol.h" +#include "index/SymbolID.h" #include "support/Threading.h" #include "clang/Frontend/CompilerInvocation.h" #include "clang/Frontend/Utils.h" #include "clang/Index/IndexSymbol.h" #include "clang/Lex/Preprocessor.h" #include "clang/Tooling/CompilationDatabase.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/Support/Allocator.h" #include "gmock/gmock.h" #include "gtest/gtest.h" #include <utility> +#include <vector> using ::testing::_; using ::testing::AllOf; using ::testing::Contains; using ::testing::ElementsAre; +using ::testing::Gt; using ::testing::IsEmpty; using ::testing::Pair; using ::testing::UnorderedElementsAre; @@ -88,6 +93,13 @@ std::unique_ptr<RefSlab> refSlab(const SymbolID &ID, const char *Path) { return std::make_unique<RefSlab>(std::move(Slab).build()); } +std::unique_ptr<RelationSlab> relSlab(llvm::ArrayRef<const Relation> Rels) { + RelationSlab::Builder RelBuilder; + for (auto &Rel : Rels) + RelBuilder.insert(Rel); + return std::make_unique<RelationSlab>(std::move(RelBuilder).build()); +} + TEST(FileSymbolsTest, UpdateAndGet) { FileSymbols FS; EXPECT_THAT(runFuzzyFind(*FS.buildIndex(IndexType::Light), ""), IsEmpty()); @@ -643,6 +655,50 @@ TEST(FileShardedIndexTest, Sharding) { EXPECT_TRUE(Shard->Cmd.hasValue()); } } + +TEST(FileIndexTest, Profile) { + FileIndex FI; + + auto FileName = testPath("foo.cpp"); + auto AST = TestTU::withHeaderCode("int a;").build(); + FI.updateMain(FileName, AST); + FI.updatePreamble(FileName, "v1", AST.getASTContext(), + AST.getPreprocessorPtr(), AST.getCanonicalIncludes()); + + llvm::BumpPtrAllocator Alloc; + MemoryTree MT(&Alloc); + FI.profile(MT); + ASSERT_THAT(MT.children(), + UnorderedElementsAre(Pair("preamble", _), Pair("main_file", _))); + + ASSERT_THAT(MT.child("preamble").children(), + UnorderedElementsAre(Pair("index", _), Pair("symbols", _))); + ASSERT_THAT(MT.child("main_file").children(), + UnorderedElementsAre(Pair("index", _), Pair("symbols", _))); + + ASSERT_THAT(MT.child("preamble").child("index").total(), Gt(0U)); + ASSERT_THAT(MT.child("main_file").child("index").total(), Gt(0U)); +} + +TEST(FileSymbolsTest, Profile) { + FileSymbols FS; + FS.update("f1", numSlab(1, 2), nullptr, nullptr, false); + FS.update("f2", nullptr, refSlab(SymbolID("1"), "f1"), nullptr, false); + FS.update("f3", nullptr, nullptr, + relSlab({{SymbolID("1"), RelationKind::BaseOf, SymbolID("2")}}), + false); + llvm::BumpPtrAllocator Alloc; + MemoryTree MT(&Alloc); + FS.profile(MT); + ASSERT_THAT(MT.children(), UnorderedElementsAre(Pair("f1", _), Pair("f2", _), + Pair("f3", _))); + EXPECT_THAT(MT.child("f1").children(), ElementsAre(Pair("symbols", _))); + EXPECT_THAT(MT.child("f1").total(), Gt(0U)); + EXPECT_THAT(MT.child("f2").children(), ElementsAre(Pair("references", _))); + EXPECT_THAT(MT.child("f2").total(), Gt(0U)); + EXPECT_THAT(MT.child("f3").children(), ElementsAre(Pair("relations", _))); + EXPECT_THAT(MT.child("f3").total(), Gt(0U)); +} } // namespace } // namespace clangd } // namespace clang _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits