sammccall updated this revision to Diff 392605.
sammccall retitled this revision from "[clangd] WIP various stdlib indexing
stuff" to "[clangd] Indexing of standard library".
sammccall edited the summary of this revision.
sammccall added a subscriber: kuhnel.
sammccall added a comment.
Herald added a subscriber: kristof.beyls.
Tests, polish, comments
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D115232/new/
https://reviews.llvm.org/D115232
Files:
clang-tools-extra/clangd/CMakeLists.txt
clang-tools-extra/clangd/ClangdServer.cpp
clang-tools-extra/clangd/Config.h
clang-tools-extra/clangd/ConfigCompile.cpp
clang-tools-extra/clangd/ConfigFragment.h
clang-tools-extra/clangd/ConfigYAML.cpp
clang-tools-extra/clangd/TUScheduler.cpp
clang-tools-extra/clangd/TUScheduler.h
clang-tools-extra/clangd/index/FileIndex.cpp
clang-tools-extra/clangd/index/FileIndex.h
clang-tools-extra/clangd/index/StdLib.cpp
clang-tools-extra/clangd/index/StdLib.h
clang-tools-extra/clangd/index/SymbolOrigin.cpp
clang-tools-extra/clangd/index/SymbolOrigin.h
clang-tools-extra/clangd/unittests/CMakeLists.txt
clang-tools-extra/clangd/unittests/StdLibTests.cpp
clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp
Index: clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp
===================================================================
--- clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp
+++ clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp
@@ -1121,7 +1121,8 @@
public:
BlockPreambleThread(llvm::StringRef BlockVersion, Notification &N)
: BlockVersion(BlockVersion), N(N) {}
- void onPreambleAST(PathRef Path, llvm::StringRef Version, ASTContext &Ctx,
+ void onPreambleAST(PathRef Path, llvm::StringRef Version,
+ const CompilerInvocation &, ASTContext &Ctx,
std::shared_ptr<clang::Preprocessor> PP,
const CanonicalIncludes &) override {
if (Version == BlockVersion)
Index: clang-tools-extra/clangd/unittests/StdLibTests.cpp
===================================================================
--- /dev/null
+++ clang-tools-extra/clangd/unittests/StdLibTests.cpp
@@ -0,0 +1,125 @@
+//===-- StdLibTests.cpp -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Compiler.h"
+#include "Config.h"
+#include "TestFS.h"
+#include "index/StdLib.h"
+#include "clang/Basic/LangOptions.h"
+#include "clang/Basic/SourceManager.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include <memory>
+
+using namespace testing;
+
+namespace clang {
+namespace clangd {
+namespace {
+
+// Check the generated header sources contains usual standard library headers.
+TEST(StdLibTests, getStdlibUmbrellaHeader) {
+ LangOptions LO;
+ LO.CPlusPlus = true;
+
+ auto CXX = getStdlibUmbrellaHeader(LO);
+ EXPECT_THAT(CXX, HasSubstr("#include <string>"));
+ EXPECT_THAT(CXX, HasSubstr("#include <cstdio>"));
+ EXPECT_THAT(CXX, Not(HasSubstr("#include <stdio.h>")));
+
+ LO.CPlusPlus = false;
+ auto C = getStdlibUmbrellaHeader(LO);
+ EXPECT_THAT(C, Not(HasSubstr("#include <string>")));
+ EXPECT_THAT(C, Not(HasSubstr("#include <cstdio>")));
+ EXPECT_THAT(C, HasSubstr("#include <stdio.h>"));
+}
+
+MATCHER_P(Named, Name, "") { return arg.Name == Name; }
+
+// Build an index, and check if it contains the right symbols.
+TEST(StdLibTests, indexStandardLibrary) {
+ MockFS FS;
+ FS.Files["std/foo.h"] = R"cpp(
+ #include <platform_stuff.h>
+ #if __cplusplus >= 201703L
+ int foo17();
+ #elif __cplusplus >= 201402L
+ int foo14();
+ #else
+ bool foo98();
+ #endif
+ )cpp";
+ FS.Files["nonstd/platform_stuff.h"] = "int magic = 42;";
+
+ ParseInputs OriginalInputs;
+ OriginalInputs.TFS = &FS;
+ OriginalInputs.CompileCommand.Filename = testPath("main.cc");
+ OriginalInputs.CompileCommand.CommandLine = {"clang++", testPath("main.cc"),
+ "-isystemstd/",
+ "-isystemnonstd/", "-std=c++14"};
+ OriginalInputs.CompileCommand.Directory = testRoot();
+ IgnoreDiagnostics Diags;
+ auto CI = buildCompilerInvocation(OriginalInputs, Diags);
+ ASSERT_TRUE(CI);
+
+ StdLibLocation Loc;
+ Loc.Paths.push_back(testPath("std/"));
+
+ auto Symbols =
+ indexStandardLibrary("#include <foo.h>", std::move(CI), Loc, FS);
+ EXPECT_THAT(Symbols, ElementsAre(Named("foo14")));
+}
+
+TEST(StdLibTests, StdLibSet) {
+ StdLibSet Set;
+ MockFS FS;
+ FS.Files["std/_"] = "";
+ FS.Files["libc/_"] = "";
+
+ auto Add = [&](const LangOptions &LO,
+ std::vector<llvm::StringRef> SearchPath) {
+ SourceManagerForFile SM("scratch", "");
+ SM.get().getFileManager().setVirtualFileSystem(FS.view(llvm::None));
+ HeaderSearch HS(/*HSOpts=*/nullptr, SM.get(), SM.get().getDiagnostics(), LO,
+ /*Target=*/nullptr);
+ for (auto P : SearchPath)
+ HS.AddSearchPath(
+ DirectoryLookup(
+ cantFail(SM.get().getFileManager().getDirectoryRef(testPath(P))),
+ SrcMgr::C_System, /*isFramework=*/false),
+ true);
+ return Set.add(LO, HS);
+ };
+
+ Config Cfg;
+ Cfg.Index.StandardLibrary = false;
+ WithContextValue Disabled(Config::Key, std::move(Cfg));
+
+ LangOptions LO;
+ LO.CPlusPlus = true;
+ EXPECT_FALSE(Add(LO, {"std"})) << "No <vector> found";
+ FS.Files["std/vector"] = "class vector;";
+ EXPECT_FALSE(Add(LO, {"std"})) << "Disabled in config";
+
+ Cfg = Config();
+ Cfg.Index.StandardLibrary = true;
+ WithContextValue Enabled(Config::Key, std::move(Cfg));
+
+ EXPECT_TRUE(Add(LO, {"std"})) << "Indexing as C++98";
+ EXPECT_FALSE(Add(LO, {"std"})) << "Don't reindex";
+ LO.CPlusPlus11 = true;
+ EXPECT_TRUE(Add(LO, {"std"})) << "Indexing as C++11";
+ LO.CPlusPlus = false;
+ EXPECT_FALSE(Add(LO, {"libc"})) << "No <stdio.h>";
+ FS.Files["libc/stdio.h"] = true;
+ EXPECT_TRUE(Add(LO, {"libc"})) << "Indexing as C";
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
Index: clang-tools-extra/clangd/unittests/CMakeLists.txt
===================================================================
--- clang-tools-extra/clangd/unittests/CMakeLists.txt
+++ clang-tools-extra/clangd/unittests/CMakeLists.txt
@@ -80,6 +80,7 @@
SemanticSelectionTests.cpp
SerializationTests.cpp
SourceCodeTests.cpp
+ StdLibTests.cpp
SymbolCollectorTests.cpp
SymbolInfoTests.cpp
SyncAPI.cpp
Index: clang-tools-extra/clangd/index/SymbolOrigin.h
===================================================================
--- clang-tools-extra/clangd/index/SymbolOrigin.h
+++ clang-tools-extra/clangd/index/SymbolOrigin.h
@@ -26,6 +26,7 @@
Merge = 1 << 3, // A non-trivial index merge was performed.
Identifier = 1 << 4, // Raw identifiers in file.
Remote = 1 << 5, // Remote index.
+ StdLib = 1 << 6, // Standard library index.
// Remaining bits reserved for index implementations.
};
Index: clang-tools-extra/clangd/index/SymbolOrigin.cpp
===================================================================
--- clang-tools-extra/clangd/index/SymbolOrigin.cpp
+++ clang-tools-extra/clangd/index/SymbolOrigin.cpp
@@ -14,7 +14,7 @@
llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, SymbolOrigin O) {
if (O == SymbolOrigin::Unknown)
return OS << "unknown";
- constexpr static char Sigils[] = "ADSMIR67";
+ constexpr static char Sigils[] = "ADSMIRL7";
for (unsigned I = 0; I < sizeof(Sigils); ++I)
if (static_cast<uint8_t>(O) & 1u << I)
OS << Sigils[I];
Index: clang-tools-extra/clangd/index/StdLib.h
===================================================================
--- /dev/null
+++ clang-tools-extra/clangd/index/StdLib.h
@@ -0,0 +1,107 @@
+//===--- StdLib.h - Index the C and C++ standard library ---------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Eagerly indexing the standard library gives a much friendlier "warm start"
+// with working code completion in a standalone file or small project.
+//
+// We act as if we saw a file which included the whole standard library:
+// #include <array>
+// #include <bitset>
+// #include <chrono>
+// ...
+// We index this TU and feed the result into the dynamic index.
+//
+// This happens within the context of some particular open file, and we reuse
+// its CompilerInvocation. Matching its include path, LangOpts etc ensures that
+// we see the standard library and configuration that matches the project.
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_STDLIB_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_STDLIB_H
+
+#include "index/Symbol.h"
+#include "support/ThreadsafeFS.h"
+#include "llvm/ADT/StringRef.h"
+#include <string>
+
+namespace clang {
+class CompilerInvocation;
+class LangOptions;
+class HeaderSearch;
+namespace clangd {
+
+// The filesystem location where a standard library was found.
+//
+// This is the directory containing <vector> or <stdio.h>.
+// It's used to ensure we only index files that are in the standard library.
+struct StdLibLocation {
+ llvm::SmallVector<std::string> Paths;
+};
+
+// Tracks the state of standard library indexing within a particular index.
+//
+// In general, we don't want to index the standard library multiple times.
+// In most cases, this class just acts as a flag to ensure we only do it once.
+//
+// However, if we first open a C++11 file, and then a C++20 file, we *do*
+// want the index to be upgraded to include the extra symbols.
+// Similarly, the C and C++ standard library can coexist.
+class StdLibSet {
+ std::atomic<int> Best[2] = {{-1}, {-1}};
+
+public:
+ // Determines if we should index the standard library in a configuration.
+ //
+ // This is true if:
+ // - standard library indexing is enabled for the file
+ // - the language version is higher than any previous add() for the language
+ // - the standard library headers exist on the search path
+ // Returns the location where the standard library was found.
+ //
+ // This function is threadsafe.
+ llvm::Optional<StdLibLocation> add(const LangOptions &, const HeaderSearch &);
+
+ // Indicates whether we a built index should be used.
+ // It should not be used if a newer version has subsequently been added.
+ //
+ // Intended pattern is:
+ // if (add()) {
+ // symbols = indexStandardLibrary();
+ // if (isBest())
+ // index.update(symbols);
+ // }
+ //
+ // This is still technically racy: we could return true here, then another
+ // thread could add->index->update a better library before we can update.
+ // We'd then overwrite it with the older version.
+ // However, it's very unlikely: indexing takes a long time.
+ bool isBest(const LangOptions &) const;
+};
+
+// Index a standard library and return the discovered symbols.
+//
+// The compiler invocation should describe the file whose config we're reusing.
+// We overwrite its virtual buffer with a lot of #include statements.
+SymbolSlab indexStandardLibrary(std::unique_ptr<CompilerInvocation> Invocation,
+ const StdLibLocation &Loc,
+ const ThreadsafeFS &TFS);
+
+// Variant that allows the umbrella header source to be specified.
+// Exposed for testing.
+SymbolSlab indexStandardLibrary(llvm::StringRef HeaderSources,
+ std::unique_ptr<CompilerInvocation> CI,
+ const StdLibLocation &Loc,
+ const ThreadsafeFS &TFS);
+
+// Generate header containing #includes for all standard library headers.
+// Exposed for testing.
+llvm::StringRef getStdlibUmbrellaHeader(const LangOptions &);
+
+} // namespace clangd
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_STDLIB_H
Index: clang-tools-extra/clangd/index/StdLib.cpp
===================================================================
--- /dev/null
+++ clang-tools-extra/clangd/index/StdLib.cpp
@@ -0,0 +1,357 @@
+//===-- StdLib.cpp ----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "StdLib.h"
+#include <fstream>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "Compiler.h"
+#include "Config.h"
+#include "SymbolCollector.h"
+#include "index/IndexAction.h"
+#include "support/Logger.h"
+#include "support/ThreadsafeFS.h"
+#include "support/Trace.h"
+#include "clang/Basic/LangOptions.h"
+#include "clang/Frontend/CompilerInvocation.h"
+#include "clang/Lex/PreprocessorOptions.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/None.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+enum Lang { C, CXX };
+
+Lang langFromOpts(const LangOptions &LO) { return LO.CPlusPlus ? CXX : C; }
+llvm::StringLiteral mandatoryHeader(Lang L) {
+ switch (L) {
+ case C:
+ return "stdio.h";
+ case CXX:
+ return "vector";
+ }
+ return L == CXX ? llvm::StringLiteral("vector") : "stdio.h";
+}
+
+LangStandard::Kind standardFromOpts(const LangOptions &LO) {
+ if (LO.CPlusPlus) {
+ return !LO.CPlusPlus11 ? LangStandard::lang_cxx98
+ : !LO.CPlusPlus14 ? LangStandard::lang_cxx11
+ : !LO.CPlusPlus17 ? LangStandard::lang_cxx14
+ : !LO.CPlusPlus20 ? LangStandard::lang_cxx17
+ : !LO.CPlusPlus2b ? LangStandard::lang_cxx20
+ : LangStandard::lang_cxx2b;
+ }
+ return !LO.C11 ? LangStandard::lang_c99
+ // C17 has no new features, so treat C14 as C17.
+ : !LO.C2x ? LangStandard::lang_c17
+ : LangStandard::lang_c2x;
+}
+
+std::string buildUmbrella(llvm::StringLiteral Mandatory,
+ std::vector<llvm::StringLiteral> Headers) {
+ std::string Result;
+ llvm::raw_string_ostream OS(Result);
+
+ // We __has_include guard all our #includes to avoid errors when using older
+ // stdlib version that don't have headers for the newest language standards.
+ // But make sure we get *some* error if things are totally broken.
+ OS << llvm::formatv(
+ "#if !__has_include(<{0}>)\n"
+ "#error Mandatory header <{0}> not found in standard library!\n"
+ "#endif\n",
+ Mandatory);
+
+ llvm::sort(Headers.begin(), Headers.end());
+ auto Last = std::unique(Headers.begin(), Headers.end());
+ for (auto Header = Headers.begin(); Header != Last; ++Header) {
+ OS << llvm::formatv("#if __has_include({0})\n"
+ "#include {0}\n"
+ "#endif\n",
+ *Header);
+ }
+ OS.flush();
+ return Result;
+}
+
+} // namespace
+
+llvm::StringRef getStdlibUmbrellaHeader(const LangOptions &LO) {
+ // The umbrella header is the same for all versions of each language.
+ // Headers that are unsupported in old lang versions are usually guarded by
+ // #if. Some headers may be not present in old stdlib versions, the umbrella
+ // header guards with __has_include for this purpose.
+ Lang L = langFromOpts(LO);
+ switch (L) {
+ case CXX:
+ static std::string *UmbrellaCXX =
+ new std::string(buildUmbrella(mandatoryHeader(L), {
+#define SYMBOL(Name, NameSpace, Header) #Header,
+#include "StdSymbolMap.inc"
+#undef SYMBOL
+ }));
+ return *UmbrellaCXX;
+ case C:
+ static std::string *UmbrellaC =
+ new std::string(buildUmbrella(mandatoryHeader(L), {
+#define SYMBOL(Name, NameSpace, Header) #Header,
+#include "CSymbolMap.inc"
+#undef SYMBOL
+ }));
+ return *UmbrellaC;
+ }
+}
+
+namespace {
+
+// Including the standard library leaks unwanted transitively included symbols.
+//
+// We want to drop these, they're a bit tricky to identify:
+// - we don't want to limit to symbols our our list, as our list has only
+// top-level symbols (and there may be legitimate stdlib extensions).
+// - we can't limit to only symbols defined in known stdlib headers, as stdlib
+// internal structure is murky
+// - we can't strictly require symbols to come from a particular path, e.g.
+// libstdc++ is mostly under /usr/include/c++/10/...
+// but std::ctype_base is under /usr/include/<platform>/c++/10/...
+// We require the symbol to come from a header that is *either* from
+// the standard library path (as identified by the location of <vector>), or
+// another header that defines a symbol from our stdlib list.
+static SymbolSlab filter(SymbolSlab Slab, const StdLibLocation &Loc) {
+ SymbolSlab::Builder Result;
+
+ static auto &StandardHeaders = *[] {
+ auto Set = new llvm::DenseSet<llvm::StringRef>();
+ for (llvm::StringRef Name : {
+#define SYMBOL(Name, NameSpace, Header) #Header,
+#include "CSymbolMap.inc"
+#include "StdSymbolMap.inc"
+#undef SYMBOL
+ })
+ Set->insert(Name);
+ return Set;
+ }();
+
+ // Form prefixes like file:///usr/include/c++/10/
+ // These can be trivially prefix-compared with URIs in the indexed symbols.
+ llvm::SmallVector<std::string> StdLibURIPrefixes;
+ for (const auto &Path : Loc.Paths) {
+ StdLibURIPrefixes.push_back(URI::create(Path).toString());
+ if (StdLibURIPrefixes.back().back() != '/')
+ StdLibURIPrefixes.back().push_back('/');
+ }
+ // For each header URI, is it *either* prefixed by StdLibURIPrefixes *or*
+ // owner of a symbol whose insertable header is in StandardHeaders?
+ // Pointer key because strings in a SymbolSlab are interned.
+ llvm::DenseMap<const char *, bool> GoodHeader;
+ for (const Symbol &S : Slab) {
+ if (!S.IncludeHeaders.empty() &&
+ StandardHeaders.contains(S.IncludeHeaders.front().IncludeHeader)) {
+ GoodHeader[S.CanonicalDeclaration.FileURI] = true;
+ GoodHeader[S.Definition.FileURI] = true;
+ continue;
+ }
+ for (const char *URI :
+ {S.CanonicalDeclaration.FileURI, S.Definition.FileURI}) {
+ auto R = GoodHeader.try_emplace(URI, false);
+ if (R.second) {
+ R.first->second = llvm::any_of(
+ StdLibURIPrefixes,
+ [&, URIStr(llvm::StringRef(URI))](const std::string &Prefix) {
+ return URIStr.startswith(Prefix);
+ });
+ }
+ }
+ }
+ for (const auto &Good : GoodHeader)
+ if (Good.second)
+ dlog("Stdlib header: {0}", Good.first);
+ // Empty URIs aren't considered good. (Definition can be blank).
+ auto IsGoodHeader = [&](const char *C) { return *C && GoodHeader.lookup(C); };
+
+ for (const Symbol &S : Slab) {
+ if (!(IsGoodHeader(S.CanonicalDeclaration.FileURI) ||
+ IsGoodHeader(S.Definition.FileURI))) {
+ dlog("Ignoring wrong-header symbol {0}{1} in {2}", S.Scope, S.Name,
+ S.CanonicalDeclaration.FileURI);
+ continue;
+ }
+ Result.insert(S);
+ }
+
+ return std::move(Result).build();
+}
+
+} // namespace
+
+SymbolSlab indexStandardLibrary(llvm::StringRef HeaderSources,
+ std::unique_ptr<CompilerInvocation> CI,
+ const StdLibLocation &Loc,
+ const ThreadsafeFS &TFS) {
+ if (CI->getFrontendOpts().Inputs.size() != 1 ||
+ !CI->getPreprocessorOpts().ImplicitPCHInclude.empty()) {
+ elog("Indexing standard library failed: bad CompilerInvocation");
+ assert(false && "indexing stdlib with a dubious CompilerInvocation!");
+ return SymbolSlab();
+ }
+ const FrontendInputFile &Input = CI->getFrontendOpts().Inputs.front();
+ trace::Span Tracer("StandardLibraryIndex");
+ LangStandard::Kind LangStd = standardFromOpts(*CI->getLangOpts());
+ log("Indexing {0} standard library in the context of {1}",
+ LangStandard::getLangStandardForKind(LangStd).getName(), Input.getFile());
+
+ SymbolSlab Symbols;
+ IgnoreDiagnostics IgnoreDiags;
+ CI->getPreprocessorOpts().clearRemappedFiles();
+ auto Clang = prepareCompilerInstance(
+ std::move(CI), /*Preamble=*/nullptr,
+ llvm::MemoryBuffer::getMemBuffer(HeaderSources, Input.getFile()),
+ TFS.view(/*CWD=*/llvm::None), IgnoreDiags);
+ if (!Clang) {
+ elog("Standard Library Index: Couldn't build compiler instance");
+ return Symbols;
+ }
+
+ SymbolCollector::Options IndexOpts;
+ IndexOpts.Origin = SymbolOrigin::StdLib;
+ IndexOpts.CollectMainFileSymbols = false;
+ IndexOpts.CollectMainFileRefs = false;
+ IndexOpts.CollectMacro = true;
+ IndexOpts.StoreAllDocumentation = true;
+ // Sadly we can't use IndexOpts.FileFilter to restrict indexing scope.
+ // Files from outside the location may define true std symbols anyway.
+ // We end up "blessing" such headers, and can only do that by indexing
+ // everything first.
+
+ // Refs, relations, containers in the stdlib mostly aren't useful.
+ auto Action = createStaticIndexingAction(
+ IndexOpts, [&](SymbolSlab S) { Symbols = std::move(S); }, nullptr,
+ nullptr, nullptr);
+
+ if (!Action->BeginSourceFile(*Clang, Input)) {
+ elog("Standard Library Index: BeginSourceFile() failed");
+ return Symbols;
+ }
+
+ if (llvm::Error Err = Action->Execute()) {
+ elog("Standard Library Index: Execute failed: {0}", std::move(Err));
+ return Symbols;
+ }
+
+ Action->EndSourceFile();
+ bool HadErrors = Clang->hasDiagnostics() &&
+ Clang->getDiagnostics().hasUncompilableErrorOccurred();
+ if (HadErrors) {
+ log("Errors when generating the standard library index, index may be "
+ "incomplete");
+ }
+
+ unsigned SymbolsBeforeFilter = Symbols.size();
+ Symbols = filter(std::move(Symbols), Loc);
+ log("Indexed {0} standard library: ({1} symbols, {2} filtered)",
+ LangStandard::getLangStandardForKind(LangStd).getName(), Symbols.size(),
+ SymbolsBeforeFilter - Symbols.size());
+ SPAN_ATTACH(Tracer, "symbols", int(Symbols.size()));
+ return Symbols;
+}
+
+SymbolSlab indexStandardLibrary(std::unique_ptr<CompilerInvocation> Invocation,
+ const StdLibLocation &Loc,
+ const ThreadsafeFS &TFS) {
+ return indexStandardLibrary(
+ getStdlibUmbrellaHeader(*Invocation->getLangOpts()),
+ std::move(Invocation), Loc, TFS);
+}
+
+bool StdLibSet::isBest(const LangOptions &LO) const {
+ return standardFromOpts(LO) >=
+ Best[langFromOpts(LO)].load(std::memory_order_acquire);
+}
+
+llvm::Optional<StdLibLocation> StdLibSet::add(const LangOptions &LO,
+ const HeaderSearch &HS) {
+ Lang L = langFromOpts(LO);
+ int OldVersion = Best[L].load(std::memory_order_acquire);
+ int NewVersion = standardFromOpts(LO);
+ dlog("Index stdlib? {0}",
+ LangStandard::getLangStandardForKind(standardFromOpts(LO)).getName());
+
+ if (NewVersion <= OldVersion) {
+ dlog("No: have {0}, {1}>={2}",
+ LangStandard::getLangStandardForKind(
+ static_cast<LangStandard::Kind>(NewVersion))
+ .getName(),
+ OldVersion, NewVersion);
+ return llvm::None;
+ }
+
+ if (!Config::current().Index.StandardLibrary) {
+ dlog("No: disabled in config");
+ return llvm::None;
+ }
+
+ // We'd like to index a standard library here if there is one.
+ // Check for the existence of <vector> on the search path.
+ // We could cache this, but we only get here repeatedly when there's no
+ // stdlib, and even then only once per preamble build.
+ llvm::StringLiteral ProbeHeader = mandatoryHeader(L);
+ llvm::SmallString<256> Path; // Scratch space.
+ llvm::SmallVector<std::string> SearchPaths;
+ auto RecordHeaderPath = [&](llvm::StringRef HeaderPath) {
+ llvm::StringRef DirPath = llvm::sys::path::parent_path(HeaderPath);
+ if (!HS.getFileMgr().getVirtualFileSystem().getRealPath(DirPath, Path))
+ SearchPaths.emplace_back(Path);
+ };
+ for (const auto &DL :
+ llvm::make_range(HS.search_dir_begin(), HS.search_dir_end())) {
+ switch (DL.getLookupType()) {
+ case DirectoryLookup::LT_NormalDir: {
+ Path = DL.getDir()->getName();
+ llvm::sys::path::append(Path, ProbeHeader);
+ llvm::vfs::Status Stat;
+ if (!HS.getFileMgr().getNoncachedStatValue(Path, Stat) &&
+ Stat.isRegularFile())
+ RecordHeaderPath(Path);
+ break;
+ }
+ case DirectoryLookup::LT_Framework:
+ // stdlib can't be a framework (framework includes bust have a slash)
+ continue;
+ case DirectoryLookup::LT_HeaderMap:
+ llvm::StringRef Target =
+ DL.getHeaderMap()->lookupFilename(ProbeHeader, Path);
+ if (!Target.empty())
+ RecordHeaderPath(Target);
+ break;
+ }
+ }
+ if (SearchPaths.empty()) {
+ dlog("No: didn't find <{0}>)", ProbeHeader);
+ return llvm::None;
+ }
+ dlog("Found standard library in {0}", llvm::join(SearchPaths, ", "));
+
+ while (!Best[L].compare_exchange_weak(OldVersion, NewVersion,
+ std::memory_order_acq_rel))
+ if (OldVersion >= NewVersion) {
+ dlog("No: lost the race");
+ return llvm::None; // Another thread won the race while we were checking.
+ }
+
+ dlog("Yes, index stdlib!");
+ return StdLibLocation{std::move(SearchPaths)};
+}
+
+} // namespace clangd
+} // namespace clang
Index: clang-tools-extra/clangd/index/FileIndex.h
===================================================================
--- clang-tools-extra/clangd/index/FileIndex.h
+++ clang-tools-extra/clangd/index/FileIndex.h
@@ -117,6 +117,7 @@
void updatePreamble(PathRef Path, llvm::StringRef Version, ASTContext &AST,
std::shared_ptr<Preprocessor> PP,
const CanonicalIncludes &Includes);
+ void updatePreamble(IndexFileIn);
/// Update symbols and references from main file \p Path with
/// `indexMainDecls`.
Index: clang-tools-extra/clangd/index/FileIndex.cpp
===================================================================
--- clang-tools-extra/clangd/index/FileIndex.cpp
+++ clang-tools-extra/clangd/index/FileIndex.cpp
@@ -423,13 +423,7 @@
MainFileSymbols(IndexContents::All),
MainFileIndex(std::make_unique<MemIndex>()) {}
-void FileIndex::updatePreamble(PathRef Path, llvm::StringRef Version,
- ASTContext &AST,
- std::shared_ptr<Preprocessor> PP,
- const CanonicalIncludes &Includes) {
- IndexFileIn IF;
- std::tie(IF.Symbols, std::ignore, IF.Relations) =
- indexHeaderSymbols(Version, AST, std::move(PP), Includes);
+void FileIndex::updatePreamble(IndexFileIn IF) {
FileShardedIndex ShardedIndex(std::move(IF));
for (auto Uri : ShardedIndex.getAllSources()) {
auto IF = ShardedIndex.getShard(Uri);
@@ -460,6 +454,16 @@
}
}
+void FileIndex::updatePreamble(PathRef Path, llvm::StringRef Version,
+ ASTContext &AST,
+ std::shared_ptr<Preprocessor> PP,
+ const CanonicalIncludes &Includes) {
+ IndexFileIn IF;
+ std::tie(IF.Symbols, std::ignore, IF.Relations) =
+ indexHeaderSymbols(Version, AST, std::move(PP), Includes);
+ updatePreamble(std::move(IF));
+}
+
void FileIndex::updateMain(PathRef Path, ParsedAST &AST) {
auto Contents = indexMainDecls(AST);
MainFileSymbols.update(
Index: clang-tools-extra/clangd/TUScheduler.h
===================================================================
--- clang-tools-extra/clangd/TUScheduler.h
+++ clang-tools-extra/clangd/TUScheduler.h
@@ -134,7 +134,7 @@
/// contains only AST nodes from the #include directives at the start of the
/// file. AST node in the current file should be observed on onMainAST call.
virtual void onPreambleAST(PathRef Path, llvm::StringRef Version,
- ASTContext &Ctx,
+ const CompilerInvocation &CI, ASTContext &Ctx,
std::shared_ptr<clang::Preprocessor> PP,
const CanonicalIncludes &) {}
Index: clang-tools-extra/clangd/TUScheduler.cpp
===================================================================
--- clang-tools-extra/clangd/TUScheduler.cpp
+++ clang-tools-extra/clangd/TUScheduler.cpp
@@ -977,11 +977,10 @@
LatestBuild = clang::clangd::buildPreamble(
FileName, *Req.CI, Inputs, StoreInMemory,
- [this, Version(Inputs.Version)](ASTContext &Ctx,
- std::shared_ptr<clang::Preprocessor> PP,
- const CanonicalIncludes &CanonIncludes) {
- Callbacks.onPreambleAST(FileName, Version, Ctx, std::move(PP),
- CanonIncludes);
+ [&](ASTContext &Ctx, std::shared_ptr<clang::Preprocessor> PP,
+ const CanonicalIncludes &CanonIncludes) {
+ Callbacks.onPreambleAST(FileName, Inputs.Version, *Req.CI, Ctx,
+ std::move(PP), CanonIncludes);
});
if (LatestBuild && isReliable(LatestBuild->CompileCommand))
HeaderIncluders.update(FileName, LatestBuild->Includes.allHeaders());
Index: clang-tools-extra/clangd/ConfigYAML.cpp
===================================================================
--- clang-tools-extra/clangd/ConfigYAML.cpp
+++ clang-tools-extra/clangd/ConfigYAML.cpp
@@ -167,6 +167,10 @@
F.External.emplace(std::move(External));
F.External->Range = N.getSourceRange();
});
+ Dict.handle("StandardLibrary", [&](Node &N) {
+ if (auto StandardLibrary = boolValue(N, "StandardLibrary"))
+ F.StandardLibrary = *StandardLibrary;
+ });
Dict.parse(N);
}
@@ -194,12 +198,8 @@
void parse(Fragment::CompletionBlock &F, Node &N) {
DictParser Dict("Completion", this);
Dict.handle("AllScopes", [&](Node &N) {
- if (auto Value = scalarValue(N, "AllScopes")) {
- if (auto AllScopes = llvm::yaml::parseBool(**Value))
- F.AllScopes = *AllScopes;
- else
- warning("AllScopes should be a boolean", N);
- }
+ if (auto AllScopes = boolValue(N, "AllScopes"))
+ F.AllScopes = *AllScopes;
});
Dict.parse(N);
}
@@ -334,6 +334,16 @@
return Result;
}
+ llvm::Optional<Located<bool>> boolValue(Node &N, llvm::StringRef Desc) {
+ if (auto Scalar = scalarValue(N, Desc)) {
+ if (auto StandardLibrary = llvm::yaml::parseBool(**Scalar))
+ return Located<bool>(*StandardLibrary, Scalar->Range);
+ else
+ warning(Desc + " should be a boolean", N);
+ }
+ return llvm::None;
+ }
+
// Report a "hard" error, reflecting a config file that can never be valid.
void error(const llvm::Twine &Msg, llvm::SMRange Range) {
HadError = true;
Index: clang-tools-extra/clangd/ConfigFragment.h
===================================================================
--- clang-tools-extra/clangd/ConfigFragment.h
+++ clang-tools-extra/clangd/ConfigFragment.h
@@ -191,6 +191,9 @@
llvm::Optional<Located<std::string>> MountPoint;
};
llvm::Optional<Located<ExternalBlock>> External;
+ // Whether the standard library visible from this file should be indexed.
+ // This makes all standard library symbols available, included or not.
+ llvm::Optional<Located<bool>> StandardLibrary;
};
IndexBlock Index;
Index: clang-tools-extra/clangd/ConfigCompile.cpp
===================================================================
--- clang-tools-extra/clangd/ConfigCompile.cpp
+++ clang-tools-extra/clangd/ConfigCompile.cpp
@@ -321,6 +321,11 @@
}
if (F.External)
compile(std::move(**F.External), F.External->Range);
+ if (F.StandardLibrary)
+ Out.Apply.push_back(
+ [Val(**F.StandardLibrary)](const Params &, Config &C) {
+ C.Index.StandardLibrary = Val;
+ });
}
void compile(Fragment::IndexBlock::ExternalBlock &&External,
Index: clang-tools-extra/clangd/Config.h
===================================================================
--- clang-tools-extra/clangd/Config.h
+++ clang-tools-extra/clangd/Config.h
@@ -79,11 +79,12 @@
/// forward-slashes.
std::string MountPoint;
};
- /// Controls background-index behavior.
+ /// Controls index behavior.
struct {
- /// Whether this TU should be indexed.
+ /// Whether this TU should be background-indexed.
BackgroundPolicy Background = BackgroundPolicy::Build;
ExternalIndexSpec External;
+ bool StandardLibrary = false;
} Index;
enum UnusedIncludesPolicy { Strict, None };
Index: clang-tools-extra/clangd/ClangdServer.cpp
===================================================================
--- clang-tools-extra/clangd/ClangdServer.cpp
+++ clang-tools-extra/clangd/ClangdServer.cpp
@@ -14,7 +14,6 @@
#include "FindSymbols.h"
#include "Format.h"
#include "HeaderSourceSwitch.h"
-#include "Headers.h"
#include "InlayHints.h"
#include "ParsedAST.h"
#include "Preamble.h"
@@ -27,15 +26,14 @@
#include "index/CanonicalIncludes.h"
#include "index/FileIndex.h"
#include "index/Merge.h"
+#include "index/StdLib.h"
#include "refactor/Rename.h"
#include "refactor/Tweak.h"
#include "support/Logger.h"
-#include "support/Markup.h"
#include "support/MemoryTree.h"
#include "support/ThreadsafeFS.h"
#include "support/Trace.h"
#include "clang/Format/Format.h"
-#include "clang/Frontend/CompilerInstance.h"
#include "clang/Frontend/CompilerInvocation.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Tooling/CompilationDatabase.h"
@@ -43,14 +41,9 @@
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/ScopeExit.h"
-#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringRef.h"
-#include "llvm/Support/Errc.h"
#include "llvm/Support/Error.h"
-#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
-#include "llvm/Support/ScopedPrinter.h"
#include "llvm/Support/raw_ostream.h"
#include <algorithm>
#include <chrono>
@@ -67,16 +60,39 @@
// Update the FileIndex with new ASTs and plumb the diagnostics responses.
struct UpdateIndexCallbacks : public ParsingCallbacks {
UpdateIndexCallbacks(FileIndex *FIndex,
- ClangdServer::Callbacks *ServerCallbacks)
- : FIndex(FIndex), ServerCallbacks(ServerCallbacks) {}
+ ClangdServer::Callbacks *ServerCallbacks,
+ const ThreadsafeFS &TFS, bool Sync)
+ : FIndex(FIndex), ServerCallbacks(ServerCallbacks), TFS(TFS) {
+ if (!Sync)
+ Tasks.emplace();
+ }
- void onPreambleAST(PathRef Path, llvm::StringRef Version, ASTContext &Ctx,
+ void onPreambleAST(PathRef Path, llvm::StringRef Version,
+ const CompilerInvocation &CI, ASTContext &Ctx,
std::shared_ptr<clang::Preprocessor> PP,
const CanonicalIncludes &CanonIncludes) override {
+ // If this preamble uses a standard library we haven't seen yet, index it.
+ if (auto Loc = Stdlib.add(*CI.getLangOpts(), PP->getHeaderSearchInfo()))
+ indexStdlib(CI, std::move(*Loc));
+
if (FIndex)
FIndex->updatePreamble(Path, Version, Ctx, std::move(PP), CanonIncludes);
}
+ void indexStdlib(const CompilerInvocation &CI, StdLibLocation Loc) {
+ auto Task = [this, LO(*CI.getLangOpts()), Loc(std::move(Loc)),
+ CI(std::make_unique<CompilerInvocation>(CI))]() mutable {
+ IndexFileIn IF;
+ IF.Symbols = indexStandardLibrary(std::move(CI), Loc, TFS);
+ if (Stdlib.isBest(LO))
+ FIndex->updatePreamble(std::move(IF));
+ };
+ if (Tasks)
+ Tasks->runAsync("IndexStdlib", std::move(Task));
+ else
+ Task();
+ }
+
void onMainAST(PathRef Path, ParsedAST &AST, PublishFn Publish) override {
if (FIndex)
FIndex->updateMain(Path, AST);
@@ -111,6 +127,9 @@
private:
FileIndex *FIndex;
ClangdServer::Callbacks *ServerCallbacks;
+ const ThreadsafeFS &TFS;
+ StdLibSet Stdlib;
+ llvm::Optional<AsyncTaskRunner> Tasks;
};
class DraftStoreFS : public ThreadsafeFS {
@@ -163,9 +182,10 @@
// Pass a callback into `WorkScheduler` to extract symbols from a newly
// parsed file and rebuild the file index synchronously each time an AST
// is parsed.
- WorkScheduler.emplace(
- CDB, TUScheduler::Options(Opts),
- std::make_unique<UpdateIndexCallbacks>(DynamicIdx.get(), Callbacks));
+ WorkScheduler.emplace(CDB, TUScheduler::Options(Opts),
+ std::make_unique<UpdateIndexCallbacks>(
+ DynamicIdx.get(), Callbacks, TFS,
+ /*Sync=*/Opts.AsyncThreadsCount == 0));
// Adds an index to the stack, at higher priority than existing indexes.
auto AddIndex = [&](SymbolIndex *Idx) {
if (this->Index != nullptr) {
@@ -895,7 +915,7 @@
// It's safe to pass in the TU, as dumpAST() does not
// deserialize the preamble.
auto Node = DynTypedNode::create(
- *Inputs->AST.getASTContext().getTranslationUnitDecl());
+ *Inputs->AST.getASTContext().getTranslationUnitDecl());
return CB(dumpAST(Node, Inputs->AST.getTokens(),
Inputs->AST.getASTContext()));
}
Index: clang-tools-extra/clangd/CMakeLists.txt
===================================================================
--- clang-tools-extra/clangd/CMakeLists.txt
+++ clang-tools-extra/clangd/CMakeLists.txt
@@ -119,6 +119,7 @@
index/Ref.cpp
index/Relation.cpp
index/Serialization.cpp
+ index/StdLib.cpp
index/Symbol.cpp
index/SymbolCollector.cpp
index/SymbolID.cpp
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
