https://github.com/Bigcheese updated
https://github.com/llvm/llvm-project/pull/144000
>From d07ccf298fd3755036ec90e4cd7f1d1408c4506e Mon Sep 17 00:00:00 2001
From: Michael Spencer <bigchees...@gmail.com>
Date: Thu, 12 Jun 2025 17:13:52 -0700
Subject: [PATCH] [clang][scan-deps] Add option to disable caching stat
failures
While the source code isn't supposed to change during a build, in some
environments it does. This adds an option that disables caching of
stat failures, meaning that source files can be added to the build
during scanning.
This adds a `-no-cache-negative-stats` option to clang-scan-deps to
enable this behavior. There are no tests for clang-scan-deps as
there's no reliable way to do so from it. A unit test has been added
that modifies the filesystem between scans to test it.
---
.../DependencyScanningFilesystem.h | 24 ++++-----
.../DependencyScanningService.h | 6 ++-
.../DependencyScanningFilesystem.cpp | 40 +++++++++++---
.../DependencyScanningService.cpp | 3 +-
.../DependencyScanningWorker.cpp | 3 +-
clang/tools/clang-scan-deps/ClangScanDeps.cpp | 8 ++-
clang/tools/clang-scan-deps/Opts.td | 1 +
.../DependencyScannerTest.cpp | 52 +++++++++++++++++++
.../DependencyScanningFilesystemTest.cpp | 37 +++++++------
9 files changed, 132 insertions(+), 42 deletions(-)
diff --git
a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
index a20a89a4c2b76..da83220babea3 100644
---
a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
+++
b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
@@ -23,6 +23,8 @@ namespace clang {
namespace tooling {
namespace dependencies {
+class DependencyScanningService;
+
using DependencyDirectivesTy =
SmallVector<dependency_directives_scan::Directive, 20>;
@@ -349,7 +351,7 @@ class DependencyScanningWorkerFilesystem
static const char ID;
DependencyScanningWorkerFilesystem(
- DependencyScanningFilesystemSharedCache &SharedCache,
+ DependencyScanningService &Service,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS);
llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override;
@@ -435,10 +437,7 @@ class DependencyScanningWorkerFilesystem
/// Returns entry associated with the unique ID in the shared cache or
nullptr
/// if none is found.
const CachedFileSystemEntry *
- findSharedEntryByUID(llvm::vfs::Status Stat) const {
- return SharedCache.getShardForUID(Stat.getUniqueID())
- .findEntryByUID(Stat.getUniqueID());
- }
+ findSharedEntryByUID(llvm::vfs::Status Stat) const;
/// Associates the given entry with the filename in the local cache and
/// returns it.
@@ -452,20 +451,14 @@ class DependencyScanningWorkerFilesystem
/// some. Otherwise, constructs new one with the given error code, associates
/// it with the filename and returns the result.
const CachedFileSystemEntry &
- getOrEmplaceSharedEntryForFilename(StringRef Filename, std::error_code EC) {
- return SharedCache.getShardForFilename(Filename)
- .getOrEmplaceEntryForFilename(Filename, EC);
- }
+ getOrEmplaceSharedEntryForFilename(StringRef Filename, std::error_code EC);
/// Returns entry associated with the filename in the shared cache if there
is
/// some. Otherwise, associates the given entry with the filename and returns
/// it.
const CachedFileSystemEntry &
getOrInsertSharedEntryForFilename(StringRef Filename,
- const CachedFileSystemEntry &Entry) {
- return SharedCache.getShardForFilename(Filename)
- .getOrInsertEntryForFilename(Filename, Entry);
- }
+ const CachedFileSystemEntry &Entry);
void printImpl(raw_ostream &OS, PrintType Type,
unsigned IndentLevel) const override {
@@ -478,8 +471,9 @@ class DependencyScanningWorkerFilesystem
/// VFS.
bool shouldBypass(StringRef Path) const;
- /// The global cache shared between worker threads.
- DependencyScanningFilesystemSharedCache &SharedCache;
+ /// The service associated with this VFS.
+ DependencyScanningService &Service;
+
/// The local cache is used by the worker thread to cache file system queries
/// locally instead of querying the global cache every time.
DependencyScanningFilesystemLocalCache LocalCache;
diff --git
a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
index 4e97c7bc9f36e..ceaf3c2279e7f 100644
--- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
+++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
@@ -87,7 +87,8 @@ class DependencyScanningService {
ScanningOptimizations OptimizeArgs = ScanningOptimizations::Default,
bool EagerLoadModules = false, bool TraceVFS = false,
std::time_t BuildSessionTimestamp =
- llvm::sys::toTimeT(std::chrono::system_clock::now()));
+ llvm::sys::toTimeT(std::chrono::system_clock::now()),
+ bool CacheNegativeStats = true);
ScanningMode getMode() const { return Mode; }
@@ -99,6 +100,8 @@ class DependencyScanningService {
bool shouldTraceVFS() const { return TraceVFS; }
+ bool shouldCacheNegativeStats() const { return CacheNegativeStats; }
+
DependencyScanningFilesystemSharedCache &getSharedCache() {
return SharedCache;
}
@@ -116,6 +119,7 @@ class DependencyScanningService {
const bool EagerLoadModules;
/// Whether to trace VFS accesses.
const bool TraceVFS;
+ const bool CacheNegativeStats;
/// The global file system cache.
DependencyScanningFilesystemSharedCache SharedCache;
/// The global module cache entries.
diff --git
a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
index 140833050f4e9..2868522f80018 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -7,6 +7,7 @@
//===----------------------------------------------------------------------===//
#include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
+#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Threading.h"
#include <optional>
@@ -232,19 +233,19 @@ bool
DependencyScanningWorkerFilesystem::shouldBypass(StringRef Path) const {
}
DependencyScanningWorkerFilesystem::DependencyScanningWorkerFilesystem(
- DependencyScanningFilesystemSharedCache &SharedCache,
+ DependencyScanningService &Service,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
: llvm::RTTIExtends<DependencyScanningWorkerFilesystem,
llvm::vfs::ProxyFileSystem>(std::move(FS)),
- SharedCache(SharedCache),
- WorkingDirForCacheLookup(llvm::errc::invalid_argument) {
+ Service(Service), WorkingDirForCacheLookup(llvm::errc::invalid_argument)
{
updateWorkingDirForCacheLookup();
}
const CachedFileSystemEntry &
DependencyScanningWorkerFilesystem::getOrEmplaceSharedEntryForUID(
TentativeEntry TEntry) {
- auto &Shard = SharedCache.getShardForUID(TEntry.Status.getUniqueID());
+ auto &Shard =
+ Service.getSharedCache().getShardForUID(TEntry.Status.getUniqueID());
return Shard.getOrEmplaceEntryForUID(TEntry.Status.getUniqueID(),
std::move(TEntry.Status),
std::move(TEntry.Contents));
@@ -255,18 +256,44 @@
DependencyScanningWorkerFilesystem::findEntryByFilenameWithWriteThrough(
StringRef Filename) {
if (const auto *Entry = LocalCache.findEntryByFilename(Filename))
return Entry;
- auto &Shard = SharedCache.getShardForFilename(Filename);
+ auto &Shard = Service.getSharedCache().getShardForFilename(Filename);
if (const auto *Entry = Shard.findEntryByFilename(Filename))
return &LocalCache.insertEntryForFilename(Filename, *Entry);
return nullptr;
}
+const CachedFileSystemEntry *
+DependencyScanningWorkerFilesystem::findSharedEntryByUID(
+ llvm::vfs::Status Stat) const {
+ return Service.getSharedCache()
+ .getShardForUID(Stat.getUniqueID())
+ .findEntryByUID(Stat.getUniqueID());
+}
+
+const CachedFileSystemEntry &
+DependencyScanningWorkerFilesystem::getOrEmplaceSharedEntryForFilename(
+ StringRef Filename, std::error_code EC) {
+ return Service.getSharedCache()
+ .getShardForFilename(Filename)
+ .getOrEmplaceEntryForFilename(Filename, EC);
+}
+
+const CachedFileSystemEntry &
+DependencyScanningWorkerFilesystem::getOrInsertSharedEntryForFilename(
+ StringRef Filename, const CachedFileSystemEntry &Entry) {
+ return Service.getSharedCache()
+ .getShardForFilename(Filename)
+ .getOrInsertEntryForFilename(Filename, Entry);
+}
+
llvm::ErrorOr<const CachedFileSystemEntry &>
DependencyScanningWorkerFilesystem::computeAndStoreResult(
StringRef OriginalFilename, StringRef FilenameForLookup) {
llvm::ErrorOr<llvm::vfs::Status> Stat =
getUnderlyingFS().status(OriginalFilename);
if (!Stat) {
+ if (!Service.shouldCacheNegativeStats())
+ return Stat.getError();
const auto &Entry =
getOrEmplaceSharedEntryForFilename(FilenameForLookup, Stat.getError());
return insertLocalEntryForFilename(FilenameForLookup, Entry);
@@ -420,7 +447,8 @@ DependencyScanningWorkerFilesystem::getRealPath(const Twine
&Path,
return HandleCachedRealPath(*RealPath);
// If we have the result in the shared cache, cache it locally.
- auto &Shard = SharedCache.getShardForFilename(*FilenameForLookup);
+ auto &Shard =
+ Service.getSharedCache().getShardForFilename(*FilenameForLookup);
if (const auto *ShardRealPath =
Shard.findRealPathByFilename(*FilenameForLookup)) {
const auto &RealPath = LocalCache.insertRealPathForFilename(
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
b/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
index 7f40c99f07287..c2f3cdbb02e37 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
@@ -15,7 +15,8 @@ using namespace dependencies;
DependencyScanningService::DependencyScanningService(
ScanningMode Mode, ScanningOutputFormat Format,
ScanningOptimizations OptimizeArgs, bool EagerLoadModules, bool TraceVFS,
- std::time_t BuildSessionTimestamp)
+ std::time_t BuildSessionTimestamp, bool CacheNegativeStats)
: Mode(Mode), Format(Format), OptimizeArgs(OptimizeArgs),
EagerLoadModules(EagerLoadModules), TraceVFS(TraceVFS),
+ CacheNegativeStats(CacheNegativeStats),
BuildSessionTimestamp(BuildSessionTimestamp) {}
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
index 9bd85479d9810..d9820ca3c584e 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
@@ -611,8 +611,7 @@ DependencyScanningWorker::DependencyScanningWorker(
switch (Service.getMode()) {
case ScanningMode::DependencyDirectivesScan:
- DepFS =
- new DependencyScanningWorkerFilesystem(Service.getSharedCache(), FS);
+ DepFS = new DependencyScanningWorkerFilesystem(Service, FS);
BaseFS = DepFS;
break;
case ScanningMode::CanonicalPreprocessing:
diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
index 921ba7aadd67d..ce0770a51d65f 100644
--- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -85,6 +85,7 @@ static ScanningOutputFormat Format =
ScanningOutputFormat::Make;
static ScanningOptimizations OptimizeArgs;
static std::string ModuleFilesDir;
static bool EagerLoadModules;
+static bool CacheNegativeStats = true;
static unsigned NumThreads = 0;
static std::string CompilationDB;
static std::optional<std::string> ModuleName;
@@ -191,6 +192,8 @@ static void ParseArgs(int argc, char **argv) {
EagerLoadModules = Args.hasArg(OPT_eager_load_pcm);
+ CacheNegativeStats = !Args.hasArg(OPT_no_cache_negative_stats);
+
if (const llvm::opt::Arg *A = Args.getLastArg(OPT_j)) {
StringRef S{A->getValue()};
if (!llvm::to_integer(S, NumThreads, 0)) {
@@ -1080,8 +1083,9 @@ int clang_scan_deps_main(int argc, char **argv, const
llvm::ToolContext &) {
});
};
- DependencyScanningService Service(ScanMode, Format, OptimizeArgs,
- EagerLoadModules, /*TraceVFS=*/Verbose);
+ DependencyScanningService Service(
+ ScanMode, Format, OptimizeArgs, EagerLoadModules, /*TraceVFS=*/Verbose,
+ llvm::sys::toTimeT(std::chrono::system_clock::now()),
CacheNegativeStats);
llvm::Timer T;
T.startTimer();
diff --git a/clang/tools/clang-scan-deps/Opts.td
b/clang/tools/clang-scan-deps/Opts.td
index 9cccbb3aaf0c8..582ae60851e1e 100644
--- a/clang/tools/clang-scan-deps/Opts.td
+++ b/clang/tools/clang-scan-deps/Opts.td
@@ -22,6 +22,7 @@ defm module_files_dir : Eq<"module-files-dir",
def optimize_args_EQ : CommaJoined<["-", "--"], "optimize-args=">,
HelpText<"Which command-line arguments of modules to optimize">;
def eager_load_pcm : F<"eager-load-pcm", "Load PCM files eagerly (instead of
lazily on import)">;
+def no_cache_negative_stats : F<"no-cache-negative-stats", "Don't cache stat
failures">;
def j : Arg<"j", "Number of worker threads to use (default: use all concurrent
threads)">;
diff --git
a/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp
b/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp
index 683d9070b1dcf..d194b2877ad8f 100644
--- a/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp
+++ b/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp
@@ -384,3 +384,55 @@ TEST(DependencyScanner, ScanDepsWithDiagConsumer) {
EXPECT_TRUE(DiagConsumer.Finished);
}
}
+
+TEST(DependencyScanner, NoNegativeCache) {
+ StringRef CWD = "/root";
+
+ auto VFS = new llvm::vfs::InMemoryFileSystem();
+ VFS->setCurrentWorkingDirectory(CWD);
+ auto Sept = llvm::sys::path::get_separator();
+ std::string HeaderPath =
+ std::string(llvm::formatv("{0}root{0}header.h", Sept));
+ std::string Test0Path =
+ std::string(llvm::formatv("{0}root{0}test0.cpp", Sept));
+ std::string Test1Path =
+ std::string(llvm::formatv("{0}root{0}test1.cpp", Sept));
+
+ VFS->addFile(Test0Path, 0,
+ llvm::MemoryBuffer::getMemBuffer(
+ "#if __has_include(\"header.h\")\n#endif"));
+ VFS->addFile(Test1Path, 0,
+ llvm::MemoryBuffer::getMemBuffer("#include \"header.h\""));
+
+ DependencyScanningService Service(
+ ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make,
+ ScanningOptimizations::All, false, false,
+ llvm::sys::toTimeT(std::chrono::system_clock::now()), false);
+ DependencyScanningTool ScanTool(Service, VFS);
+
+ std::vector<std::string> CommandLine0 = {"clang",
+ "-target",
+ "x86_64-apple-macosx10.7",
+ "-c",
+ "test0.cpp",
+ "-o"
+ "test0.cpp.o"};
+ std::vector<std::string> CommandLine1 = {"clang",
+ "-target",
+ "x86_64-apple-macosx10.7",
+ "-c",
+ "test1.cpp",
+ "-o"
+ "test1.cpp.o"};
+
+ std::string Result;
+ ASSERT_THAT_ERROR(
+ ScanTool.getDependencyFile(CommandLine0, CWD).moveInto(Result),
+ llvm::Succeeded());
+
+ VFS->addFile(HeaderPath, 0, llvm::MemoryBuffer::getMemBuffer(""));
+
+ ASSERT_THAT_ERROR(
+ ScanTool.getDependencyFile(CommandLine1, CWD).moveInto(Result),
+ llvm::Succeeded());
+}
diff --git
a/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
b/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
index 7420743c97a2a..a68ea72d3816c 100644
---
a/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
+++
b/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
@@ -7,6 +7,7 @@
//===----------------------------------------------------------------------===//
#include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
+#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/Support/VirtualFileSystem.h"
#include "gtest/gtest.h"
@@ -19,9 +20,10 @@ TEST(DependencyScanningWorkerFilesystem,
CacheStatusFailures) {
auto InstrumentingFS =
llvm::makeIntrusiveRefCnt<llvm::vfs::TracingFileSystem>(InMemoryFS);
- DependencyScanningFilesystemSharedCache SharedCache;
- DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS);
- DependencyScanningWorkerFilesystem DepFS2(SharedCache, InstrumentingFS);
+ DependencyScanningService Service(ScanningMode::DependencyDirectivesScan,
+ ScanningOutputFormat::Make);
+ DependencyScanningWorkerFilesystem DepFS(Service, InstrumentingFS);
+ DependencyScanningWorkerFilesystem DepFS2(Service, InstrumentingFS);
DepFS.status("/foo.c");
EXPECT_EQ(InstrumentingFS->NumStatusCalls, 1u);
@@ -45,9 +47,10 @@ TEST(DependencyScanningFilesystem, CacheGetRealPath) {
auto InstrumentingFS =
llvm::makeIntrusiveRefCnt<llvm::vfs::TracingFileSystem>(InMemoryFS);
- DependencyScanningFilesystemSharedCache SharedCache;
- DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS);
- DependencyScanningWorkerFilesystem DepFS2(SharedCache, InstrumentingFS);
+ DependencyScanningService Service(ScanningMode::DependencyDirectivesScan,
+ ScanningOutputFormat::Make);
+ DependencyScanningWorkerFilesystem DepFS(Service, InstrumentingFS);
+ DependencyScanningWorkerFilesystem DepFS2(Service, InstrumentingFS);
{
llvm::SmallString<128> Result;
@@ -80,8 +83,9 @@ TEST(DependencyScanningFilesystem,
RealPathAndStatusInvariants) {
InMemoryFS->addFile("/foo.c", 0, llvm::MemoryBuffer::getMemBuffer(""));
InMemoryFS->addFile("/bar.c", 0, llvm::MemoryBuffer::getMemBuffer(""));
- DependencyScanningFilesystemSharedCache SharedCache;
- DependencyScanningWorkerFilesystem DepFS(SharedCache, InMemoryFS);
+ DependencyScanningService Service(ScanningMode::DependencyDirectivesScan,
+ ScanningOutputFormat::Make);
+ DependencyScanningWorkerFilesystem DepFS(Service, InMemoryFS);
// Success.
{
@@ -133,8 +137,9 @@ TEST(DependencyScanningFilesystem, CacheStatOnExists) {
InMemoryFS->setCurrentWorkingDirectory("/");
InMemoryFS->addFile("/foo", 0, llvm::MemoryBuffer::getMemBuffer(""));
InMemoryFS->addFile("/bar", 0, llvm::MemoryBuffer::getMemBuffer(""));
- DependencyScanningFilesystemSharedCache SharedCache;
- DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS);
+ DependencyScanningService Service(ScanningMode::DependencyDirectivesScan,
+ ScanningOutputFormat::Make);
+ DependencyScanningWorkerFilesystem DepFS(Service, InstrumentingFS);
DepFS.status("/foo");
DepFS.status("/foo");
@@ -156,8 +161,9 @@ TEST(DependencyScanningFilesystem, CacheStatFailures) {
auto InstrumentingFS =
llvm::makeIntrusiveRefCnt<llvm::vfs::TracingFileSystem>(InMemoryFS);
- DependencyScanningFilesystemSharedCache SharedCache;
- DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS);
+ DependencyScanningService Service(ScanningMode::DependencyDirectivesScan,
+ ScanningOutputFormat::Make);
+ DependencyScanningWorkerFilesystem DepFS(Service, InstrumentingFS);
DepFS.status("/dir");
DepFS.status("/dir");
@@ -183,8 +189,9 @@ TEST(DependencyScanningFilesystem,
DiagnoseStaleStatFailures) {
auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
InMemoryFS->setCurrentWorkingDirectory("/");
- DependencyScanningFilesystemSharedCache SharedCache;
- DependencyScanningWorkerFilesystem DepFS(SharedCache, InMemoryFS);
+ DependencyScanningService Service(ScanningMode::DependencyDirectivesScan,
+ ScanningOutputFormat::Make);
+ DependencyScanningWorkerFilesystem DepFS(Service, InMemoryFS);
bool Path1Exists = DepFS.exists("/path1.suffix");
EXPECT_EQ(Path1Exists, false);
@@ -197,7 +204,7 @@ TEST(DependencyScanningFilesystem,
DiagnoseStaleStatFailures) {
EXPECT_EQ(Path1Exists, false);
std::vector<llvm::StringRef> InvalidPaths =
- SharedCache.getInvalidNegativeStatCachedPaths(*InMemoryFS);
+ Service.getSharedCache().getInvalidNegativeStatCachedPaths(*InMemoryFS);
EXPECT_EQ(InvalidPaths.size(), 1u);
ASSERT_STREQ("/path1.suffix", InvalidPaths[0].str().c_str());
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
