https://github.com/Bigcheese updated https://github.com/llvm/llvm-project/pull/190067
>From f4e3438b70f40ac08f5e8d8daab81e37f7e5b284 Mon Sep 17 00:00:00 2001 From: Michael Spencer <[email protected]> Date: Wed, 1 Apr 2026 14:36:33 -0700 Subject: [PATCH 1/3] [libclang] Add clang_ModuleCache_prune This allows a build system to direct Clang to prune a module cache directory using the same method Clang does internally. This also changes `clang::maybePruneImpl` to clean up files directly in the directory, not just subdirectories. --- clang/include/clang-c/BuildSystem.h | 18 +++++++ clang/lib/Serialization/ModuleCache.cpp | 57 +++++++++++--------- clang/tools/libclang/BuildSystem.cpp | 7 +++ clang/tools/libclang/libclang.map | 5 ++ clang/unittests/libclang/LibclangTest.cpp | 66 +++++++++++++++++++++++ 5 files changed, 127 insertions(+), 26 deletions(-) diff --git a/clang/include/clang-c/BuildSystem.h b/clang/include/clang-c/BuildSystem.h index 57e16af20a700..3bd088c185ca8 100644 --- a/clang/include/clang-c/BuildSystem.h +++ b/clang/include/clang-c/BuildSystem.h @@ -18,6 +18,7 @@ #include "clang-c/CXString.h" #include "clang-c/ExternC.h" #include "clang-c/Platform.h" +#include <time.h> LLVM_CLANG_C_EXTERN_C_BEGIN @@ -143,6 +144,23 @@ clang_ModuleMapDescriptor_writeToBuffer(CXModuleMapDescriptor, unsigned options, */ CINDEX_LINKAGE void clang_ModuleMapDescriptor_dispose(CXModuleMapDescriptor); +/** + * Prune module files in the module cache directory that haven't been accessed + * in a long time. + * + * \param Path the path to the module cache directory. + * + * \param PruneInterval the minimum time in seconds between two prune + * operations. If the timestamp file is newer than this, pruning is skipped. + * + * \param PruneAfter the time in seconds after which unused module files are + * removed. + * + */ +CINDEX_LINKAGE void clang_ModuleCache_prune(const char *Path, + time_t PruneInterval, + time_t PruneAfter); + /** * @} */ diff --git a/clang/lib/Serialization/ModuleCache.cpp b/clang/lib/Serialization/ModuleCache.cpp index 658da6e3b7145..48b9f030a292d 100644 --- a/clang/lib/Serialization/ModuleCache.cpp +++ b/clang/lib/Serialization/ModuleCache.cpp @@ -59,38 +59,43 @@ void clang::maybePruneImpl(StringRef Path, time_t PruneInterval, // Walk the entire module cache, looking for unused module files and module // indices. std::error_code EC; + auto TryPruneFile = [&](StringRef FilePath) { + // We only care about module and global module index files. + StringRef Extension = llvm::sys::path::extension(FilePath); + if (Extension != ".pcm" && Extension != ".timestamp" && + llvm::sys::path::filename(FilePath) != "modules.idx") + return; + + // Look at this file. If we can't stat it, there's nothing interesting + // there. + if (llvm::sys::fs::status(FilePath, StatBuf)) + return; + + // If the file has been used recently enough, leave it there. + time_t FileAccessTime = llvm::sys::toTimeT(StatBuf.getLastAccessedTime()); + if (CurrentTime - FileAccessTime <= PruneAfter) + return; + + // Remove the file. + llvm::sys::fs::remove(FilePath); + + // Remove the timestamp file. + std::string TimpestampFilename = FilePath.str() + ".timestamp"; + llvm::sys::fs::remove(TimpestampFilename); + }; + for (llvm::sys::fs::directory_iterator Dir(Path, EC), DirEnd; Dir != DirEnd && !EC; Dir.increment(EC)) { - // If we don't have a directory, there's nothing to look into. - if (!llvm::sys::fs::is_directory(Dir->path())) + // If we don't have a directory, try to prune it as a file in the root. + if (!llvm::sys::fs::is_directory(Dir->path())) { + TryPruneFile(Dir->path()); continue; + } // Walk all the files within this directory. for (llvm::sys::fs::directory_iterator File(Dir->path(), EC), FileEnd; - File != FileEnd && !EC; File.increment(EC)) { - // We only care about module and global module index files. - StringRef Extension = llvm::sys::path::extension(File->path()); - if (Extension != ".pcm" && Extension != ".timestamp" && - llvm::sys::path::filename(File->path()) != "modules.idx") - continue; - - // Look at this file. If we can't stat it, there's nothing interesting - // there. - if (llvm::sys::fs::status(File->path(), StatBuf)) - continue; - - // If the file has been used recently enough, leave it there. - time_t FileAccessTime = llvm::sys::toTimeT(StatBuf.getLastAccessedTime()); - if (CurrentTime - FileAccessTime <= PruneAfter) - continue; - - // Remove the file. - llvm::sys::fs::remove(File->path()); - - // Remove the timestamp file. - std::string TimpestampFilename = File->path() + ".timestamp"; - llvm::sys::fs::remove(TimpestampFilename); - } + File != FileEnd && !EC; File.increment(EC)) + TryPruneFile(File->path()); // If we removed all the files in the directory, remove the directory // itself. diff --git a/clang/tools/libclang/BuildSystem.cpp b/clang/tools/libclang/BuildSystem.cpp index 2f638ee8700d9..e81f69d9960f2 100644 --- a/clang/tools/libclang/BuildSystem.cpp +++ b/clang/tools/libclang/BuildSystem.cpp @@ -12,6 +12,7 @@ #include "clang-c/BuildSystem.h" #include "CXString.h" +#include "clang/Serialization/ModuleCache.h" #include "llvm/ADT/SmallString.h" #include "llvm/Support/CBindingWrapping.h" #include "llvm/Support/Chrono.h" @@ -150,3 +151,9 @@ clang_ModuleMapDescriptor_writeToBuffer(CXModuleMapDescriptor MMD, unsigned, void clang_ModuleMapDescriptor_dispose(CXModuleMapDescriptor MMD) { delete MMD; } + +void clang_ModuleCache_prune(const char *Path, time_t PruneInterval, + time_t PruneAfter) { + if (Path) + clang::maybePruneImpl(Path, PruneInterval, PruneAfter); +} diff --git a/clang/tools/libclang/libclang.map b/clang/tools/libclang/libclang.map index 3d9d2e268a611..2691602d432f6 100644 --- a/clang/tools/libclang/libclang.map +++ b/clang/tools/libclang/libclang.map @@ -457,6 +457,11 @@ LLVM_21 { clang_Cursor_isGCCAssemblyVolatile; }; +LLVM_23 { + global: + clang_ModuleCache_prune; +}; + # Example of how to add a new symbol version entry. If you do add a new symbol # version, please update the example to depend on the version you added. # LLVM_X { diff --git a/clang/unittests/libclang/LibclangTest.cpp b/clang/unittests/libclang/LibclangTest.cpp index b2a87d240e56e..2c676aae4dcb1 100644 --- a/clang/unittests/libclang/LibclangTest.cpp +++ b/clang/unittests/libclang/LibclangTest.cpp @@ -7,9 +7,11 @@ //===----------------------------------------------------------------------===// #include "TestUtils.h" +#include "clang-c/BuildSystem.h" #include "clang-c/Index.h" #include "clang-c/Rewrite.h" #include "llvm/ADT/StringRef.h" +#include "llvm/Support/Chrono.h" #include "llvm/Support/Debug.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" @@ -356,6 +358,70 @@ TEST(libclang, ModuleMapDescriptor) { clang_ModuleMapDescriptor_dispose(MMD); } +TEST(libclang, ModuleCache_prune) { + llvm::SmallString<256> CacheDir; + ASSERT_FALSE( + llvm::sys::fs::createUniqueDirectory("libclang-module-cache", CacheDir)); + + auto writeFile = [](const llvm::Twine &Path, llvm::StringRef Content = "") { + std::error_code EC; + llvm::raw_fd_ostream OS(Path.str(), EC); + ASSERT_FALSE(EC); + OS << Content; + }; + + auto setOldAccessTime = [](const llvm::Twine &Path) { + int FD; + std::error_code EC = llvm::sys::fs::openFileForReadWrite( + Path, FD, llvm::sys::fs::CD_OpenExisting, llvm::sys::fs::OF_None); + ASSERT_FALSE(EC); + // Set the access time to be old enough to get pruned. + time_t OldTime = time(nullptr) - 86400; + EC = llvm::sys::fs::setLastAccessAndModificationTime( + FD, llvm::sys::toTimePoint(OldTime)); + ASSERT_FALSE(EC); + ::close(FD); + }; + + // Create a subdirectory with a .pcm file and a .timestamp file. + llvm::SmallString<256> SubDir(CacheDir); + llvm::sys::path::append(SubDir, "ABCDEF"); + ASSERT_FALSE(llvm::sys::fs::create_directory(SubDir)); + + llvm::SmallString<256> PCMFile(SubDir); + llvm::sys::path::append(PCMFile, "Foo.pcm"); + writeFile(PCMFile, "fake pcm"); + + // Also create a .pcm file in the root of the cache directory. + llvm::SmallString<256> RootPCMFile(CacheDir); + llvm::sys::path::append(RootPCMFile, "Bar.pcm"); + writeFile(RootPCMFile, "fake pcm"); + + setOldAccessTime(PCMFile); + setOldAccessTime(RootPCMFile); + + // Create the modules.timestamp file with an old modification time so the + // prune interval check passes. + llvm::SmallString<256> ModulesTimestamp(CacheDir); + llvm::sys::path::append(ModulesTimestamp, "modules.timestamp"); + writeFile(ModulesTimestamp); + setOldAccessTime(ModulesTimestamp); + + // Verify the files exist before pruning. + EXPECT_TRUE(llvm::sys::fs::exists(PCMFile)); + EXPECT_TRUE(llvm::sys::fs::exists(RootPCMFile)); + + clang_ModuleCache_prune(CacheDir.c_str(), /*PruneInterval=*/1, + /*PruneAfter=*/1); + + // The old .pcm files should have been removed from both the subdirectory + // and the root. + EXPECT_FALSE(llvm::sys::fs::exists(PCMFile)); + EXPECT_FALSE(llvm::sys::fs::exists(RootPCMFile)); + + llvm::sys::fs::remove_directories(CacheDir); +} + TEST_F(LibclangParseTest, GlobalOptions) { EXPECT_EQ(clang_CXIndex_getGlobalOptions(Index), CXGlobalOpt_None); } >From ceb6176cc89a610e1adeaa4225dfa1d544445bc2 Mon Sep 17 00:00:00 2001 From: Michael Spencer <[email protected]> Date: Wed, 1 Apr 2026 18:58:17 -0700 Subject: [PATCH 2/3] Don't remove modules.timestamp. --- clang/lib/Serialization/ModuleCache.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/clang/lib/Serialization/ModuleCache.cpp b/clang/lib/Serialization/ModuleCache.cpp index 48b9f030a292d..99d02ec740690 100644 --- a/clang/lib/Serialization/ModuleCache.cpp +++ b/clang/lib/Serialization/ModuleCache.cpp @@ -88,7 +88,9 @@ void clang::maybePruneImpl(StringRef Path, time_t PruneInterval, Dir != DirEnd && !EC; Dir.increment(EC)) { // If we don't have a directory, try to prune it as a file in the root. if (!llvm::sys::fs::is_directory(Dir->path())) { - TryPruneFile(Dir->path()); + // Don't prune the timestamp file at the top level. + if (llvm::sys::path::filename(Dir->path()) != "modules.timestamp") + TryPruneFile(Dir->path()); continue; } >From 5dacc2786f2102165e67d21178eeec8b8e3869e2 Mon Sep 17 00:00:00 2001 From: Michael Spencer <[email protected]> Date: Wed, 1 Apr 2026 22:11:46 -0700 Subject: [PATCH 3/3] Address review comments. --- clang/lib/Serialization/ModuleCache.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/clang/lib/Serialization/ModuleCache.cpp b/clang/lib/Serialization/ModuleCache.cpp index 99d02ec740690..e7a1cea335f0d 100644 --- a/clang/lib/Serialization/ModuleCache.cpp +++ b/clang/lib/Serialization/ModuleCache.cpp @@ -61,9 +61,14 @@ void clang::maybePruneImpl(StringRef Path, time_t PruneInterval, std::error_code EC; auto TryPruneFile = [&](StringRef FilePath) { // We only care about module and global module index files. + StringRef Filename = llvm::sys::path::filename(FilePath); StringRef Extension = llvm::sys::path::extension(FilePath); if (Extension != ".pcm" && Extension != ".timestamp" && - llvm::sys::path::filename(FilePath) != "modules.idx") + Filename != "modules.idx") + return; + + // Don't prune the pruning timestamp file. + if (Filename == "modules.timestamp") return; // Look at this file. If we can't stat it, there's nothing interesting @@ -79,7 +84,7 @@ void clang::maybePruneImpl(StringRef Path, time_t PruneInterval, // Remove the file. llvm::sys::fs::remove(FilePath); - // Remove the timestamp file. + // Remove the timestamp file created by implicit module builds. std::string TimpestampFilename = FilePath.str() + ".timestamp"; llvm::sys::fs::remove(TimpestampFilename); }; @@ -88,9 +93,7 @@ void clang::maybePruneImpl(StringRef Path, time_t PruneInterval, Dir != DirEnd && !EC; Dir.increment(EC)) { // If we don't have a directory, try to prune it as a file in the root. if (!llvm::sys::fs::is_directory(Dir->path())) { - // Don't prune the timestamp file at the top level. - if (llvm::sys::path::filename(Dir->path()) != "modules.timestamp") - TryPruneFile(Dir->path()); + TryPruneFile(Dir->path()); continue; } _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
