https://github.com/jansvoboda11 updated https://github.com/llvm/llvm-project/pull/188877
>From 196117b66a6bf6cf282394673c63d5ebd8ef2c82 Mon Sep 17 00:00:00 2001 From: Jan Svoboda <[email protected]> Date: Fri, 20 Mar 2026 20:54:01 -0700 Subject: [PATCH 1/2] [clang] Introduce `ModuleCache::write()` --- .../clang/Basic/DiagnosticCommonKinds.td | 1 + .../include/clang/Frontend/CompilerInstance.h | 10 +-- .../include/clang/Frontend/FrontendActions.h | 12 ++++ .../include/clang/Serialization/ModuleCache.h | 10 ++- .../InProcessModuleCache.cpp | 6 ++ clang/lib/Frontend/CompilerInstance.cpp | 61 ++++++++++++++----- clang/lib/Frontend/FrontendActions.cpp | 3 +- clang/lib/Serialization/ModuleCache.cpp | 40 ++++++++++++ 8 files changed, 121 insertions(+), 22 deletions(-) diff --git a/clang/include/clang/Basic/DiagnosticCommonKinds.td b/clang/include/clang/Basic/DiagnosticCommonKinds.td index cb267e3ee05c1..b5f99606789fe 100644 --- a/clang/include/clang/Basic/DiagnosticCommonKinds.td +++ b/clang/include/clang/Basic/DiagnosticCommonKinds.td @@ -103,6 +103,7 @@ def err_deleted_non_function : Error< "only functions can have deleted definitions">; def err_module_not_found : Error<"module '%0' not found">, DefaultFatal; def err_module_not_built : Error<"could not build module '%0'">, DefaultFatal; +def err_module_not_written : Error<"could not write module file for '%0' to '%1': %2">, DefaultFatal; def err_module_build_disabled: Error< "module '%0' is needed but has not been provided, and implicit use of module " "files is disabled">, DefaultFatal; diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h index f206d012eacc9..be44817aa5a1b 100644 --- a/clang/include/clang/Frontend/CompilerInstance.h +++ b/clang/include/clang/Frontend/CompilerInstance.h @@ -934,12 +934,14 @@ class CompilerInstance : public ModuleLoader { std::optional<ThreadSafeCloneConfig> ThreadSafeConfig = std::nullopt); /// Compile a module file for the given module, using the options - /// provided by the importing compiler instance. Returns true if the module - /// was built without errors. + /// provided by the importing compiler instance. Returns the PCM file in + /// a buffer. // FIXME: This should be private, but it's called from static non-member // functions in the implementation file. - bool compileModule(SourceLocation ImportLoc, StringRef ModuleName, - StringRef ModuleFileName, CompilerInstance &Instance); + std::unique_ptr<llvm::MemoryBuffer> compileModule(SourceLocation ImportLoc, + StringRef ModuleName, + StringRef ModuleFileName, + CompilerInstance &Instance); ModuleLoadResult loadModule(SourceLocation ImportLoc, ModuleIdPath Path, Module::NameVisibilityKind Visibility, diff --git a/clang/include/clang/Frontend/FrontendActions.h b/clang/include/clang/Frontend/FrontendActions.h index 87a9f0d4cb06c..c5aff7ae1a713 100644 --- a/clang/include/clang/Frontend/FrontendActions.h +++ b/clang/include/clang/Frontend/FrontendActions.h @@ -114,6 +114,15 @@ class GeneratePCHAction : public ASTFrontendAction { }; class GenerateModuleAction : public ASTFrontendAction { +public: + /// When \c OS is non-null, uses it for outputting the PCM file instead of + /// automatically creating an output file. + explicit GenerateModuleAction(std::unique_ptr<raw_pwrite_stream> OS = nullptr) + : OS(std::move(OS)) {} + +private: + std::unique_ptr<raw_pwrite_stream> OS; + virtual std::unique_ptr<raw_pwrite_stream> CreateOutputFile(CompilerInstance &CI, StringRef InFile) = 0; @@ -145,6 +154,9 @@ class GenerateInterfaceStubsAction : public ASTFrontendAction { }; class GenerateModuleFromModuleMapAction : public GenerateModuleAction { +public: + using GenerateModuleAction::GenerateModuleAction; + private: bool BeginSourceFileAction(CompilerInstance &CI) override; diff --git a/clang/include/clang/Serialization/ModuleCache.h b/clang/include/clang/Serialization/ModuleCache.h index c6795c5dc358a..4fced900bbdcb 100644 --- a/clang/include/clang/Serialization/ModuleCache.h +++ b/clang/include/clang/Serialization/ModuleCache.h @@ -14,6 +14,7 @@ #include <ctime> namespace llvm { +class MemoryBufferRef; class AdvisoryLock; } // namespace llvm @@ -52,7 +53,11 @@ class ModuleCache { virtual InMemoryModuleCache &getInMemoryModuleCache() = 0; virtual const InMemoryModuleCache &getInMemoryModuleCache() const = 0; - // TODO: Virtualize writing/reading PCM files, etc. + /// Write the PCM contents to the given path in the module cache. + virtual std::error_code write(StringRef Path, + llvm::MemoryBufferRef Buffer) = 0; + + // TODO: Virtualize reading PCM files, etc. virtual ~ModuleCache() = default; }; @@ -65,6 +70,9 @@ std::shared_ptr<ModuleCache> createCrossProcessModuleCache(); /// Shared implementation of `ModuleCache::maybePrune()`. void maybePruneImpl(StringRef Path, time_t PruneInterval, time_t PruneAfter); + +/// Shared implementation of `ModuleCache::write()`. +std::error_code writeImpl(StringRef Path, llvm::MemoryBufferRef Buffer); } // namespace clang #endif diff --git a/clang/lib/DependencyScanning/InProcessModuleCache.cpp b/clang/lib/DependencyScanning/InProcessModuleCache.cpp index cd7385c8f38c2..7bdfae8f3e567 100644 --- a/clang/lib/DependencyScanning/InProcessModuleCache.cpp +++ b/clang/lib/DependencyScanning/InProcessModuleCache.cpp @@ -127,6 +127,12 @@ class InProcessModuleCache : public ModuleCache { maybePruneImpl(Path, PruneInterval, PruneAfter); } + std::error_code write(StringRef Path, llvm::MemoryBufferRef Buffer) override { + // FIXME: This could use an in-memory cache to avoid IO, and only write to + // disk at the end of the scan. + return writeImpl(Path, Buffer); + } + InMemoryModuleCache &getInMemoryModuleCache() override { return InMemory; } const InMemoryModuleCache &getInMemoryModuleCache() const override { return InMemory; diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp index 1f1b6701c38df..262bf3484e6a0 100644 --- a/clang/lib/Frontend/CompilerInstance.cpp +++ b/clang/lib/Frontend/CompilerInstance.cpp @@ -56,6 +56,7 @@ #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/Signals.h" +#include "llvm/Support/SmallVectorMemoryBuffer.h" #include "llvm/Support/TimeProfiler.h" #include "llvm/Support/Timer.h" #include "llvm/Support/VirtualFileSystem.h" @@ -1238,10 +1239,10 @@ class PrettyStackTraceBuildModule : public llvm::PrettyStackTraceEntry { }; } // namespace -bool CompilerInstance::compileModule(SourceLocation ImportLoc, - StringRef ModuleName, - StringRef ModuleFileName, - CompilerInstance &Instance) { +std::unique_ptr<llvm::MemoryBuffer> +CompilerInstance::compileModule(SourceLocation ImportLoc, StringRef ModuleName, + StringRef ModuleFileName, + CompilerInstance &Instance) { PrettyStackTraceBuildModule CrashInfo(ModuleName, ModuleFileName); llvm::TimeTraceScope TimeScope("Module Compile", ModuleName); @@ -1250,18 +1251,22 @@ bool CompilerInstance::compileModule(SourceLocation ImportLoc, if (getModuleCache().getInMemoryModuleCache().isPCMFinal(ModuleFileName)) { getDiagnostics().Report(ImportLoc, diag::err_module_rebuild_finalized) << ModuleName; - return false; + return nullptr; } getDiagnostics().Report(ImportLoc, diag::remark_module_build) << ModuleName << ModuleFileName; + SmallString<0> Buffer; + // Execute the action to actually build the module in-place. Use a separate // thread so that we get a stack large enough. bool Crashed = !llvm::CrashRecoveryContext().RunSafelyOnNewStack( [&]() { + auto OS = std::make_unique<llvm::raw_svector_ostream>(Buffer); + std::unique_ptr<FrontendAction> Action = - std::make_unique<GenerateModuleFromModuleMapAction>(); + std::make_unique<GenerateModuleFromModuleMapAction>(std::move(OS)); if (auto WrapGenModuleAction = Instance.getGenModuleActionWrapper()) Action = WrapGenModuleAction(Instance.getFrontendOpts(), @@ -1297,10 +1302,17 @@ bool CompilerInstance::compileModule(SourceLocation ImportLoc, setBuildGlobalModuleIndex(true); } - // If \p AllowPCMWithCompilerErrors is set return 'success' even if errors + if (Crashed) + return nullptr; + + // Unless \p AllowPCMWithCompilerErrors is set, return 'failure' if errors // occurred. - return !Instance.getDiagnostics().hasErrorOccurred() || - Instance.getFrontendOpts().AllowPCMWithCompilerErrors; + if (Instance.getDiagnostics().hasErrorOccurred() && + !Instance.getFrontendOpts().AllowPCMWithCompilerErrors) + return nullptr; + + return std::make_unique<llvm::SmallVectorMemoryBuffer>( + std::move(Buffer), Instance.getFrontendOpts().OutputFile); } static OptionalFileEntryRef getPublicModuleMap(FileEntryRef File, @@ -1442,13 +1454,17 @@ static bool compileModuleImpl(CompilerInstance &ImportingInstance, SourceLocation ImportLoc, SourceLocation ModuleNameLoc, Module *Module, ModuleFileName ModuleFileName) { + std::unique_ptr<llvm::MemoryBuffer> Buffer; + { auto Instance = ImportingInstance.cloneForModuleCompile( ModuleNameLoc, Module, ModuleFileName); - if (!ImportingInstance.compileModule(ModuleNameLoc, - Module->getTopLevelModuleName(), - ModuleFileName, *Instance)) { + Buffer = ImportingInstance.compileModule(ModuleNameLoc, + Module->getTopLevelModuleName(), + ModuleFileName, *Instance); + + if (!Buffer) { ImportingInstance.getDiagnostics().Report(ModuleNameLoc, diag::err_module_not_built) << Module->Name << SourceRange(ImportLoc, ModuleNameLoc); @@ -1456,6 +1472,16 @@ static bool compileModuleImpl(CompilerInstance &ImportingInstance, } } + std::error_code EC = + ImportingInstance.getModuleCache().write(ModuleFileName, *Buffer); + if (EC) { + ImportingInstance.getDiagnostics().Report(ModuleNameLoc, + diag::err_module_not_written) + << Module->Name << ModuleFileName << EC.message() + << SourceRange(ImportLoc, ModuleNameLoc); + return false; + } + // The module is built successfully, we can update its timestamp now. if (ImportingInstance.getPreprocessor() .getHeaderSearchInfo() @@ -2196,8 +2222,9 @@ void CompilerInstance::createModuleFromSource(SourceLocation ImportLoc, // output is nondeterministic (as .pcm files refer to each other by name). // Can this affect the output in any way? SmallString<128> ModuleFileName; + int FD; if (std::error_code EC = llvm::sys::fs::createTemporaryFile( - CleanModuleName, "pcm", ModuleFileName)) { + CleanModuleName, "pcm", FD, ModuleFileName)) { getDiagnostics().Report(ImportLoc, diag::err_fe_unable_to_open_output) << ModuleFileName << EC.message(); return; @@ -2225,12 +2252,14 @@ void CompilerInstance::createModuleFromSource(SourceLocation ImportLoc, Other->DeleteBuiltModules = false; // Build the module, inheriting any modules that we've built locally. - bool Success = compileModule(ImportLoc, ModuleName, ModuleFileName, *Other); - + std::unique_ptr<llvm::MemoryBuffer> Buffer = + compileModule(ImportLoc, ModuleName, ModuleFileName, *Other); BuiltModules = std::move(Other->BuiltModules); - if (Success) { + if (Buffer) { + llvm::raw_fd_ostream OS(FD, /*shouldClose=*/true); BuiltModules[std::string(ModuleName)] = std::string(ModuleFileName); + OS << Buffer->getBuffer(); llvm::sys::RemoveFileOnSignal(ModuleFileName); } } diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp index e5eaab0da7adb..42f1ae3d83ed3 100644 --- a/clang/lib/Frontend/FrontendActions.cpp +++ b/clang/lib/Frontend/FrontendActions.cpp @@ -188,7 +188,8 @@ bool GeneratePCHAction::BeginSourceFileAction(CompilerInstance &CI) { std::vector<std::unique_ptr<ASTConsumer>> GenerateModuleAction::CreateMultiplexConsumer(CompilerInstance &CI, StringRef InFile) { - std::unique_ptr<raw_pwrite_stream> OS = CreateOutputFile(CI, InFile); + if (!OS) + OS = CreateOutputFile(CI, InFile); if (!OS) return {}; diff --git a/clang/lib/Serialization/ModuleCache.cpp b/clang/lib/Serialization/ModuleCache.cpp index 658da6e3b7145..6a1fe5e635cd8 100644 --- a/clang/lib/Serialization/ModuleCache.cpp +++ b/clang/lib/Serialization/ModuleCache.cpp @@ -101,6 +101,39 @@ void clang::maybePruneImpl(StringRef Path, time_t PruneInterval, } } +std::error_code clang::writeImpl(StringRef Path, llvm::MemoryBufferRef Buffer) { + StringRef Extension = llvm::sys::path::extension(Path); + SmallString<128> ModelPath = StringRef(Path).drop_back(Extension.size()); + ModelPath += "-%%%%%%%%"; + ModelPath += Extension; + ModelPath += ".tmp"; + + std::error_code EC; + int FD; + SmallString<128> TmpPath; + if ((EC = llvm::sys::fs::createUniqueFile(ModelPath, FD, TmpPath))) { + if (EC != std::errc::no_such_file_or_directory) + return EC; + + StringRef Dir = llvm::sys::path::parent_path(Path); + if (std::error_code InnerEC = llvm::sys::fs::create_directories(Dir)) + return InnerEC; + + if ((EC = llvm::sys::fs::createUniqueFile(ModelPath, FD, TmpPath))) + return EC; + } + + { + llvm::raw_fd_ostream OS(FD, /*shouldClose=*/true); + OS << Buffer.getBuffer(); + } + + if ((EC = llvm::sys::fs::rename(TmpPath, Path))) + return EC; + + return {}; +} + namespace { class CrossProcessModuleCache : public ModuleCache { InMemoryModuleCache InMemory; @@ -157,6 +190,13 @@ class CrossProcessModuleCache : public ModuleCache { maybePruneImpl(Path, PruneInterval, PruneAfter); } + std::error_code write(StringRef Path, llvm::MemoryBufferRef Buffer) override { + // This is a compiler-internal input/output, let's bypass the sandbox. + auto BypassSandbox = llvm::sys::sandbox::scopedDisable(); + + return writeImpl(Path, Buffer); + } + InMemoryModuleCache &getInMemoryModuleCache() override { return InMemory; } const InMemoryModuleCache &getInMemoryModuleCache() const override { return InMemory; >From eaa8b5e9260c7e4303500865935e7604f03751b0 Mon Sep 17 00:00:00 2001 From: Jan Svoboda <[email protected]> Date: Fri, 27 Mar 2026 09:48:29 -0700 Subject: [PATCH 2/2] Add missing include --- clang/include/clang/Serialization/ModuleCache.h | 1 + 1 file changed, 1 insertion(+) diff --git a/clang/include/clang/Serialization/ModuleCache.h b/clang/include/clang/Serialization/ModuleCache.h index 4fced900bbdcb..9ea4d84380660 100644 --- a/clang/include/clang/Serialization/ModuleCache.h +++ b/clang/include/clang/Serialization/ModuleCache.h @@ -12,6 +12,7 @@ #include "clang/Basic/LLVM.h" #include <ctime> +#include <system_error> namespace llvm { class MemoryBufferRef; _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
