https://github.com/c-rhodes updated https://github.com/llvm/llvm-project/pull/181059
>From 86407b6e65fcf2fe230f27cd0a2a71ec71a178d1 Mon Sep 17 00:00:00 2001 From: Ben Dunbobbin <[email protected]> Date: Tue, 10 Feb 2026 16:44:02 +0000 Subject: [PATCH 1/4] [Windows][Support] Add helper to expand short 8.3 form paths (#178480) Windows supports short 8.3 form filenames (for example, `compile_commands.json` -> `COMPIL~1.JSO`) for legacy reasons. See: https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file#short-vs-long-names Such paths are not unusual because, on Windows, the system temporary directory is commonly derived from the `TMP`/`TEMP` environment variables. For historical compatibility reasons, these variables are often set to short 8.3 form paths on systems where user names exceed eight characters. Introduce `windows::makeLongFormPath()` to convert paths to their long form by expanding any 8.3 components via `GetLongPathNameW`. As part of this change: - Extended-length path prefix handling is centralized by adding `stripExtendedPrefix()` and reusing it in `realPathFromHandle()`. - `widenPath()` is cleaned up to use shared prefix constants. This was split out from #178303 at the request of the codeowner so that the Windows support parts can be reviewed separately. (cherry picked from commit e6f5e4910df519a3f14e0db86d24abe8fd25082b) --- .../llvm/Support/Windows/WindowsSupport.h | 5 + llvm/lib/Support/Windows/Path.inc | 92 ++++++++-- llvm/unittests/Support/Path.cpp | 159 ++++++++++++++++++ 3 files changed, 241 insertions(+), 15 deletions(-) diff --git a/llvm/include/llvm/Support/Windows/WindowsSupport.h b/llvm/include/llvm/Support/Windows/WindowsSupport.h index f35e7b55cb8d3..50a2540dba687 100644 --- a/llvm/include/llvm/Support/Windows/WindowsSupport.h +++ b/llvm/include/llvm/Support/Windows/WindowsSupport.h @@ -249,6 +249,11 @@ LLVM_ABI std::error_code widenPath(const Twine &Path8, /// ensuring we're not retrieving a malicious injected module but a module /// loaded from the system path. LLVM_ABI HMODULE loadSystemModuleSecure(LPCWSTR lpModuleName); + +/// Convert a UTF-8 path to a long form UTF-8 path expanding any short 8.3 form +/// components. +LLVM_ABI std::error_code makeLongFormPath(const Twine &Path8, + llvm::SmallVectorImpl<char> &Result8); } // end namespace windows } // end namespace sys } // end namespace llvm. diff --git a/llvm/lib/Support/Windows/Path.inc b/llvm/lib/Support/Windows/Path.inc index c03b85b2f4bb3..97dd75a74d18c 100644 --- a/llvm/lib/Support/Windows/Path.inc +++ b/llvm/lib/Support/Windows/Path.inc @@ -60,6 +60,34 @@ static bool is_separator(const wchar_t value) { } } +// Long path path prefix constants (UTF-8). +static constexpr llvm::StringLiteral LongPathPrefix8 = R"(\\?\)"; +static constexpr llvm::StringLiteral LongPathUNCPrefix8 = R"(\\?\UNC\)"; + +// Long path prefix constants (UTF-16). +static constexpr wchar_t LongPathPrefix16[] = LR"(\\?\)"; +static constexpr wchar_t LongPathUNCPrefix16[] = LR"(\\?\UNC\)"; + +static constexpr DWORD LongPathPrefix16Len = + static_cast<DWORD>(std::size(LongPathPrefix16) - 1); +static constexpr DWORD LongPathUNCPrefix16Len = + static_cast<DWORD>(std::size(LongPathUNCPrefix16) - 1); + +static void stripLongPathPrefix(wchar_t *&Data, DWORD &CountChars) { + if (CountChars >= LongPathUNCPrefix16Len && + ::wmemcmp(Data, LongPathUNCPrefix16, LongPathUNCPrefix16Len) == 0) { + // Convert \\?\UNC\foo\bar to \\foo\bar + CountChars -= 6; + Data += 6; + Data[0] = L'\\'; + } else if (CountChars >= LongPathPrefix16Len && + ::wmemcmp(Data, LongPathPrefix16, LongPathPrefix16Len) == 0) { + // Convert \\?\C:\foo to C:\foo + CountChars -= 4; + Data += 4; + } +} + namespace llvm { namespace sys { namespace windows { @@ -95,10 +123,8 @@ std::error_code widenPath(const Twine &Path8, SmallVectorImpl<wchar_t> &Path16, return mapWindowsError(::GetLastError()); } - const char *const LongPathPrefix = "\\\\?\\"; - if ((Path16.size() + CurPathLen) < MaxPathLen || - Path8Str.starts_with(LongPathPrefix)) + Path8Str.starts_with(LongPathPrefix8)) return std::error_code(); if (!IsAbsolute) { @@ -116,17 +142,62 @@ std::error_code widenPath(const Twine &Path8, SmallVectorImpl<wchar_t> &Path16, assert(!RootName.empty() && "Root name cannot be empty for an absolute path!"); - SmallString<2 * MAX_PATH> FullPath(LongPathPrefix); + SmallString<2 * MAX_PATH> FullPath; if (RootName[1] != ':') { // Check if UNC. - FullPath.append("UNC\\"); + FullPath.append(LongPathUNCPrefix8); FullPath.append(Path8Str.begin() + 2, Path8Str.end()); } else { + FullPath.append(LongPathPrefix8); FullPath.append(Path8Str); } return UTF8ToUTF16(FullPath, Path16); } +std::error_code makeLongFormPath(const Twine &Path8, + llvm::SmallVectorImpl<char> &Result8) { + SmallString<128> PathStorage; + StringRef PathStr = Path8.toStringRef(PathStorage); + bool HadPrefix = PathStr.starts_with(LongPathPrefix8); + + SmallVector<wchar_t, 128> Path16; + if (std::error_code EC = widenPath(PathStr, Path16)) + return EC; + + // Start with a buffer equal to input. + llvm::SmallVector<wchar_t, 128> Long16; + DWORD Len = static_cast<DWORD>(Path16.size()); + + // Loop instead of a double call to be defensive against TOCTOU races. + do { + Long16.resize_for_overwrite(Len); + + Len = ::GetLongPathNameW(Path16.data(), Long16.data(), Len); + + // A zero return value indicates a failure other than insufficient space. + if (Len == 0) + return mapWindowsError(::GetLastError()); + + // If there's insufficient space, the return value is the required size in + // characters *including* the null terminator, and therefore greater than + // the buffer size we provided. Equality would imply success with no room + // for the terminator and should not occur for this API. + assert(Len != Long16.size()); + } while (Len > Long16.size()); + + // On success, GetLongPathNameW returns the number of characters not + // including the null-terminator. + Long16.truncate(Len); + + // Strip \\?\ or \\?\UNC\ long length prefix if it wasn't part of the + // original path. + wchar_t *Data = Long16.data(); + if (!HadPrefix) + stripLongPathPrefix(Data, Len); + + return sys::windows::UTF16ToUTF8(Data, Len, Result8); +} + } // end namespace windows namespace fs { @@ -407,16 +478,7 @@ static std::error_code realPathFromHandle(HANDLE H, // paths don't get canonicalized by file APIs. wchar_t *Data = Buffer.data(); DWORD CountChars = Buffer.size(); - if (CountChars >= 8 && ::memcmp(Data, L"\\\\?\\UNC\\", 16) == 0) { - // Convert \\?\UNC\foo\bar to \\foo\bar - CountChars -= 6; - Data += 6; - Data[0] = '\\'; - } else if (CountChars >= 4 && ::memcmp(Data, L"\\\\?\\", 8) == 0) { - // Convert \\?\c:\foo to c:\foo - CountChars -= 4; - Data += 4; - } + stripLongPathPrefix(Data, CountChars); // Convert the result from UTF-16 to UTF-8. if (std::error_code EC = UTF16ToUTF8(Data, CountChars, RealPath)) diff --git a/llvm/unittests/Support/Path.cpp b/llvm/unittests/Support/Path.cpp index 7f22b7c2edb4b..08c0c55404d29 100644 --- a/llvm/unittests/Support/Path.cpp +++ b/llvm/unittests/Support/Path.cpp @@ -32,6 +32,8 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/Support/Chrono.h" #include "llvm/Support/Windows/WindowsSupport.h" +#include "llvm/Support/WindowsError.h" +#include <fileapi.h> #include <windows.h> #include <winerror.h> #endif @@ -2485,6 +2487,163 @@ TEST_F(FileSystemTest, widenPath) { #endif #ifdef _WIN32 +/// Checks whether short 8.3 form names are enabled in the given UTF-8 path. +static llvm::Expected<bool> areShortNamesEnabled(llvm::StringRef Path8) { + // Create a directory under Path8 with a name long enough that Windows will + // provide a short 8.3 form name, if short 8.3 form names are enabled. + SmallString<MAX_PATH> Dir(Path8); + path::append(Dir, "verylongdir"); + if (std::error_code EC = fs::create_directories(Dir)) + return llvm::errorCodeToError(EC); + scope_exit Close([&] { fs::remove_directories(Dir); }); + + SmallVector<wchar_t, MAX_PATH> Path16; + if (std::error_code EC = sys::windows::widenPath(Dir, Path16)) + return llvm::errorCodeToError(EC); + + WIN32_FIND_DATAW Data; + HANDLE H = ::FindFirstFileW(Path16.data(), &Data); + if (H == INVALID_HANDLE_VALUE) + return llvm::errorCodeToError(llvm::mapWindowsError(::GetLastError())); + ::FindClose(H); + + return (Data.cAlternateFileName[0] != L'\0'); +} + +/// Returns the short 8.3 form path for the given UTF-8 path, or an empty string +/// on failure. Uses Win32 GetShortPathNameW. +static std::string getShortPathName(llvm::StringRef Path8) { + // Convert UTF-8 to UTF-16. + SmallVector<wchar_t, MAX_PATH> Path16; + if (std::error_code EC = sys::windows::widenPath(Path8, Path16)) + return {}; + + // Get the required buffer size for the short 8.3 form path (includes null + // terminator). + DWORD Required = ::GetShortPathNameW(Path16.data(), nullptr, 0); + if (Required == 0) + return {}; + + SmallVector<wchar_t, MAX_PATH> ShortPath; + ShortPath.resize_for_overwrite(Required); + + DWORD Written = + ::GetShortPathNameW(Path16.data(), ShortPath.data(), Required); + if (Written == 0 || Written >= Required) + return {}; + + ShortPath.truncate(Written); + + SmallString<MAX_PATH> Utf8Result; + if (std::error_code EC = sys::windows::UTF16ToUTF8( + ShortPath.data(), ShortPath.size(), Utf8Result)) + return {}; + + return std::string(Utf8Result); +} + +/// Returns true if the two paths refer to the same file or directory by +/// comparing their UniqueIDs. +static bool sameEntity(llvm::StringRef P1, llvm::StringRef P2) { + fs::UniqueID ID1, ID2; + return !fs::getUniqueID(P1, ID1) && !fs::getUniqueID(P2, ID2) && ID1 == ID2; +} + +/// Removes the Windows long path path prefix (\\?\ or \\?\UNC\) from the given +/// UTF-8 path, if present. +static std::string stripPrefix(llvm::StringRef P) { + if (P.starts_with(R"(\\?\UNC\)")) + return "\\" + P.drop_front(7).str(); + if (P.starts_with(R"(\\?\)")) + return P.drop_front(4).str(); + return P.str(); +} + +TEST_F(FileSystemTest, makeLongFormPath) { + auto Enabled = areShortNamesEnabled(TestDirectory.str()); + ASSERT_TRUE(static_cast<bool>(Enabled)) + << llvm::toString(Enabled.takeError()); + if (!*Enabled) + GTEST_SKIP() << "Short 8.3 form names not enabled in: " << TestDirectory; + + // Setup: A test directory longer than 8 characters for which a distinct + // short 8.3 form name will be created on Windows. Typically, 123456~1. + constexpr const char *OneDir = "\\123456789"; // >8 chars + + // Setup: Create a path where even if all components were reduced to short 8.3 + // form names, the total length would exceed MAX_PATH. + SmallString<MAX_PATH * 2> Deep(TestDirectory); + const size_t NLevels = (MAX_PATH / 8) + 1; + for (size_t I = 0; I < NLevels; ++I) + Deep.append(OneDir); + + ASSERT_NO_ERROR(fs::create_directories(Deep)); + + // Setup: Create prefixed and non-prefixed short 8.3 form paths from the deep + // test path we just created. + std::string DeepShortWithPrefix = getShortPathName(Deep); + ASSERT_TRUE(StringRef(DeepShortWithPrefix).starts_with(R"(\\?\)")) + << "Expected prefixed short 8.3 form path, got: " << DeepShortWithPrefix; + std::string DeepShort = stripPrefix(DeepShortWithPrefix); + + // Setup: Create a short 8.3 form path for the first-level directory. + SmallString<MAX_PATH> FirstLevel(TestDirectory); + FirstLevel.append(OneDir); + std::string Short = getShortPathName(FirstLevel); + ASSERT_FALSE(Short.empty()) + << "Expected short 8.3 form path for test directory."; + + // Setup: Create a short 8.3 form path with . and .. components for the + // first-level directory. + llvm::SmallString<MAX_PATH> WithDots(FirstLevel); + llvm::sys::path::append(WithDots, ".", "..", OneDir); + std::string DotAndDotDot = getShortPathName(WithDots); + ASSERT_FALSE(DotAndDotDot.empty()) + << "Expected short 8.3 form path for test directory."; + auto ContainsDotAndDotDot = [](llvm::StringRef S) { + return S.contains("\\.\\") && S.contains("\\..\\"); + }; + ASSERT_TRUE(ContainsDotAndDotDot(DotAndDotDot)) + << "Expected '.' and '..' components in: " << DotAndDotDot; + + // Case 1: Non-existent short 8.3 form path. + SmallString<MAX_PATH> NoExist("NotEre~1"); + ASSERT_FALSE(fs::exists(NoExist)); + SmallString<MAX_PATH> NoExistResult; + EXPECT_TRUE(windows::makeLongFormPath(NoExist, NoExistResult)); + EXPECT_TRUE(NoExistResult.empty()); + + // Case 2: Valid short 8.3 form path. + SmallString<MAX_PATH> ShortResult; + ASSERT_FALSE(windows::makeLongFormPath(Short, ShortResult)); + EXPECT_TRUE(sameEntity(Short, ShortResult)); + + // Case 3: Valid . and .. short 8.3 form path. + SmallString<MAX_PATH> DotAndDotDotResult; + ASSERT_FALSE(windows::makeLongFormPath(DotAndDotDot, DotAndDotDotResult)); + EXPECT_TRUE(sameEntity(DotAndDotDot, DotAndDotDotResult)); + // Assert that '.' and '..' remain as path components. + ASSERT_TRUE(ContainsDotAndDotDot(DotAndDotDotResult)); + + // Case 4: Deep short 8.3 form path without \\?\ prefix. + SmallString<MAX_PATH> DeepResult; + ASSERT_FALSE(windows::makeLongFormPath(DeepShort, DeepResult)); + EXPECT_TRUE(sameEntity(DeepShort, DeepResult)); + EXPECT_FALSE(StringRef(DeepResult).starts_with(R"(\\?\)")) + << "Expected unprefixed result, got: " << DeepResult; + + // Case 5: Deep short 8.3 form path with \\?\ prefix. + SmallString<MAX_PATH> DeepPrefixedResult; + ASSERT_FALSE( + windows::makeLongFormPath(DeepShortWithPrefix, DeepPrefixedResult)); + EXPECT_TRUE(sameEntity(DeepShortWithPrefix, DeepPrefixedResult)); + EXPECT_TRUE(StringRef(DeepPrefixedResult).starts_with(R"(\\?\)")) + << "Expected prefixed result, got: " << DeepPrefixedResult; + + // Cleanup. + ASSERT_NO_ERROR(fs::remove_directories(TestDirectory.str())); +} + // Windows refuses lock request if file region is already locked by the same // process. POSIX system in this case updates the existing lock. TEST_F(FileSystemTest, FileLocker) { >From 68a700cc0fd5c0432ac593b220e63e8f109c78ea Mon Sep 17 00:00:00 2001 From: Ben Dunbobbin <[email protected]> Date: Wed, 11 Feb 2026 09:02:19 +0000 Subject: [PATCH 2/4] [DTLTO][NFC] Minor improvements to the input file preparation class (#180824) This change performs a small set of NFC refactors to improve clarity. In particular, we make it clear that the responsibilities of this class now extend beyond its original archive-handling role. (cherry picked from commit 8b57b9730222cdc098d6a8708099e5b4986b1e2c) --- llvm/include/llvm/DTLTO/DTLTO.h | 42 +++++---- llvm/include/llvm/LTO/LTO.h | 2 +- llvm/lib/DTLTO/DTLTO.cpp | 159 ++++++++++++++------------------ llvm/lib/LTO/LTO.cpp | 2 +- 4 files changed, 99 insertions(+), 106 deletions(-) diff --git a/llvm/include/llvm/DTLTO/DTLTO.h b/llvm/include/llvm/DTLTO/DTLTO.h index 02b098a68aec5..e80577aa12834 100644 --- a/llvm/include/llvm/DTLTO/DTLTO.h +++ b/llvm/include/llvm/DTLTO/DTLTO.h @@ -6,8 +6,8 @@ // //===---------------------------------------------------------------------===// -#ifndef LLVM_DTLTO_H -#define LLVM_DTLTO_H +#ifndef LLVM_DTLTO_DTLTO_H +#define LLVM_DTLTO_DTLTO_H #include "llvm/LTO/LTO.h" #include "llvm/Support/MemoryBuffer.h" @@ -15,6 +15,20 @@ namespace llvm { namespace lto { +// The purpose of this class is to prepare inputs so that distributed ThinLTO +// backend compilations can succeed. +// +// For distributed compilation, each input must exist as an individual bitcode +// file on disk and be loadable via its ModuleID. This requirement is not met +// for archive members, as an archive is a collection of files rather than a +// standalone file. Similarly, for FatLTO objects, the bitcode is stored in a +// section of the containing ELF object file. To address this, the class ensures +// that an individual bitcode file exists for each input (by writing it out if +// necessary) and that the ModuleID is updated to point to it. +// +// The class ensures that lto::InputFile objects are preserved until enough of +// the LTO pipeline has executed to determine the required per-module +// information, such as whether a module will participate in ThinLTO. class DTLTO : public LTO { using Base = LTO; @@ -33,7 +47,10 @@ class DTLTO : public LTO { addInput(std::unique_ptr<InputFile> InputPtr) override; protected: - LLVM_ABI llvm::Error handleArchiveInputs() override; + // Save the contents of ThinLTO-enabled input files that must be serialized + // for distribution, such as archive members and FatLTO objects, to individual + // bitcode files named after the module ID. + LLVM_ABI llvm::Error serializeInputsForDistribution() override; LLVM_ABI void cleanup() override; @@ -48,24 +65,17 @@ class DTLTO : public LTO { /// Controls preservation of any created temporary files. bool SaveTemps; - // Determines if a file at the given path is a thin archive file. - Expected<bool> isThinArchive(const StringRef ArchivePath); - - // Write the archive member content to a file named after the module ID. - Error saveInputArchiveMember(lto::InputFile *Input); - - // Iterates through all input files and saves their content - // to files if they are regular archive members. - Error saveInputArchiveMembers(); - // Array of input bitcode files for LTO. std::vector<std::shared_ptr<lto::InputFile>> InputFiles; - // A cache to avoid repeatedly reading the same archive file. - StringMap<bool> ArchiveFiles; + // Cache of whether a path refers to a thin archive. + StringMap<bool> ArchiveIsThinCache; + + // Determines if the file at the given path is a thin archive. + Expected<bool> isThinArchive(const StringRef ArchivePath); }; } // namespace lto } // namespace llvm -#endif // LLVM_DTLTO_H +#endif // LLVM_DTLTO_DTLTO_H diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index 9846d84e02383..2f09e0d91e8fa 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -458,7 +458,7 @@ class LTO { protected: // Called at the start of run(). - virtual Error handleArchiveInputs() { return Error::success(); } + virtual Error serializeInputsForDistribution() { return Error::success(); } // Called before returning from run(). virtual void cleanup() {} diff --git a/llvm/lib/DTLTO/DTLTO.cpp b/llvm/lib/DTLTO/DTLTO.cpp index 4a1107e76e47b..92b5bb362ba4c 100644 --- a/llvm/lib/DTLTO/DTLTO.cpp +++ b/llvm/lib/DTLTO/DTLTO.cpp @@ -8,7 +8,7 @@ // // \file // This file implements support functions for Distributed ThinLTO, focusing on -// archive file handling. +// preparing input files for distribution. // //===----------------------------------------------------------------------===// @@ -34,31 +34,35 @@ using namespace llvm; namespace { -// Writes the content of a memory buffer into a file. -llvm::Error saveBuffer(StringRef FileBuffer, StringRef FilePath) { +// Saves the content of Buffer to Path overwriting any existing file. +Error save(StringRef Buffer, StringRef Path) { std::error_code EC; - raw_fd_ostream OS(FilePath.str(), EC, sys::fs::OpenFlags::OF_None); - if (EC) { + raw_fd_ostream OS(Path.str(), EC, sys::fs::OpenFlags::OF_None); + if (EC) return createStringError(inconvertibleErrorCode(), - "Failed to create file %s: %s", FilePath.data(), + "Failed to create file %s: %s", Path.data(), EC.message().c_str()); - } - OS.write(FileBuffer.data(), FileBuffer.size()); - if (OS.has_error()) { + OS.write(Buffer.data(), Buffer.size()); + if (OS.has_error()) return createStringError(inconvertibleErrorCode(), - "Failed writing to file %s", FilePath.data()); - } + "Failed writing to file %s", Path.data()); return Error::success(); } +// Saves the content of Input to Path overwriting any existing file. +Error save(lto::InputFile *Input, StringRef Path) { + MemoryBufferRef MB = Input->getFileBuffer(); + return save(MB.getBuffer(), Path); +} + // Compute the file path for a thin archive member. // // For thin archives, an archive member name is typically a file path relative // to the archive file's directory. This function resolves that path. -SmallString<64> computeThinArchiveMemberPath(const StringRef ArchivePath, - const StringRef MemberName) { +SmallString<256> computeThinArchiveMemberPath(StringRef ArchivePath, + StringRef MemberName) { assert(!ArchivePath.empty() && "An archive file path must be non empty."); - SmallString<64> MemberPath; + SmallString<256> MemberPath; if (sys::path::is_relative(MemberName)) { MemberPath = sys::path::parent_path(ArchivePath); sys::path::append(MemberPath, MemberName); @@ -77,12 +81,11 @@ SmallString<64> computeThinArchiveMemberPath(const StringRef ArchivePath, // the archive type. Expected<bool> lto::DTLTO::isThinArchive(const StringRef ArchivePath) { // Return cached result if available. - auto Cached = ArchiveFiles.find(ArchivePath); - if (Cached != ArchiveFiles.end()) + auto Cached = ArchiveIsThinCache.find(ArchivePath); + if (Cached != ArchiveIsThinCache.end()) return Cached->second; uint64_t FileSize = -1; - bool IsThin = false; std::error_code EC = sys::fs::file_size(ArchivePath, FileSize); if (EC) return createStringError(inconvertibleErrorCode(), @@ -94,43 +97,45 @@ Expected<bool> lto::DTLTO::isThinArchive(const StringRef ArchivePath) { ArchivePath.data()); // Read only the first few bytes containing the magic signature. - ErrorOr<std::unique_ptr<MemoryBuffer>> MemBufferOrError = - MemoryBuffer::getFileSlice(ArchivePath, sizeof(object::ThinArchiveMagic), - 0); - - if ((EC = MemBufferOrError.getError())) + ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr = MemoryBuffer::getFileSlice( + ArchivePath, sizeof(object::ThinArchiveMagic), 0); + if ((EC = MBOrErr.getError())) return createStringError(inconvertibleErrorCode(), "Failed to read from archive %s: %s", ArchivePath.data(), EC.message().c_str()); - StringRef MemBuf = (*MemBufferOrError.get()).getBuffer(); - if (file_magic::archive != identify_magic(MemBuf)) + StringRef Buf = (*MBOrErr)->getBuffer(); + if (file_magic::archive != identify_magic(Buf)) return createStringError(inconvertibleErrorCode(), "Unknown format for archive %s", ArchivePath.data()); - IsThin = MemBuf.starts_with(object::ThinArchiveMagic); + bool IsThin = Buf.starts_with(object::ThinArchiveMagic); + + // Cache the result. + ArchiveIsThinCache[ArchivePath] = IsThin; - // Cache the result - ArchiveFiles[ArchivePath] = IsThin; return IsThin; } +// Add an input file and prepare it for distribution. +// // This function performs the following tasks: -// 1. Adds the input file to the LTO object's list of input files. -// 2. For thin archive members, generates a new module ID which is a path to a -// thin archive member file. -// 3. For regular archive members, generates a new unique module ID. -// 4. Updates the bitcode module's identifier. +// 1. Add the input file to the LTO object's list of input files. +// 2. For thin archive members, overwrite the module ID with the path to the +// member file on disk. +// 3. For archive members and FatLTO objects, overwrite the module ID with a +// unique path naming a file that will contain the member content. The file +// is created and populated later (see serializeInputs()). Expected<std::shared_ptr<lto::InputFile>> -lto::DTLTO::addInput(std::unique_ptr<lto::InputFile> InputPtr) { +lto::DTLTO::addInput(std::unique_ptr<InputFile> InputPtr) { TimeTraceScope TimeScope("Add input for DTLTO"); // Add the input file to the LTO object. InputFiles.emplace_back(InputPtr.release()); - std::shared_ptr<lto::InputFile> &Input = InputFiles.back(); + auto &Input = InputFiles.back(); + BitcodeModule &BM = Input->getPrimaryBitcodeModule(); - StringRef ModuleId = Input->getName(); StringRef ArchivePath = Input->getArchivePath(); // In most cases, the module ID already points to an individual bitcode file @@ -138,82 +143,60 @@ lto::DTLTO::addInput(std::unique_ptr<lto::InputFile> InputPtr) { if (ArchivePath.empty() && !Input->isFatLTOObject()) return Input; - SmallString<64> NewModuleId; - BitcodeModule &BM = Input->getPrimaryBitcodeModule(); - // For a member of a thin archive that is not a FatLTO object, there is an // existing file on disk that can be used, so we can avoid having to - // materialize. + // serialize. Expected<bool> UseThinMember = Input->isFatLTOObject() ? false : isThinArchive(ArchivePath); if (!UseThinMember) return UseThinMember.takeError(); - if (*UseThinMember) { - // For thin archives, use the path to the actual file. - NewModuleId = + // For thin archives, use the path to the actual member file on disk. + auto MemberPath = computeThinArchiveMemberPath(ArchivePath, Input->getMemberName()); - } else { - // For regular archives and FatLTO objects, generate a unique name. - Input->setSerializeForDistribution(true); - - // Create unique identifier using process ID and sequence number. - std::string PID = utohexstr(sys::Process::getProcessId()); - std::string Seq = std::to_string(InputFiles.size()); - - NewModuleId = sys::path::parent_path(LinkerOutputFile); - sys::path::append(NewModuleId, sys::path::filename(ModuleId) + "." + Seq + - "." + PID + ".o"); + BM.setModuleIdentifier(Saver.save(MemberPath.str())); + return Input; } - // Update the module identifier and save it. - BM.setModuleIdentifier(Saver.save(NewModuleId.str())); - + // A new file on disk will be needed for archive members and FatLTO objects. + Input->setSerializeForDistribution(true); + + // Create a unique path by including the process ID and sequence number in the + // filename. + SmallString<256> Id(sys::path::parent_path(LinkerOutputFile)); + sys::path::append(Id, + Twine(sys::path::filename(Input->getName())) + "." + + std::to_string(InputFiles.size()) /*Sequence number*/ + + "." + utohexstr(sys::Process::getProcessId()) + ".o"); + BM.setModuleIdentifier(Saver.save(Id.str())); return Input; } -// Write the archive member content to a file named after the module ID. -// If a file with that name already exists, it's likely a leftover from a -// previously terminated linker process and can be safely overwritten. -Error lto::DTLTO::saveInputArchiveMember(lto::InputFile *Input) { - StringRef ModuleId = Input->getName(); - if (Input->getSerializeForDistribution()) { +// Save the contents of ThinLTO-enabled input files that must be serialized for +// distribution, such as archive members and FatLTO objects, to individual +// bitcode files named after the module ID. +// +// Must be called after all input files are added but before optimization +// begins. If a file with that name already exists, it is likely a leftover from +// a previously terminated linker process and can be safely overwritten. +llvm::Error lto::DTLTO::serializeInputsForDistribution() { + for (auto &Input : InputFiles) { + if (!Input->isThinLTO() || !Input->getSerializeForDistribution()) + continue; + // Save the content of the input file to a file named after the module ID. + StringRef ModuleId = Input->getName(); TimeTraceScope TimeScope("Serialize bitcode input for DTLTO", ModuleId); // Cleanup this file on abnormal process exit. if (!SaveTemps) llvm::sys::RemoveFileOnSignal(ModuleId); - MemoryBufferRef MemoryBufferRef = Input->getFileBuffer(); - if (Error EC = saveBuffer(MemoryBufferRef.getBuffer(), ModuleId)) + if (Error EC = save(Input.get(), ModuleId)) return EC; } - return Error::success(); -} - -// Iterates through all ThinLTO-enabled input files and saves their content -// to separate files if they are regular archive members. -Error lto::DTLTO::saveInputArchiveMembers() { - for (auto &Input : InputFiles) { - if (!Input->isThinLTO()) - continue; - if (Error EC = saveInputArchiveMember(Input.get())) - return EC; - } - return Error::success(); -} - -// Entry point for DTLTO archives support. -// -// Sets up the temporary file remover and processes archive members. -// Must be called after all inputs are added but before optimization begins. -llvm::Error lto::DTLTO::handleArchiveInputs() { - // Process and save archive members to separate files if needed. - if (Error EC = saveInputArchiveMembers()) - return EC; return Error::success(); } -// Remove temporary archive member files created to enable distribution. +// Remove serialized inputs created to enable distribution. void lto::DTLTO::cleanup() { if (!SaveTemps) { TimeTraceScope TimeScope("Remove temporary inputs for DTLTO"); diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index 955a19db48607..2766d4c06fdb5 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -1219,7 +1219,7 @@ Error LTO::checkPartiallySplit() { Error LTO::run(AddStreamFn AddStream, FileCache Cache) { llvm::scope_exit CleanUp([this]() { cleanup(); }); - if (Error EC = handleArchiveInputs()) + if (Error EC = serializeInputsForDistribution()) return EC; // Compute "dead" symbols, we don't want to import/export these! >From 5739f43a5602b29ff3205cc29f9751c0c67245ca Mon Sep 17 00:00:00 2001 From: Ben Dunbobbin <[email protected]> Date: Wed, 11 Feb 2026 21:20:25 +0000 Subject: [PATCH 3/4] [DTLTO][Windows] Expand short 8.3 form paths in ThinLTO module IDs (#178303) Windows supports short 8.3 form filenames (e.g. `compile_commands.json` -> `COMPIL~1.JSO`) for legacy reasons. See: https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file#short-vs-long-names. Short 8.3 form paths are undesirable in distributed compilation scenarios because they are local mappings tied to a specific directory layout on a specific machine. As a result, they can break or defeat sandboxing and path-based isolation mechanisms used by distributed build systems. We have observed such failures with DTLTO even in simple scenarios. For example, on Windows, running: ``` clang.exe hello.c -flto=thin -fuse-ld=lld -fthinlto-distributor=fastbuild.exe -### ``` on my development machine reveals a short 8.3 form path being passed to LLD (output paraphrased): ``` ld.lld -o a.out -plugin-opt=thinlto --thinlto-distributor=fastbuild.exe \ --thinlto-remote-compiler=clang.exe C:\Users\DUNBOB~1\AppData\Local\Temp\hello-380d65.o ``` This behavior occurs because, on Windows, the system temporary directory is commonly derived from the `TMP`/`TEMP` environment variables. For historical compatibility reasons, these variables are often set to short 8.3 form paths, particularly on systems where user names exceed eight characters. Whilst it's possible for users to work around these issues, in practice, especially in automated and CI environments, users often have limited control over their environment. DTLTO generally tries to avoid embedding distribution-specific logic in the LLVM source tree, and this principle also applies to path normalization. However, on Windows, such short 8.3 form paths are undesirable for any distribution system. This normalization also cannot be delegated to distributors, as the ThinLTO module ID must be finalized early during LTO and cannot be modified later. Given this, normalizing away short 8.3 paths on Windows is a pragmatic, targeted improvement, even though path normalization is not something a toolchain would typically perform in the general case. SIE internal tracker: TOOLCHAIN-19185 (cherry picked from commit 5710e418c335c99f1d1ea619a4622837553b53d5) --- cross-project-tests/dtlto/path.test | 92 ++++++++++++++++++++++++++ cross-project-tests/lit.site.cfg.py.in | 15 ++++- llvm/include/llvm/DTLTO/DTLTO.h | 7 +- llvm/lib/DTLTO/DTLTO.cpp | 68 ++++++++++++++++--- 4 files changed, 171 insertions(+), 11 deletions(-) create mode 100644 cross-project-tests/dtlto/path.test diff --git a/cross-project-tests/dtlto/path.test b/cross-project-tests/dtlto/path.test new file mode 100644 index 0000000000000..e435fc6b0b581 --- /dev/null +++ b/cross-project-tests/dtlto/path.test @@ -0,0 +1,92 @@ +# REQUIRES: ld.lld,llvm-ar,tempshortpaths + +# Check that any short 8.3 form paths (containing '~' characters) in ThinLTO +# Module IDs are expanded prior to being passed to the distributor. +# +# This test is Windows-specific, as enforced by `tempshortpaths`. + +RUN: rm -rf %t && split-file %s %t && cd %t + +# Create an archive and a thin archive to confirm member paths are expanded. +RUN: %clang --target=x86_64-linux-gnu -c foo.c moo.c start.c -flto=thin -O2 +RUN: prospero-llvm-ar rcs foo.a foo.o +RUN: prospero-llvm-ar --thin rcs moo.a moo.o + +# Build with DTLTO. +RUN: %python in-83-dir.py \ +RUN: %clang --target=x86_64-linux-gnu -flto=thin -fuse-ld=lld -nostdlib \ +RUN: foo.a moo.a start.o -Wl,--save-temps \ +RUN: -fthinlto-distributor=%python \ +RUN: -Xthinlto-distributor=%llvm_src_root/utils/dtlto/local.py + +# Check that all short 8.3 form paths have been expanded. +RUN: FileCheck --input-file a.*.dist-file.json %s --check-prefix=TILDE +TILDE-NOT: ~ + +# It is important that cross-module inlining occurs for this test to show that +# Clang can successfully load the bitcode file dependencies recorded in the +# summary indices. Explicitly check that the expected importing has occurred. +RUN: llvm-dis *.1.*.thinlto.bc -o - | \ +RUN: FileCheck %s --check-prefixes=FOO,MOO,START +RUN: llvm-dis *.2.*.thinlto.bc -o - | \ +RUN: FileCheck %s --check-prefixes=FOO,MOO,START +RUN: llvm-dis *.3.*.thinlto.bc -o - | \ +RUN: FileCheck %s --check-prefixes=FOO,MOO,START +FOO-DAG: foo.o +MOO-DAG: moo.o +START-DAG: start.o + +#--- foo.c +extern int moo(int), _start(int); +__attribute__((retain)) int foo(int x) { return x + moo(x) + _start(x); } + +#--- moo.c +extern int foo(int), _start(int); +__attribute__((retain)) int moo(int x) { return x + foo(x) + _start(x); } + +#--- start.c +extern int foo(int), moo(int); +__attribute__((retain)) int _start(int x) { return x + foo(x) + moo(x); } + +#--- in-83-dir.py +import os, shutil, sys, uuid, subprocess +from pathlib import Path +import ctypes +from ctypes import wintypes + + +def get_short_path(p): + g = ctypes.windll.kernel32.GetShortPathNameW + g.argtypes = [wintypes.LPCWSTR, wintypes.LPWSTR, wintypes.DWORD] + g.restype = wintypes.DWORD + n = g(os.path.abspath(p), None, 0) # First call gets required buffer size. + b = ctypes.create_unicode_buffer(n) + if g(os.path.abspath(p), b, n): # Second call fills buffer. + return b.value + raise ctypes.WinError() + + +temp = Path(os.environ["TEMP"]) +assert temp.is_dir() + +# Copy the CWD to a unique directory inside TEMP and ensure that one of the path +# components is long enough to have a distinct short 8.3 form. +# TEMP is likely to be on a drive that supports short 8.3 form paths. +d = (temp / str(uuid.uuid4()) / "veryverylong").resolve() +d.parent.mkdir(parents=True) +try: + shutil.copytree(Path.cwd(), d) + + # Replace the arguments of the command that name files with equivalents in + # our temp directory, prefixed with the short 8.3 form. + cmd = [ + a if Path(a).is_absolute() or not (d / a).is_file() else get_short_path(d / a) + for a in sys.argv[1:] + ] + + print(cmd) + + sys.exit(subprocess.run(cmd).returncode) + +finally: + shutil.rmtree(d.parent) diff --git a/cross-project-tests/lit.site.cfg.py.in b/cross-project-tests/lit.site.cfg.py.in index 39458dfc79afd..b8992b6dca45e 100644 --- a/cross-project-tests/lit.site.cfg.py.in +++ b/cross-project-tests/lit.site.cfg.py.in @@ -1,7 +1,8 @@ @LIT_SITE_CFG_IN_HEADER@ -import sys +import shutil, sys, os, uuid import lit.util +from pathlib import Path config.targets_to_build = "@TARGETS_TO_BUILD@".split() config.llvm_src_root = "@LLVM_SOURCE_DIR@" @@ -22,7 +23,19 @@ config.mlir_src_root = "@MLIR_SOURCE_DIR@" config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@" import lit.llvm + lit.llvm.initialize(lit_config, config) # Let the main config do the real work. lit_config.load_config(config, "@CROSS_PROJECT_TESTS_SOURCE_DIR@/lit.cfg.py") + +def are_short_names_enabled(path): + d = Path(path) / str(uuid.uuid4()) / "verylongdir" + d.mkdir(parents=True) + try: + return (d.parent / "verylo~1").exists() + finally: + shutil.rmtree(d.parent) + +if os.name == "nt" and are_short_names_enabled(config.environment.get("TEMP")): + config.available_features.add("tempshortpaths") diff --git a/llvm/include/llvm/DTLTO/DTLTO.h b/llvm/include/llvm/DTLTO/DTLTO.h index e80577aa12834..5a8566b71bc16 100644 --- a/llvm/include/llvm/DTLTO/DTLTO.h +++ b/llvm/include/llvm/DTLTO/DTLTO.h @@ -24,7 +24,9 @@ namespace lto { // standalone file. Similarly, for FatLTO objects, the bitcode is stored in a // section of the containing ELF object file. To address this, the class ensures // that an individual bitcode file exists for each input (by writing it out if -// necessary) and that the ModuleID is updated to point to it. +// necessary) and that the ModuleID is updated to point to it. Module IDs are +// also normalized on Windows to remove short 8.3 form paths that cannot be +// loaded on remote machines. // // The class ensures that lto::InputFile objects are preserved until enough of // the LTO pipeline has executed to determine the required per-module @@ -62,6 +64,9 @@ class DTLTO : public LTO { /// The output file to which this LTO invocation will contribute. StringRef LinkerOutputFile; + /// The normalized output directory, derived from LinkerOutputFile. + StringRef LinkerOutputDir; + /// Controls preservation of any created temporary files. bool SaveTemps; diff --git a/llvm/lib/DTLTO/DTLTO.cpp b/llvm/lib/DTLTO/DTLTO.cpp index 92b5bb362ba4c..b5e233fe0ef54 100644 --- a/llvm/lib/DTLTO/DTLTO.cpp +++ b/llvm/lib/DTLTO/DTLTO.cpp @@ -27,6 +27,9 @@ #include "llvm/Support/Signals.h" #include "llvm/Support/TimeProfiler.h" #include "llvm/Support/raw_ostream.h" +#ifdef _WIN32 +#include "llvm/Support/Windows/WindowsSupport.h" +#endif #include <string> @@ -55,6 +58,25 @@ Error save(lto::InputFile *Input, StringRef Path) { return save(MB.getBuffer(), Path); } +// Normalize and save a path. Aside from expanding Windows 8.3 short paths, +// no other normalization is currently required here. These paths are +// machine-local and break distribution systems; other normalization is +// handled by the DTLTO distributors. +Expected<StringRef> normalizePath(StringRef Path, StringSaver &Saver) { +#if defined(_WIN32) + if (Path.empty()) + return Path; + SmallString<256> Expanded; + if (std::error_code EC = llvm::sys::windows::makeLongFormPath(Path, Expanded)) + return createStringError(inconvertibleErrorCode(), + "Normalization failed for path %s: %s", + Path.str().c_str(), EC.message().c_str()); + return Saver.save(Expanded.str()); +#else + return Saver.save(Path); +#endif +} + // Compute the file path for a thin archive member. // // For thin archives, an archive member name is typically a file path relative @@ -122,11 +144,14 @@ Expected<bool> lto::DTLTO::isThinArchive(const StringRef ArchivePath) { // // This function performs the following tasks: // 1. Add the input file to the LTO object's list of input files. -// 2. For thin archive members, overwrite the module ID with the path to the -// member file on disk. -// 3. For archive members and FatLTO objects, overwrite the module ID with a -// unique path naming a file that will contain the member content. The file -// is created and populated later (see serializeInputs()). +// 2. For individual bitcode file inputs on Windows only, overwrite the module +// ID with a normalized path to remove short 8.3 form components. +// 3. For thin archive members, overwrite the module ID with the path +// (normalized on Windows) to the member file on disk. +// 4. For archive members and FatLTO objects, overwrite the module ID with a +// unique path (normalized on Windows) naming a file that will contain the +// member content. The file is created and populated later (see +// serializeInputs()). Expected<std::shared_ptr<lto::InputFile>> lto::DTLTO::addInput(std::unique_ptr<InputFile> InputPtr) { TimeTraceScope TimeScope("Add input for DTLTO"); @@ -136,12 +161,28 @@ lto::DTLTO::addInput(std::unique_ptr<InputFile> InputPtr) { auto &Input = InputFiles.back(); BitcodeModule &BM = Input->getPrimaryBitcodeModule(); + auto setIdFromPath = [&](StringRef Path) -> Error { + auto N = normalizePath(Path, Saver); + if (!N) + return N.takeError(); + BM.setModuleIdentifier(*N); + return Error::success(); + }; + StringRef ArchivePath = Input->getArchivePath(); // In most cases, the module ID already points to an individual bitcode file - // on disk, so no further preparation for distribution is required. - if (ArchivePath.empty() && !Input->isFatLTOObject()) + // on disk, so no further preparation for distribution is required. However, + // on Windows we overwite the module ID to expand Windows 8.3 short form + // paths. These paths are machine-local and break distribution systems; other + // normalization is handled by the DTLTO distributors. + if (ArchivePath.empty() && !Input->isFatLTOObject()) { +#if defined(_WIN32) + if (Error E = setIdFromPath(Input->getName())) + return std::move(E); +#endif return Input; + } // For a member of a thin archive that is not a FatLTO object, there is an // existing file on disk that can be used, so we can avoid having to @@ -154,16 +195,25 @@ lto::DTLTO::addInput(std::unique_ptr<InputFile> InputPtr) { // For thin archives, use the path to the actual member file on disk. auto MemberPath = computeThinArchiveMemberPath(ArchivePath, Input->getMemberName()); - BM.setModuleIdentifier(Saver.save(MemberPath.str())); + if (Error E = setIdFromPath(MemberPath)) + return std::move(E); return Input; } // A new file on disk will be needed for archive members and FatLTO objects. Input->setSerializeForDistribution(true); + // Get the normalized output directory, if we haven't already. + if (LinkerOutputDir.empty()) { + auto N = normalizePath(sys::path::parent_path(LinkerOutputFile), Saver); + if (!N) + return N.takeError(); + LinkerOutputDir = *N; + } + // Create a unique path by including the process ID and sequence number in the // filename. - SmallString<256> Id(sys::path::parent_path(LinkerOutputFile)); + SmallString<256> Id(LinkerOutputDir); sys::path::append(Id, Twine(sys::path::filename(Input->getName())) + "." + std::to_string(InputFiles.size()) /*Sequence number*/ + >From 147dcd4efd3b89093a24ec91572c8d9b3fa30aaa Mon Sep 17 00:00:00 2001 From: Dunbobbin <[email protected]> Date: Wed, 11 Feb 2026 22:04:01 +0000 Subject: [PATCH 4/4] [TEST][FIX] Fix typo in tool name: 'llvm-ar' (cherry picked from commit 5c114d80f88ef418b5f3ba3c919d300e21c4e677) --- cross-project-tests/dtlto/path.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cross-project-tests/dtlto/path.test b/cross-project-tests/dtlto/path.test index e435fc6b0b581..ec0dd1a59e6c0 100644 --- a/cross-project-tests/dtlto/path.test +++ b/cross-project-tests/dtlto/path.test @@ -9,8 +9,8 @@ RUN: rm -rf %t && split-file %s %t && cd %t # Create an archive and a thin archive to confirm member paths are expanded. RUN: %clang --target=x86_64-linux-gnu -c foo.c moo.c start.c -flto=thin -O2 -RUN: prospero-llvm-ar rcs foo.a foo.o -RUN: prospero-llvm-ar --thin rcs moo.a moo.o +RUN: llvm-ar rcs foo.a foo.o +RUN: llvm-ar --thin rcs moo.a moo.o # Build with DTLTO. RUN: %python in-83-dir.py \ _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
