https://github.com/azhan92 updated https://github.com/llvm/llvm-project/pull/206519
>From 792ce0ab221b94a0cc51f34f86b977c2c1c5d3c0 Mon Sep 17 00:00:00 2001 From: alisonzhang <[email protected]> Date: Sun, 28 Jun 2026 15:47:43 -0400 Subject: [PATCH 1/8] Get canonical names for encodings --- llvm/include/llvm/Support/TextEncoding.h | 11 +++++++++++ llvm/lib/Support/TextEncoding.cpp | 13 ++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/llvm/include/llvm/Support/TextEncoding.h b/llvm/include/llvm/Support/TextEncoding.h index 8a304910aa5dd..09e24000594db 100644 --- a/llvm/include/llvm/Support/TextEncoding.h +++ b/llvm/include/llvm/Support/TextEncoding.h @@ -105,6 +105,17 @@ class TextEncodingConverter { LLVM_ABI static ErrorOr<TextEncodingConverter> create(StringRef From, StringRef To); + /// Maps the encoding name to enum constant if possible. + /// Uses normalized charset name matching. + /// \param[in] Name the character encoding name + /// \return the TextEncoding enum value if known, std::nullopt otherwise + LLVM_ABI static std::optional<TextEncoding> getKnownEncoding(StringRef Name); + + /// Returns the canonical name for a known encoding. + /// \param[in] Encoding the TextEncoding enum value + /// \return the canonical name for the encoding (e.g., "UTF-8" or "IBM-1047") + LLVM_ABI static StringRef getKnownEncodingName(TextEncoding Encoding); + TextEncodingConverter(const TextEncodingConverter &) = delete; TextEncodingConverter &operator=(const TextEncodingConverter &) = delete; diff --git a/llvm/lib/Support/TextEncoding.cpp b/llvm/lib/Support/TextEncoding.cpp index d36f02c1300b9..8e9653dab38ec 100644 --- a/llvm/lib/Support/TextEncoding.cpp +++ b/llvm/lib/Support/TextEncoding.cpp @@ -48,7 +48,7 @@ static void normalizeCharSetName(StringRef CSName, } // Maps the encoding name to enum constant if possible. -static std::optional<TextEncoding> getKnownEncoding(StringRef Name) { +std::optional<TextEncoding> TextEncodingConverter::getKnownEncoding(StringRef Name) { SmallString<16> Normalized; normalizeCharSetName(Name, Normalized); if (Normalized.equals("utf8")) @@ -58,6 +58,17 @@ static std::optional<TextEncoding> getKnownEncoding(StringRef Name) { return std::nullopt; } +// Returns the canonical name for a known encoding. +StringRef TextEncodingConverter::getKnownEncodingName(TextEncoding Encoding) { + switch (Encoding) { + case TextEncoding::UTF8: + return "UTF-8"; + case TextEncoding::IBM1047: + return "IBM-1047"; + } + llvm_unreachable("Invalid TextEncoding value"); +} + [[maybe_unused]] static void HandleOverflow(size_t &Capacity, char *&Output, size_t &OutputLength, SmallVectorImpl<char> &Result) { >From 41a96501462d2ff871731a727f923ec89ea81268 Mon Sep 17 00:00:00 2001 From: alisonzhang <[email protected]> Date: Sun, 28 Jun 2026 15:50:57 -0400 Subject: [PATCH 2/8] Text mode mismatch checking --- clang/lib/Basic/FileManager.cpp | 17 +++++++++----- llvm/include/llvm/Support/VirtualFileSystem.h | 9 ++++++++ llvm/lib/Support/VirtualFileSystem.cpp | 22 ++++++++++++++++--- 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/clang/lib/Basic/FileManager.cpp b/clang/lib/Basic/FileManager.cpp index 8fb3ba0a27aad..94fc4c15f51d2 100644 --- a/clang/lib/Basic/FileManager.cpp +++ b/clang/lib/Basic/FileManager.cpp @@ -539,15 +539,22 @@ FileManager::getBufferForFile(FileEntryRef FE, bool isVolatile, FileSize = -1; StringRef Filename = FE.getName(); - // If the file is already open, use the open file descriptor. + // If the file is already open, check if the mode matches. if (Entry->File) { - auto Result = Entry->File->getBuffer(Filename, FileSize, - RequiresNullTerminator, isVolatile); + // Check if the cached file's mode matches the requested mode + // Only perform mismatch recovery for real files + if (!Entry->File->realFileCheckTextModeMismatch(IsText)) { + // Mode matches, use the cached file descriptor + auto Result = Entry->File->getBuffer(Filename, FileSize, + RequiresNullTerminator, isVolatile); + Entry->closeFile(); + return Result; + } + // Mode mismatch - close the cached file and reopen with correct mode Entry->closeFile(); - return Result; } - // Otherwise, open the file. + // Open the file with the requested mode. return getBufferForFileImpl(Filename, FileSize, isVolatile, RequiresNullTerminator, IsText); } diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h index d22c534228331..9cc35b6f5fb6d 100644 --- a/llvm/include/llvm/Support/VirtualFileSystem.h +++ b/llvm/include/llvm/Support/VirtualFileSystem.h @@ -137,6 +137,15 @@ class LLVM_ABI File { /// Closes the file. virtual std::error_code close() = 0; + /// Checks if this is a real file and the requested text mode differs + /// from the current mode. For real files with a text mode mismatch where + /// the buffer was previously requested, this will call llvm::report_fatal_error. + /// Always returns false for non-real files. + /// Default implementation returns false for non-real files. + virtual bool realFileCheckTextModeMismatch(bool RequestedIsText) const { + return false; + } + // Get the same file with a different path. static ErrorOr<std::unique_ptr<File>> getWithPath(ErrorOr<std::unique_ptr<File>> Result, const Twine &P); diff --git a/llvm/lib/Support/VirtualFileSystem.cpp b/llvm/lib/Support/VirtualFileSystem.cpp index 42e8bb4f9958e..1c1f9e9dd15b6 100644 --- a/llvm/lib/Support/VirtualFileSystem.cpp +++ b/llvm/lib/Support/VirtualFileSystem.cpp @@ -194,11 +194,15 @@ class RealFile : public File { file_t FD; Status S; std::string RealName; + bool IsTextMode; + bool BufferWasRequested; - RealFile(file_t RawFD, StringRef NewName, StringRef NewRealPathName) + RealFile(file_t RawFD, StringRef NewName, StringRef NewRealPathName, + bool IsText) : FD(RawFD), S(NewName, {}, {}, {}, {}, {}, llvm::sys::fs::file_type::status_error, {}), - RealName(NewRealPathName.str()) { + RealName(NewRealPathName.str()), IsTextMode(IsText), + BufferWasRequested(false) { assert(FD != kInvalidFile && "Invalid or inactive file descriptor"); } @@ -213,6 +217,16 @@ class RealFile : public File { bool IsVolatile) override; std::error_code close() override; void setPath(const Twine &Path) override; + bool realFileCheckTextModeMismatch(bool RequestedIsText) const override { + bool HasMismatch = IsTextMode != RequestedIsText; + if (HasMismatch && BufferWasRequested) { + llvm::report_fatal_error( + "Text mode mismatch: file was previously opened with " + + Twine(IsTextMode ? "text" : "binary") + " mode, now requested with " + + Twine(RequestedIsText ? "text" : "binary") + " mode"); + } + return HasMismatch; + } }; } // namespace @@ -242,6 +256,7 @@ RealFile::getBuffer(const Twine &Name, int64_t FileSize, auto BypassSandbox = sys::sandbox::scopedDisable(); assert(FD != kInvalidFile && "cannot get buffer for closed file"); + BufferWasRequested = true; return MemoryBuffer::getOpenFile(FD, Name, FileSize, RequiresNullTerminator, IsVolatile); } @@ -320,8 +335,9 @@ class RealFileSystem : public FileSystem { adjustPath(Name, Storage), Flags, &RealName); if (!FDOrErr) return errorToErrorCode(FDOrErr.takeError()); + bool IsText = (Flags & sys::fs::OF_Text) != sys::fs::OF_None; return std::unique_ptr<File>( - new RealFile(*FDOrErr, Name.str(), RealName.str())); + new RealFile(*FDOrErr, Name.str(), RealName.str(), IsText)); } struct WorkingDirectory { >From 568ba3dc979a4ffb60bb48279568daba5c84bf0f Mon Sep 17 00:00:00 2001 From: alisonzhang <[email protected]> Date: Sun, 28 Jun 2026 15:51:57 -0400 Subject: [PATCH 3/8] getEncodingNameFromFileTag --- llvm/include/llvm/Support/AutoConvert.h | 26 +++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/llvm/include/llvm/Support/AutoConvert.h b/llvm/include/llvm/Support/AutoConvert.h index d68b0e8b515e0..b437b157b7725 100644 --- a/llvm/include/llvm/Support/AutoConvert.h +++ b/llvm/include/llvm/Support/AutoConvert.h @@ -18,6 +18,7 @@ #include <_Ccsid.h> #endif #ifdef __cplusplus +#include "llvm/ADT/SmallString.h" #include "llvm/ADT/Twine.h" #include "llvm/Support/Error.h" #include <system_error> @@ -105,6 +106,31 @@ inline ErrorOr<bool> needConversion(const Twine &FileName, const int FD = -1) { return false; } +inline ErrorOr<SmallString<32>> +getEncodingNameFromFileTag(const Twine &FileName, const int FD = -1) { +#ifdef __MVS__ + ErrorOr<__ccsid_t> TagOrErr = getzOSFileTag(FileName, FD); + if (!TagOrErr) + return TagOrErr.getError(); + + __ccsid_t Tag = *TagOrErr; + if (Tag == 0) + return SmallString<32>(); // Return empty string for no tag + + if (Tag == 1208) + return SmallString<32>("utf-8"); + + if (Tag == 1047) + return SmallString<32>("ibm-1047"); + + SmallString<32> Result; + raw_svector_ostream(Result) << Tag; + return Result; +#else + return SmallString<32>(); // Return empty string for non-MVS platforms +#endif +} + } /* namespace llvm */ #endif /* __cplusplus */ >From e76575e1dec9f16899b9e339a929e639ebb86df3 Mon Sep 17 00:00:00 2001 From: alisonzhang <[email protected]> Date: Sun, 28 Jun 2026 16:02:38 -0400 Subject: [PATCH 4/8] Add cache for text encoding converters --- clang/include/clang/Basic/SourceManager.h | 15 ++++++++++ clang/lib/Basic/SourceManager.cpp | 34 +++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h index 1939d1aa4915e..2b5cf0b1e084e 100644 --- a/clang/include/clang/Basic/SourceManager.h +++ b/clang/include/clang/Basic/SourceManager.h @@ -50,6 +50,7 @@ #include "llvm/Support/Allocator.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/TextEncoding.h" #include <cassert> #include <cstddef> #include <map> @@ -846,6 +847,11 @@ class SourceManager : public RefCountedBase<SourceManager> { /// we can add a cc1-level option to do so. SmallVector<std::pair<std::string, FullSourceLoc>, 2> StoredModuleBuildStack; + /// Cache of all text encoding converters used by this SourceManager. + /// This includes both the input charset converter and file tag converters. + /// Maps from "source_encoding:target_encoding" to the converter. + llvm::StringMap<std::unique_ptr<llvm::TextEncodingConverter>> ConverterCache; + public: SourceManager(DiagnosticsEngine &Diag, FileManager &FileMgr, bool UserFilesAreVolatile = false); @@ -863,6 +869,15 @@ class SourceManager : public RefCountedBase<SourceManager> { FileManager &getFileManager() const { return FileMgr; } + /// Get or create a text encoding converter from the cache. + /// This method manages all converters (input charset and file tag converters) + /// in a single cache owned by SourceManager. + /// \param SourceEncoding the source character encoding name + /// \return pointer to the converter or an error code + /// The target encoding is always UTF-8. + llvm::ErrorOr<llvm::TextEncodingConverter *> + getOrCreateConverter(llvm::StringRef SourceEncoding); + /// Set true if the SourceManager should report the original file name /// for contents of files that were overridden by other files. Defaults to /// true. diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp index 5540aade05ef5..2b2c58ef851d8 100644 --- a/clang/lib/Basic/SourceManager.cpp +++ b/clang/lib/Basic/SourceManager.cpp @@ -448,6 +448,40 @@ ContentCache &SourceManager::createMemBufferContentCache( return *Entry; } +llvm::ErrorOr<llvm::TextEncodingConverter *> +SourceManager::getOrCreateConverter(llvm::StringRef SourceEncoding) { + // Use getKnownEncoding to get normalized encoding names + std::optional<llvm::TextEncoding> SourceKnown = + llvm::TextEncodingConverter::getKnownEncoding(SourceEncoding); + + if (SourceKnown && *SourceKnown == llvm::TextEncoding::UTF8) + return nullptr; + + // Create a cache key using canonical encoding name + llvm::StringRef CacheKey = SourceKnown + ? llvm::TextEncodingConverter::getKnownEncodingName(*SourceKnown) + : SourceEncoding; + + // Check if converter already exists in cache + auto It = ConverterCache.find(CacheKey); + if (It != ConverterCache.end()) + return It->second.get(); + + // Create a new converter + llvm::ErrorOr<llvm::TextEncodingConverter> NewConverter = + llvm::TextEncodingConverter::create(SourceEncoding, "UTF-8"); + + if (!NewConverter) + return NewConverter.getError(); + + // Store the converter in the cache + auto Inserted = ConverterCache.insert( + std::make_pair(CacheKey, std::make_unique<llvm::TextEncodingConverter>( + std::move(*NewConverter)))); + + return Inserted.first->second.get(); +} + const SrcMgr::SLocEntry &SourceManager::loadSLocEntry(unsigned Index, bool *Invalid) const { return const_cast<SourceManager *>(this)->loadSLocEntry(Index, Invalid); >From add43de1f7cd34d08f758f6f104c5951bd09701e Mon Sep 17 00:00:00 2001 From: alisonzhang <[email protected]> Date: Sun, 28 Jun 2026 16:20:56 -0400 Subject: [PATCH 5/8] Add CLANG_DEFAULT_INPUT_ENCODING_IBM1047 flag --- clang/CMakeLists.txt | 3 +++ clang/include/clang/Config/config.h.cmake | 3 +++ clang/lib/Driver/ToolChains/Clang.cpp | 15 ++++++++++++++- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt index e920e83a537d4..0d56dcc200563 100644 --- a/clang/CMakeLists.txt +++ b/clang/CMakeLists.txt @@ -275,6 +275,9 @@ set(ENABLE_X86_RELAX_RELOCATIONS ON CACHE BOOL set(PPC_LINUX_DEFAULT_IEEELONGDOUBLE OFF CACHE BOOL "Enable IEEE binary128 as default long double format on PowerPC Linux.") +set(CLANG_DEFAULT_INPUT_ENCODING_IBM1047 OFF CACHE BOOL + "Set IBM-1047 as the default input encoding") + set(CLANG_SPAWN_CC1 OFF CACHE BOOL "Whether clang should use a new process for the CC1 invocation") diff --git a/clang/include/clang/Config/config.h.cmake b/clang/include/clang/Config/config.h.cmake index 11b4096726f67..fbafafc710afe 100644 --- a/clang/include/clang/Config/config.h.cmake +++ b/clang/include/clang/Config/config.h.cmake @@ -75,6 +75,9 @@ /* Enable IEEE binary128 as default long double format on PowerPC Linux. */ #cmakedefine01 PPC_LINUX_DEFAULT_IEEELONGDOUBLE +/* Set IBM-1047 as the default input encoding */ +#cmakedefine01 CLANG_DEFAULT_INPUT_ENCODING_IBM1047 + /* Enable each functionality of modules */ #cmakedefine01 CLANG_ENABLE_OBJC_REWRITER #cmakedefine01 CLANG_ENABLE_STATIC_ANALYZER diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index a3a3954bc464e..103efece4f30a 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -7973,10 +7973,23 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, // -finput_charset=UTF-8 is default. Reject others if (Arg *inputCharset = Args.getLastArg(options::OPT_finput_charset_EQ)) { StringRef value = inputCharset->getValue(); - if (!value.equals_insensitive("utf-8")) +#if CLANG_DEFAULT_INPUT_ENCODING_IBM1047 + bool isValid = value.equals_insensitive("ibm-1047") || + value.equals_insensitive("ibm1047"); +#else + bool isValid = value.equals_insensitive("utf-8"); +#endif + if (!isValid) D.Diag(diag::err_drv_invalid_value) << inputCharset->getAsString(Args) << value; } +#if CLANG_DEFAULT_INPUT_ENCODING_IBM1047 + else { + // When IBM-1047 default is enabled and no explicit charset is specified, + // set IBM-1047 as the default + CmdArgs.push_back("-finput-charset=IBM-1047"); + } +#endif // -fexec_charset=UTF-8 is default. Reject others if (Arg *execCharset = Args.getLastArg(options::OPT_fexec_charset_EQ)) { >From e9fdd11488aa8150948f790f539d2477aa4fb654 Mon Sep 17 00:00:00 2001 From: alisonzhang <[email protected]> Date: Sun, 28 Jun 2026 22:29:36 -0400 Subject: [PATCH 6/8] Changes for finput-charset --- .../clang/Basic/DiagnosticCommonKinds.td | 5 + clang/include/clang/Basic/LangOptions.h | 3 + clang/include/clang/Basic/SourceManager.h | 41 +++++- .../include/clang/Frontend/CompilerInstance.h | 1 + clang/lib/Basic/SourceManager.cpp | 121 ++++++++++++++---- clang/lib/Frontend/CompilerInstance.cpp | 10 +- .../lib/Frontend/VerifyDiagnosticConsumer.cpp | 4 +- clang/lib/Lex/ModuleMap.cpp | 7 +- clang/lib/Lex/PPDirectives.cpp | 6 +- clang/lib/Lex/Preprocessor.cpp | 4 +- clang/lib/Serialization/ASTReader.cpp | 5 +- 11 files changed, 175 insertions(+), 32 deletions(-) diff --git a/clang/include/clang/Basic/DiagnosticCommonKinds.td b/clang/include/clang/Basic/DiagnosticCommonKinds.td index f2ed2f4698b8d..55e71020c9c91 100644 --- a/clang/include/clang/Basic/DiagnosticCommonKinds.td +++ b/clang/include/clang/Basic/DiagnosticCommonKinds.td @@ -417,6 +417,11 @@ def note_file_sloc_usage : Note< "%plural{0:|: plus %2B (%human2B) for macro expansions}2">; def note_file_misc_sloc_usage : Note< "%0 additional files entered using a total of %1B (%human1B) of space">; +def warn_encoding_conversion_failed : Warning< + "conversion from source encoding failed for '%0': %1; interpreting as %2">, + InGroup<DiagGroup<"encoding-conversion-failed">>; +def err_encoding_conversion_failed : Error< + "conversion from source encoding failed for '%0': %1">; // Modules def err_module_format_unhandled : Error< diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h index 53c4c1084784a..5256e55380236 100644 --- a/clang/include/clang/Basic/LangOptions.h +++ b/clang/include/clang/Basic/LangOptions.h @@ -621,6 +621,9 @@ class LangOptions : public LangOptionsBase { /// The allocation token mode. std::optional<llvm::AllocTokenMode> AllocTokenMode; + /// Name of the input encoding to convert to the internal encoding. + std::string InputEncoding; + LangOptions(); /// Set language defaults for the given input language and diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h index 2b5cf0b1e084e..fbb6a3be69c1b 100644 --- a/clang/include/clang/Basic/SourceManager.h +++ b/clang/include/clang/Basic/SourceManager.h @@ -157,6 +157,13 @@ class alignas(8) ContentCache { /// FIXME: Remove this once OrigEntry is a FileEntryRef with a stable name. StringRef Filename; + /// Information on whether this is associated with a FileID for a file (as + /// opposed to a buffer) and, if so, what conversion (if any) was requested. + /// The integer part uses 2 bits: bit 0 indicates if used by FileID, + /// bit 1 indicates if the file was tagged. + llvm::PointerIntPair<llvm::TextEncodingConverter *, 2u, unsigned> + FileIDConverterInfo; + /// A bump pointer allocated array of offsets for each source line. /// /// This is lazily computed. The lines are owned by the SourceManager @@ -273,6 +280,36 @@ class alignas(8) ContentCache { // If BufStr has an invalid BOM, returns the BOM name; otherwise, returns // nullptr static const char *getInvalidBOM(StringRef BufStr); + + /// Helper methods for FileIDConverterInfo bit manipulation. + /// Bit 0: Used by FileID flag + /// Bit 1: File tagged flag + + bool isUsedByFileID() const { + return FileIDConverterInfo.getInt() & 0x1; + } + + void setUsedByFileID(bool Used) { + unsigned Flags = FileIDConverterInfo.getInt(); + if (Used) + Flags |= 0x1; + else + Flags &= ~0x1; + FileIDConverterInfo.setInt(Flags); + } + + bool isFileTagged() const { + return FileIDConverterInfo.getInt() & 0x2; + } + + void setFileTagged(bool Tagged) { + unsigned Flags = FileIDConverterInfo.getInt(); + if (Tagged) + Flags |= 0x2; + else + Flags &= ~0x2; + FileIDConverterInfo.setInt(Flags); + } }; // Assert that the \c ContentCache objects will always be 8-byte aligned so @@ -939,6 +976,7 @@ class SourceManager : public RefCountedBase<SourceManager> { /// being \#included from the specified IncludePosition. FileID createFileID(FileEntryRef SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, + llvm::StringRef InputEncodingName = {}, int LoadedID = 0, SourceLocation::UIntTy LoadedOffset = 0); @@ -963,7 +1001,8 @@ class SourceManager : public RefCountedBase<SourceManager> { /// Get the FileID for \p SourceFile if it exists. Otherwise, create a /// new FileID for the \p SourceFile. FileID getOrCreateFileID(FileEntryRef SourceFile, - SrcMgr::CharacteristicKind FileCharacter); + SrcMgr::CharacteristicKind FileCharacter, + llvm::StringRef InputEncodingName = {}); /// Creates an expansion SLocEntry for the substitution of an argument into a /// function-like macro's body. Returns the start of the expansion. diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h index 24488e053c628..6bc5661d605d0 100644 --- a/clang/include/clang/Frontend/CompilerInstance.h +++ b/clang/include/clang/Frontend/CompilerInstance.h @@ -869,6 +869,7 @@ class CompilerInstance : public ModuleLoader { /// /// \return True on success. static bool InitializeSourceManager(const FrontendInputFile &Input, + llvm::StringRef InputEncodingName, DiagnosticsEngine &Diags, FileManager &FileMgr, SourceManager &SourceMgr); diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp index 2b2c58ef851d8..e30537c751005 100644 --- a/clang/lib/Basic/SourceManager.cpp +++ b/clang/lib/Basic/SourceManager.cpp @@ -10,6 +10,7 @@ // //===----------------------------------------------------------------------===// +#include "clang/Config/config.h" #include "clang/Basic/SourceManager.h" #include "clang/Basic/Diagnostic.h" #include "clang/Basic/FileManager.h" @@ -32,6 +33,7 @@ #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Support/SmallVectorMemoryBuffer.h" #include <algorithm> #include <cassert> #include <cstddef> @@ -136,8 +138,6 @@ ContentCache::getBufferOrNone(DiagnosticsEngine &Diag, FileManager &FM, // return paths. IsBufferInvalid = true; - auto BufferOrError = FM.getBufferForFile(*ContentsEntry, IsFileVolatile); - // If we were unable to open the file, then we are in an inconsistent // situation where the content cache referenced a file which no longer // exists. Most likely, we were using a stat cache with an invalid entry but @@ -152,6 +152,55 @@ ContentCache::getBufferOrNone(DiagnosticsEngine &Diag, FileManager &FM, Buffer = std::move(*BufferOrError); + // Unless this is a named pipe (in which case we can handle a mismatch), + // check that the file's size is the same as in the file entry (which may + // have come from a stat cache). + assert(Buffer->getBufferSize() >= (size_t)ContentsEntry->getSize()); + if (!ContentsEntry->isNamedPipe() && + Buffer->getBufferSize() < (size_t)ContentsEntry->getSize()) { + Diag.Report(Loc, diag::err_file_modified) << ContentsEntry->getName(); + + return std::nullopt; + } + + // Convert source from the input charset to UTF-8 if necessary. + llvm::TextEncodingConverter *Converter = FileIDConverterInfo.getPointer(); + if (Converter) { + StringRef OriginalBuf = Buffer->getBuffer(); + llvm::SmallString<0> UTF8Buf; + UTF8Buf.reserve(OriginalBuf.size() + 1); + + std::error_code EC = Converter->convert(OriginalBuf, UTF8Buf); + if (EC) { + // For tagged files, conversion failure is an error and we don't fall back + if (isFileTagged()) { + Diag.Report(Loc, diag::err_encoding_conversion_failed) + << ContentsEntry->getName() << EC.message(); + return std::nullopt; + } + + // If conversion fails, emit a warning and fall back to interpreting the + // file as the default encoding. + // + // This allows the compiler to accept system or third-party headers that + // are encoded in the default encoding even if conversion to the + // option-specified input encoding failed. + // + // TODO: Add input byte offset information. + // + // TODO: Consider adjusting the message to omit the recovery description + // if the warning has been upgraded to an error. + const char *FallbackEncoding = CLANG_DEFAULT_INPUT_ENCODING_IBM1047 ? "IBM-1047" : "UTF-8"; + Diag.Report(Loc, diag::warn_encoding_conversion_failed) + << ContentsEntry->getName() << EC.message() << FallbackEncoding; + } else { + // TODO: Reclaim memory if the buffer size exceeds the content. + auto NewBuf = std::make_unique<llvm::SmallVectorMemoryBuffer>( + std::move(UTF8Buf), Buffer->getBufferIdentifier()); + Buffer = std::move(NewBuf); + } + } + // Check that the file's size fits in an 'unsigned' (with room for a // past-the-end value). This is deeply regrettable, but various parts of // Clang (including elsewhere in this file!) use 'unsigned' to represent file @@ -167,22 +216,15 @@ ContentCache::getBufferOrNone(DiagnosticsEngine &Diag, FileManager &FM, return std::nullopt; } - // Unless this is a named pipe (in which case we can handle a mismatch), - // check that the file's size is the same as in the file entry (which may - // have come from a stat cache). - // The buffer will always be larger than the file size on z/OS in the presence - // of characters outside the base character set. - assert(Buffer->getBufferSize() >= (size_t)ContentsEntry->getSize()); - if (!ContentsEntry->isNamedPipe() && - Buffer->getBufferSize() < (size_t)ContentsEntry->getSize()) { - Diag.Report(Loc, diag::err_file_modified) << ContentsEntry->getName(); - - return std::nullopt; - } - - // If the buffer is valid, check to see if it has a UTF Byte Order Mark - // (BOM). We only support UTF-8 with and without a BOM right now. See - // http://en.wikipedia.org/wiki/Byte_order_mark for more information. + // If the buffer is valid, check to see if it has a UTF Byte Order Mark (BOM) + // Note that any conversion requested using `-finput-charset` (if successful) + // has already occurred, so we are expecting UTF-8 with or without a BOM. + // + // In theory, if we see a non-UTF-8 BOM, we can assume that an appropriate + // conversion was not supplied via `-finput-charset` and we could try to + // convert based on the BOM. + // + // See http://en.wikipedia.org/wiki/Byte_order_mark for more information. StringRef BufStr = Buffer->getBuffer(); const char *InvalidBOM = getInvalidBOM(BufStr); @@ -591,6 +633,7 @@ FileID SourceManager::getNextFileID(FileID FID) const { FileID SourceManager::createFileID(FileEntryRef SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, + llvm::StringRef InputEncodingName, int LoadedID, SourceLocation::UIntTy LoadedOffset) { SrcMgr::ContentCache &IR = getOrCreateContentCache(SourceFile, @@ -604,9 +647,38 @@ FileID SourceManager::createFileID(FileEntryRef SourceFile, FileIDContentCaches.push_back(Cache); } + llvm::ErrorOr<llvm::TextEncodingConverter *> Converter = nullptr; + if (!InputEncodingName.empty()) { + // No file tag but -finput-charset conversion is desired. + // Get the converter from the cache using the input encoding name. + Converter = getOrCreateConverter(InputEncodingName); + if (!Converter) { + llvm::report_fatal_error( + "Cannot create converter for file '" + SourceFile.getName() + "': " + + Converter.getError().message()); + } + } + + #ifndef NDEBUG + // Either the content cache has never been used for a FileID (and, if we are + // being asked to use a converter, there should be no valid buffer set up for + // it) or the conversion (or lack thereof) should be the same as that used + // previously. + llvm::TextEncodingConverter *CacheConverter = IR.FileIDConverterInfo.getPointer(); + bool CacheUsedByFileID = IR.isUsedByFileID(); + llvm::TextEncodingConverter *ConverterPtr = Converter ? *Converter : nullptr; + if (CacheUsedByFileID) + assert(CacheConverter == ConverterPtr); + else + assert(!ConverterPtr || IR.IsBufferInvalid || !IR.getBufferIfLoaded()); +#endif + IR.setUsedByFileID(true); + IR.FileIDConverterInfo.setPointer(Converter ? *Converter : nullptr); + // If this is a named pipe, immediately load the buffer to ensure subsequent // calls to ContentCache::getSize() are accurate. - if (Cache->ContentsEntry->isNamedPipe()) + // Do the same if character-encoding conversion was requested. + if (Cache->ContentsEntry->isNamedPipe() || Converter) (void)Cache->getBufferOrNone(Diag, getFileManager(), SourceLocation()); return createFileIDImpl(*Cache, Filename, IncludePos, FileCharacter, LoadedID, @@ -644,10 +716,12 @@ FileID SourceManager::createFileID(const llvm::MemoryBufferRef &Buffer, /// new FileID for the \p SourceFile. FileID SourceManager::getOrCreateFileID(FileEntryRef SourceFile, - SrcMgr::CharacteristicKind FileCharacter) { + SrcMgr::CharacteristicKind FileCharacter, + llvm::StringRef InputEncodingName) { FileID ID = translateFile(SourceFile); - return ID.isValid() ? ID : createFileID(SourceFile, SourceLocation(), - FileCharacter); + return ID.isValid() ? ID + : createFileID(SourceFile, SourceLocation(), FileCharacter, + InputEncodingName); } /// createFileID - Create a new FileID for the specified ContentCache and @@ -2403,7 +2477,8 @@ SourceManagerForFile::SourceManagerForFile(StringRef FileName, SourceMgr = std::make_unique<SourceManager>(*Diagnostics, *FileMgr); FileEntryRef FE = llvm::cantFail(FileMgr->getFileRef(FileName)); FileID ID = - SourceMgr->createFileID(FE, SourceLocation(), clang::SrcMgr::C_User); + SourceMgr->createFileID(FE, SourceLocation(), clang::SrcMgr::C_User, + /*InputEncodingName=*/{}); assert(ID.isValid()); SourceMgr->setMainFileID(ID); } diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp index 8aee45b5dc644..6be01e4563e44 100644 --- a/clang/lib/Frontend/CompilerInstance.cpp +++ b/clang/lib/Frontend/CompilerInstance.cpp @@ -912,12 +912,15 @@ CompilerInstance::createOutputFileImpl(StringRef OutputPath, bool Binary, // Initialization Utilities bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input){ - return InitializeSourceManager(Input, getDiagnostics(), getFileManager(), - getSourceManager()); + StringRef InputEncodingName = + hasPreprocessor() ? llvm::StringRef(getLangOpts().InputEncoding) : llvm::StringRef(); + return InitializeSourceManager(Input, InputEncodingName, getDiagnostics(), + getFileManager(), getSourceManager()); } // static bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input, + llvm::StringRef InputEncodingName, DiagnosticsEngine &Diags, FileManager &FileMgr, SourceManager &SourceMgr) { @@ -950,7 +953,8 @@ bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input, } SourceMgr.setMainFileID( - SourceMgr.createFileID(*FileOrErr, SourceLocation(), Kind)); + SourceMgr.createFileID(*FileOrErr, SourceLocation(), Kind, + InputEncodingName)); assert(SourceMgr.getMainFileID().isValid() && "Couldn't establish MainFileID!"); diff --git a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp index 1bfe644b2525a..691bc5a5fd31d 100644 --- a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp +++ b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp @@ -610,8 +610,10 @@ static bool ParseDirective(StringRef S, ExpectedData *ED, SourceManager &SM, } FileID FID = SM.translateFile(*File); + // FIXME: Figure out character-encoding converter treatment. if (FID.isInvalid()) - FID = SM.createFileID(*File, Pos, SrcMgr::C_User); + FID = SM.createFileID(*File, Pos, SrcMgr::C_User, + /*InputEncodingName=*/{}); if (PH.Next(Line) && Line > 0) ExpectedLoc = SM.translateLineCol(FID, Line, 1); diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp index 6c07386f89010..3e5c5b317d580 100644 --- a/clang/lib/Lex/ModuleMap.cpp +++ b/clang/lib/Lex/ModuleMap.cpp @@ -1473,7 +1473,12 @@ bool ModuleMap::parseModuleMapFile(FileEntryRef File, bool IsSystem, if (LocalFID.isInvalid()) { auto FileCharacter = IsSystem ? SrcMgr::C_System_ModuleMap : SrcMgr::C_User_ModuleMap; - LocalFID = SourceMgr.createFileID(File, ExternModuleLoc, FileCharacter); + // Module map files are textual "source files". Use input charset converter + // if available, and file tag converters are handled by SourceManager's cache. + // Get input encoding from LangOptions for charset conversion + llvm::StringRef InputEncodingName = LangOpts.InputEncoding; + LocalFID = SourceMgr.createFileID(File, ExternModuleLoc, FileCharacter, + InputEncodingName); } ID = LocalFID; } diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp index eb21a510dcf83..f989c2d1d4b96 100644 --- a/clang/lib/Lex/PPDirectives.cpp +++ b/clang/lib/Lex/PPDirectives.cpp @@ -2796,7 +2796,11 @@ Preprocessor::ImportAction Preprocessor::HandleHeaderIncludeOrImport( // position on the file where it will be included and after the expansions. if (IncludePos.isMacroID()) IncludePos = SourceMgr.getExpansionRange(IncludePos).getEnd(); - FileID FID = SourceMgr.createFileID(*File, IncludePos, FileCharacter); + // Use the SourceManager's input charset converter for non-tagged files + // by passing the input encoding name + llvm::StringRef InputEncodingName = getLangOpts().InputEncoding; + FileID FID = SourceMgr.createFileID(*File, IncludePos, FileCharacter, + InputEncodingName); if (!FID.isValid()) { TheModuleLoader.HadFatalFailure = true; return ImportAction::Failure; diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp index c69d084d6514f..610d3c703c693 100644 --- a/clang/lib/Lex/Preprocessor.cpp +++ b/clang/lib/Lex/Preprocessor.cpp @@ -651,8 +651,10 @@ void Preprocessor::EnterMainSourceFile() { << PPOpts.PCHThroughHeader; return; } + // FIXME: Figure out character-encoding converter treatment. setPCHThroughHeaderFileID( - SourceMgr.createFileID(*File, SourceLocation(), SrcMgr::C_User)); + SourceMgr.createFileID(*File, SourceLocation(), SrcMgr::C_User, + /*InputEncodingName=*/{})); } // Skip tokens from the Predefines and if needed the main file. diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index f8a6a38bb9b5c..52b60df62977d 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -2002,7 +2002,10 @@ bool ASTReader::ReadSLocEntry(int ID) { } SrcMgr::CharacteristicKind FileCharacter = (SrcMgr::CharacteristicKind)Record[2]; - FileID FID = SourceMgr.createFileID(*File, IncludeLoc, FileCharacter, ID, + // Note: If conversion was originally necessary, OverriddenBuffer should be + // true and the associated handling will trigger. + FileID FID = SourceMgr.createFileID(*File, IncludeLoc, FileCharacter, + /*InputEncodingName=*/{}, ID, BaseOffset + Record[0]); SrcMgr::FileInfo &FileInfo = SourceMgr.getSLocEntry(FID).getFile(); FileInfo.NumCreatedFIDs = Record[5]; >From 236c44859bad044a9e9a8810a3e66d67e81d3ae0 Mon Sep 17 00:00:00 2001 From: alisonzhang <[email protected]> Date: Sun, 28 Jun 2026 21:53:49 -0400 Subject: [PATCH 7/8] Use converters for tagged files --- clang/lib/Basic/SourceManager.cpp | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp index e30537c751005..1a186a6a4d98e 100644 --- a/clang/lib/Basic/SourceManager.cpp +++ b/clang/lib/Basic/SourceManager.cpp @@ -138,6 +138,18 @@ ContentCache::getBufferOrNone(DiagnosticsEngine &Diag, FileManager &FM, // return paths. IsBufferInvalid = true; + // If a converter is set, open the file in binary mode to get raw bytes + // and avoid platform-specific auto-conversion (e.g., EBCDIC->ASCII on z/OS, + // CRLF->LF on Windows). The explicit converter will handle all transformations. + bool NeedsExplicitConversion = FileIDConverterInfo.getPointer() != nullptr; + bool IsText = !NeedsExplicitConversion; + + auto BufferOrError = FM.getBufferForFile(*ContentsEntry, IsFileVolatile, + /*RequiresNullTerminator=*/true, + /*MaybeLimit=*/std::nullopt, + IsText); + +>>>>>>> de18010595aa (Use converters for tagged files) // If we were unable to open the file, then we are in an inconsistent // situation where the content cache referenced a file which no longer // exists. Most likely, we were using a stat cache with an invalid entry but @@ -648,7 +660,24 @@ FileID SourceManager::createFileID(FileEntryRef SourceFile, } llvm::ErrorOr<llvm::TextEncodingConverter *> Converter = nullptr; - if (!InputEncodingName.empty()) { + llvm::ErrorOr<llvm::SmallString<32>> Ccsid = + llvm::getEncodingNameFromFileTag(SourceFile.getName()); + if (!Ccsid) { + Diag.Report(SourceLocation(), diag::err_cannot_open_file) + << SourceFile.getName() << Ccsid.getError().message(); + return FileID(); + } + if (!Ccsid->empty()) { + // File has a tag, use the converter from SourceManager's cache + Converter = getOrCreateConverter(*Ccsid); + if (!Converter) { + Diag.Report(SourceLocation(), diag::err_cannot_open_file) + << SourceFile.getName() + << (llvm::Twine("cannot create converter from encoding '") + *Ccsid + "'"); + return FileID(); + } + IR.setFileTagged(true); + } else if (!InputEncodingName.empty()) { // No file tag but -finput-charset conversion is desired. // Get the converter from the cache using the input encoding name. Converter = getOrCreateConverter(InputEncodingName); >From 32671c0c189d99830d4084999a92f0ccb32011ee Mon Sep 17 00:00:00 2001 From: alisonzhang <[email protected]> Date: Fri, 3 Jul 2026 09:00:47 -0400 Subject: [PATCH 8/8] Clang format --- clang/include/clang/Basic/LangOptions.h | 2 +- clang/include/clang/Basic/SourceManager.h | 13 ++--- .../include/clang/Frontend/CompilerInstance.h | 2 +- clang/lib/Basic/SourceManager.cpp | 52 ++++++++++--------- clang/lib/Frontend/CompilerInstance.cpp | 20 ++++--- .../lib/Frontend/VerifyDiagnosticConsumer.cpp | 4 +- clang/lib/Lex/ModuleMap.cpp | 9 ++-- clang/lib/Lex/Preprocessor.cpp | 6 +-- clang/lib/Serialization/ASTReader.cpp | 4 +- llvm/include/llvm/Support/VirtualFileSystem.h | 6 +-- llvm/lib/Support/TextEncoding.cpp | 3 +- 11 files changed, 60 insertions(+), 61 deletions(-) diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h index 5256e55380236..3200d73f0381f 100644 --- a/clang/include/clang/Basic/LangOptions.h +++ b/clang/include/clang/Basic/LangOptions.h @@ -623,7 +623,7 @@ class LangOptions : public LangOptionsBase { /// Name of the input encoding to convert to the internal encoding. std::string InputEncoding; - + LangOptions(); /// Set language defaults for the given input language and diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h index fbb6a3be69c1b..16dd446371540 100644 --- a/clang/include/clang/Basic/SourceManager.h +++ b/clang/include/clang/Basic/SourceManager.h @@ -285,9 +285,7 @@ class alignas(8) ContentCache { /// Bit 0: Used by FileID flag /// Bit 1: File tagged flag - bool isUsedByFileID() const { - return FileIDConverterInfo.getInt() & 0x1; - } + bool isUsedByFileID() const { return FileIDConverterInfo.getInt() & 0x1; } void setUsedByFileID(bool Used) { unsigned Flags = FileIDConverterInfo.getInt(); @@ -298,9 +296,7 @@ class alignas(8) ContentCache { FileIDConverterInfo.setInt(Flags); } - bool isFileTagged() const { - return FileIDConverterInfo.getInt() & 0x2; - } + bool isFileTagged() const { return FileIDConverterInfo.getInt() & 0x2; } void setFileTagged(bool Tagged) { unsigned Flags = FileIDConverterInfo.getInt(); @@ -976,8 +972,7 @@ class SourceManager : public RefCountedBase<SourceManager> { /// being \#included from the specified IncludePosition. FileID createFileID(FileEntryRef SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, - llvm::StringRef InputEncodingName = {}, - int LoadedID = 0, + llvm::StringRef InputEncodingName = {}, int LoadedID = 0, SourceLocation::UIntTy LoadedOffset = 0); /// Create a new FileID that represents the specified memory buffer. @@ -1002,7 +997,7 @@ class SourceManager : public RefCountedBase<SourceManager> { /// new FileID for the \p SourceFile. FileID getOrCreateFileID(FileEntryRef SourceFile, SrcMgr::CharacteristicKind FileCharacter, - llvm::StringRef InputEncodingName = {}); + llvm::StringRef InputEncodingName = {}); /// Creates an expansion SLocEntry for the substitution of an argument into a /// function-like macro's body. Returns the start of the expansion. diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h index 6bc5661d605d0..0ebc50d99b86d 100644 --- a/clang/include/clang/Frontend/CompilerInstance.h +++ b/clang/include/clang/Frontend/CompilerInstance.h @@ -869,7 +869,7 @@ class CompilerInstance : public ModuleLoader { /// /// \return True on success. static bool InitializeSourceManager(const FrontendInputFile &Input, - llvm::StringRef InputEncodingName, + llvm::StringRef InputEncodingName, DiagnosticsEngine &Diags, FileManager &FileMgr, SourceManager &SourceMgr); diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp index 1a186a6a4d98e..bf4016531a7c5 100644 --- a/clang/lib/Basic/SourceManager.cpp +++ b/clang/lib/Basic/SourceManager.cpp @@ -10,13 +10,13 @@ // //===----------------------------------------------------------------------===// -#include "clang/Config/config.h" #include "clang/Basic/SourceManager.h" #include "clang/Basic/Diagnostic.h" #include "clang/Basic/FileManager.h" #include "clang/Basic/LLVM.h" #include "clang/Basic/SourceLocation.h" #include "clang/Basic/SourceManagerInternals.h" +#include "clang/Config/config.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/MapVector.h" #include "llvm/ADT/STLExtras.h" @@ -32,8 +32,8 @@ #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" -#include "llvm/Support/raw_ostream.h" #include "llvm/Support/SmallVectorMemoryBuffer.h" +#include "llvm/Support/raw_ostream.h" #include <algorithm> #include <cassert> #include <cstddef> @@ -140,14 +140,14 @@ ContentCache::getBufferOrNone(DiagnosticsEngine &Diag, FileManager &FM, // If a converter is set, open the file in binary mode to get raw bytes // and avoid platform-specific auto-conversion (e.g., EBCDIC->ASCII on z/OS, - // CRLF->LF on Windows). The explicit converter will handle all transformations. + // CRLF->LF on Windows). The explicit converter will handle all + // transformations. bool NeedsExplicitConversion = FileIDConverterInfo.getPointer() != nullptr; bool IsText = !NeedsExplicitConversion; auto BufferOrError = FM.getBufferForFile(*ContentsEntry, IsFileVolatile, /*RequiresNullTerminator=*/true, - /*MaybeLimit=*/std::nullopt, - IsText); + /*MaybeLimit=*/std::nullopt, IsText); >>>>>>> de18010595aa (Use converters for tagged files) // If we were unable to open the file, then we are in an inconsistent @@ -202,14 +202,15 @@ ContentCache::getBufferOrNone(DiagnosticsEngine &Diag, FileManager &FM, // // TODO: Consider adjusting the message to omit the recovery description // if the warning has been upgraded to an error. - const char *FallbackEncoding = CLANG_DEFAULT_INPUT_ENCODING_IBM1047 ? "IBM-1047" : "UTF-8"; + const char *FallbackEncoding = + CLANG_DEFAULT_INPUT_ENCODING_IBM1047 ? "IBM-1047" : "UTF-8"; Diag.Report(Loc, diag::warn_encoding_conversion_failed) << ContentsEntry->getName() << EC.message() << FallbackEncoding; } else { // TODO: Reclaim memory if the buffer size exceeds the content. auto NewBuf = std::make_unique<llvm::SmallVectorMemoryBuffer>( std::move(UTF8Buf), Buffer->getBufferIdentifier()); - Buffer = std::move(NewBuf); + Buffer = std::move(NewBuf); } } @@ -512,9 +513,10 @@ SourceManager::getOrCreateConverter(llvm::StringRef SourceEncoding) { return nullptr; // Create a cache key using canonical encoding name - llvm::StringRef CacheKey = SourceKnown - ? llvm::TextEncodingConverter::getKnownEncodingName(*SourceKnown) - : SourceEncoding; + llvm::StringRef CacheKey = + SourceKnown + ? llvm::TextEncodingConverter::getKnownEncodingName(*SourceKnown) + : SourceEncoding; // Check if converter already exists in cache auto It = ConverterCache.find(CacheKey); @@ -529,9 +531,9 @@ SourceManager::getOrCreateConverter(llvm::StringRef SourceEncoding) { return NewConverter.getError(); // Store the converter in the cache - auto Inserted = ConverterCache.insert( - std::make_pair(CacheKey, std::make_unique<llvm::TextEncodingConverter>( - std::move(*NewConverter)))); + auto Inserted = ConverterCache.insert(std::make_pair( + CacheKey, + std::make_unique<llvm::TextEncodingConverter>(std::move(*NewConverter)))); return Inserted.first->second.get(); } @@ -645,7 +647,7 @@ FileID SourceManager::getNextFileID(FileID FID) const { FileID SourceManager::createFileID(FileEntryRef SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, - llvm::StringRef InputEncodingName, + llvm::StringRef InputEncodingName, int LoadedID, SourceLocation::UIntTy LoadedOffset) { SrcMgr::ContentCache &IR = getOrCreateContentCache(SourceFile, @@ -673,7 +675,8 @@ FileID SourceManager::createFileID(FileEntryRef SourceFile, if (!Converter) { Diag.Report(SourceLocation(), diag::err_cannot_open_file) << SourceFile.getName() - << (llvm::Twine("cannot create converter from encoding '") + *Ccsid + "'"); + << (llvm::Twine("cannot create converter from encoding '") + *Ccsid + + "'"); return FileID(); } IR.setFileTagged(true); @@ -682,18 +685,19 @@ FileID SourceManager::createFileID(FileEntryRef SourceFile, // Get the converter from the cache using the input encoding name. Converter = getOrCreateConverter(InputEncodingName); if (!Converter) { - llvm::report_fatal_error( - "Cannot create converter for file '" + SourceFile.getName() + "': " + - Converter.getError().message()); + llvm::report_fatal_error("Cannot create converter for file '" + + SourceFile.getName() + + "': " + Converter.getError().message()); } } - #ifndef NDEBUG +#ifndef NDEBUG // Either the content cache has never been used for a FileID (and, if we are // being asked to use a converter, there should be no valid buffer set up for // it) or the conversion (or lack thereof) should be the same as that used // previously. - llvm::TextEncodingConverter *CacheConverter = IR.FileIDConverterInfo.getPointer(); + llvm::TextEncodingConverter *CacheConverter = + IR.FileIDConverterInfo.getPointer(); bool CacheUsedByFileID = IR.isUsedByFileID(); llvm::TextEncodingConverter *ConverterPtr = Converter ? *Converter : nullptr; if (CacheUsedByFileID) @@ -746,11 +750,11 @@ FileID SourceManager::createFileID(const llvm::MemoryBufferRef &Buffer, FileID SourceManager::getOrCreateFileID(FileEntryRef SourceFile, SrcMgr::CharacteristicKind FileCharacter, - llvm::StringRef InputEncodingName) { + llvm::StringRef InputEncodingName) { FileID ID = translateFile(SourceFile); return ID.isValid() ? ID - : createFileID(SourceFile, SourceLocation(), FileCharacter, - InputEncodingName); + : createFileID(SourceFile, SourceLocation(), + FileCharacter, InputEncodingName); } /// createFileID - Create a new FileID for the specified ContentCache and @@ -2507,7 +2511,7 @@ SourceManagerForFile::SourceManagerForFile(StringRef FileName, FileEntryRef FE = llvm::cantFail(FileMgr->getFileRef(FileName)); FileID ID = SourceMgr->createFileID(FE, SourceLocation(), clang::SrcMgr::C_User, - /*InputEncodingName=*/{}); + /*InputEncodingName=*/{}); assert(ID.isValid()); SourceMgr->setMainFileID(ID); } diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp index 6be01e4563e44..ccb987df1083e 100644 --- a/clang/lib/Frontend/CompilerInstance.cpp +++ b/clang/lib/Frontend/CompilerInstance.cpp @@ -913,17 +913,16 @@ CompilerInstance::createOutputFileImpl(StringRef OutputPath, bool Binary, bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input){ StringRef InputEncodingName = - hasPreprocessor() ? llvm::StringRef(getLangOpts().InputEncoding) : llvm::StringRef(); - return InitializeSourceManager(Input, InputEncodingName, getDiagnostics(), - getFileManager(), getSourceManager()); + hasPreprocessor() ? llvm::StringRef(getLangOpts().InputEncoding) + : llvm::StringRef(); + return InitializeSourceManager(Input, InputEncodingName, getDiagnostics(), + getFileManager(), getSourceManager()); } // static -bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input, - llvm::StringRef InputEncodingName, - DiagnosticsEngine &Diags, - FileManager &FileMgr, - SourceManager &SourceMgr) { +bool CompilerInstance::InitializeSourceManager( + const FrontendInputFile &Input, llvm::StringRef InputEncodingName, + DiagnosticsEngine &Diags, FileManager &FileMgr, SourceManager &SourceMgr) { SrcMgr::CharacteristicKind Kind = Input.getKind().getFormat() == InputKind::ModuleMap ? Input.isSystem() ? SrcMgr::C_System_ModuleMap @@ -952,9 +951,8 @@ bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input, return false; } - SourceMgr.setMainFileID( - SourceMgr.createFileID(*FileOrErr, SourceLocation(), Kind, - InputEncodingName)); + SourceMgr.setMainFileID(SourceMgr.createFileID(*FileOrErr, SourceLocation(), + Kind, InputEncodingName)); assert(SourceMgr.getMainFileID().isValid() && "Couldn't establish MainFileID!"); diff --git a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp index 691bc5a5fd31d..1ae2337adbb1f 100644 --- a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp +++ b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp @@ -610,10 +610,10 @@ static bool ParseDirective(StringRef S, ExpectedData *ED, SourceManager &SM, } FileID FID = SM.translateFile(*File); - // FIXME: Figure out character-encoding converter treatment. + // FIXME: Figure out character-encoding converter treatment. if (FID.isInvalid()) FID = SM.createFileID(*File, Pos, SrcMgr::C_User, - /*InputEncodingName=*/{}); + /*InputEncodingName=*/{}); if (PH.Next(Line) && Line > 0) ExpectedLoc = SM.translateLineCol(FID, Line, 1); diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp index 3e5c5b317d580..b9750e7a8c197 100644 --- a/clang/lib/Lex/ModuleMap.cpp +++ b/clang/lib/Lex/ModuleMap.cpp @@ -1473,12 +1473,13 @@ bool ModuleMap::parseModuleMapFile(FileEntryRef File, bool IsSystem, if (LocalFID.isInvalid()) { auto FileCharacter = IsSystem ? SrcMgr::C_System_ModuleMap : SrcMgr::C_User_ModuleMap; - // Module map files are textual "source files". Use input charset converter - // if available, and file tag converters are handled by SourceManager's cache. - // Get input encoding from LangOptions for charset conversion + // Module map files are textual "source files". Use input charset + // converter if available, and file tag converters are handled by + // SourceManager's cache. Get input encoding from LangOptions for charset + // conversion llvm::StringRef InputEncodingName = LangOpts.InputEncoding; LocalFID = SourceMgr.createFileID(File, ExternModuleLoc, FileCharacter, - InputEncodingName); + InputEncodingName); } ID = LocalFID; } diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp index 610d3c703c693..6ff3652760de7 100644 --- a/clang/lib/Lex/Preprocessor.cpp +++ b/clang/lib/Lex/Preprocessor.cpp @@ -652,9 +652,9 @@ void Preprocessor::EnterMainSourceFile() { return; } // FIXME: Figure out character-encoding converter treatment. - setPCHThroughHeaderFileID( - SourceMgr.createFileID(*File, SourceLocation(), SrcMgr::C_User, - /*InputEncodingName=*/{})); + setPCHThroughHeaderFileID(SourceMgr.createFileID(*File, SourceLocation(), + SrcMgr::C_User, + /*InputEncodingName=*/{})); } // Skip tokens from the Predefines and if needed the main file. diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index 52b60df62977d..998cea077d836 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -2004,8 +2004,8 @@ bool ASTReader::ReadSLocEntry(int ID) { FileCharacter = (SrcMgr::CharacteristicKind)Record[2]; // Note: If conversion was originally necessary, OverriddenBuffer should be // true and the associated handling will trigger. - FileID FID = SourceMgr.createFileID(*File, IncludeLoc, FileCharacter, - /*InputEncodingName=*/{}, ID, + FileID FID = SourceMgr.createFileID(*File, IncludeLoc, FileCharacter, + /*InputEncodingName=*/{}, ID, BaseOffset + Record[0]); SrcMgr::FileInfo &FileInfo = SourceMgr.getSLocEntry(FID).getFile(); FileInfo.NumCreatedFIDs = Record[5]; diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h index 9cc35b6f5fb6d..274886f920fc6 100644 --- a/llvm/include/llvm/Support/VirtualFileSystem.h +++ b/llvm/include/llvm/Support/VirtualFileSystem.h @@ -139,9 +139,9 @@ class LLVM_ABI File { /// Checks if this is a real file and the requested text mode differs /// from the current mode. For real files with a text mode mismatch where - /// the buffer was previously requested, this will call llvm::report_fatal_error. - /// Always returns false for non-real files. - /// Default implementation returns false for non-real files. + /// the buffer was previously requested, this will call + /// llvm::report_fatal_error. Always returns false for non-real files. Default + /// implementation returns false for non-real files. virtual bool realFileCheckTextModeMismatch(bool RequestedIsText) const { return false; } diff --git a/llvm/lib/Support/TextEncoding.cpp b/llvm/lib/Support/TextEncoding.cpp index 8e9653dab38ec..99f2c8099d966 100644 --- a/llvm/lib/Support/TextEncoding.cpp +++ b/llvm/lib/Support/TextEncoding.cpp @@ -48,7 +48,8 @@ static void normalizeCharSetName(StringRef CSName, } // Maps the encoding name to enum constant if possible. -std::optional<TextEncoding> TextEncodingConverter::getKnownEncoding(StringRef Name) { +std::optional<TextEncoding> +TextEncodingConverter::getKnownEncoding(StringRef Name) { SmallString<16> Normalized; normalizeCharSetName(Name, Normalized); if (Normalized.equals("utf8")) _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
