https://github.com/mzukovec created https://github.com/llvm/llvm-project/pull/206317
## Summary This fixes an MS ABI RTTI `type_info` name-size explosion for deeply nested template types that contain local lambda closure types. See the https://github.com/llvm/llvm-project/issues/206313 Clang already hashes long MSVC-mangled RTTI symbol names, but the raw RTTI name stored in the `TypeDescriptor` payload was still emitted through a normal mangler. For nested template instantiations involving local lambdas, that path could recursively spell the lambda’s enclosing template/function context into writable `.data` COMDAT sections. This patch adds an RTTI-shortening mode to the MS mangler and uses it for both RTTI descriptor symbols and RTTI raw-name payloads. In that mode: - long RTTI raw-name payloads are hashed using the existing MSVC MD5 mangling threshold - local lambda closure names avoid recursively spelling their enclosing template/function context - temporary “extra” manglers used for template-specialization source-name/backref keys inherit the same RTTI-shortening behavior This is intended to bound RTTI `TypeDescriptor` growth and avoid large writable `.data` COMDATs. It does not try to exactly reverse-engineer MSVC’s legacy hashed local-lambda spelling. ## Tests Added coverage for the nested-template/local-lambda RTTI case, including an implicit check that the RTTI output does not recursively include the `make_layer` template context. Extended the existing MS MD5 mangling test to cover RTTI `TypeDescriptor` payload hashing at the 4096-byte threshold. >From a6e7a4e01b8b372e909e6f38935a006a83425c31 Mon Sep 17 00:00:00 2001 From: mzukovec <[email protected]> Date: Sun, 28 Jun 2026 09:51:02 +0200 Subject: [PATCH] [clang] Avoid MS RTTI type_info name explosion --- clang/lib/AST/MicrosoftMangle.cpp | 57 ++++++++++-------- clang/test/CodeGenCXX/mangle-ms-md5.cpp | 17 ++++++ .../test/CodeGenCXX/mangle-ms-rtti-lambda.cpp | 58 +++++++++++++++++++ 3 files changed, 109 insertions(+), 23 deletions(-) create mode 100644 clang/test/CodeGenCXX/mangle-ms-rtti-lambda.cpp diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp index adfe260a0d091..9115c2918c16e 100644 --- a/clang/lib/AST/MicrosoftMangle.cpp +++ b/clang/lib/AST/MicrosoftMangle.cpp @@ -332,6 +332,7 @@ class MicrosoftCXXNameMangler { ASTContext &getASTContext() const { return Context.getASTContext(); } const bool PointersAre64Bit; + const bool ShortenRTTINames; DiagnosticBuilder Error(SourceLocation, StringRef, StringRef); DiagnosticBuilder Error(SourceLocation, StringRef); @@ -341,25 +342,31 @@ class MicrosoftCXXNameMangler { enum QualifierMangleMode { QMM_Drop, QMM_Mangle, QMM_Escape, QMM_Result }; enum class TplArgKind { ClassNTTP, StructuralValue }; - MicrosoftCXXNameMangler(MicrosoftMangleContextImpl &C, raw_ostream &Out_) + MicrosoftCXXNameMangler(MicrosoftMangleContextImpl &C, raw_ostream &Out_, + bool ShortenRTTINamesIn = false) : Context(C), Out(Out_), Structor(nullptr), StructorType(-1), TemplateArgStringStorage(TemplateArgStringStorageAlloc), PointersAre64Bit(C.getASTContext().getTargetInfo().getPointerWidth( - LangAS::Default) == 64) {} + LangAS::Default) == 64), + ShortenRTTINames(ShortenRTTINamesIn) {} MicrosoftCXXNameMangler(MicrosoftMangleContextImpl &C, raw_ostream &Out_, - const CXXConstructorDecl *D, CXXCtorType Type) + const CXXConstructorDecl *D, CXXCtorType Type, + bool ShortenRTTINamesIn = false) : Context(C), Out(Out_), Structor(getStructor(D)), StructorType(Type), TemplateArgStringStorage(TemplateArgStringStorageAlloc), PointersAre64Bit(C.getASTContext().getTargetInfo().getPointerWidth( - LangAS::Default) == 64) {} + LangAS::Default) == 64), + ShortenRTTINames(ShortenRTTINamesIn) {} MicrosoftCXXNameMangler(MicrosoftMangleContextImpl &C, raw_ostream &Out_, - const CXXDestructorDecl *D, CXXDtorType Type) + const CXXDestructorDecl *D, CXXDtorType Type, + bool ShortenRTTINamesIn = false) : Context(C), Out(Out_), Structor(getStructor(D)), StructorType(Type), TemplateArgStringStorage(TemplateArgStringStorageAlloc), PointersAre64Bit(C.getASTContext().getTargetInfo().getPointerWidth( - LangAS::Default) == 64) {} + LangAS::Default) == 64), + ShortenRTTINames(ShortenRTTINamesIn) {} raw_ostream &getStream() const { return Out; } @@ -963,7 +970,9 @@ void MicrosoftCXXNameMangler::mangleName(GlobalDecl GD) { // Always start with the unqualified name. mangleUnqualifiedName(GD); - mangleNestedName(GD); + const auto *RD = dyn_cast<CXXRecordDecl>(GD.getDecl()); + if (!ShortenRTTINames || !RD || !RD->isLambda()) + mangleNestedName(GD); // Terminate the whole name with an '@'. Out << '@'; @@ -1119,7 +1128,7 @@ void MicrosoftCXXNameMangler::mangleUnqualifiedName(GlobalDecl GD, // Mangle full template name into temporary buffer. llvm::SmallString<64> TemplateMangling; llvm::raw_svector_ostream Stream(TemplateMangling); - MicrosoftCXXNameMangler Extra(Context, Stream); + MicrosoftCXXNameMangler Extra(Context, Stream, ShortenRTTINames); Extra.mangleTemplateInstantiationName(TD, *TemplateArgs); // Use the string backref vector to possibly get a back reference. @@ -1267,7 +1276,8 @@ void MicrosoftCXXNameMangler::mangleUnqualifiedName(GlobalDecl GD, // If the context is a variable or a class member and not a parameter, // it is encoded in a qualified name. - if (LambdaManglingNumber && LambdaContextDecl) { + if (!ShortenRTTINames && LambdaManglingNumber && + LambdaContextDecl) { if ((isa<VarDecl>(LambdaContextDecl) || isa<FieldDecl>(LambdaContextDecl)) && !isa<ParmVarDecl>(LambdaContextDecl)) { @@ -2173,7 +2183,7 @@ void MicrosoftCXXNameMangler::mangleTemplateArgValue(QualType T, void MicrosoftCXXNameMangler::mangleObjCProtocol(const ObjCProtocolDecl *PD) { llvm::SmallString<64> TemplateMangling; llvm::raw_svector_ostream Stream(TemplateMangling); - MicrosoftCXXNameMangler Extra(Context, Stream); + MicrosoftCXXNameMangler Extra(Context, Stream, ShortenRTTINames); Stream << "?$"; Extra.mangleSourceName("Protocol"); @@ -2187,7 +2197,7 @@ void MicrosoftCXXNameMangler::mangleObjCLifetime(const QualType Type, SourceRange Range) { llvm::SmallString<64> TemplateMangling; llvm::raw_svector_ostream Stream(TemplateMangling); - MicrosoftCXXNameMangler Extra(Context, Stream); + MicrosoftCXXNameMangler Extra(Context, Stream, ShortenRTTINames); Stream << "?$"; switch (Quals.getObjCLifetime()) { @@ -2216,7 +2226,7 @@ void MicrosoftCXXNameMangler::mangleObjCKindOfType(const ObjCObjectType *T, SourceRange Range) { llvm::SmallString<64> TemplateMangling; llvm::raw_svector_ostream Stream(TemplateMangling); - MicrosoftCXXNameMangler Extra(Context, Stream); + MicrosoftCXXNameMangler Extra(Context, Stream, ShortenRTTINames); Stream << "?$"; Extra.mangleSourceName("KindOf"); @@ -2468,7 +2478,7 @@ void MicrosoftCXXNameMangler::mangleAddressSpaceType(QualType T, assert(Quals.hasAddressSpace() && "Not valid without address space"); llvm::SmallString<32> ASMangling; llvm::raw_svector_ostream Stream(ASMangling); - MicrosoftCXXNameMangler Extra(Context, Stream); + MicrosoftCXXNameMangler Extra(Context, Stream, ShortenRTTINames); Stream << "?$"; LangAS AS = Quals.getAddressSpace(); @@ -3658,7 +3668,7 @@ void MicrosoftCXXNameMangler::mangleType(const ComplexType *T, Qualifiers, llvm::SmallString<64> TemplateMangling; llvm::raw_svector_ostream Stream(TemplateMangling); - MicrosoftCXXNameMangler Extra(Context, Stream); + MicrosoftCXXNameMangler Extra(Context, Stream, ShortenRTTINames); Stream << "?$"; Extra.mangleSourceName("_Complex"); Extra.mangleType(ElementType, Range, QMM_Escape); @@ -3723,7 +3733,7 @@ void MicrosoftCXXNameMangler::mangleType(const VectorType *T, Qualifiers Quals, llvm::SmallString<64> TemplateMangling; llvm::raw_svector_ostream Stream(TemplateMangling); - MicrosoftCXXNameMangler Extra(Context, Stream); + MicrosoftCXXNameMangler Extra(Context, Stream, ShortenRTTINames); Stream << "?$"; Extra.mangleSourceName("__vector"); Extra.mangleType(QualType(ET ? static_cast<const Type *>(ET) : BitIntTy, 0), @@ -3755,7 +3765,7 @@ void MicrosoftCXXNameMangler::mangleType(const ConstantMatrixType *T, llvm::SmallString<64> TemplateMangling; llvm::raw_svector_ostream Stream(TemplateMangling); - MicrosoftCXXNameMangler Extra(Context, Stream); + MicrosoftCXXNameMangler Extra(Context, Stream, ShortenRTTINames); Stream << "?$"; @@ -3905,7 +3915,7 @@ void MicrosoftCXXNameMangler::mangleType(const AtomicType *T, Qualifiers, llvm::SmallString<64> TemplateMangling; llvm::raw_svector_ostream Stream(TemplateMangling); - MicrosoftCXXNameMangler Extra(Context, Stream); + MicrosoftCXXNameMangler Extra(Context, Stream, ShortenRTTINames); Stream << "?$"; Extra.mangleSourceName("_Atomic"); Extra.mangleType(ValueType, Range, QMM_Escape); @@ -3919,7 +3929,7 @@ void MicrosoftCXXNameMangler::mangleType(const PipeType *T, Qualifiers, llvm::SmallString<64> TemplateMangling; llvm::raw_svector_ostream Stream(TemplateMangling); - MicrosoftCXXNameMangler Extra(Context, Stream); + MicrosoftCXXNameMangler Extra(Context, Stream, ShortenRTTINames); Stream << "?$"; Extra.mangleSourceName("ocl_pipe"); Extra.mangleType(ElementType, Range, QMM_Escape); @@ -3957,7 +3967,7 @@ void MicrosoftCXXNameMangler::mangleType(const BitIntType *T, Qualifiers, SourceRange Range) { llvm::SmallString<64> TemplateMangling; llvm::raw_svector_ostream Stream(TemplateMangling); - MicrosoftCXXNameMangler Extra(Context, Stream); + MicrosoftCXXNameMangler Extra(Context, Stream, ShortenRTTINames); Stream << "?$"; if (T->isUnsigned()) Extra.mangleSourceName("_UBitInt"); @@ -3989,7 +3999,7 @@ void MicrosoftCXXNameMangler::mangleType(const OverflowBehaviorType *T, llvm::SmallString<64> TemplateMangling; llvm::raw_svector_ostream Stream(TemplateMangling); - MicrosoftCXXNameMangler Extra(Context, Stream); + MicrosoftCXXNameMangler Extra(Context, Stream, ShortenRTTINames); Stream << "?$"; if (T->isWrapKind()) { Extra.mangleSourceName("ObtWrap_"); @@ -4185,7 +4195,7 @@ void MicrosoftMangleContextImpl::mangleCXXVBTable( void MicrosoftMangleContextImpl::mangleCXXRTTI(QualType T, raw_ostream &Out) { msvc_hashing_ostream MHO(Out); - MicrosoftCXXNameMangler Mangler(*this, MHO); + MicrosoftCXXNameMangler Mangler(*this, MHO, /*ShortenRTTINames=*/true); Mangler.getStream() << "??_R0"; Mangler.mangleType(T, SourceRange(), MicrosoftCXXNameMangler::QMM_Result); Mangler.getStream() << "@8"; @@ -4193,8 +4203,9 @@ void MicrosoftMangleContextImpl::mangleCXXRTTI(QualType T, raw_ostream &Out) { void MicrosoftMangleContextImpl::mangleCXXRTTIName( QualType T, raw_ostream &Out, bool NormalizeIntegers = false) { - MicrosoftCXXNameMangler Mangler(*this, Out); - Mangler.getStream() << '.'; + Out << '.'; + msvc_hashing_ostream MHO(Out); + MicrosoftCXXNameMangler Mangler(*this, MHO, /*ShortenRTTINames=*/true); Mangler.mangleType(T, SourceRange(), MicrosoftCXXNameMangler::QMM_Result); } diff --git a/clang/test/CodeGenCXX/mangle-ms-md5.cpp b/clang/test/CodeGenCXX/mangle-ms-md5.cpp index 6cd145d9166b6..bfc41156e64d8 100644 --- a/clang/test/CodeGenCXX/mangle-ms-md5.cpp +++ b/clang/test/CodeGenCXX/mangle-ms-md5.cpp @@ -34,6 +34,7 @@ struct Y4095 { Y4095::Y4095() {} // CHECK-DAG: @"??@a6a285da2eea70dba6b578022be61d81@??_R4@" = linkonce_odr constant %rtti.CompleteObjectLocator // CHECK-DAG: @"??@a6a285da2eea70dba6b578022be61d81@" = alias +// CHECK-DAG: @"??@c14087f0ec22b387aea7c59083f4f546@" = linkonce_odr global %rtti.TypeDescriptor37 { ptr @"??_7type_info@@6B@", ptr null, [38 x i8] c".??@5ca87b4c7c2322ca1e21f8b01a23e135@\00" }, comdat // RUN: %clang_cc1 -DTHROW -fcxx-exceptions -fms-compatibility-version=18.0 -emit-llvm -o - -triple i686-pc-win32 %s | FileCheck --check-prefix=HAVECTOR %s // RUN: %clang_cc1 -DTHROW -fcxx-exceptions -fms-compatibility-version=19.0 -emit-llvm -o - -triple i686-pc-win32 %s | FileCheck --check-prefix=OMITCTOR %s @@ -71,3 +72,19 @@ int X4088(z) = 1515; // Use initialization to verify mangled name association in the il int X4089(z) = 1717; // CHECK-DAG: @"??@0269945400a3474730d6880df0967d8f@" = dso_local global i32 1717, align 4 + +#define R4090 X4089(r) +struct R4090 { + R4090(); + virtual void f(); +}; +R4090::R4090() {} +// CHECK-DAG: @"??@{{[0-9a-f]+}}@" = linkonce_odr global %rtti.TypeDescriptor4096 { ptr @"??_7type_info@@6B@", ptr null, [4097 x i8] c".?AU{{r+}}@@\00" }, comdat + +#define R4091 C2(X4089(s), s) +struct R4091 { + R4091(); + virtual void f(); +}; +R4091::R4091() {} +// CHECK-DAG: @"??@{{[0-9a-f]+}}@" = linkonce_odr global %rtti.TypeDescriptor37 { ptr @"??_7type_info@@6B@", ptr null, [38 x i8] c".??@c9d63594821ede4ea693d109deaa7a17@\00" }, comdat diff --git a/clang/test/CodeGenCXX/mangle-ms-rtti-lambda.cpp b/clang/test/CodeGenCXX/mangle-ms-rtti-lambda.cpp new file mode 100644 index 0000000000000..a6a7e4b9e1ee1 --- /dev/null +++ b/clang/test/CodeGenCXX/mangle-ms-rtti-lambda.cpp @@ -0,0 +1,58 @@ +// RUN: %clang_cc1 -triple x86_64-pc-windows-msvc -std=c++17 -fms-compatibility -fms-extensions -emit-llvm -o - %s | FileCheck --implicit-check-not='?$make_layer' %s + +namespace std { +class type_info { +public: + const char *name() const; +}; +} + +using size_t = decltype(sizeof(0)); + +template <typename... Ts> struct pack {}; + +template <typename A, typename B> struct pair_like { + using first = A; + using second = B; +}; + +template <typename A, typename B, typename C, typename D> struct quad_like { + using a = A; + using b = B; + using c = C; + using d = D; +}; + +template <typename T> struct holder { + T *value; +}; + +template <typename T> auto make_layer() { + auto local_lambda = [](const T *, size_t) noexcept { return sizeof(T); }; + using L = decltype(local_lambda); + using PairTL = pair_like<T, L>; + using PairLT = pair_like<L, T>; + using Quad = quad_like<PairTL, PairLT, holder<T>, holder<L>>; + return pack<T, L, PairTL, PairLT, Quad, holder<Quad>>{}; +} + +struct seed { + int value; +}; + +using t01 = decltype(make_layer<seed>()); +using t02 = decltype(make_layer<t01>()); +using t03 = decltype(make_layer<t02>()); + +extern "C" const std::type_info *repro_type_infos[] = { + &typeid(t01), + &typeid(t02), + &typeid(t03), +}; + +extern "C" const char *repro_last_name() { + return typeid(t03).name(); +} + +// CHECK-DAG: global %rtti.TypeDescriptor{{[0-9]+}} { ptr @"??_7type_info@@6B@", ptr null, [{{[0-9]+}} x i8] c".?AU?$pack@Useed@@V<lambda_1>@@U?$pair_like@Useed@@V<lambda_1>@@@@ +// CHECK-DAG: global %rtti.TypeDescriptor37 { ptr @"??_7type_info@@6B@", ptr null, [38 x i8] c".??@{{[0-9a-f]+}}@\00" }, comdat _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
