llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-clang-codegen Author: Peter Collingbourne (pcc) <details> <summary>Changes</summary> Pointer field protection is a use-after-free vulnerability mitigation that works by changing how data structures' pointer fields are stored in memory. For more information, see the RFC: https://discourse.llvm.org/t/rfc-structure-protection-a-family-of-uaf-mitigation-techniques/85555 TODO: - Fix test failure. - Add more tests. - Add documentation. --- Patch is 82.31 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/133538.diff 52 Files Affected: - (modified) clang/include/clang/AST/ASTContext.h (+22) - (modified) clang/include/clang/Basic/Attr.td (+6) - (modified) clang/include/clang/Basic/Features.def (+3) - (modified) clang/include/clang/Basic/LangOptions.def (+3) - (modified) clang/include/clang/Basic/LangOptions.h (+11) - (modified) clang/include/clang/Basic/TokenKinds.def (+1) - (modified) clang/include/clang/Driver/Options.td (+6) - (modified) clang/lib/AST/ASTContext.cpp (+95) - (modified) clang/lib/AST/ExprConstant.cpp (+1) - (modified) clang/lib/AST/Type.cpp (+3-1) - (modified) clang/lib/AST/TypePrinter.cpp (+3) - (modified) clang/lib/CodeGen/CGCall.cpp (+108-6) - (modified) clang/lib/CodeGen/CGClass.cpp (+45-7) - (modified) clang/lib/CodeGen/CGExpr.cpp (+7-1) - (modified) clang/lib/CodeGen/CGExprAgg.cpp (+1-1) - (modified) clang/lib/CodeGen/CGExprCXX.cpp (+10) - (modified) clang/lib/CodeGen/CGExprConstant.cpp (+38-1) - (modified) clang/lib/CodeGen/CodeGenFunction.cpp (+38-5) - (modified) clang/lib/CodeGen/CodeGenFunction.h (+6-2) - (modified) clang/lib/CodeGen/CodeGenModule.cpp (+39) - (modified) clang/lib/CodeGen/CodeGenModule.h (+7) - (modified) clang/lib/CodeGen/ItaniumCXXABI.cpp (+1) - (modified) clang/lib/CodeGen/MicrosoftCXXABI.cpp (+1) - (modified) clang/lib/Driver/ToolChains/Clang.cpp (+4) - (modified) clang/lib/Sema/SemaDeclAttr.cpp (+8) - (modified) clang/lib/Sema/SemaExprCXX.cpp (+5) - (added) clang/test/CodeGen/pfp-attribute-disable.cpp (+33) - (added) clang/test/CodeGen/pfp-load-store.cpp (+40) - (added) clang/test/CodeGen/pfp-memcpy.cpp (+19) - (added) clang/test/CodeGen/pfp-null-init.cpp (+16) - (added) clang/test/CodeGen/pfp-struct-gep.cpp (+25) - (modified) clang/test/CodeGenCXX/trivial_abi.cpp (+1-3) - (modified) libcxx/include/__config (+23) - (modified) libcxx/include/__functional/function.h (+1-1) - (modified) libcxx/include/__memory/shared_ptr.h (+2-2) - (modified) libcxx/include/__memory/unique_ptr.h (+2-2) - (modified) libcxx/include/__tree (+1-1) - (modified) libcxx/include/__type_traits/is_trivially_relocatable.h (+6-2) - (modified) libcxx/include/__vector/vector.h (+1-1) - (modified) libcxx/include/typeinfo (+1-1) - (modified) libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp (+9) - (modified) libcxxabi/include/__cxxabi_config.h (+10) - (modified) libcxxabi/src/private_typeinfo.h (+3-3) - (modified) llvm/include/llvm/Analysis/PtrUseVisitor.h (+15) - (modified) llvm/include/llvm/IR/Intrinsics.td (+23) - (modified) llvm/include/llvm/Transforms/Utils/Local.h (+2) - (modified) llvm/lib/Analysis/PtrUseVisitor.cpp (+2-1) - (modified) llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp (+246) - (modified) llvm/lib/Target/AArch64/AArch64InstrFormats.td (+3) - (modified) llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp (+2-4) - (modified) llvm/lib/Transforms/Scalar/SROA.cpp (+35-5) - (modified) llvm/lib/Transforms/Utils/SimplifyCFG.cpp (+20-5) ``````````diff diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h index af8c49e99a7ce..abba83e1ff9c4 100644 --- a/clang/include/clang/AST/ASTContext.h +++ b/clang/include/clang/AST/ASTContext.h @@ -183,6 +183,12 @@ struct TypeInfoChars { } }; +struct PFPField { + CharUnits structOffset; + CharUnits offset; + FieldDecl *field; +}; + /// Holds long-lived AST nodes (such as types and decls) that can be /// referred to throughout the semantic analysis of a file. class ASTContext : public RefCountedBase<ASTContext> { @@ -3618,6 +3624,22 @@ OPT_LIST(V) StringRef getCUIDHash() const; + bool isPFPStruct(const RecordDecl *rec) const; + void findPFPFields(QualType Ty, CharUnits Offset, + std::vector<PFPField> &Fields, bool IncludeVBases) const; + bool hasPFPFields(QualType ty) const; + bool isPFPField(const FieldDecl *field) const; + + /// Returns whether this record's PFP fields (if any) are trivially + /// relocatable (i.e. may be memcpy'd). This may also return true if the + /// record does not have any PFP fields, so it may be necessary for the caller + /// to check for PFP fields, e.g. by calling hasPFPFields(). + bool arePFPFieldsTriviallyRelocatable(const RecordDecl *RD) const; + + llvm::SetVector<const FieldDecl *> PFPFieldsWithEvaluatedOffset; + void recordMemberDataPointerEvaluation(const ValueDecl *VD); + void recordOffsetOfEvaluation(const OffsetOfExpr *E); + private: /// All OMPTraitInfo objects live in this collection, one per /// `pragma omp [begin] declare variant` directive. diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td index 0999d8065e9f5..3d26c2f001812 100644 --- a/clang/include/clang/Basic/Attr.td +++ b/clang/include/clang/Basic/Attr.td @@ -2460,6 +2460,12 @@ def CountedByOrNull : DeclOrTypeAttr { let LangOpts = [COnly]; } +def NoPointerFieldProtection : DeclOrTypeAttr { + let Spellings = [Clang<"no_field_protection">]; + let Subjects = SubjectList<[Field], ErrorDiag>; + let Documentation = [Undocumented]; +} + def SizedBy : DeclOrTypeAttr { let Spellings = [Clang<"sized_by">]; let Subjects = SubjectList<[Field], ErrorDiag>; diff --git a/clang/include/clang/Basic/Features.def b/clang/include/clang/Basic/Features.def index 05ce214935fad..d4ded24c5a87e 100644 --- a/clang/include/clang/Basic/Features.def +++ b/clang/include/clang/Basic/Features.def @@ -262,6 +262,9 @@ FEATURE(shadow_call_stack, FEATURE(tls, PP.getTargetInfo().isTLSSupported()) FEATURE(underlying_type, LangOpts.CPlusPlus) FEATURE(experimental_library, LangOpts.ExperimentalLibrary) +FEATURE(pointer_field_protection, + LangOpts.getPointerFieldProtection() != + LangOptions::PointerFieldProtectionKind::None) // C11 features supported by other languages as extensions. EXTENSION(c_alignas, true) diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def index 3879cc7942877..8eacb6e066007 100644 --- a/clang/include/clang/Basic/LangOptions.def +++ b/clang/include/clang/Basic/LangOptions.def @@ -501,6 +501,9 @@ LANGOPT(RelativeCXXABIVTables, 1, 0, LANGOPT(OmitVTableRTTI, 1, 0, "Use an ABI-incompatible v-table layout that omits the RTTI component") +ENUM_LANGOPT(PointerFieldProtection, PointerFieldProtectionKind, 2, PointerFieldProtectionKind::None, + "Encode struct pointer fields to protect against UAF vulnerabilities") + LANGOPT(VScaleMin, 32, 0, "Minimum vscale value") LANGOPT(VScaleMax, 32, 0, "Maximum vscale value") diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h index e925e0f3b5d85..797a14038ba4b 100644 --- a/clang/include/clang/Basic/LangOptions.h +++ b/clang/include/clang/Basic/LangOptions.h @@ -365,6 +365,17 @@ class LangOptionsBase { BKey }; + enum class PointerFieldProtectionKind { + /// Pointer field protection disabled + None, + /// Pointer field protection enabled, allocator does not tag heap + /// allocations. + Untagged, + /// Pointer field protection enabled, allocator is expected to tag heap + /// allocations. + Tagged, + }; + enum class ThreadModelKind { /// POSIX Threads. POSIX, diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def index 1bf9f43f80986..142e13b3ee784 100644 --- a/clang/include/clang/Basic/TokenKinds.def +++ b/clang/include/clang/Basic/TokenKinds.def @@ -542,6 +542,7 @@ TYPE_TRAIT_2(__is_pointer_interconvertible_base_of, IsPointerInterconvertibleBas // Clang-only C++ Type Traits TYPE_TRAIT_1(__is_trivially_relocatable, IsTriviallyRelocatable, KEYCXX) +TYPE_TRAIT_1(__has_non_relocatable_fields, HasNonRelocatableFields, KEYCXX) TYPE_TRAIT_1(__is_trivially_equality_comparable, IsTriviallyEqualityComparable, KEYCXX) TYPE_TRAIT_1(__is_bounded_array, IsBoundedArray, KEYCXX) TYPE_TRAIT_1(__is_unbounded_array, IsUnboundedArray, KEYCXX) diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index a7fcb160d3867..c903e0554319e 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -2957,6 +2957,12 @@ defm experimental_omit_vtable_rtti : BoolFOption<"experimental-omit-vtable-rtti" NegFlag<SetFalse, [], [CC1Option], "Do not omit">, BothFlags<[], [CC1Option], " the RTTI component from virtual tables">>; +def experimental_pointer_field_protection_EQ : Joined<["-"], "fexperimental-pointer-field-protection=">, Group<f_Group>, + Visibility<[ClangOption, CC1Option]>, + Values<"none,untagged,tagged">, NormalizedValuesScope<"LangOptions::PointerFieldProtectionKind">, + NormalizedValues<["None", "Untagged", "Tagged"]>, + MarshallingInfoEnum<LangOpts<"PointerFieldProtection">, "None">; + def fcxx_abi_EQ : Joined<["-"], "fc++-abi=">, Group<f_clang_Group>, Visibility<[ClangOption, CC1Option]>, HelpText<"C++ ABI to use. This will override the target C++ ABI.">; diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp index c9d1bea4c623a..c3cbfec2c93d3 100644 --- a/clang/lib/AST/ASTContext.cpp +++ b/clang/lib/AST/ASTContext.cpp @@ -79,6 +79,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/Frontend/OpenMP/OMPIRBuilder.h" #include "llvm/Support/Capacity.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MD5.h" @@ -14948,3 +14949,97 @@ bool ASTContext::useAbbreviatedThunkName(GlobalDecl VirtualMethodDecl, ThunksToBeAbbreviated[VirtualMethodDecl] = std::move(SimplifiedThunkNames); return Result; } + +bool ASTContext::arePFPFieldsTriviallyRelocatable(const RecordDecl *RD) const { + if (getLangOpts().getPointerFieldProtection() == + LangOptions::PointerFieldProtectionKind::Tagged) + return !isa<CXXRecordDecl>(RD) || + cast<CXXRecordDecl>(RD)->hasTrivialDestructor(); + return true; +} + +bool ASTContext::isPFPStruct(const RecordDecl *rec) const { + if (getLangOpts().getPointerFieldProtection() != + LangOptions::PointerFieldProtectionKind::None) + if (auto *cxxRec = dyn_cast<CXXRecordDecl>(rec)) + return !cxxRec->isStandardLayout(); + return false; +} + +void ASTContext::findPFPFields(QualType Ty, CharUnits Offset, + std::vector<PFPField> &Fields, + bool IncludeVBases) const { + if (auto *AT = getAsConstantArrayType(Ty)) { + if (auto *ElemDecl = AT->getElementType()->getAsCXXRecordDecl()) { + const ASTRecordLayout &ElemRL = getASTRecordLayout(ElemDecl); + for (unsigned i = 0; i != AT->getSize(); ++i) { + findPFPFields(AT->getElementType(), Offset + i * ElemRL.getSize(), + Fields, true); + } + } + } + auto *Decl = Ty->getAsCXXRecordDecl(); + if (!Decl) + return; + const ASTRecordLayout &RL = getASTRecordLayout(Decl); + for (FieldDecl *field : Decl->fields()) { + CharUnits fieldOffset = + Offset + toCharUnitsFromBits(RL.getFieldOffset(field->getFieldIndex())); + if (isPFPField(field)) + Fields.push_back({Offset, fieldOffset, field}); + findPFPFields(field->getType(), fieldOffset, Fields, true); + } + for (auto &Base : Decl->bases()) { + if (Base.isVirtual()) + continue; + CharUnits BaseOffset = + Offset + RL.getBaseClassOffset(Base.getType()->getAsCXXRecordDecl()); + findPFPFields(Base.getType(), BaseOffset, Fields, false); + } + if (IncludeVBases) { + for (auto &Base : Decl->vbases()) { + CharUnits BaseOffset = + Offset + RL.getVBaseClassOffset(Base.getType()->getAsCXXRecordDecl()); + findPFPFields(Base.getType(), BaseOffset, Fields, false); + } + } +} + +bool ASTContext::hasPFPFields(QualType ty) const { + std::vector<PFPField> pfpFields; + findPFPFields(ty, CharUnits::Zero(), pfpFields, true); + return !pfpFields.empty(); +} + +bool ASTContext::isPFPField(const FieldDecl *field) const { + if (!isPFPStruct(field->getParent())) + return false; + return field->getType()->isPointerType() && + !field->hasAttr<NoPointerFieldProtectionAttr>(); +} + +void ASTContext::recordMemberDataPointerEvaluation(const ValueDecl *VD) { + if (getLangOpts().getPointerFieldProtection() == + LangOptions::PointerFieldProtectionKind::None) + return; + auto *FD = dyn_cast<FieldDecl>(VD); + if (!FD) + FD = cast<FieldDecl>(cast<IndirectFieldDecl>(VD)->chain().back()); + if (!isPFPField(FD)) + return; + PFPFieldsWithEvaluatedOffset.insert(FD); +} + +void ASTContext::recordOffsetOfEvaluation(const OffsetOfExpr *E) { + if (getLangOpts().getPointerFieldProtection() == + LangOptions::PointerFieldProtectionKind::None || + E->getNumComponents() == 0) + return; + OffsetOfNode Comp = E->getComponent(E->getNumComponents() - 1); + if (Comp.getKind() != OffsetOfNode::Field) + return; + FieldDecl *FD = Comp.getField(); + if (!isPFPField(FD)) + return; + PFPFieldsWithEvaluatedOffset.insert(FD); +} diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp index 95da7b067b459..51f319fd26f20 100644 --- a/clang/lib/AST/ExprConstant.cpp +++ b/clang/lib/AST/ExprConstant.cpp @@ -14932,6 +14932,7 @@ bool IntExprEvaluator::VisitUnaryExprOrTypeTraitExpr( } bool IntExprEvaluator::VisitOffsetOfExpr(const OffsetOfExpr *OOE) { + Info.Ctx.recordOffsetOfEvaluation(OOE); CharUnits Result; unsigned n = OOE->getNumComponents(); if (n == 0) diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp index 08798219c0b83..ca7ccc7f2bb5a 100644 --- a/clang/lib/AST/Type.cpp +++ b/clang/lib/AST/Type.cpp @@ -2852,7 +2852,9 @@ bool QualType::isTriviallyRelocatableType(const ASTContext &Context) const { } else if (!BaseElementType->isObjectType()) { return false; } else if (const auto *RD = BaseElementType->getAsRecordDecl()) { - return RD->canPassInRegisters(); + return RD->canPassInRegisters() && + (Context.arePFPFieldsTriviallyRelocatable(RD) || + !Context.hasPFPFields(BaseElementType)); } else if (BaseElementType.isTriviallyCopyableType(Context)) { return true; } else { diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp index 3982ca3b50604..1382b4a9edfd4 100644 --- a/clang/lib/AST/TypePrinter.cpp +++ b/clang/lib/AST/TypePrinter.cpp @@ -2096,6 +2096,9 @@ void TypePrinter::printAttributedAfter(const AttributedType *T, case attr::ExtVectorType: OS << "ext_vector_type"; break; + case attr::NoPointerFieldProtection: + OS << "no_field_protection"; + break; } OS << "))"; } diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp index 7aa77e55dbfcc..9d824231d02cb 100644 --- a/clang/lib/CodeGen/CGCall.cpp +++ b/clang/lib/CodeGen/CGCall.cpp @@ -1298,7 +1298,8 @@ static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val, /// This safely handles the case when the src type is smaller than the /// destination type; in this situation the values of bits which not /// present in the src are undefined. -static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty, +static llvm::Value *CreateCoercedLoad(Address Src, QualType SrcFETy, + llvm::Type *Ty, CodeGenFunction &CGF) { llvm::Type *SrcTy = Src.getElementType(); @@ -1306,6 +1307,57 @@ static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty, if (SrcTy == Ty) return CGF.Builder.CreateLoad(Src); + // Coercion directly through memory does not work if the structure has pointer + // field protection because the struct in registers has a different bit + // pattern to the struct in memory, so we must read the elements one by one + // and use them to form the coerced structure. + std::vector<PFPField> PFPFields; + CGF.getContext().findPFPFields(SrcFETy, CharUnits::Zero(), PFPFields, true); + if (!PFPFields.empty()) { + auto LoadCoercedField = [&](CharUnits Offset, + llvm::Type *FieldType) -> llvm::Value * { + if (!PFPFields.empty() && PFPFields[0].offset == Offset) { + auto fieldAddr = CGF.EmitAddressOfPFPField(Src, PFPFields[0]); + llvm::Value *FieldVal = CGF.Builder.CreateLoad(fieldAddr); + if (isa<llvm::IntegerType>(FieldType)) + FieldVal = CGF.Builder.CreatePtrToInt(FieldVal, FieldType); + PFPFields.erase(PFPFields.begin()); + return FieldVal; + } + auto FieldAddr = CGF.Builder + .CreateConstInBoundsByteGEP( + Src.withElementType(CGF.Int8Ty), Offset) + .withElementType(FieldType); + return CGF.Builder.CreateLoad(FieldAddr); + }; + if (isa<llvm::IntegerType>(Ty) || isa<llvm::PointerType>(Ty)) { + auto Addr = CGF.EmitAddressOfPFPField(Src, PFPFields[0]); + llvm::Value *Val = CGF.Builder.CreateLoad(Addr); + if (isa<llvm::IntegerType>(Ty)) + Val = CGF.Builder.CreatePtrToInt(Val, Ty); + return Val; + } + if (auto *AT = dyn_cast<llvm::ArrayType>(Ty)) { + auto *ET = AT->getElementType(); + CharUnits wordSize = CGF.getContext().toCharUnitsFromBits( + CGF.CGM.getDataLayout().getTypeSizeInBits(ET)); + CharUnits Offset = CharUnits::Zero(); + llvm::Value *Val = llvm::UndefValue::get(AT); + for (unsigned i = 0; i != AT->getNumElements(); ++i, Offset += wordSize) + Val = CGF.Builder.CreateInsertValue(Val, LoadCoercedField(Offset, ET), i); + return Val; + } + auto *ST = cast<llvm::StructType>(Ty); + llvm::Value *Val = llvm::UndefValue::get(ST); + auto *SL = CGF.CGM.getDataLayout().getStructLayout(ST); + for (unsigned i = 0; i != ST->getNumElements(); ++i) { + CharUnits Offset = CharUnits::fromQuantity(SL->getElementOffset(i)); + Val = CGF.Builder.CreateInsertValue( + Val, LoadCoercedField(Offset, ST->getElementType(i)), i); + } + return Val; + } + llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty); if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) { @@ -1374,7 +1426,9 @@ static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty, return CGF.Builder.CreateLoad(Tmp); } -void CodeGenFunction::CreateCoercedStore(llvm::Value *Src, Address Dst, +void CodeGenFunction::CreateCoercedStore(llvm::Value *Src, + QualType SrcFETy, + Address Dst, llvm::TypeSize DstSize, bool DstIsVolatile) { if (!DstSize) @@ -1395,6 +1449,52 @@ void CodeGenFunction::CreateCoercedStore(llvm::Value *Src, Address Dst, } } + // Coercion directly through memory does not work if the structure has pointer + // field protection because the struct passed by value has a different bit + // pattern to the struct in memory, so we must read the elements one by one + // and use them to form the coerced structure. + std::vector<PFPField> PFPFields; + getContext().findPFPFields(SrcFETy, CharUnits::Zero(), PFPFields, true); + if (!PFPFields.empty()) { + auto StoreCoercedField = [&](CharUnits Offset, llvm::Value *FieldVal) { + if (!PFPFields.empty() && PFPFields[0].offset == Offset) { + auto fieldAddr = EmitAddressOfPFPField(Dst, PFPFields[0]); + if (isa<llvm::IntegerType>(FieldVal->getType())) + FieldVal = Builder.CreateIntToPtr(FieldVal, VoidPtrTy); + Builder.CreateStore(FieldVal, fieldAddr); + PFPFields.erase(PFPFields.begin()); + } else { + auto fieldAddr = + Builder + .CreateConstInBoundsByteGEP(Dst.withElementType(Int8Ty), Offset) + .withElementType(FieldVal->getType()); + Builder.CreateStore(FieldVal, fieldAddr); + } + }; + + if (isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy)) { + if (isa<llvm::IntegerType>(SrcTy)) + Src = Builder.CreateIntToPtr(Src, VoidPtrTy); + auto Addr = EmitAddressOfPFPField(Dst, PFPFields[0]); + Builder.CreateStore(Src, Addr); + } else if (auto *at = dyn_cast<llvm::ArrayType>(SrcTy)) { + auto *et = at->getElementType(); + CharUnits wordSize = getContext().toCharUnitsFromBits( + CGM.getDataLayout().getTypeSizeInBits(et)); + CharUnits Offset = CharUnits::Zero(); + for (unsigned i = 0; i != at->getNumElements(); ++i, Offset += wordSize) + StoreCoercedField(Offset, Builder.CreateExtractValue(Src, i)); + } else { + auto *ST = cast<llvm::StructType>(SrcTy); + auto *SL = CGM.getDataLayout().getStructLayout(ST); + for (unsigned i = 0; i != ST->getNumElements(); ++i) { + CharUnits Offset = CharUnits::fromQuantity(SL->getElementOffset(i)); + StoreCoercedField(Offset, Builder.CreateExtractValue(Src, i)); + } + } + return; + } + if (SrcSize.isScalable() || SrcSize <= DstSize) { if (SrcTy->isIntegerTy() && Dst.getElementType()->isPointerTy() && SrcSize == CGM.getDataLayout().getTypeAllocSize(Dst.getElementType())) { @@ -3347,7 +3447,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, auto AI = Fn->getArg(FirstIRArg); AI->setName(Arg->getName() + ".coerce"); CreateCoercedStore( - AI, Ptr, + AI, Ty, Ptr, llvm::TypeSize::getFixed( getContext().getTypeSizeInChars(Ty).getQuantity() - ArgI.getDirectOffset()), @@ -3969,7 +4069,7 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI, // If the value is offset in memory, apply the offset now. Address V = emitAddressAtOffset(*this, ReturnValue, RetAI); - RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this); + RV = CreateCoercedLoad(V, RetTy, RetAI.getCoerceToType(), *this); } // In ARC, end functions that return a retainable type with a call @@ -4020,6 +4120,7 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI, auto eltAddr = Builder.CreateStructGEP(addr, i); llvm::Value *elt = CreateCoercedLoad( eltAddr, + RetTy, unpaddedStruct ? unpaddedStruct->getElementType(unpaddedIndex++) : unpaddedCoercionType, *this); @@ -5550,7 +5651,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // In the simple case, just pass the coerced loaded value. assert(NumIRArgs == 1); llvm::Value *Load = - CreateCoercedLoad(Src, ArgInfo.getCoerceToType(), *this); + CreateCoercedLoad(Src, I->Ty, ArgInfo.getCoerceToType(), *this); if (CallInfo.isCmseNSCall()) { // For certain parameter types, clear padding bits, as they may reveal @@ -5611,6 +5712,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, Address eltAddr = Builder.CreateStructGEP(addr, i); llvm::Value *elt = CreateCoercedLoad( eltAddr, + I->Ty, unpaddedStruct ? unpaddedStruct->getElementType(unpaddedIndex++) : unpaddedCoercionType, *this); @@ -6105,7 +6207,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo... [truncated] `````````` </details> https://github.com/llvm/llvm-project/pull/133538 _______________________________________________ llvm-branch-commits mailing list llvm-branch-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
