Author: Andy Kaylor Date: 2026-10-09T09:58:02-07:00 New Revision: 8e242e6d18d94463092b27d52d022275fbdb13c8
URL: https://github.com/llvm/llvm-project/commit/8e242e6d18d94463092b27d52d022275fbdb13c8 DIFF: https://github.com/llvm/llvm-project/commit/8e242e6d18d94463092b27d52d022275fbdb13c8.diff LOG: [LLVMABI][AARCH64] Implement Pure Scalable Type handling (#227504) This change adds support for classifying Pure Scalable Type arguments for AArch64 targets in the LLVM ABI library. Pure Scalable Types are passed in registers, expanded if necessary, unless the argument is unnamed or there are not sufficient registers available, in which case they are passed indirectly. Pure Scalable Types are treated as single named arguments when used as return types. Assisted-by: Cursor / various models Added: clang/test/CodeGen/AArch64/abi-classify-pure-scalable.c Modified: clang/lib/CIR/Dialect/Transforms/CallConvLoweringPass.cpp clang/lib/CodeGen/CGCall.cpp llvm/include/llvm/ABI/FunctionInfo.h llvm/include/llvm/ABI/TargetInfo.h llvm/lib/ABI/IRTypeMapper.cpp llvm/lib/ABI/TargetInfo.cpp llvm/lib/ABI/Targets/AArch64.cpp llvm/unittests/ABI/AArch64TargetInfoTest.cpp llvm/unittests/ABI/IRTypeMapperTest.cpp llvm/unittests/ABI/TargetInfoTest.cpp Removed: ################################################################################ diff --git a/clang/lib/CIR/Dialect/Transforms/CallConvLoweringPass.cpp b/clang/lib/CIR/Dialect/Transforms/CallConvLoweringPass.cpp index 9777f9bb1b5861..85e3f52bb1587d 100644 --- a/clang/lib/CIR/Dialect/Transforms/CallConvLoweringPass.cpp +++ b/clang/lib/CIR/Dialect/Transforms/CallConvLoweringPass.cpp @@ -630,6 +630,10 @@ convertABIArgInfo(const llvm::abi::ArgInfo &info, MLIRContext *ctx, if (info.isIndirect()) return ArgClassification::getIndirect(info.getIndirectAlign(), info.getIndirectByVal()); + // AArch64 pure scalable aggregates use CoerceAndExpand. This bridge lowers + // x86_64 classifications only. + assert(!info.isCoerceAndExpand() && + "CoerceAndExpand is not expected for x86_64"); assert(info.isIgnore() && "Unexpected classification"); return ArgClassification::getIgnore(); } diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp index a8fc26b447208d..1f4fd55060d39d 100644 --- a/clang/lib/CodeGen/CGCall.cpp +++ b/clang/lib/CodeGen/CGCall.cpp @@ -991,6 +991,16 @@ void CodeGenModule::computeABIInfoUsingLib(CGFunctionInfo &FI) { << ")"; }); break; + case ABIArgInfo::CoerceAndExpand: + Check(Target.getUnpaddedCoerceAndExpandType() == + Res.getUnpaddedCoerceAndExpandType(), + [&]() { + llvm::dbgs() << "UnpaddedCoerceAndExpandType mismatch (expected: " + << *Target.getUnpaddedCoerceAndExpandType() + << ", given: " + << *Res.getUnpaddedCoerceAndExpandType() << ")"; + }); + break; default: break; } @@ -1064,6 +1074,13 @@ ABIArgInfo CodeGenModule::convertABIArgInfo(const llvm::abi::ArgInfo &AbiInfo, } case llvm::abi::ArgInfo::Ignore: return ABIArgInfo::getIgnore(); + case llvm::abi::ArgInfo::CoerceAndExpand: { + llvm::StructType *CoercedType = llvm::cast<llvm::StructType>( + AbiReverseMapper->convertType(AbiInfo.getCoerceToType())); + llvm::Type *UnpaddedType = + AbiReverseMapper->convertType(AbiInfo.getUnpaddedCoerceAndExpandType()); + return ABIArgInfo::getCoerceAndExpand(CoercedType, UnpaddedType); + } } llvm_unreachable("Unexpected llvm::abi::ArgInfo kind"); } diff --git a/clang/test/CodeGen/AArch64/abi-classify-pure-scalable.c b/clang/test/CodeGen/AArch64/abi-classify-pure-scalable.c new file mode 100644 index 00000000000000..847930d98388b0 --- /dev/null +++ b/clang/test/CodeGen/AArch64/abi-classify-pure-scalable.c @@ -0,0 +1,204 @@ +// RUN: %clang_cc1 -triple aarch64-linux-gnu -target-feature +sve -mvscale-min=1 -mvscale-max=1 -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK128 +// RUN: %clang_cc1 -triple aarch64-linux-gnu -target-feature +sve -mvscale-min=1 -mvscale-max=1 -fexperimental-abi-lowering -emit-llvm -o - %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK128 --implicit-check-not="not yet implemented" +// RUN: %clang_cc1 -triple aarch64_be-linux-gnu -target-feature +sve -mvscale-min=1 -mvscale-max=1 -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK128 +// RUN: %clang_cc1 -triple aarch64_be-linux-gnu -target-feature +sve -mvscale-min=1 -mvscale-max=1 -fexperimental-abi-lowering -emit-llvm -o - %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK128 --implicit-check-not="not yet implemented" +// RUN: %clang_cc1 -triple aarch64-linux-gnu -target-feature +sve -mvscale-min=2 -mvscale-max=2 -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK256 +// RUN: %clang_cc1 -triple aarch64-linux-gnu -target-feature +sve -mvscale-min=2 -mvscale-max=2 -fexperimental-abi-lowering -emit-llvm -o - %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK256 --implicit-check-not="not yet implemented" +// RUN: %clang_cc1 -triple aarch64_be-linux-gnu -target-feature +sve -mvscale-min=2 -mvscale-max=2 -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK256 +// RUN: %clang_cc1 -triple aarch64_be-linux-gnu -target-feature +sve -mvscale-min=2 -mvscale-max=2 -fexperimental-abi-lowering -emit-llvm -o - %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK256 --implicit-check-not="not yet implemented" +// RUN: %clang_cc1 -triple aarch64-linux-gnu -target-feature +sve -mvscale-min=4 -mvscale-max=4 -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK512 +// RUN: %clang_cc1 -triple aarch64-linux-gnu -target-feature +sve -mvscale-min=4 -mvscale-max=4 -fexperimental-abi-lowering -emit-llvm -o - %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK512 --implicit-check-not="not yet implemented" +// RUN: %clang_cc1 -triple aarch64_be-linux-gnu -target-feature +sve -mvscale-min=4 -mvscale-max=4 -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK512 +// RUN: %clang_cc1 -triple aarch64_be-linux-gnu -target-feature +sve -mvscale-min=4 -mvscale-max=4 -fexperimental-abi-lowering -emit-llvm -o - %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK512 --implicit-check-not="not yet implemented" + +// A Pure Scalable Type is passed expanded into one scalable vector argument +// per SVE vector or predicate member. A returned Pure Scalable Type with more +// than one member is a packed struct of those vectors. The type is passed +// indirectly if it is unnamed or if it does not fit in the remaining Z (8) or +// P (4) argument registers. +// +// The register sequence is the same at every vector length, so those checks +// are shared. The in-memory layout the sequence is coerced from is not, so a +// check that names it is per-length. + +#define N __ARM_FEATURE_SVE_BITS + +typedef __SVInt32_t fixed_int32_t __attribute__((arm_sve_vector_bits(N))); +typedef __SVFloat64_t fixed_float64_t __attribute__((arm_sve_vector_bits(N))); +typedef __SVBool_t fixed_bool_t __attribute__((arm_sve_vector_bits(N))); + +typedef struct { fixed_int32_t x; } One; +typedef struct { fixed_int32_t a; fixed_float64_t b; } Two; +typedef struct { fixed_bool_t p; fixed_int32_t v; } PredAndData; +typedef struct { fixed_bool_t a; fixed_bool_t b; } TwoPred; +typedef struct { fixed_bool_t p; } OnePred; +typedef struct { fixed_int32_t x[2]; } Arr2; +typedef struct { Two inner; } Nested; +typedef struct { fixed_int32_t x[9]; } Nine; +typedef struct { fixed_int32_t x; union {} u; } WithEmpty; +typedef struct { fixed_int32_t x; int zla[0]; } WithZero; + +// CHECK-LABEL: define{{.*}} void @arg_one(<vscale x 4 x i32> %{{.*}}) +void arg_one(One s) {} + +// CHECK-LABEL: define{{.*}} <vscale x 4 x i32> @ret_one(<vscale x 4 x i32> %{{.*}}) +One ret_one(One s) { return s; } + +// CHECK-LABEL: define{{.*}} void @arg_two(<vscale x 4 x i32> %{{.*}}, <vscale x 2 x double> %{{.*}}) +void arg_two(Two s) {} + +// CHECK-LABEL: define{{.*}} <{ <vscale x 4 x i32>, <vscale x 2 x double> }> @ret_two(<vscale x 4 x i32> %{{.*}}, <vscale x 2 x double> %{{.*}}) +Two ret_two(Two s) { return s; } + +// Fixed-length predicates are passed as <vscale x 16 x i1>. + +// CHECK-LABEL: define{{.*}} void @arg_pred(<vscale x 16 x i1> %{{.*}}, <vscale x 4 x i32> %{{.*}}) +void arg_pred(PredAndData s) {} + +// CHECK-LABEL: define{{.*}} <{ <vscale x 16 x i1>, <vscale x 4 x i32> }> @ret_pred(<vscale x 16 x i1> %{{.*}}, <vscale x 4 x i32> %{{.*}}) +PredAndData ret_pred(PredAndData s) { return s; } + +// Arrays and nested records are flattened. + +// CHECK-LABEL: define{{.*}} void @arg_arr(<vscale x 4 x i32> %{{.*}}, <vscale x 4 x i32> %{{.*}}) +void arg_arr(Arr2 a) {} + +// CHECK-LABEL: define{{.*}} <{ <vscale x 4 x i32>, <vscale x 4 x i32> }> @ret_arr(<vscale x 4 x i32> %{{.*}}, <vscale x 4 x i32> %{{.*}}) +Arr2 ret_arr(Arr2 a) { return a; } + +// CHECK-LABEL: define{{.*}} void @arg_nested(<vscale x 4 x i32> %{{.*}}, <vscale x 2 x double> %{{.*}}) +void arg_nested(Nested s) {} + +// A zero-length array is empty and does not disqualify a Pure Scalable Type. + +// CHECK-LABEL: define{{.*}} void @arg_zero(<vscale x 4 x i32> %{{.*}}) +void arg_zero(WithZero s) {} + +// CHECK-LABEL: define{{.*}} <vscale x 4 x i32> @ret_zero(<vscale x 4 x i32> %{{.*}}) +WithZero ret_zero(WithZero s) { return s; } + +// An empty C union member is skipped. + +// CHECK-LABEL: define{{.*}} void @arg_empty(<vscale x 4 x i32> %{{.*}}) +void arg_empty(WithEmpty s) {} + +// CHECK-LABEL: define{{.*}} <vscale x 4 x i32> @ret_empty(<vscale x 4 x i32> %{{.*}}) +WithEmpty ret_empty(WithEmpty s) { return s; } + +// A type with more than 8 data vectors never fits in registers. + +// CHECK128-LABEL: define{{.*}} void @arg_nine(ptr {{.*}}align 16 dead_on_return dereferenceable(144) %{{.*}}) +// CHECK256-LABEL: define{{.*}} void @arg_nine(ptr {{.*}}align 16 dead_on_return dereferenceable(288) %{{.*}}) +// CHECK512-LABEL: define{{.*}} void @arg_nine(ptr {{.*}}align 16 dead_on_return dereferenceable(576) %{{.*}}) +void arg_nine(Nine s) {} + +// CHECK-LABEL: define{{.*}} void @ret_nine(ptr {{.*}}sret(%struct.Nine) align 16 %{{.*}}) +Nine ret_nine(void) { Nine s = {}; return s; } + +// A Pure Scalable Type that does not fit in the remaining registers is passed +// indirectly and does not consume registers, so a later one can still fit. + +// CHECK128-LABEL: define{{.*}} void @budget(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d, <vscale x 4 x i32> %e, <vscale x 4 x i32> %f, <vscale x 4 x i32> %g, ptr {{.*}}align 16 dead_on_return dereferenceable(32) %two, <vscale x 4 x i32> %{{.*}}) +// CHECK256-LABEL: define{{.*}} void @budget(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d, <vscale x 4 x i32> %e, <vscale x 4 x i32> %f, <vscale x 4 x i32> %g, ptr {{.*}}align 16 dead_on_return dereferenceable(64) %two, <vscale x 4 x i32> %{{.*}}) +// CHECK512-LABEL: define{{.*}} void @budget(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d, <vscale x 4 x i32> %e, <vscale x 4 x i32> %f, <vscale x 4 x i32> %g, ptr {{.*}}align 16 dead_on_return dereferenceable(128) %two, <vscale x 4 x i32> %{{.*}}) +void budget(__SVInt32_t a, __SVInt32_t b, __SVInt32_t c, __SVInt32_t d, + __SVInt32_t e, __SVInt32_t f, __SVInt32_t g, Two two, One one) {} + +// CHECK128-LABEL: define{{.*}} void @pred_budget(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b, <vscale x 16 x i1> %c, ptr {{.*}}align 2 dead_on_return dereferenceable(4) %both, <vscale x 16 x i1> %{{.*}}) +// CHECK256-LABEL: define{{.*}} void @pred_budget(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b, <vscale x 16 x i1> %c, ptr {{.*}}align 2 dead_on_return dereferenceable(8) %both, <vscale x 16 x i1> %{{.*}}) +// CHECK512-LABEL: define{{.*}} void @pred_budget(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b, <vscale x 16 x i1> %c, ptr {{.*}}align 2 dead_on_return dereferenceable(16) %both, <vscale x 16 x i1> %{{.*}}) +void pred_budget(__SVBool_t a, __SVBool_t b, __SVBool_t c, + TwoPred both, OnePred one) {} + +// An unnamed Pure Scalable Type is passed indirectly. + +void variadic_callee(One named, ...); + +// CHECK-LABEL: define{{.*}} void @test_variadic( +// CHECK128: call void (<vscale x 4 x i32>, ...) @variadic_callee(<vscale x 4 x i32> %{{.*}}, ptr {{.*}}align 16 dead_on_return dereferenceable(16) %{{.*}}) +// CHECK256: call void (<vscale x 4 x i32>, ...) @variadic_callee(<vscale x 4 x i32> %{{.*}}, ptr {{.*}}align 16 dead_on_return dereferenceable(32) %{{.*}}) +// CHECK512: call void (<vscale x 4 x i32>, ...) @variadic_callee(<vscale x 4 x i32> %{{.*}}, ptr {{.*}}align 16 dead_on_return dereferenceable(64) %{{.*}}) +void test_variadic(One *a, One *b) { variadic_callee(*a, *b); } + +// An SVE tuple is passed directly if it fits in the remaining registers and +// indirectly otherwise. + +// CHECK-LABEL: define{{.*}} void @arg_tuple(<vscale x 4 x i32> %{{.*}}, <vscale x 4 x i32> %{{.*}}) +void arg_tuple(__clang_svint32x2_t v) {} + +// CHECK-LABEL: define{{.*}} { <vscale x 4 x i32>, <vscale x 4 x i32> } @ret_tuple(<vscale x 4 x i32> %{{.*}}, <vscale x 4 x i32> %{{.*}}) +__clang_svint32x2_t ret_tuple(__clang_svint32x2_t v) { return v; } + +// CHECK-LABEL: define{{.*}} void @tuple_budget(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d, <vscale x 4 x i32> %e, <vscale x 4 x i32> %f, <vscale x 4 x i32> %g, ptr {{.*}}align 16 dead_on_return %{{.*}}) +void tuple_budget(__SVInt32_t a, __SVInt32_t b, __SVInt32_t c, __SVInt32_t d, + __SVInt32_t e, __SVInt32_t f, __SVInt32_t g, + __clang_svint32x2_t t) {} + +// CHECK-LABEL: define{{.*}} void @pred_tuple(<vscale x 16 x i1> %{{.*}}, <vscale x 16 x i1> %{{.*}}, <vscale x 16 x i1> %{{.*}}, <vscale x 16 x i1> %{{.*}}, ptr {{.*}}align 2 dead_on_return %{{.*}}) +void pred_tuple(__clang_svboolx4_t a, __clang_svboolx2_t b) {} + +// A fixed-length SVE type is 16-byte aligned however wide it is, but the +// vector it lowers to is aligned to its size rounded up to a power of two. A +// record holding one wider than 16 bytes is therefore packed once lowered, and +// every gap in a packed record is an explicit array. + +// A packed record is lowered with explicit padding for every gap, including +// one that falls at an offset the source alignment already reaches. The +// coerce type carries those arrays. The return type is the unpadded register +// sequence. + +typedef struct __attribute__((packed)) { + fixed_bool_t p; + fixed_int32_t misaligned; + fixed_int32_t natural __attribute__((aligned(16))); + fixed_bool_t tail; +} PackedGap; + +// CHECK-LABEL: define{{.*}} <{ <vscale x 16 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 16 x i1> }> @ret_packed_gap() +// CHECK128: getelementptr inbounds nuw { <2 x i8>, <4 x i32>, [14 x i8], <4 x i32>, <2 x i8>, [14 x i8] }, ptr %retval, i32 0, i32 0 +// CHECK256: getelementptr inbounds nuw { <4 x i8>, <8 x i32>, [12 x i8], <8 x i32>, <4 x i8>, [12 x i8] }, ptr %retval, i32 0, i32 0 +// CHECK512: getelementptr inbounds nuw { <8 x i8>, <16 x i32>, [8 x i8], <16 x i32>, <8 x i8>, [8 x i8] }, ptr %retval, i32 0, i32 0 +PackedGap ret_packed_gap(void) { PackedGap s = {}; return s; } + +// CHECK-LABEL: define{{.*}} void @arg_packed_gap(<vscale x 16 x i1> %{{.*}}, <vscale x 4 x i32> %{{.*}}, <vscale x 4 x i32> %{{.*}}, <vscale x 16 x i1> %{{.*}}) +// CHECK128: getelementptr inbounds nuw { <2 x i8>, <4 x i32>, [14 x i8], <4 x i32>, <2 x i8>, [14 x i8] }, ptr %s, i32 0, i32 0 +// CHECK256: getelementptr inbounds nuw { <4 x i8>, <8 x i32>, [12 x i8], <8 x i32>, <4 x i8>, [12 x i8] }, ptr %s, i32 0, i32 0 +// CHECK512: getelementptr inbounds nuw { <8 x i8>, <16 x i32>, [8 x i8], <16 x i32>, <8 x i8>, [8 x i8] }, ptr %s, i32 0, i32 0 +void arg_packed_gap(PackedGap s) {} + +// A trailing predicate leaves the record with tail padding. A 16-byte data +// vector rounds the members up to the record size and the coerce type has no +// array for the tail. A wider one packs the record, and then the tail is an +// explicit array. + +typedef struct { fixed_int32_t v; fixed_bool_t p; } DataThenPred; + +// CHECK-LABEL: define{{.*}} <{ <vscale x 4 x i32>, <vscale x 16 x i1> }> @ret_data_then_pred(<vscale x 4 x i32> %{{.*}}, <vscale x 16 x i1> %{{.*}}) +// CHECK128: getelementptr inbounds nuw { <4 x i32>, <2 x i8> }, ptr %s, i32 0, i32 0 +// CHECK256: getelementptr inbounds nuw { <8 x i32>, <4 x i8>, [12 x i8] }, ptr %s, i32 0, i32 0 +// CHECK512: getelementptr inbounds nuw { <16 x i32>, <8 x i8>, [8 x i8] }, ptr %s, i32 0, i32 0 +DataThenPred ret_data_then_pred(DataThenPred s) { return s; } + +// An over-aligned record is not packed. Its tail padding is an explicit array +// in the coerce type where the member alignment does not account for it, and +// a data vector at least as wide as the record alignment leaves no tail at +// all. + +typedef struct __attribute__((aligned(32))) { fixed_int32_t x; } OverAligned; + +// CHECK-LABEL: define{{.*}} <vscale x 4 x i32> @ret_overaligned(<vscale x 4 x i32> %{{.*}}) +// CHECK128: getelementptr inbounds nuw { <4 x i32>, [16 x i8] }, ptr %s, i32 0, i32 0 +// CHECK256: getelementptr inbounds nuw { <8 x i32> }, ptr %s, i32 0, i32 0 +// CHECK512: getelementptr inbounds nuw { <16 x i32> }, ptr %s, i32 0, i32 0 +OverAligned ret_overaligned(OverAligned s) { return s; } + +typedef struct __attribute__((aligned(64))) { + fixed_int32_t v; + fixed_bool_t p; +} OverAlignedTail; + +// CHECK-LABEL: define{{.*}} <{ <vscale x 4 x i32>, <vscale x 16 x i1> }> @ret_overaligned_tail(<vscale x 4 x i32> %{{.*}}, <vscale x 16 x i1> %{{.*}}) +// CHECK128: getelementptr inbounds nuw { <4 x i32>, <2 x i8>, [46 x i8] }, ptr %s, i32 0, i32 0 +// CHECK256: getelementptr inbounds nuw { <8 x i32>, <4 x i8>, [28 x i8] }, ptr %s, i32 0, i32 0 +// CHECK512: getelementptr inbounds nuw { <16 x i32>, <8 x i8>, [56 x i8] }, ptr %s, i32 0, i32 0 +OverAlignedTail ret_overaligned_tail(OverAlignedTail s) { return s; } diff --git a/llvm/include/llvm/ABI/FunctionInfo.h b/llvm/include/llvm/ABI/FunctionInfo.h index 1450694ea06ae9..9edd588c480018 100644 --- a/llvm/include/llvm/ABI/FunctionInfo.h +++ b/llvm/include/llvm/ABI/FunctionInfo.h @@ -44,10 +44,16 @@ class ArgInfo { IndirectAliased, /// Ignore the argument (treat as void). Useful for void and empty structs. Ignore, + + /// Only valid for aggregate argument types. The value is expanded into + /// one argument per non-padding element of CoerceToType. Arrays of i8 are + /// assumed to be padding. + CoerceAndExpand, }; private: const Type *CoercionType = nullptr; + const Type *UnpaddedCoerceAndExpandType = nullptr; // Alignment is optional for direct arguments, but required for indirect // arguments. This invariant is enforced by the methods of this class. // @@ -148,6 +154,24 @@ class ArgInfo { static ArgInfo getIgnore() { return ArgInfo(Ignore); } + /// \param CoerceToType Aggregate type whose non-padding elements are the + /// in-memory pieces of the value, in order. Padding elements are + /// arrays of i8. + /// \param UnpaddedCoerceToType \p CoerceToType with padding elements + /// removed. A single element is stored as that element; two or more + /// are a packed record. + static ArgInfo getCoerceAndExpand(const Type *CoerceToType, + const Type *UnpaddedCoerceToType) { + assert(CoerceToType && UnpaddedCoerceToType && + "coerce-and-expand types cannot be null"); + assert(isa<RecordType>(CoerceToType) && + "coerce-and-expand coerce type must be a record"); + ArgInfo AI(CoerceAndExpand); + AI.CoercionType = CoerceToType; + AI.UnpaddedCoerceAndExpandType = UnpaddedCoerceToType; + return AI; + } + ArgInfo &setSignExt(bool SignExtend = true) { this->SignExt = SignExtend; if (SignExtend) @@ -175,6 +199,7 @@ class ArgInfo { bool isIndirectAliased() const { return TheKind == IndirectAliased; } bool isIgnore() const { return TheKind == Ignore; } bool isExtend() const { return TheKind == Extend; } + bool isCoerceAndExpand() const { return TheKind == CoerceAndExpand; } unsigned getDirectOffset() const { assert((isDirect() || isExtend()) && "Not a direct or extend kind"); @@ -244,9 +269,26 @@ class ArgInfo { } const Type *getCoerceToType() const { - assert((isDirect() || isExtend()) && "Invalid Kind!"); + assert((isDirect() || isExtend() || isCoerceAndExpand()) && + "Invalid Kind!"); return CoercionType; } + + const Type *getUnpaddedCoerceAndExpandType() const { + assert(isCoerceAndExpand() && "Invalid Kind!"); + return UnpaddedCoerceAndExpandType; + } + + /// True if \p Ty is an array of i8, the padding element of a + /// coerce-and-expand type. A matrix is an array in this type system + /// and lowers to a vector, so it is not padding. + static bool isPaddingForCoerceAndExpand(const Type *Ty) { + const auto *AT = dyn_cast<ArrayType>(Ty); + if (!AT || AT->isMatrixType()) + return false; + const auto *Elt = dyn_cast<IntegerType>(AT->getElementType()); + return Elt && Elt->getSizeInBits().getFixedValue() == 8; + } }; struct ArgEntry { diff --git a/llvm/include/llvm/ABI/TargetInfo.h b/llvm/include/llvm/ABI/TargetInfo.h index f7820caff567e4..859eca06ba785a 100644 --- a/llvm/include/llvm/ABI/TargetInfo.h +++ b/llvm/include/llvm/ABI/TargetInfo.h @@ -90,6 +90,11 @@ class TargetInfo { /// Width of `long long` in bits. Default 64. virtual unsigned getLongLongWidth() const { return 64; } + /// Alignment of the widest integer the target's data layout gives an + /// alignment to. An integer wider than that one is aligned to this. Default + /// 16, the alignment of i128 on AArch64 and x86-64. + virtual llvm::Align getMaxIntegerAlign() const { return llvm::Align(16); } + protected: LLVM_ABI RecordArgABI getRecordArgABI(const RecordType *RT) const; LLVM_ABI RecordArgABI getRecordArgABI(const Type *Ty) const; @@ -101,6 +106,28 @@ class TargetInfo { } LLVM_ABI ArgInfo getNaturalAlignIndirect(const Type *Ty, unsigned AddrSpace, bool ByVal = true) const; + + /// An array of \p NumBytes i8 elements, the padding element of a + /// coerce-and-expand type. + LLVM_ABI const Type *getI8Array(uint64_t NumBytes) const; + + /// In-memory form of \p Ty. A record becomes one field list: non-empty + /// bases and fields in offset order. Padding is an array of i8. The result + /// is packed when a member offset or the record size is not a multiple of + /// the converted member alignment, and then every gap is an explicit array. + /// Any other record has an array only where that alignment does not already + /// produce the gap, and for the tail that alignment is capped at + /// getMaxIntegerAlign(). A record with a virtual base has no layout here. An + /// array is rebuilt when its element type changes. Any other type is + /// returned unchanged. + LLVM_ABI const Type *convertTypeForMem(const Type *Ty) const; + + /// A record with one field per element of \p Elems, each at offset 0. + /// \p Packed selects a packed record, which has no alignment padding + /// between fields. + LLVM_ABI const Type *getStructOfTypes(ArrayRef<const Type *> Elems, + bool Packed) const; + LLVM_ABI bool isAggregateTypeForABI(const Type *Ty) const; /// If Ty is a transparent union, return its first field type; otherwise diff --git a/llvm/lib/ABI/IRTypeMapper.cpp b/llvm/lib/ABI/IRTypeMapper.cpp index 0045344ba58b5b..61e0c54d8a2a5c 100644 --- a/llvm/lib/ABI/IRTypeMapper.cpp +++ b/llvm/lib/ABI/IRTypeMapper.cpp @@ -111,6 +111,15 @@ llvm::Type *IRTypeMapper::convertTupleType(const abi::TupleType *TT) { } llvm::Type *IRTypeMapper::convertRecordType(const abi::RecordType *RT) { + // A packed record is a sequence of fields with no alignment padding. Pure + // scalable aggregates use one as the unpadded coerce-and-expand type, and + // its fields may be scalable vectors. + if (RT->getPacking() == StructPacking::Packed) { + SmallVector<llvm::Type *, 8> FieldTypes; + for (const FieldInfo &Field : RT->getFields()) + FieldTypes.push_back(convertType(Field.FieldType)); + return llvm::StructType::get(Context, FieldTypes, /*isPacked=*/true); + } return createStructFromFields(RT->getFields(), RT->getSizeInBits(), RT->getAlignment(), RT->isUnion()); } diff --git a/llvm/lib/ABI/TargetInfo.cpp b/llvm/lib/ABI/TargetInfo.cpp index eda9c903453a6d..2597597057320b 100644 --- a/llvm/lib/ABI/TargetInfo.cpp +++ b/llvm/lib/ABI/TargetInfo.cpp @@ -7,7 +7,10 @@ //===----------------------------------------------------------------------===// #include "llvm/ABI/TargetInfo.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/Support/Casting.h" +#include "llvm/Support/MathExtras.h" #include <algorithm> #include <cstdint> @@ -47,6 +50,152 @@ ArgInfo TargetInfo::getNaturalAlignIndirect(const Type *Ty, unsigned AddrSpace, return ArgInfo::getIndirect(Ty->getAlignment(), ByVal, AddrSpace); } +const Type *TargetInfo::getI8Array(uint64_t NumBytes) const { + assert(NumBytes != 0 && "empty padding"); + const Type *I8 = TB.getIntegerType(8, llvm::Align(1), /*Signed=*/false); + return TB.getArrayType(I8, NumBytes, NumBytes * 8); +} + +const Type *TargetInfo::getStructOfTypes(llvm::ArrayRef<const Type *> Elems, + bool Packed) const { + assert(!Elems.empty() && "empty coerce sequence"); + llvm::SmallVector<FieldInfo, 8> Fields; + Fields.reserve(Elems.size()); + for (const Type *Elt : Elems) + Fields.emplace_back(Elt, /*OffsetInBits=*/0); + + StructPacking Pack = Packed ? StructPacking::Packed : StructPacking::Default; + return TB.getRecordType(Fields, llvm::TypeSize::getFixed(0), llvm::Align(1), + /*UnadjustedAlign=*/llvm::Align(1), Pack); +} + +// Returns the alignment of Ty, a type returned by convertTypeForMem, as a +// member of the struct built there. A packed record has alignment 1. A record +// that is not packed has the alignment of its most-aligned member. An array has +// the alignment of its element type. A fixed-length vector is aligned to its +// size rounded up to a power of two, which for an AArch64 fixed-length SVE type +// is wider than the 2 or 16 bytes the source language gives it. +static llvm::Align getConvertedAlign(const Type *Ty) { + if (const auto *AT = dyn_cast<ArrayType>(Ty)) + return getConvertedAlign(AT->getElementType()); + + if (const auto *VT = dyn_cast<VectorType>(Ty); VT && VT->isFixedLength()) + return llvm::Align(VT->getABISizeInBits() / 8); + + const auto *RT = dyn_cast<RecordType>(Ty); + if (!RT) + return Ty->getAlignment(); + if (RT->getPacking() == StructPacking::Packed) + return llvm::Align(1); + + llvm::Align MaxAlign(1); + for (llvm::ArrayRef<FieldInfo> Members : + {RT->getFields(), RT->getBaseClasses()}) { + for (const FieldInfo &Member : Members) { + if (!Member.isEmpty()) + MaxAlign = std::max(MaxAlign, getConvertedAlign(Member.FieldType)); + } + } + return MaxAlign; +} + +const Type *TargetInfo::convertTypeForMem(const Type *Ty) const { + if (const auto *AT = dyn_cast<ArrayType>(Ty)) { + if (AT->isMatrixType()) + return Ty; + const Type *Elt = convertTypeForMem(AT->getElementType()); + if (Elt == AT->getElementType()) + return Ty; + assert(AT->getSizeInBits().isFixed() && + "converted array element changes a scalable size"); + return TB.getArrayType(Elt, AT->getNumElements(), + AT->getSizeInBits().getFixedValue()); + } + + const auto *RT = dyn_cast<RecordType>(Ty); + if (!RT || RT->isUnion()) + return Ty; + + // Current callers can't get here with virtual bases. If we need to handle + // virtual bases in the future, we'll need explicit handling for that below. + assert(RT->getNumVirtualBaseClasses() == 0 && "record has a virtual base"); + + struct ConvertedMember { + const Type *Ty; + uint64_t Offset; + llvm::Align Alignment; + }; + llvm::SmallVector<ConvertedMember, 8> Members; + // The record is packed when a member offset or the record size is not a + // multiple of the converted alignment. + bool Packed = false; + llvm::Align MaxAlign(1); + auto addMember = [&](const Type *MemberTy, uint64_t Offset) { + const Type *ConvertedTy = convertTypeForMem(MemberTy); + assert(!ConvertedTy->getSizeInBits().isScalable() && + "scalable member has no fixed offset"); + llvm::Align Alignment = getConvertedAlign(ConvertedTy); + if (Offset % (Alignment.value() * 8) != 0) + Packed = true; + MaxAlign = std::max(MaxAlign, Alignment); + Members.push_back({ConvertedTy, Offset, Alignment}); + }; + for (const FieldInfo &Base : RT->getBaseClasses()) { + if (!Base.FieldType->isEmptyRecord()) + addMember(Base.FieldType, Base.OffsetInBits); + } + for (const FieldInfo &Field : RT->getFields()) { + if (!Field.isEmpty()) + addMember(Field.FieldType, Field.OffsetInBits); + } + llvm::stable_sort(Members, + [](const ConvertedMember &A, const ConvertedMember &B) { + return A.Offset < B.Offset; + }); + + llvm::TypeSize RecordSize = RT->getSizeInBits(); + if (RecordSize.isFixed() && + RecordSize.getFixedValue() % (MaxAlign.value() * 8) != 0) + Packed = true; + + llvm::SmallVector<FieldInfo, 8> Fields; + uint64_t Current = 0; + // Padding in a packed record is explicit for every gap. Padding in any + // other record is explicit only where the converted alignment does not place + // the next member. + auto needsPadding = [&](uint64_t Offset, llvm::Align Alignment) { + uint64_t AlignBits = Packed ? 8 : Alignment.value() * 8; + return Offset != llvm::alignTo(Current, AlignBits); + }; + for (const ConvertedMember &Member : Members) { + if (Member.Offset > Current && + needsPadding(Member.Offset, Member.Alignment)) { + uint64_t PadBits = Member.Offset - Current; + assert(PadBits % 8 == 0 && "padding is not a whole number of bytes"); + Fields.emplace_back(getI8Array(PadBits / 8), Current); + } + Fields.emplace_back(Member.Ty, Member.Offset); + Current = std::max(Current, Member.Offset + + Member.Ty->getSizeInBits().getFixedValue()); + } + + // The tail is placed by an integer as wide as the most aligned member, so + // the alignment that reaches it is capped at the widest integer's. + if (RecordSize.isFixed()) { + uint64_t Size = RecordSize.getFixedValue(); + llvm::Align TailAlign = std::min(MaxAlign, getMaxIntegerAlign()); + if (Size > Current && needsPadding(Size, TailAlign)) { + uint64_t PadBits = Size - Current; + assert(PadBits % 8 == 0 && "tail padding is not a whole number of bytes"); + Fields.emplace_back(getI8Array(PadBits / 8), Current); + } + } + + StructPacking Pack = Packed ? StructPacking::Packed : StructPacking::Default; + return TB.getRecordType(Fields, RecordSize, RT->getAlignment(), + RT->getUnadjustedAlignment(), Pack); +} + RecordArgABI TargetInfo::getRecordArgABI(const RecordType *RT) const { if (RT && !RT->canPassInRegisters()) return RAA_Indirect; diff --git a/llvm/lib/ABI/Targets/AArch64.cpp b/llvm/lib/ABI/Targets/AArch64.cpp index 9573f31e1a4053..c7bf77f55f5b30 100644 --- a/llvm/lib/ABI/Targets/AArch64.cpp +++ b/llvm/lib/ABI/Targets/AArch64.cpp @@ -9,6 +9,8 @@ #include "llvm/ABI/FunctionInfo.h" #include "llvm/ABI/TargetInfo.h" #include "llvm/ABI/Types.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/Support/Casting.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" @@ -61,10 +63,19 @@ class AArch64TargetInfo : public TargetInfo { ArgInfo coerceIllegalVector(const VectorType *VT, unsigned &NSRN, unsigned &NPRN) const; + ArgInfo coerceAndExpandPureScalableAggregate( + const Type *Ty, bool IsNamedArg, unsigned NVec, unsigned NPred, + const SmallVectorImpl<const Type *> &UnpaddedCoerceToSeq, unsigned &NSRN, + unsigned &NPRN) const; bool isIllegalVectorType(const Type *Ty) const; bool passAsAggregateType(const Type *Ty) const; + bool passAsPureScalableType(const Type *Ty, unsigned &NV, unsigned &NP, + SmallVectorImpl<const Type *> &CoerceToSeq) const; + + void flattenType(const Type *Ty, + SmallVectorImpl<const Type *> &Flattened) const; bool isHomogeneousAggregateBaseType(const Type *Ty) const override; bool isHomogeneousAggregateSmallEnough(const Type *Base, @@ -126,6 +137,20 @@ ArgInfo AArch64TargetInfo::classifyReturnType(const Type *RetTy, return ArgInfo::getDirect(); } + // In AAPCS return values of a Pure Scalable type are treated as a single + // named argument and passed expanded in registers, or indirectly if there are + // not enough registers. + if (Opts.Kind == AArch64ABIKind::AAPCS) { + unsigned NSRN = 0, NPRN = 0; + unsigned NVec = 0, NPred = 0; + SmallVector<const Type *> UnpaddedCoerceToSeq; + if (passAsPureScalableType(RetTy, NVec, NPred, UnpaddedCoerceToSeq) && + (NVec + NPred) > 0) + return coerceAndExpandPureScalableAggregate( + RetTy, /*IsNamedArg=*/true, NVec, NPred, UnpaddedCoerceToSeq, NSRN, + NPRN); + } + reportNYI("Aggregate return type handling"); return ArgInfo::getIgnore(); } @@ -227,6 +252,17 @@ ArgInfo AArch64TargetInfo::classifyArgumentType( return ArgInfo::getDirect(CoerceTy, /*Offset=*/0, llvm::Align(TyAlign)); } + // In AAPCS, named arguments of a Pure Scalable Type are passed expanded + // in registers, or indirectly if there are not enough registers. + if (Opts.Kind == AArch64ABIKind::AAPCS) { + unsigned NVec = 0, NPred = 0; + SmallVector<const Type *> UnpaddedCoerceToSeq; + if (passAsPureScalableType(Ty, NVec, NPred, UnpaddedCoerceToSeq) && + (NVec + NPred) > 0) + return coerceAndExpandPureScalableAggregate( + Ty, IsNamedArg, NVec, NPred, UnpaddedCoerceToSeq, NSRN, NPRN); + } + reportNYI("Aggregate argument type handling"); return ArgInfo::getIgnore(); } @@ -339,6 +375,180 @@ bool AArch64TargetInfo::isIllegalVectorType(const Type *Ty) const { return false; } +// Expand a memory type into a sequence with an element for each non-record, +// non-array member of the type, with the exception of the padding types, which +// are retained. +void AArch64TargetInfo::flattenType( + const Type *Ty, SmallVectorImpl<const Type *> &Flattened) const { + if (ArgInfo::isPaddingForCoerceAndExpand(Ty)) { + Flattened.push_back(Ty); + return; + } + + if (const auto *AT = dyn_cast<ArrayType>(Ty)) { + uint64_t NElt = AT->getNumElements(); + if (NElt == 0) + return; + + SmallVector<const Type *, 4> EltFlattened; + flattenType(AT->getElementType(), EltFlattened); + + for (uint64_t I = 0; I < NElt; ++I) + llvm::append_range(Flattened, EltFlattened); + return; + } + + if (const auto *RT = dyn_cast<RecordType>(Ty)) { + for (const FieldInfo &Field : RT->getFields()) + flattenType(Field.FieldType, Flattened); + return; + } + + Flattened.push_back(Ty); +} + +ArgInfo AArch64TargetInfo::coerceAndExpandPureScalableAggregate( + const Type *Ty, bool IsNamedArg, unsigned NVec, unsigned NPred, + const SmallVectorImpl<const Type *> &UnpaddedCoerceToSeq, unsigned &NSRN, + unsigned &NPRN) const { + // An unnamed argument, or one that does not fit in the remaining Z or P + // registers, is passed indirectly and does not consume those registers. + if (!IsNamedArg || NSRN + NVec > 8 || NPRN + NPred > 4) + return getNaturalAlignIndirect(Ty, getAllocaAddrSpace(), /*ByVal=*/false); + + NSRN += NVec; + NPRN += NPred; + + // A sizeless SVE tuple is already one register per member. + if (Ty->isSVESizelessType()) + return ArgInfo::getDirect(); + + assert(!UnpaddedCoerceToSeq.empty() && "pure scalable type has no members"); + const Type *UnpaddedCoerceToType = + UnpaddedCoerceToSeq.size() == 1 + ? UnpaddedCoerceToSeq[0] + : getStructOfTypes(UnpaddedCoerceToSeq, /*Packed=*/true); + + SmallVector<const Type *, 8> CoerceToSeq; + flattenType(convertTypeForMem(Ty), CoerceToSeq); + return ArgInfo::getCoerceAndExpand( + getStructOfTypes(CoerceToSeq, /*Packed=*/false), UnpaddedCoerceToType); +} + +// A Pure Scalable Type (AAPCS64) is passed in Z and P registers. On success +// NVec and NPred are how many of each it needs, and CoerceToSeq has one +// scalable vector per register. A sequence longer than 12 is rejected so the +// caller treats the type as a large composite. +bool AArch64TargetInfo::passAsPureScalableType( + const Type *Ty, unsigned &NVec, unsigned &NPred, + SmallVectorImpl<const Type *> &CoerceToSeq) const { + if (const auto *AT = dyn_cast<ArrayType>(Ty)) { + if (AT->isMatrixType()) + return false; + + uint64_t NElt = AT->getNumElements(); + if (NElt == 0) + return false; + + unsigned NV = 0, NP = 0; + SmallVector<const Type *, 4> EltCoerceToSeq; + if (!passAsPureScalableType(AT->getElementType(), NV, NP, EltCoerceToSeq)) + return false; + + if (CoerceToSeq.size() + NElt * EltCoerceToSeq.size() > 12) + return false; + + for (uint64_t I = 0; I < NElt; ++I) + llvm::append_range(CoerceToSeq, EltCoerceToSeq); + + NVec += NElt * NV; + NPred += NElt * NP; + return true; + } + + if (const auto *RT = dyn_cast<RecordType>(Ty)) { + if (getRecordArgABI(RT) != RAA_Default) + return false; + // Pure scalable types are never unions and never contain unions. + if (RT->isUnion()) + return false; + + // A flexible array member is lowered as a zero-length array, which the + // field walk below skips. The member disqualifies a pure scalable type. + if (RT->hasFlexibleArrayMember()) + return false; + + // Direct virtual bases are not in getBaseClasses(). A record that has + // one cannot be passed in registers, and getRecordArgABI rejected it + // above. + for (const FieldInfo &Base : RT->getBaseClasses()) { + if (Base.FieldType->isEmptyRecord()) + continue; + if (!passAsPureScalableType(Base.FieldType, NVec, NPred, CoerceToSeq)) + return false; + } + for (const FieldInfo &Field : RT->getFields()) { + if (Field.isEmpty()) + continue; + if (!passAsPureScalableType(Field.FieldType, NVec, NPred, CoerceToSeq)) + return false; + } + return true; + } + + if (const auto *TT = dyn_cast<TupleType>(Ty)) { + const VectorType *VT = TT->getVectorType(); + if (!VT->isScalable() || VT->isSVECount()) + return false; + + unsigned N = TT->getNumVectors(); + if (CoerceToSeq.size() + N > 12) + return false; + + bool IsPred = VT->isSVEPredicate(); + if (!IsPred && !VT->isSVEData()) + return false; + if (IsPred) + NPred += N; + else + NVec += N; + + for (unsigned I = 0; I < N; ++I) + CoerceToSeq.push_back(VT); + return true; + } + + if (const auto *VT = dyn_cast<VectorType>(Ty)) { + const Type *Coerced = nullptr; + bool IsPred = false; + if (VT->isFixedLengthSVEPredicate()) { + IsPred = true; + Coerced = convertFixedToScalableVectorType(VT); + } else if (VT->isFixedLengthSVEData()) { + Coerced = convertFixedToScalableVectorType(VT); + } else if (VT->isScalable() && VT->isSVEPredicate()) { + IsPred = true; + Coerced = VT; + } else if (VT->isScalable() && VT->isSVEData()) { + Coerced = VT; + } else { + return false; + } + + if (CoerceToSeq.size() + 1 > 12) + return false; + + if (IsPred) + ++NPred; + else + ++NVec; + CoerceToSeq.push_back(Coerced); + return true; + } + + return false; +} + bool AArch64TargetInfo::isHomogeneousAggregateBaseType(const Type *Ty) const { // Soft-float ABI: no types are homogeneous aggregates. if (isSoftFloat()) diff --git a/llvm/unittests/ABI/AArch64TargetInfoTest.cpp b/llvm/unittests/ABI/AArch64TargetInfoTest.cpp index bbb9a52eb48282..1f51f18954ad39 100644 --- a/llvm/unittests/ABI/AArch64TargetInfoTest.cpp +++ b/llvm/unittests/ABI/AArch64TargetInfoTest.cpp @@ -1275,4 +1275,420 @@ TEST_F(AArch64TargetInfoTest, ClassifyEmptyAndZeroSizeIgnore) { } } +static void expectScalableData(const ABIType *Ty, const ABIType *Elt, + unsigned MinElts) { + const auto *VT = llvm::dyn_cast<llvm::abi::VectorType>(Ty); + ASSERT_NE(VT, nullptr); + EXPECT_TRUE(VT->isScalable()); + EXPECT_TRUE(VT->isSVEData()); + EXPECT_EQ(VT->getElementType(), Elt); + EXPECT_EQ(VT->getNumElements().getKnownMinValue(), MinElts); +} + +static void expectScalablePredicate(const ABIType *Ty) { + const auto *VT = llvm::dyn_cast<llvm::abi::VectorType>(Ty); + ASSERT_NE(VT, nullptr); + EXPECT_TRUE(VT->isScalable()); + EXPECT_TRUE(VT->isSVEPredicate()); + EXPECT_EQ(VT->getElementType()->getSizeInBits().getFixedValue(), 1u); + EXPECT_EQ(VT->getNumElements().getKnownMinValue(), 16u); +} + +static const llvm::abi::RecordType *expectPaddedCoerce(const ArgInfo &Info, + unsigned NFields) { + EXPECT_TRUE(Info.isCoerceAndExpand()); + const auto *RT = + llvm::dyn_cast<llvm::abi::RecordType>(Info.getCoerceToType()); + EXPECT_NE(RT, nullptr); + if (!RT) + return nullptr; + EXPECT_EQ(RT->getPacking(), StructPacking::Default); + EXPECT_EQ(RT->getNumFields(), NFields); + return RT; +} + +// A fixed-length SVE vector inside a record is a Pure Scalable Type under +// AAPCS. It is expanded to one scalable vector per register. One member stays +// a single vector; several members become a packed record. A hole that +// alignment does not explain is an i8 array in the padded coerce type. +TEST_F(AArch64TargetInfoTest, ClassifyPureScalableStructCoerceAndExpand) { + std::unique_ptr<TargetInfo> TI = + createAArch64TargetInfo(TB, AArch64ABIOptions(AArch64ABIKind::AAPCS)); + + const ABIType *One = + makeRecord({FieldInfo(FixedSVInt32, 0)}, 256, llvm::Align(16), + /*UnadjustedAlign=*/llvm::Align(16)); + const ABIType *Two = + makeRecord({FieldInfo(FixedSVInt32, 0), FieldInfo(FixedSVFloat64, 256)}, + 512, llvm::Align(16), /*UnadjustedAlign=*/llvm::Align(16)); + // The data vector is 8 bytes past its naturally aligned offset, so the + // record is packed. The gap before it and the tail are both i8 arrays. + const ABIType *Gapped = + makeRecord({FieldInfo(FixedSVBool, 0), FieldInfo(FixedSVInt32, 320)}, 640, + llvm::Align(16), /*UnadjustedAlign=*/llvm::Align(16)); + const ABIType *EmptyUnion = + TB.getUnionType({}, llvm::TypeSize::getFixed(0), llvm::Align(1), + /*UnadjustedAlign=*/llvm::Align(1), + StructPacking::Default, RecordFlags::CanPassInRegisters); + const ABIType *WithEmpty = + makeRecord({FieldInfo(FixedSVInt32, 0), FieldInfo(EmptyUnion, 256)}, 256, + llvm::Align(16), /*UnadjustedAlign=*/llvm::Align(16)); + + { + std::unique_ptr<FunctionInfo> FI = + FunctionInfo::create(llvm::CallingConv::C, One, {One}); + TI->computeInfo(*FI); + for (const ArgInfo &Info : {FI->getReturnInfo(), FI->getArgInfo(0).Info}) { + const llvm::abi::RecordType *Padded = expectPaddedCoerce(Info, 1); + ASSERT_NE(Padded, nullptr); + EXPECT_EQ(Padded->getFields()[0].FieldType, FixedSVInt32); + expectScalableData(Info.getUnpaddedCoerceAndExpandType(), I32, 4); + } + } + + { + std::unique_ptr<FunctionInfo> FI = + FunctionInfo::create(llvm::CallingConv::C, Two, {Two}); + TI->computeInfo(*FI); + const llvm::abi::RecordType *Padded = + expectPaddedCoerce(FI->getArgInfo(0).Info, 2); + ASSERT_NE(Padded, nullptr); + EXPECT_EQ(Padded->getFields()[0].FieldType, FixedSVInt32); + EXPECT_EQ(Padded->getFields()[1].FieldType, FixedSVFloat64); + + const auto *Unpadded = llvm::dyn_cast<llvm::abi::RecordType>( + FI->getArgInfo(0).Info.getUnpaddedCoerceAndExpandType()); + ASSERT_NE(Unpadded, nullptr); + EXPECT_EQ(Unpadded->getPacking(), StructPacking::Packed); + ASSERT_EQ(Unpadded->getNumFields(), 2u); + expectScalableData(Unpadded->getFields()[0].FieldType, I32, 4); + expectScalableData(Unpadded->getFields()[1].FieldType, F64, 2); + } + + { + std::unique_ptr<FunctionInfo> FI = + FunctionInfo::create(llvm::CallingConv::C, Void, {Gapped}); + TI->computeInfo(*FI); + const llvm::abi::RecordType *Padded = + expectPaddedCoerce(FI->getArgInfo(0).Info, 4); + ASSERT_NE(Padded, nullptr); + EXPECT_EQ(Padded->getFields()[0].FieldType, FixedSVBool); + const auto *Pad = + llvm::dyn_cast<llvm::abi::ArrayType>(Padded->getFields()[1].FieldType); + ASSERT_NE(Pad, nullptr); + EXPECT_EQ(Pad->getNumElements(), 8u); + EXPECT_EQ(Pad->getElementType()->getSizeInBits().getFixedValue(), 8u); + EXPECT_EQ(Padded->getFields()[2].FieldType, FixedSVInt32); + const auto *Tail = + llvm::dyn_cast<llvm::abi::ArrayType>(Padded->getFields()[3].FieldType); + ASSERT_NE(Tail, nullptr); + EXPECT_EQ(Tail->getNumElements(), 8u); + + const auto *Unpadded = llvm::dyn_cast<llvm::abi::RecordType>( + FI->getArgInfo(0).Info.getUnpaddedCoerceAndExpandType()); + ASSERT_NE(Unpadded, nullptr); + ASSERT_EQ(Unpadded->getNumFields(), 2u); + expectScalablePredicate(Unpadded->getFields()[0].FieldType); + expectScalableData(Unpadded->getFields()[1].FieldType, I32, 4); + } + + { + std::unique_ptr<FunctionInfo> FI = + FunctionInfo::create(llvm::CallingConv::C, WithEmpty, {WithEmpty}); + TI->computeInfo(*FI); + const llvm::abi::RecordType *Padded = + expectPaddedCoerce(FI->getReturnInfo(), 1); + ASSERT_NE(Padded, nullptr); + EXPECT_EQ(Padded->getFields()[0].FieldType, FixedSVInt32); + } +} + +// A packed pure scalable record keeps a naturally aligned gap as an explicit +// i8 array. An over-aligned record keeps its tail the same way. +TEST_F(AArch64TargetInfoTest, ClassifyPureScalablePackedGapAndOveralign) { + std::unique_ptr<TargetInfo> TI = + createAArch64TargetInfo(TB, AArch64ABIOptions(AArch64ABIKind::AAPCS)); + + const llvm::abi::VectorType *Pred = + TB.getVectorType(U8, llvm::ElementCount::getFixed(2), llvm::Align(2), + llvm::abi::VectorKind::SVEPredicate); + // Predicate at 0, misaligned data vector at byte 2, naturally aligned data + // vector at byte 48, tail predicate at byte 80. Size 96. + const ABIType *PackedGap = makeRecord( + {FieldInfo(Pred, 0), FieldInfo(FixedSVInt32, 16), + FieldInfo(FixedSVInt32, 384), FieldInfo(Pred, 640)}, + /*SizeBits=*/768, llvm::Align(16), /*UnadjustedAlign=*/llvm::Align(2)); + const ABIType *Over = makeRecord({FieldInfo(FixedSVInt32, 0)}, + /*SizeBits=*/512, llvm::Align(32), + /*UnadjustedAlign=*/llvm::Align(16)); + + { + std::unique_ptr<FunctionInfo> FI = + FunctionInfo::create(llvm::CallingConv::C, PackedGap, {PackedGap}); + TI->computeInfo(*FI); + const llvm::abi::RecordType *Padded = + expectPaddedCoerce(FI->getArgInfo(0).Info, 6); + ASSERT_NE(Padded, nullptr); + EXPECT_EQ(Padded->getFields()[0].FieldType, Pred); + EXPECT_EQ(Padded->getFields()[1].FieldType, FixedSVInt32); + const auto *Gap = + llvm::dyn_cast<llvm::abi::ArrayType>(Padded->getFields()[2].FieldType); + ASSERT_NE(Gap, nullptr); + EXPECT_EQ(Gap->getNumElements(), 14u); + EXPECT_EQ(Padded->getFields()[3].FieldType, FixedSVInt32); + EXPECT_EQ(Padded->getFields()[4].FieldType, Pred); + const auto *Tail = + llvm::dyn_cast<llvm::abi::ArrayType>(Padded->getFields()[5].FieldType); + ASSERT_NE(Tail, nullptr); + EXPECT_EQ(Tail->getNumElements(), 14u); + + const auto *Unpadded = llvm::dyn_cast<llvm::abi::RecordType>( + FI->getReturnInfo().getUnpaddedCoerceAndExpandType()); + ASSERT_NE(Unpadded, nullptr); + EXPECT_EQ(Unpadded->getPacking(), StructPacking::Packed); + EXPECT_EQ(Unpadded->getNumFields(), 4u); + } + + { + std::unique_ptr<FunctionInfo> FI = + FunctionInfo::create(llvm::CallingConv::C, Over, {Over}); + TI->computeInfo(*FI); + const llvm::abi::RecordType *Padded = + expectPaddedCoerce(FI->getReturnInfo(), 2); + ASSERT_NE(Padded, nullptr); + EXPECT_EQ(Padded->getFields()[0].FieldType, FixedSVInt32); + const auto *Tail = + llvm::dyn_cast<llvm::abi::ArrayType>(Padded->getFields()[1].FieldType); + ASSERT_NE(Tail, nullptr); + EXPECT_EQ(Tail->getNumElements(), 32u); + expectScalableData(FI->getReturnInfo().getUnpaddedCoerceAndExpandType(), + I32, 4); + } +} + +// A fixed-length SVE vector is 16-byte aligned however wide it is, but it +// converts with the alignment of its size. A record holding a 256-bit vector +// is therefore packed, and the gap or tail beside a narrower predicate is an +// explicit i8 array. +TEST_F(AArch64TargetInfoTest, ClassifyPureScalableWideVectorPadsNarrowMember) { + std::unique_ptr<TargetInfo> TI = + createAArch64TargetInfo(TB, AArch64ABIOptions(AArch64ABIKind::AAPCS)); + + // A 256-bit predicate is 4 bytes wide. + const llvm::abi::VectorType *Pred = + TB.getVectorType(U8, llvm::ElementCount::getFixed(4), llvm::Align(2), + llvm::abi::VectorKind::SVEPredicate); + // Predicate at 0, data vector at byte 16. Size 48. + const ABIType *PredThenData = + makeRecord({FieldInfo(Pred, 0), FieldInfo(FixedSVInt32, 128)}, + /*SizeBits=*/384, llvm::Align(16), + /*UnadjustedAlign=*/llvm::Align(16)); + // Data vector at 0, predicate at byte 32. Size 48. + const ABIType *DataThenPred = + makeRecord({FieldInfo(FixedSVInt32, 0), FieldInfo(Pred, 256)}, + /*SizeBits=*/384, llvm::Align(16), + /*UnadjustedAlign=*/llvm::Align(16)); + + { + std::unique_ptr<FunctionInfo> FI = + FunctionInfo::create(llvm::CallingConv::C, Void, {PredThenData}); + TI->computeInfo(*FI); + const llvm::abi::RecordType *Padded = + expectPaddedCoerce(FI->getArgInfo(0).Info, 3); + ASSERT_NE(Padded, nullptr); + EXPECT_EQ(Padded->getFields()[0].FieldType, Pred); + const auto *Gap = + llvm::dyn_cast<llvm::abi::ArrayType>(Padded->getFields()[1].FieldType); + ASSERT_NE(Gap, nullptr); + EXPECT_EQ(Gap->getNumElements(), 12u); + EXPECT_EQ(Padded->getFields()[2].FieldType, FixedSVInt32); + + const auto *Unpadded = llvm::dyn_cast<llvm::abi::RecordType>( + FI->getArgInfo(0).Info.getUnpaddedCoerceAndExpandType()); + ASSERT_NE(Unpadded, nullptr); + ASSERT_EQ(Unpadded->getNumFields(), 2u); + expectScalablePredicate(Unpadded->getFields()[0].FieldType); + expectScalableData(Unpadded->getFields()[1].FieldType, I32, 4); + } + + { + std::unique_ptr<FunctionInfo> FI = + FunctionInfo::create(llvm::CallingConv::C, Void, {DataThenPred}); + TI->computeInfo(*FI); + const llvm::abi::RecordType *Padded = + expectPaddedCoerce(FI->getArgInfo(0).Info, 3); + ASSERT_NE(Padded, nullptr); + EXPECT_EQ(Padded->getFields()[0].FieldType, FixedSVInt32); + EXPECT_EQ(Padded->getFields()[1].FieldType, Pred); + const auto *Tail = + llvm::dyn_cast<llvm::abi::ArrayType>(Padded->getFields()[2].FieldType); + ASSERT_NE(Tail, nullptr); + EXPECT_EQ(Tail->getNumElements(), 12u); + } +} + +// Arrays and base classes expand to the same register sequence as their +// members. +TEST_F(AArch64TargetInfoTest, ClassifyPureScalableArrayAndBase) { + std::unique_ptr<TargetInfo> TI = + createAArch64TargetInfo(TB, AArch64ABIOptions(AArch64ABIKind::AAPCS)); + + const ABIType *Pair = + makeRecord({FieldInfo(FixedSVInt32, 0), FieldInfo(FixedSVInt32, 256)}, + 512, llvm::Align(16), /*UnadjustedAlign=*/llvm::Align(16)); + const ABIType *Arr = + TB.getArrayType(FixedSVInt32, /*NumElements=*/2, /*SizeInBits=*/512); + const ABIType *Base = makeRecord( + {FieldInfo(FixedSVInt32, 0)}, 256, llvm::Align(16), + /*UnadjustedAlign=*/llvm::Align(16), passableRecordFlags(/*IsCXX=*/true)); + const ABIType *Derived = + makeRecord({FieldInfo(FixedSVFloat64, 256)}, 512, llvm::Align(16), + /*UnadjustedAlign=*/llvm::Align(16), + passableRecordFlags(/*IsCXX=*/true), {FieldInfo(Base, 0)}); + const ABIType *Nested = makeRecord({FieldInfo(Pair, 0)}, 512, llvm::Align(16), + /*UnadjustedAlign=*/llvm::Align(16)); + + for (const ABIType *Ty : {Arr, Derived, Nested}) { + std::unique_ptr<FunctionInfo> FI = + FunctionInfo::create(llvm::CallingConv::C, Ty, {Ty}); + TI->computeInfo(*FI); + const llvm::abi::RecordType *Padded = + expectPaddedCoerce(FI->getArgInfo(0).Info, 2); + ASSERT_NE(Padded, nullptr); + const auto *Unpadded = llvm::dyn_cast<llvm::abi::RecordType>( + FI->getArgInfo(0).Info.getUnpaddedCoerceAndExpandType()); + ASSERT_NE(Unpadded, nullptr); + EXPECT_EQ(Unpadded->getPacking(), StructPacking::Packed); + ASSERT_EQ(Unpadded->getNumFields(), 2u); + } + + std::unique_ptr<FunctionInfo> FI = + FunctionInfo::create(llvm::CallingConv::C, Void, {Derived}); + TI->computeInfo(*FI); + const llvm::abi::RecordType *Padded = + expectPaddedCoerce(FI->getArgInfo(0).Info, 2); + ASSERT_NE(Padded, nullptr); + EXPECT_EQ(Padded->getFields()[0].FieldType, FixedSVInt32); + EXPECT_EQ(Padded->getFields()[1].FieldType, FixedSVFloat64); +} + +// A zero-length array is empty and does not disqualify a Pure Scalable Type. +TEST_F(AArch64TargetInfoTest, ClassifyPureScalableZeroLengthArray) { + std::unique_ptr<TargetInfo> TI = + createAArch64TargetInfo(TB, AArch64ABIOptions(AArch64ABIKind::AAPCS)); + + const ABIType *ZeroArray = + TB.getArrayType(I32, /*NumElements=*/0, /*SizeInBits=*/0); + const ABIType *WithZero = + makeRecord({FieldInfo(FixedSVInt32, 0), FieldInfo(ZeroArray, 256)}, 256, + llvm::Align(16), /*UnadjustedAlign=*/llvm::Align(16)); + + std::unique_ptr<FunctionInfo> FI = + FunctionInfo::create(llvm::CallingConv::C, WithZero, {WithZero}); + TI->computeInfo(*FI); + for (const ArgInfo &Info : {FI->getReturnInfo(), FI->getArgInfo(0).Info}) { + const llvm::abi::RecordType *Padded = expectPaddedCoerce(Info, 1); + ASSERT_NE(Padded, nullptr); + EXPECT_EQ(Padded->getFields()[0].FieldType, FixedSVInt32); + expectScalableData(Info.getUnpaddedCoerceAndExpandType(), I32, 4); + } +} + +// A Pure Scalable Type that does not fit in the remaining registers is +// indirect and leaves those registers available. An unnamed argument is +// indirect even when registers remain. More than 8 data vectors never fits. +TEST_F(AArch64TargetInfoTest, ClassifyPureScalableRegisterBudget) { + std::unique_ptr<TargetInfo> TI = + createAArch64TargetInfo(TB, AArch64ABIOptions(AArch64ABIKind::AAPCS)); + + const ABIType *One = + makeRecord({FieldInfo(FixedSVInt32, 0)}, 256, llvm::Align(16), + /*UnadjustedAlign=*/llvm::Align(16)); + const ABIType *Two = + makeRecord({FieldInfo(FixedSVInt32, 0), FieldInfo(FixedSVInt32, 256)}, + 512, llvm::Align(16), /*UnadjustedAlign=*/llvm::Align(16)); + const ABIType *Nine = + TB.getArrayType(FixedSVInt32, /*NumElements=*/9, /*SizeInBits=*/9 * 256); + + { + const ABIType *Args[] = {SVInt32, SVInt32, SVInt32, SVInt32, SVInt32, + SVInt32, SVInt32, Two, One}; + std::unique_ptr<FunctionInfo> FI = + FunctionInfo::create(llvm::CallingConv::C, Void, Args); + TI->computeInfo(*FI); + expectAlignedIndirect(FI->getArgInfo(7).Info, llvm::Align(16), + /*ByVal=*/false); + EXPECT_TRUE(FI->getArgInfo(8).Info.isCoerceAndExpand()); + } + + { + const ABIType *Pred = + makeRecord({FieldInfo(FixedSVBool, 0)}, 256, llvm::Align(2), + /*UnadjustedAlign=*/llvm::Align(2)); + const ABIType *TwoPred = + makeRecord({FieldInfo(FixedSVBool, 0), FieldInfo(FixedSVBool, 256)}, + 512, llvm::Align(2), /*UnadjustedAlign=*/llvm::Align(2)); + const ABIType *Args[] = {SVBool, SVBool, SVBool, TwoPred, Pred}; + std::unique_ptr<FunctionInfo> FI = + FunctionInfo::create(llvm::CallingConv::C, Void, Args); + TI->computeInfo(*FI); + expectAlignedIndirect(FI->getArgInfo(3).Info, llvm::Align(2), + /*ByVal=*/false); + EXPECT_TRUE(FI->getArgInfo(4).Info.isCoerceAndExpand()); + } + + { + std::unique_ptr<FunctionInfo> FI = FunctionInfo::create( + llvm::CallingConv::C, Void, {One, One}, RequiredArgs(1)); + TI->computeInfo(*FI); + EXPECT_TRUE(FI->getArgInfo(0).Info.isCoerceAndExpand()); + expectAlignedIndirect(FI->getArgInfo(1).Info, llvm::Align(16), + /*ByVal=*/false); + } + + { + std::unique_ptr<FunctionInfo> FI = + FunctionInfo::create(llvm::CallingConv::C, Nine, {Nine}); + TI->computeInfo(*FI); + expectAlignedIndirect(FI->getReturnInfo(), llvm::Align(16), + /*ByVal=*/false); + expectAlignedIndirect(FI->getArgInfo(0).Info, llvm::Align(16), + /*ByVal=*/false); + } +} + +// Under AAPCS an SVE tuple is a Pure Scalable Type. It is passed directly +// when it fits in registers and indirectly when it does not. +TEST_F(AArch64TargetInfoTest, ClassifySVETupleAAPCS) { + std::unique_ptr<TargetInfo> TI = + createAArch64TargetInfo(TB, AArch64ABIOptions(AArch64ABIKind::AAPCS)); + + { + std::unique_ptr<FunctionInfo> FI = + FunctionInfo::create(llvm::CallingConv::C, SVInt32x2, {SVBoolx4}); + TI->computeInfo(*FI); + expectUncoercedDirect(FI->getReturnInfo()); + expectUncoercedDirect(FI->getArgInfo(0).Info); + } + + { + const ABIType *Args[] = {SVInt32, SVInt32, SVInt32, SVInt32, + SVInt32, SVInt32, SVInt32, SVInt32x2}; + std::unique_ptr<FunctionInfo> FI = + FunctionInfo::create(llvm::CallingConv::C, Void, Args); + TI->computeInfo(*FI); + expectAlignedIndirect(FI->getArgInfo(7).Info, llvm::Align(16), + /*ByVal=*/false); + } + + { + std::unique_ptr<FunctionInfo> FI = + FunctionInfo::create(llvm::CallingConv::C, Void, {SVBoolx4, SVBoolx2}); + TI->computeInfo(*FI); + expectUncoercedDirect(FI->getArgInfo(0).Info); + expectAlignedIndirect(FI->getArgInfo(1).Info, llvm::Align(2), + /*ByVal=*/false); + } +} + } // namespace diff --git a/llvm/unittests/ABI/IRTypeMapperTest.cpp b/llvm/unittests/ABI/IRTypeMapperTest.cpp index 0cf2061f520d2a..051e5561b0142d 100644 --- a/llvm/unittests/ABI/IRTypeMapperTest.cpp +++ b/llvm/unittests/ABI/IRTypeMapperTest.cpp @@ -99,6 +99,26 @@ TEST_F(IRTypeMapperTest, SVEDataTupleMapsToStructOfVectors) { EXPECT_EQ(Struct->getElementType(I), ExpectedVec); } +TEST_F(IRTypeMapperTest, PackedRecordMapsToPackedStruct) { + const llvm::abi::Type *I32 = + TB.getIntegerType(32, llvm::Align(4), /*Signed=*/true); + const llvm::abi::VectorType *SVInt32 = + TB.getVectorType(I32, llvm::ElementCount::getScalable(4), llvm::Align(16), + llvm::abi::VectorKind::SVEData); + const llvm::abi::RecordType *Packed = TB.getRecordType( + {llvm::abi::FieldInfo(SVInt32, 0), llvm::abi::FieldInfo(SVInt32, 0)}, + llvm::TypeSize::getFixed(0), llvm::Align(1), + /*UnadjustedAlign=*/llvm::Align(1), llvm::abi::StructPacking::Packed); + + auto *Struct = llvm::dyn_cast<llvm::StructType>(Mapper.convertType(Packed)); + ASSERT_NE(Struct, nullptr); + EXPECT_TRUE(Struct->isPacked()); + ASSERT_EQ(Struct->getNumElements(), 2u); + llvm::Type *Vec = Mapper.convertType(SVInt32); + EXPECT_EQ(Struct->getElementType(0), Vec); + EXPECT_EQ(Struct->getElementType(1), Vec); +} + TEST_F(IRTypeMapperTest, SameSizeAtomicMapsToValueType) { const llvm::abi::Type *F32 = TB.getFloatType(llvm::APFloat::IEEEsingle(), llvm::Align(4)); diff --git a/llvm/unittests/ABI/TargetInfoTest.cpp b/llvm/unittests/ABI/TargetInfoTest.cpp index cde1ff3542cda5..8d3ca61f773807 100644 --- a/llvm/unittests/ABI/TargetInfoTest.cpp +++ b/llvm/unittests/ABI/TargetInfoTest.cpp @@ -70,6 +70,13 @@ class AllocaAS5TargetInfo : public TargetInfo { llvm::abi::ABICompatInfo Compat; }; +// Exposes convertTypeForMem, which is protected on TargetInfo. +class MemTargetInfo : public TestTargetInfo { +public: + using TargetInfo::convertTypeForMem; + using TestTargetInfo::TestTargetInfo; +}; + // A target lacking a 128-bit integer type, so the _BitInt register threshold // is the width of `long long`. class NoInt128TargetInfo : public TestTargetInfo { @@ -384,4 +391,219 @@ TEST_F(TargetInfoTest, NaturalAlignIndirectDefaultsToZeroAddrSpace) { EXPECT_EQ(AI.getIndirectAddrSpace(), 0u); } +// --- convertTypeForMem ------------------------------------------------------- + +static const llvm::abi::RecordType *asRecord(const ABIType *Ty) { + return llvm::dyn_cast<llvm::abi::RecordType>(Ty); +} + +static const llvm::abi::ArrayType *asI8Array(const ABIType *Ty) { + const auto *AT = llvm::dyn_cast<llvm::abi::ArrayType>(Ty); + if (!AT || AT->getElementType()->getSizeInBits().getFixedValue() != 8) + return nullptr; + return AT; +} + +// A naturally aligned struct keeps the gap implicit and is not packed. +TEST_F(TargetInfoTest, ConvertTypeForMemNaturalStructIsUnpacked) { + MemTargetInfo TI(TB); + const ABIType *I8 = TB.getIntegerType(8, llvm::Align(1), /*Signed=*/true); + const ABIType *S = + recordOf({FieldInfo(I8, 0), FieldInfo(I32, 32)}, 64, llvm::Align(4)); + const llvm::abi::RecordType *Mem = asRecord(TI.convertTypeForMem(S)); + ASSERT_NE(Mem, nullptr); + EXPECT_EQ(Mem->getPacking(), StructPacking::Default); + ASSERT_EQ(Mem->getNumFields(), 2u); + EXPECT_EQ(Mem->getFields()[0].FieldType, I8); + EXPECT_EQ(Mem->getFields()[1].FieldType, I32); +} + +// The record has tail padding, but rounding the members up by the most aligned +// member reaches the record size, so a non-packed struct produces that padding +// without an array for it. +TEST_F(TargetInfoTest, ConvertTypeForMemTailCoveredByMemberAlignIsImplicit) { + MemTargetInfo TI(TB); + const ABIType *I8 = TB.getIntegerType(8, llvm::Align(1), /*Signed=*/true); + // int at 0, char at byte 4, size 8 from the int's alignment. + const ABIType *S = + recordOf({FieldInfo(I32, 0), FieldInfo(I8, 32)}, 64, llvm::Align(4)); + const llvm::abi::RecordType *Mem = asRecord(TI.convertTypeForMem(S)); + ASSERT_NE(Mem, nullptr); + EXPECT_EQ(Mem->getPacking(), StructPacking::Default); + ASSERT_EQ(Mem->getNumFields(), 2u); + EXPECT_EQ(Mem->getFields()[0].FieldType, I32); + EXPECT_EQ(Mem->getFields()[1].FieldType, I8); +} + +// alignas on the record does not raise the converted member alignment, so the +// tail is an explicit i8 array. The record stays unpacked. +TEST_F(TargetInfoTest, ConvertTypeForMemOveralignedTailIsExplicit) { + MemTargetInfo TI(TB); + const ABIType *S = recordOf({FieldInfo(I32, 0)}, 128, llvm::Align(16)); + const llvm::abi::RecordType *Mem = asRecord(TI.convertTypeForMem(S)); + ASSERT_NE(Mem, nullptr); + EXPECT_EQ(Mem->getPacking(), StructPacking::Default); + ASSERT_EQ(Mem->getNumFields(), 2u); + EXPECT_EQ(Mem->getFields()[0].FieldType, I32); + const llvm::abi::ArrayType *Tail = asI8Array(Mem->getFields()[1].FieldType); + ASSERT_NE(Tail, nullptr); + EXPECT_EQ(Tail->getNumElements(), 12u); +} + +// An over-aligned member widens the gap before it and the tail. Both are +// explicit, and the record stays unpacked. +TEST_F(TargetInfoTest, ConvertTypeForMemOveralignedMemberPadsGapAndTail) { + MemTargetInfo TI(TB); + const ABIType *I8 = TB.getIntegerType(8, llvm::Align(1), /*Signed=*/true); + // char at 0, int at byte 16, size 32 from the member's alignment. + const ABIType *S = + recordOf({FieldInfo(I8, 0), FieldInfo(I32, 128)}, 256, llvm::Align(16)); + const llvm::abi::RecordType *Mem = asRecord(TI.convertTypeForMem(S)); + ASSERT_NE(Mem, nullptr); + EXPECT_EQ(Mem->getPacking(), StructPacking::Default); + ASSERT_EQ(Mem->getNumFields(), 4u); + EXPECT_EQ(Mem->getFields()[0].FieldType, I8); + const llvm::abi::ArrayType *Gap = asI8Array(Mem->getFields()[1].FieldType); + ASSERT_NE(Gap, nullptr); + EXPECT_EQ(Gap->getNumElements(), 15u); + EXPECT_EQ(Mem->getFields()[2].FieldType, I32); + const llvm::abi::ArrayType *Tail = asI8Array(Mem->getFields()[3].FieldType); + ASSERT_NE(Tail, nullptr); + EXPECT_EQ(Tail->getNumElements(), 12u); +} + +// A misaligned field makes the record packed. A later field that sits at its +// natural alignment still gets an explicit gap, and so does the tail. +TEST_F(TargetInfoTest, ConvertTypeForMemPackedRecordKeepsNaturalGap) { + MemTargetInfo TI(TB); + const ABIType *I8 = TB.getIntegerType(8, llvm::Align(1), /*Signed=*/true); + // char, misaligned int at byte 1, naturally aligned int at byte 8, char at + // byte 12. Size 16. + const ABIType *S = recordOf({FieldInfo(I8, 0), FieldInfo(I32, 8), + FieldInfo(I32, 64), FieldInfo(I8, 96)}, + 128, llvm::Align(4)); + const llvm::abi::RecordType *Mem = asRecord(TI.convertTypeForMem(S)); + ASSERT_NE(Mem, nullptr); + EXPECT_EQ(Mem->getPacking(), StructPacking::Packed); + ASSERT_EQ(Mem->getNumFields(), 6u); + EXPECT_EQ(Mem->getFields()[0].FieldType, I8); + EXPECT_EQ(Mem->getFields()[1].FieldType, I32); + const llvm::abi::ArrayType *Gap = asI8Array(Mem->getFields()[2].FieldType); + ASSERT_NE(Gap, nullptr); + EXPECT_EQ(Gap->getNumElements(), 3u); + EXPECT_EQ(Mem->getFields()[3].FieldType, I32); + EXPECT_EQ(Mem->getFields()[4].FieldType, I8); + const llvm::abi::ArrayType *Tail = asI8Array(Mem->getFields()[5].FieldType); + ASSERT_NE(Tail, nullptr); + EXPECT_EQ(Tail->getNumElements(), 3u); +} + +// A vector is converted with the alignment of its size rounded up to a power +// of two, not the alignment the source language gave it. An AArch64 +// fixed-length SVE type wider than 16 bytes carries the smaller alignment, so +// a record laid out from it is packed and the gap before the vector is an +// explicit array. +TEST_F(TargetInfoTest, ConvertTypeForMemVectorAlignsToItsSize) { + MemTargetInfo TI(TB); + const ABIType *I8 = TB.getIntegerType(8, llvm::Align(1), /*Signed=*/true); + // A 256-bit SVE predicate and data vector, both 16-byte aligned in C. + const ABIType *Pred = + TB.getVectorType(I8, llvm::ElementCount::getFixed(4), llvm::Align(2), + llvm::abi::VectorKind::SVEPredicate); + const ABIType *Data = + TB.getVectorType(I32, llvm::ElementCount::getFixed(8), llvm::Align(16), + llvm::abi::VectorKind::SVEData); + // 4-byte predicate, 32-byte vector at byte 16. Size 48. + const ABIType *S = recordOf({FieldInfo(Pred, 0), FieldInfo(Data, 128)}, 384, + llvm::Align(16)); + const llvm::abi::RecordType *Mem = asRecord(TI.convertTypeForMem(S)); + ASSERT_NE(Mem, nullptr); + EXPECT_EQ(Mem->getPacking(), StructPacking::Packed); + ASSERT_EQ(Mem->getNumFields(), 3u); + EXPECT_EQ(Mem->getFields()[0].FieldType, Pred); + const llvm::abi::ArrayType *Gap = asI8Array(Mem->getFields()[1].FieldType); + ASSERT_NE(Gap, nullptr); + EXPECT_EQ(Gap->getNumElements(), 12u); + EXPECT_EQ(Mem->getFields()[2].FieldType, Data); +} + +// The same alignment rule applies to the tail. The record size is a multiple +// of the source alignment but not of the vector's converted alignment, so the +// record is packed and the tail padding is an explicit array. +TEST_F(TargetInfoTest, ConvertTypeForMemVectorTailIsExplicit) { + MemTargetInfo TI(TB); + const ABIType *I8 = TB.getIntegerType(8, llvm::Align(1), /*Signed=*/true); + const ABIType *Pred = + TB.getVectorType(I8, llvm::ElementCount::getFixed(4), llvm::Align(2), + llvm::abi::VectorKind::SVEPredicate); + const ABIType *Data = + TB.getVectorType(I32, llvm::ElementCount::getFixed(8), llvm::Align(16), + llvm::abi::VectorKind::SVEData); + // 32-byte vector, 4-byte predicate at byte 32. Size 48. + const ABIType *S = recordOf({FieldInfo(Data, 0), FieldInfo(Pred, 256)}, 384, + llvm::Align(16)); + const llvm::abi::RecordType *Mem = asRecord(TI.convertTypeForMem(S)); + ASSERT_NE(Mem, nullptr); + EXPECT_EQ(Mem->getPacking(), StructPacking::Packed); + ASSERT_EQ(Mem->getNumFields(), 3u); + EXPECT_EQ(Mem->getFields()[0].FieldType, Data); + EXPECT_EQ(Mem->getFields()[1].FieldType, Pred); + const llvm::abi::ArrayType *Tail = asI8Array(Mem->getFields()[2].FieldType); + ASSERT_NE(Tail, nullptr); + EXPECT_EQ(Tail->getNumElements(), 12u); +} + +// The tail is reached by an integer as wide as the most aligned member, so +// the alignment that covers it is capped at getMaxIntegerAlign(). A member +// aligned beyond that does not carry the tail, which is an explicit array even +// though the record is not packed. +TEST_F(TargetInfoTest, ConvertTypeForMemTailAlignCappedAtWidestInteger) { + MemTargetInfo TI(TB); + const ABIType *I8 = TB.getIntegerType(8, llvm::Align(1), /*Signed=*/true); + const ABIType *Pred = + TB.getVectorType(I8, llvm::ElementCount::getFixed(4), llvm::Align(2), + llvm::abi::VectorKind::SVEPredicate); + const ABIType *Data = + TB.getVectorType(I32, llvm::ElementCount::getFixed(8), llvm::Align(16), + llvm::abi::VectorKind::SVEData); + // 32-byte vector, 4-byte predicate at byte 32, alignas(64). Size 64. The + // members reach byte 36, which the vector's alignment of 32 would round to + // the record size, but the 16-byte cap only reaches byte 48. + const ABIType *S = recordOf({FieldInfo(Data, 0), FieldInfo(Pred, 256)}, 512, + llvm::Align(64)); + const llvm::abi::RecordType *Mem = asRecord(TI.convertTypeForMem(S)); + ASSERT_NE(Mem, nullptr); + EXPECT_EQ(Mem->getPacking(), StructPacking::Default); + ASSERT_EQ(Mem->getNumFields(), 3u); + EXPECT_EQ(Mem->getFields()[0].FieldType, Data); + EXPECT_EQ(Mem->getFields()[1].FieldType, Pred); + const llvm::abi::ArrayType *Tail = asI8Array(Mem->getFields()[2].FieldType); + ASSERT_NE(Tail, nullptr); + EXPECT_EQ(Tail->getNumElements(), 28u); +} + +// A packed member has converted alignment 1, so placing it at byte 1 does not +// pack the outer record, and the naturally aligned field after it has no +// explicit padding array. +TEST_F(TargetInfoTest, ConvertTypeForMemPackedMemberHasAlignOne) { + MemTargetInfo TI(TB); + const ABIType *I8 = TB.getIntegerType(8, llvm::Align(1), /*Signed=*/true); + const ABIType *Inner = + recordOf({FieldInfo(I8, 0), FieldInfo(I32, 8)}, 40, llvm::Align(1)); + // char, packed inner at byte 1, int at byte 8. Size 12. + const ABIType *Outer = + recordOf({FieldInfo(I8, 0), FieldInfo(Inner, 8), FieldInfo(I32, 64)}, 96, + llvm::Align(4)); + const llvm::abi::RecordType *Mem = asRecord(TI.convertTypeForMem(Outer)); + ASSERT_NE(Mem, nullptr); + EXPECT_EQ(Mem->getPacking(), StructPacking::Default); + ASSERT_EQ(Mem->getNumFields(), 3u); + EXPECT_EQ(Mem->getFields()[0].FieldType, I8); + const llvm::abi::RecordType *InnerMem = + asRecord(Mem->getFields()[1].FieldType); + ASSERT_NE(InnerMem, nullptr); + EXPECT_EQ(InnerMem->getPacking(), StructPacking::Packed); + EXPECT_EQ(Mem->getFields()[2].FieldType, I32); +} + } // namespace _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
