================
@@ -1511,3 +1520,380 @@ void
CIRABIRewriteContext::rewriteFunctionAddress(cir::GetGlobalOp addrOp,
cir::CastKind::bitcast, addrOp.getAddr());
addrOp.getAddr().replaceAllUsesExcept(bitcast.getResult(), bitcast);
}
+
+namespace {
+
+/// What one `va_arg` expansion needs from the op it rewrites. The x86-64
+/// cursor fields are reached through `vaFields` by index: 0 gp_offset,
+/// 1 fp_offset, 2 overflow_arg_area, 3 reg_save_area.
+struct VAArgFetch {
+ mlir::Location loc;
+ mlir::Value valist;
+ llvm::ArrayRef<mlir::Type> vaFields;
+ mlir::Type resultTy;
+ const ArgClassification ∾
+ const mlir::DataLayout &dl;
+ mlir::ModuleOp module;
+};
+
+/// The cursor fields a register fetch reads, and the predicate saying every
+/// class it needs still has room. An offset is null when the fetch needs no
+/// register of that class.
+struct RegisterCursor {
+ mlir::Value gpOffsetP;
+ mlir::Value fpOffsetP;
+ mlir::Value gpOffset;
+ mlir::Value fpOffset;
+ mlir::Value inRegs;
+};
+
+mlir::LogicalResult reportVAArgNYI(cir::VAArgOp op, llvm::StringRef what) {
+ op->emitOpError() << "va_arg of " << what
+ << " not yet implemented in CallConvLowering";
+ return mlir::failure();
+}
+
+/// Sets \p isRegPair when a two-register argument is a coerced pair, one
+/// register per member, rather than a single wide scalar, and records in
+/// \p pairIsSse which element is SSE class. Fails on a coercion that is not
+/// two eightbytes.
+mlir::LogicalResult classifyRegisterPair(cir::VAArgOp op,
+ const ArgClassification &ac,
+ std::array<bool, 2> &pairIsSse,
+ bool &isRegPair) {
+ auto pairTy = mlir::dyn_cast<cir::RecordType>(ac.coercedType);
+ if (!pairTy)
+ return mlir::success();
+
+ if (pairTy.getNumElements() != 2)
+ return reportVAArgNYI(op, "a register coercion that is not two
eightbytes");
+
+ assert(!ac.directOffset &&
+ "a pair already spans both eightbytes, so it cannot also start "
+ "partway into the value");
+ for (auto [i, memberTy] : llvm::enumerate(pairTy.getMembers()))
+ pairIsSse[i] = isSSERegisterClass(memberTy);
+ isRegPair = true;
+ return mlir::success();
+}
+
+/// The alignment an argument is placed at. A record can require more than
+/// the types of its members imply, from an `aligned` attribute on the record
+/// or on one of its fields, and neither raises the alignment of any type.
+/// Only the record layout knows, so the member-derived value alone can be too
+/// small.
+uint64_t argumentAreaAlign(mlir::Type ty, mlir::ModuleOp modOp,
+ const mlir::DataLayout &dl) {
+ uint64_t align = dl.getTypeABIAlignment(ty);
+ if (auto recTy = mlir::dyn_cast<cir::RecordType>(ty))
+ if (auto layout = cir::tryGetRecordLayout(modOp, recTy.getName()))
+ align = std::max<uint64_t>(align, layout.getRecordAlign());
+ return align;
+}
+
+mlir::Value roundPointerUpToAlignment(CIRBaseBuilderTy &b, mlir::Location loc,
+ mlir::Value bytePtr, uint64_t align,
+ const mlir::DataLayout &dl) {
+ assert(llvm::isPowerOf2_64(align) &&
+ "mask rounding needs a power-of-two alignment");
+ mlir::Value bumped =
+ b.createPtrStride(loc, bytePtr, b.getSignedInt(loc, align - 1, 32));
+ std::optional<uint64_t> indexWidth =
+ dl.getTypeIndexBitwidth(bytePtr.getType());
+ assert(indexWidth && "a pointer in the argument area has an index width");
+ mlir::Value mask = b.getSignedInt(loc, -static_cast<int64_t>(align),
+ static_cast<unsigned>(*indexWidth));
+ return cir::PtrMaskOp::create(b, loc, bytePtr.getType(), bumped, mask);
+}
+
+/// Reads the argument's address out of the overflow area, which also advances
+/// the cursor past the argument.
+mlir::Value buildOverflowAddr(CIRBaseBuilderTy &b, const VAArgFetch &f) {
+ cir::IntType byteTy = b.getUIntNTy(8);
+ mlir::Value overflowP = b.createGetMember(
+ f.loc, b.getPointerTo(f.vaFields[2]), f.valist, "overflow_arg_area", 2);
+ mlir::Value overflow = b.createLoad(f.loc, overflowP);
+ mlir::Value bytePtr = b.createPtrBitcast(overflow, byteTy);
+
+ uint64_t tyAlign = argumentAreaAlign(f.resultTy, f.module, f.dl);
+ if (tyAlign > 8)
+ bytePtr = roundPointerUpToAlignment(b, f.loc, bytePtr, tyAlign, f.dl);
+
+ uint64_t tySize = f.dl.getTypeSize(f.resultTy).getFixedValue();
+ uint64_t stride = (tySize + 7) & ~UINT64_C(7);
+ mlir::Value strideVal = b.getSignedInt(f.loc, stride, 32);
+ mlir::Value next = b.createPtrStride(f.loc, bytePtr, strideVal);
+ b.createStore(f.loc, next, overflowP);
+ return bytePtr;
+}
+
+/// Loads the cursor offsets and builds the predicate that sends the fetch to
+/// the register-save area. Both offsets count from the start of that area, so
+/// the integer limit is the six GP registers at 48 and the vector limit is
+/// those plus the eight SSE registers at 176.
+RegisterCursor buildRegisterGate(CIRBaseBuilderTy &b, const VAArgFetch &f,
+ unsigned neededInt, unsigned neededSse) {
+ RegisterCursor cursor;
+ if (neededInt) {
+ cursor.gpOffsetP = b.createGetMember(f.loc, b.getPointerTo(f.vaFields[0]),
+ f.valist, "gp_offset", 0);
+ cursor.gpOffset = b.createLoad(f.loc, cursor.gpOffsetP);
+ mlir::Value limit =
+ b.getConstantInt(f.loc, cursor.gpOffset.getType(), 48 - neededInt * 8);
+ cursor.inRegs =
+ b.createCompare(f.loc, cir::CmpOpKind::le, cursor.gpOffset, limit);
+ }
+ if (neededSse) {
+ cursor.fpOffsetP = b.createGetMember(f.loc, b.getPointerTo(f.vaFields[1]),
+ f.valist, "fp_offset", 1);
+ cursor.fpOffset = b.createLoad(f.loc, cursor.fpOffsetP);
+ mlir::Value limit = b.getConstantInt(f.loc, cursor.fpOffset.getType(),
+ 176 - neededSse * 16);
+ mlir::Value fitsInFp =
+ b.createCompare(f.loc, cir::CmpOpKind::le, cursor.fpOffset, limit);
+ cursor.inRegs = cursor.inRegs
+ ? b.createLogicalAnd(f.loc, cursor.inRegs, fitsInFp)
+ : fitsInFp;
+ }
+ return cursor;
+}
+
+/// Copies each half of a non-contiguous pair out of the register-save area
+/// into \p regPairTemp, laid out as the coerced pair.
+void reassembleRegisterPair(CIRBaseBuilderTy &b, mlir::Location loc,
+ const ArgClassification &ac,
+ const RegisterCursor &cursor,
+ llvm::ArrayRef<bool> pairIsSse,
+ mlir::Value regSaveArea, mlir::Value regPairTemp) {
+ auto pairTy = mlir::cast<cir::RecordType>(ac.coercedType);
+ // SSE slots sit 16 bytes apart within their own area, so track how many of
+ // each class came before.
+ unsigned seenOfClass[2] = {0, 0};
+ for (unsigned i = 0; i < 2; ++i) {
+ bool isSse = pairIsSse[i];
+ mlir::Value base = isSse ? cursor.fpOffset : cursor.gpOffset;
+ unsigned regSize = isSse ? 16 : 8;
+ unsigned prior = seenOfClass[isSse]++;
+ mlir::Value off = base;
+ if (prior) {
+ off = b.createAdd(loc, base,
----------------
andykaylor wrote:
I'm not sure I'm following this. It looks like `varargs_aggregate_mixed_pair`
has places where we're loading with higher alignment than OGCG and places where
we're loading with lower alignment. Requiring lower alignment should be fine,
but requiring higher alignment doesn't seem safe.
https://github.com/llvm/llvm-project/pull/222420
_______________________________________________
cfe-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
