================
@@ -1511,3 +1520,380 @@ void
CIRABIRewriteContext::rewriteFunctionAddress(cir::GetGlobalOp addrOp,
cir::CastKind::bitcast, addrOp.getAddr());
addrOp.getAddr().replaceAllUsesExcept(bitcast.getResult(), bitcast);
}
+
+namespace {
+
+/// What one `va_arg` expansion needs from the op it rewrites. The x86-64
+/// cursor fields are reached through `vaFields` by index: 0 gp_offset,
+/// 1 fp_offset, 2 overflow_arg_area, 3 reg_save_area.
+struct VAArgFetch {
+ mlir::Location loc;
+ mlir::Value valist;
+ llvm::ArrayRef<mlir::Type> vaFields;
+ mlir::Type resultTy;
+ const ArgClassification ∾
+ const mlir::DataLayout &dl;
+ mlir::ModuleOp module;
+};
+
+/// The cursor fields a register fetch reads, and the predicate saying every
+/// class it needs still has room. An offset is null when the fetch needs no
+/// register of that class.
+struct RegisterCursor {
+ mlir::Value gpOffsetP;
+ mlir::Value fpOffsetP;
+ mlir::Value gpOffset;
+ mlir::Value fpOffset;
+ mlir::Value inRegs;
+};
+
+mlir::LogicalResult reportVAArgNYI(cir::VAArgOp op, llvm::StringRef what) {
+ op->emitOpError() << "va_arg of " << what
+ << " not yet implemented in CallConvLowering";
+ return mlir::failure();
+}
+
+/// Sets \p isRegPair when a two-register argument is a coerced pair, one
+/// register per member, rather than a single wide scalar, and records in
+/// \p pairIsSse which element is SSE class. Fails on a coercion that is not
+/// two eightbytes.
+mlir::LogicalResult classifyRegisterPair(cir::VAArgOp op,
+ const ArgClassification &ac,
+ std::array<bool, 2> &pairIsSse,
+ bool &isRegPair) {
+ auto pairTy = mlir::dyn_cast<cir::RecordType>(ac.coercedType);
+ if (!pairTy)
+ return mlir::success();
+
+ if (pairTy.getNumElements() != 2)
+ return reportVAArgNYI(op, "a register coercion that is not two
eightbytes");
+
+ assert(!ac.directOffset &&
+ "a pair already spans both eightbytes, so it cannot also start "
+ "partway into the value");
+ for (auto [i, memberTy] : llvm::enumerate(pairTy.getMembers()))
+ pairIsSse[i] = isSSERegisterClass(memberTy);
+ isRegPair = true;
+ return mlir::success();
+}
+
+/// The alignment an argument is placed at. A record can require more than
+/// the types of its members imply, from an `aligned` attribute on the record
+/// or on one of its fields, and neither raises the alignment of any type.
+/// Only the record layout knows, so the member-derived value alone can be too
+/// small.
+uint64_t argumentAreaAlign(mlir::Type ty, mlir::ModuleOp modOp,
+ const mlir::DataLayout &dl) {
+ uint64_t align = dl.getTypeABIAlignment(ty);
+ if (auto recTy = mlir::dyn_cast<cir::RecordType>(ty))
+ if (auto layout = cir::tryGetRecordLayout(modOp, recTy.getName()))
+ align = std::max<uint64_t>(align, layout.getRecordAlign());
+ return align;
+}
+
+mlir::Value roundPointerUpToAlignment(CIRBaseBuilderTy &b, mlir::Location loc,
+ mlir::Value bytePtr, uint64_t align,
+ const mlir::DataLayout &dl) {
+ assert(llvm::isPowerOf2_64(align) &&
+ "mask rounding needs a power-of-two alignment");
+ mlir::Value bumped =
+ b.createPtrStride(loc, bytePtr, b.getSignedInt(loc, align - 1, 32));
+ std::optional<uint64_t> indexWidth =
+ dl.getTypeIndexBitwidth(bytePtr.getType());
+ assert(indexWidth && "a pointer in the argument area has an index width");
+ mlir::Value mask = b.getSignedInt(loc, -static_cast<int64_t>(align),
+ static_cast<unsigned>(*indexWidth));
+ return cir::PtrMaskOp::create(b, loc, bytePtr.getType(), bumped, mask);
+}
+
+/// Reads the argument's address out of the overflow area, which also advances
+/// the cursor past the argument.
+mlir::Value buildOverflowAddr(CIRBaseBuilderTy &b, const VAArgFetch &f) {
+ cir::IntType byteTy = b.getUIntNTy(8);
+ mlir::Value overflowP = b.createGetMember(
+ f.loc, b.getPointerTo(f.vaFields[2]), f.valist, "overflow_arg_area", 2);
+ mlir::Value overflow = b.createLoad(f.loc, overflowP);
+ mlir::Value bytePtr = b.createPtrBitcast(overflow, byteTy);
+
+ uint64_t tyAlign = argumentAreaAlign(f.resultTy, f.module, f.dl);
+ if (tyAlign > 8)
+ bytePtr = roundPointerUpToAlignment(b, f.loc, bytePtr, tyAlign, f.dl);
+
+ uint64_t tySize = f.dl.getTypeSize(f.resultTy).getFixedValue();
+ uint64_t stride = (tySize + 7) & ~UINT64_C(7);
+ mlir::Value strideVal = b.getSignedInt(f.loc, stride, 32);
+ mlir::Value next = b.createPtrStride(f.loc, bytePtr, strideVal);
+ b.createStore(f.loc, next, overflowP);
+ return bytePtr;
+}
+
+/// Loads the cursor offsets and builds the predicate that sends the fetch to
+/// the register-save area. Both offsets count from the start of that area, so
+/// the integer limit is the six GP registers at 48 and the vector limit is
+/// those plus the eight SSE registers at 176.
+RegisterCursor buildRegisterGate(CIRBaseBuilderTy &b, const VAArgFetch &f,
+ unsigned neededInt, unsigned neededSse) {
+ RegisterCursor cursor;
+ if (neededInt) {
+ cursor.gpOffsetP = b.createGetMember(f.loc, b.getPointerTo(f.vaFields[0]),
+ f.valist, "gp_offset", 0);
+ cursor.gpOffset = b.createLoad(f.loc, cursor.gpOffsetP);
+ mlir::Value limit =
+ b.getConstantInt(f.loc, cursor.gpOffset.getType(), 48 - neededInt * 8);
+ cursor.inRegs =
+ b.createCompare(f.loc, cir::CmpOpKind::le, cursor.gpOffset, limit);
+ }
+ if (neededSse) {
+ cursor.fpOffsetP = b.createGetMember(f.loc, b.getPointerTo(f.vaFields[1]),
+ f.valist, "fp_offset", 1);
+ cursor.fpOffset = b.createLoad(f.loc, cursor.fpOffsetP);
+ mlir::Value limit = b.getConstantInt(f.loc, cursor.fpOffset.getType(),
+ 176 - neededSse * 16);
+ mlir::Value fitsInFp =
+ b.createCompare(f.loc, cir::CmpOpKind::le, cursor.fpOffset, limit);
+ cursor.inRegs = cursor.inRegs
+ ? b.createLogicalAnd(f.loc, cursor.inRegs, fitsInFp)
+ : fitsInFp;
+ }
+ return cursor;
+}
+
+/// Copies each half of a non-contiguous pair out of the register-save area
+/// into \p regPairTemp, laid out as the coerced pair.
+void reassembleRegisterPair(CIRBaseBuilderTy &b, mlir::Location loc,
+ const ArgClassification &ac,
+ const RegisterCursor &cursor,
+ llvm::ArrayRef<bool> pairIsSse,
+ mlir::Value regSaveArea, mlir::Value regPairTemp) {
+ auto pairTy = mlir::cast<cir::RecordType>(ac.coercedType);
+ // SSE slots sit 16 bytes apart within their own area, so track how many of
+ // each class came before.
+ unsigned seenOfClass[2] = {0, 0};
+ for (unsigned i = 0; i < 2; ++i) {
+ bool isSse = pairIsSse[i];
+ mlir::Value base = isSse ? cursor.fpOffset : cursor.gpOffset;
+ unsigned regSize = isSse ? 16 : 8;
+ unsigned prior = seenOfClass[isSse]++;
+ mlir::Value off = base;
+ if (prior) {
+ off = b.createAdd(loc, base,
----------------
andykaylor wrote:
Will this always get the right alignment?
https://github.com/llvm/llvm-project/pull/222420
_______________________________________________
cfe-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
