https://github.com/adams381 updated 
https://github.com/llvm/llvm-project/pull/225010

>From 64a586fc805a208ef8cc381bfdffd5e08ef59129 Mon Sep 17 00:00:00 2001
From: Adam Smith <[email protected]>
Date: Sun, 20 Sep 2026 23:17:58 -0700
Subject: [PATCH] [CIR] Read a non-byval parameter back out of its spill slot

CallConvLowering recognizes a forwarded non-byval indirect argument by the slot
its operand was loaded from.  At -O1 and above cir-simplify folds that load
away when the slot is a constant alloca, which is what CIRGen emits for a
const-qualified by-value parameter.  The walk that gives each such parameter's
slot the alignment the ABI promises now also reads the record back out of the
slot at the spill and points the parameter's call-argument uses at that load,
so an Expand, byval or coerced argument reads it too.

Assisted-by: Cursor / claude-opus-5
---
 .../Transforms/CallConvLoweringPass.cpp       |  21 +-
 .../TargetLowering/CIRABIRewriteContext.cpp   | 159 ++++-
 .../TargetLowering/CIRABIRewriteContext.h     |  43 +-
 ...-lowering-x86_64-non-byval-thunk-mixed.cpp |  58 ++
 ...v-lowering-x86_64-non-byval-thunk-sret.cpp |  70 ++
 ...l-conv-lowering-x86_64-non-byval-thunk.cpp |  50 ++
 .../indirect-non-byval-forward-param.cir      | 655 ++++++++++++++++++
 .../abi-lowering/indirect-non-byval-nyi.cir   | 309 +++++++++
 8 files changed, 1329 insertions(+), 36 deletions(-)
 create mode 100644 
clang/test/CIR/CodeGen/call-conv-lowering-x86_64-non-byval-thunk-mixed.cpp
 create mode 100644 
clang/test/CIR/CodeGen/call-conv-lowering-x86_64-non-byval-thunk-sret.cpp
 create mode 100644 
clang/test/CIR/CodeGen/call-conv-lowering-x86_64-non-byval-thunk.cpp

diff --git a/clang/lib/CIR/Dialect/Transforms/CallConvLoweringPass.cpp 
b/clang/lib/CIR/Dialect/Transforms/CallConvLoweringPass.cpp
index 3d878fd7866ef..9fd800c1e3178 100644
--- a/clang/lib/CIR/Dialect/Transforms/CallConvLoweringPass.cpp
+++ b/clang/lib/CIR/Dialect/Transforms/CallConvLoweringPass.cpp
@@ -889,7 +889,7 @@ void CallConvLoweringPass::runOnOperation() {
   DataLayout dl(moduleOp);
   CIRABIRewriteContext rewriteCtx(moduleOp, dl);
   // A non-byval indirect parameter's slot outlives the rewrite that retypes
-  // the parameter, so that a call forwarding the parameter can still recognise
+  // the parameter, so that a call forwarding the parameter can still recognize
   // it.  Draining on scope exit collapses those slots whichever way this
   // function returns.
   llvm::scope_exit drainParamSlots(
@@ -1032,14 +1032,17 @@ void CallConvLoweringPass::runOnOperation() {
     addressTakers[callee].push_back(getGlobal);
   });
 
-  // Restate every non-byval indirect parameter's slot alignment as the one the
-  // ABI promises for that parameter, before anything reads a slot.  A call is
-  // rewritten together with its callee rather than with the function
-  // containing it, so a call forwarding such a parameter can be reached before
-  // the parameter's own function is rewritten.  Doing this up front makes the
-  // forwarding decision independent of the order the two were declared in.
-  for (auto &kv : classifications)
-    rewriteCtx.normalizeParameterSlotAlignments(kv.first, kv.second);
+  // Restate every non-byval indirect parameter's slot alignment and route any
+  // use of such a parameter as a call argument through a load of that slot,
+  // before any definition or call site is rewritten.  Doing this up front
+  // makes the forwarding decision independent of the order the callee and its
+  // caller were declared in.
+  for (auto &kv : classifications) {
+    if (failed(rewriteCtx.prepareNonByvalParameters(kv.first, kv.second))) {
+      signalPassFailure();
+      return;
+    }
+  }
 
   // Rewrite each function together with every direct call to it and every op
   // holding its address.  By the time we move on to function F+1, F's
diff --git 
a/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp 
b/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp
index 371fa67922872..e623f1cf94b0b 100644
--- a/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp
+++ b/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp
@@ -559,11 +559,11 @@ static void eraseDeadRecordLoads(ArrayRef<cir::LoadOp> 
loads) {
 }
 
 /// The store that spills non-byval indirect parameter \p blockArg, and the
-/// slot it spills into.  CIRGen spills every by-value parameter into a local
-/// alloca with a single store before any other use, and this pass runs on that
-/// CIRGen output before any alloca-promoting or splitting pass, so the block
-/// argument has exactly that one use.  Both results are null when DCE already
-/// removed a dead spill.
+/// slot it spills into.  prepareNonByvalParameters has already established
+/// that the spill is the block argument's only use and that it stores into
+/// an alloca, which is what the assertions below rest on.  Both results are
+/// null when the parameter has no spill, which is so only when nothing uses
+/// it at all.
 static std::pair<cir::StoreOp, cir::AllocaOp>
 findParamSpill(mlir::BlockArgument blockArg) {
   if (blockArg.use_empty())
@@ -582,7 +582,7 @@ findParamSpill(mlir::BlockArgument blockArg) {
 /// change the block argument's type to a pointer and insert a load at entry
 /// so the body sees a local copy of the original value type.  For each
 /// Indirect non-byval arg, change the block argument to a pointer and queue
-/// the CIRGen param-slot alloca to be replaced by it (no entry load /
+/// the param-slot alloca to be replaced by it (no entry load /
 /// byte-copy) so the body operates on the caller's storage in place.  For each
 /// Expand arg, replace the single struct block argument with N scalar block
 /// arguments (one per field) and store each field directly into the 
parameter's
@@ -790,7 +790,7 @@ void insertArgCoercion(
 
         // Pointing the slot's uses at the incoming pointer waits until every
         // call site has been rewritten.  A call that hands this parameter
-        // straight on recognises it by the slot its operand was loaded from,
+        // straight on recognizes it by the slot its operand was loaded from,
         // and collapsing the slot here would leave that call reading a block
         // argument with no defining operation to inspect.  A dead spill DCE
         // already removed leaves nothing to collapse.
@@ -1063,14 +1063,147 @@ void rewriteIndirectReturnCall(cir::CallOp call,
 
 } // namespace
 
-void CIRABIRewriteContext::normalizeParameterSlotAlignments(
+/// Bring \p funcOp's non-byval indirect parameter \p argNo into the shape the
+/// rest of the rewrite assumes.  \p claimedSlots carries the slots \p funcOp's
+/// earlier non-byval indirect parameters took.  See prepareNonByvalParameters.
+static mlir::LogicalResult
+prepareNonByvalParameter(cir::FuncOp funcOp, unsigned argNo,
+                         const ArgClassification &ac,
+                         mlir::BlockArgument blockArg, mlir::DominanceInfo 
&dom,
+                         SmallPtrSetImpl<mlir::Operation *> &claimedSlots) {
+  // The spill is the store that writes the parameter itself.  Any other use
+  // consumes the record value.  At -O1 and above such a use comes from
+  // cir-simplify: CIRGen marks a const-qualified parameter's slot const, so
+  // the load of it folds to the stored parameter.
+  cir::StoreOp spill;
+  cir::StoreOp extraSpill;
+  mlir::Operation *otherUse = nullptr;
+  SmallVector<mlir::OpOperand *> callArgs;
+  for (mlir::OpOperand &use : blockArg.getUses()) {
+    auto store = dyn_cast<cir::StoreOp>(use.getOwner());
+    if (store && store.getValue() == blockArg) {
+      // Which of two stores is the spill decides which slot stands in for the
+      // parameter, and the use list is in no particular order, so there is
+      // nothing to prefer between them.
+      if (spill)
+        extraSpill = store;
+      else
+        spill = store;
+      continue;
+    }
+    // Only an argument is served by a load of the slot.  Any other consumer
+    // belongs to a rewrite that reads the parameter its own way: a returned
+    // record, for one, is rewritten through the sret slot, which assumes the
+    // returned load names the return slot and not this one.
+    auto call = dyn_cast<cir::CIRCallOpInterface>(use.getOwner());
+    if (call && llvm::is_contained(call.getArgOperands(), blockArg))
+      callArgs.push_back(&use);
+    else if (!otherUse)
+      otherUse = use.getOwner();
+  }
+
+  if (extraSpill)
+    return extraSpill->emitOpError()
+           << "non-byval parameter " << argNo
+           << " spilled more than once is not yet implemented in "
+              "CallConvLowering";
+
+  if (otherUse)
+    return otherUse->emitOpError()
+           << "non-byval parameter " << argNo
+           << " consumed other than as a call argument is not yet implemented "
+              "in CallConvLowering";
+
+  if (!spill) {
+    // Every other kind of use was reported above, so the parameter has no
+    // uses at all and needs neither a slot nor a read.
+    if (callArgs.empty())
+      return mlir::success();
+
+    // Without a spill the parameter becomes the incoming pointer directly,
+    // which is the storage an argument taken from it has to name.  Giving it
+    // the spill it lacks lets the checks and the read below apply unchanged,
+    // and neither survives the pass: insertArgCoercion erases the store and
+    // finalizeParameterSlots replaces the slot.
+    mlir::OpBuilder builder(funcOp.getContext());
+    builder.setInsertionPointToStart(blockArg.getOwner());
+    auto synthesized = cir::AllocaOp::create(
+        builder, funcOp.getLoc(), cir::PointerType::get(blockArg.getType()),
+        builder.getStringAttr("nonbyval.param"),
+        builder.getI64IntegerAttr(ac.indirectAlign.value()));
+    spill =
+        cir::StoreOp::create(builder, funcOp.getLoc(), blockArg, synthesized);
+  }
+
+  // The incoming pointer replaces the slot itself, in the default address
+  // space.  Retargeting a cast of it would leave the allocation's other views
+  // reading storage nothing writes.  Storage in another address space, or
+  // storage that is not a local alloca at all, is not something the incoming
+  // pointer can be substituted for.
+  cir::AllocaOp slot = spill.getAddr().getDefiningOp<cir::AllocaOp>();
+  if (!slot ||
+      spill.getAddr().getType() != cir::PointerType::get(blockArg.getType()))
+    return spill->emitOpError()
+           << "non-byval parameter " << argNo
+           << " spilled to storage that cannot take the incoming pointer is "
+              "not yet implemented in CallConvLowering";
+
+  // One slot stands in for one non-byval parameter, since the incoming
+  // pointer replaces it.  Two of them spilled to the same slot are each
+  // spilled once, so the check above cannot see the collision and only
+  // comparing the slots can.
+  if (!claimedSlots.insert(slot).second)
+    return spill->emitOpError()
+           << "non-byval parameter " << argNo
+           << " sharing its spill slot with another parameter is not yet "
+              "implemented in CallConvLowering";
+
+  // CIRGen picked the slot's alignment for a local copy of the record, but the
+  // slot is about to stand in for the parameter, and a call forwarding it may
+  // only promise what the incoming pointer does.
+  slot.setAlignment(ac.indirectAlign.value());
+
+  if (callArgs.empty())
+    return mlir::success();
+
+  // A reader the spill does not dominate could read the slot before anything
+  // wrote it, and a load placed at the spill would not dominate it either.
+  for (mlir::OpOperand *callArg : callArgs)
+    if (!dom.properlyDominates(spill, callArg->getOwner()))
+      return callArg->getOwner()->emitOpError()
+             << "non-byval parameter " << argNo
+             << " read before its spill is not yet implemented in "
+                "CallConvLowering";
+
+  // Read the record back out of the slot instead, so every reader sees the
+  // shape CIRGen emits without cir-simplify: a load naming the storage the
+  // argument would have to name anyway.  Reading at the spill, not at the
+  // reader, keeps the value the parameter's own whatever the body later
+  // stores into the slot, and the load promises only the alignment the
+  // classification gives the incoming pointer.
+  //
+  // A call that forwards the argument consumes the load once it recognizes
+  // the slot.  One that wants a copy keeps it, reading the incoming pointer
+  // once finalizeParameterSlots replaces the slot.
+  mlir::OpBuilder builder(spill);
+  builder.setInsertionPointAfter(spill);
+  auto reload = cir::LoadOp::create(builder, spill.getLoc(), slot);
+  reload.setAlignment(ac.indirectAlign.value());
+  for (mlir::OpOperand *callArg : callArgs)
+    callArg->set(reload.getResult());
+  return mlir::success();
+}
+
+mlir::LogicalResult CIRABIRewriteContext::prepareNonByvalParameters(
     cir::FuncOp funcOp, const FunctionClassification &fc) {
   if (!funcOp.isDefinition())
-    return;
+    return mlir::success();
   mlir::Region &body = funcOp->getRegion(0);
   if (body.empty())
-    return;
+    return mlir::success();
   mlir::Block &entry = body.front();
+  mlir::DominanceInfo dom;
+  SmallPtrSet<mlir::Operation *, 4> claimedSlots;
 
   // No signature has been rewritten yet, so no sret pointer has been prepended
   // and no Expand argument has been split into its fields.  Every
@@ -1081,9 +1214,11 @@ void 
CIRABIRewriteContext::normalizeParameterSlotAlignments(
       continue;
     assert(idx < entry.getNumArguments() &&
            "classification count must not exceed entry block arguments");
-    if (cir::AllocaOp slot = findParamSpill(entry.getArgument(idx)).second)
-      slot.setAlignment(ac.indirectAlign.value());
+    if (failed(prepareNonByvalParameter(funcOp, idx, ac, 
entry.getArgument(idx),
+                                        dom, claimedSlots)))
+      return mlir::failure();
   }
+  return mlir::success();
 }
 
 void CIRABIRewriteContext::finalizeParameterSlots() {
diff --git 
a/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.h 
b/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.h
index b23359f8e5dee..c1b5468b80033 100644
--- a/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.h
+++ b/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.h
@@ -66,22 +66,35 @@ class CIRABIRewriteContext : public 
mlir::abi::ABIRewriteContext {
   void rewriteFunctionAddress(cir::GetGlobalOp addrOp, cir::FuncOp funcOp,
                               mlir::OpBuilder &builder);
 
-  /// Restate each non-byval indirect parameter's CIRGen slot alignment as the
-  /// alignment the ABI promises for that parameter.  CIRGen picked the slot's
-  /// alignment for a local copy of the record, but the slot is about to stand
-  /// in for the parameter, and a call forwarding it may only promise what the
-  /// incoming pointer does.  Call for every function before any call site is
-  /// rewritten, since a call is rewritten with its callee rather than with its
-  /// enclosing function and so may be reached first.  Not an override, since
-  /// this has no counterpart in the generic contract.
-  void
-  normalizeParameterSlotAlignments(cir::FuncOp funcOp,
-                                   const mlir::abi::FunctionClassification 
&fc);
+  /// Bring each non-byval indirect parameter of \p funcOp into the shape the
+  /// rest of the rewrite assumes: the parameter's only use, if it has one, is
+  /// a single store into an alloca of the matching pointer type that no other
+  /// non-byval indirect parameter spills to, and that alloca states the
+  /// alignment the ABI promises rather than the one CIRGen picked for a local
+  /// copy.  A use of the parameter as a call argument is routed through a load
+  /// of that slot, so that it names the storage an argument has to name.
+  ///
+  /// Call for every function before any definition or call site is rewritten.
+  /// findParamSpill asserts this shape while the enclosing definition is
+  /// rewritten, and a call is rewritten with its callee rather than with its
+  /// enclosing function and so may be reached first.
+  ///
+  /// A parameter read with no spill to name is given one, since it becomes
+  /// the incoming pointer directly.  A parameter spilled twice, spilled to a
+  /// slot another such parameter also spills to, consumed other than as a
+  /// call argument, spilled where the incoming pointer cannot replace the
+  /// storage, or read where the spill does not dominate it gets a diagnostic
+  /// on \p funcOp and failure.
+  ///
+  /// Not an override, since this has no counterpart in the generic contract.
+  mlir::LogicalResult
+  prepareNonByvalParameters(cir::FuncOp funcOp,
+                            const mlir::abi::FunctionClassification &fc);
 
-  /// Replace each non-byval indirect parameter's CIRGen slot with the
+  /// Replace each non-byval indirect parameter's spill slot with the
   /// incoming pointer, so the body operates on the caller's storage in place.
   /// Call once, after every function and call site has been rewritten: a call
-  /// forwarding such a parameter reads the slot to recognise it.  Not an
+  /// forwarding such a parameter reads the slot to recognize it.  Not an
   /// override, since deferring this has no counterpart in the generic
   /// contract.
   void finalizeParameterSlots();
@@ -92,10 +105,10 @@ class CIRABIRewriteContext : public 
mlir::abi::ABIRewriteContext {
   mlir::ModuleOp module;
   const mlir::DataLayout &dl;
 
-  /// CIRGen param-slot allocas that non-byval indirect parameters will
+  /// Param-slot allocas that non-byval indirect parameters will
   /// replace, paired with the incoming pointer that replaces them.  The
   /// rewrite retypes the block argument but leaves the slot standing, because
-  /// a call site recognises a forwardable parameter by the slot its operand
+  /// a call site recognizes a forwardable parameter by the slot its operand
   /// was loaded from.  finalizeParameterSlots does the replacement once every
   /// call site has been rewritten.
   llvm::SmallVector<std::pair<cir::AllocaOp, mlir::BlockArgument>>
diff --git 
a/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-non-byval-thunk-mixed.cpp 
b/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-non-byval-thunk-mixed.cpp
new file mode 100644
index 0000000000000..6cc93986484ae
--- /dev/null
+++ b/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-non-byval-thunk-mixed.cpp
@@ -0,0 +1,58 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -O2 -fclangir -emit-cir %s 
-o %t.cir
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o 
%t-O0.cir
+// RUN: FileCheck --check-prefix=CIR --input-file=%t-O0.cir %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -O2 -disable-llvm-passes 
-fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --check-prefixes=LLVM,LLVM-CIR --input-file=%t-cir.ll %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -O2 -disable-llvm-passes 
-emit-llvm %s -o %t.ll
+// RUN: FileCheck --check-prefixes=LLVM,OGCG --input-file=%t.ll %s
+
+struct Str {
+  char c;
+  Str(const Str &);
+  ~Str();
+};
+
+struct Io {
+  virtual char *stream(const Str &, Str);
+};
+
+struct Replicas {
+  virtual ~Replicas();
+};
+
+struct Proxy : Replicas, Io {
+  ~Proxy();
+};
+
+struct Namd : Proxy {
+  char *stream(const Str &, Str) override;
+};
+
+// The top-level const is written only on the definition, which is what marks
+// the parameter's spill slot const.
+char *Namd::stream(const Str &, const Str) { return nullptr; }
+
+// The by-value parameter is forwarded without a copy, from behind a reference
+// parameter and alongside a returned pointer.
+
+// CIR-LABEL: cir.func{{.*}} @_ZThn8_N4Namd6streamERK3StrS0_
+// CIR-SAME:    %{{[^ :]+}}: !cir.ptr<!rec_Namd> {llvm.noundef}
+// CIR-SAME:    %{{[^ :]+}}: !cir.ptr<!rec_Str> {llvm.align = 1 : i64, 
llvm.dereferenceable = 1 : i64, llvm.nonnull, llvm.noundef}
+// CIR-SAME:    %[[STR:[^ :]+]]: !cir.ptr<!rec_Str> {llvm.align = 1 : i64, 
llvm.dereferenceable = 1 : i64, llvm.nofreeobj, llvm.noundef}
+// CIR:         cir.call @_ZN4Namd6streamERK3StrS0_(%{{[^,)]+}}, %{{[^,)]+}}, 
%[[STR]]) : (!cir.ptr<!rec_Namd> {llvm.align = 8 : i64, llvm.dereferenceable = 
16 : i64, llvm.nonnull, llvm.noundef}, !cir.ptr<!rec_Str> {llvm.align = 1 : 
i64, llvm.dereferenceable = 1 : i64, llvm.nonnull, llvm.noundef}, 
!cir.ptr<!rec_Str> {llvm.align = 1 : i64, llvm.dereferenceable = 1 : i64, 
llvm.nofreeobj, llvm.noundef}) -> (!cir.ptr<!s8i> {llvm.noundef})
+
+// The overrider takes the same pointer without byval, which is the definition
+// side of the same contract.
+
+// LLVM-LABEL: define dso_local noundef ptr @_ZN4Namd6streamERK3StrS0_(
+// LLVM-SAME:    ptr noundef nonnull align 8 dereferenceable(16) %{{[^,]+}},
+// LLVM-SAME:    ptr noundef nonnull align 1 dereferenceable(1) %{{[^,]+}},
+// LLVM-SAME:    ptr nofreeobj noundef align 1 dereferenceable(1) %{{[^,)]+}})
+
+// LLVM-LABEL: define dso_local noundef ptr @_ZThn8_N4Namd6streamERK3StrS0_(
+// LLVM-SAME:    ptr noundef %{{[^,]+}},
+// LLVM-SAME:    ptr noundef nonnull align 1 dereferenceable(1) %{{[^,]+}},
+// LLVM-SAME:    ptr nofreeobj noundef align 1 dereferenceable(1) 
%[[STR:[^,)]+]])
+// LLVM-CIR:     call noundef ptr @_ZN4Namd6streamERK3StrS0_(ptr noundef 
nonnull align 8 dereferenceable(16) %{{[^,)]+}}, ptr noundef nonnull align 1 
dereferenceable(1) %{{[^,)]+}}, ptr nofreeobj noundef align 1 
dereferenceable(1) %[[STR]])
+// OGCG:         tail call noundef ptr @_ZN4Namd6streamERK3StrS0_(ptr noundef 
nonnull align 8 dereferenceable(16) %{{[^,)]+}}, ptr noundef nonnull align 1 
dereferenceable(1) %{{[^,)]+}}, ptr nofreeobj noundef align 1 
dereferenceable(1) %[[STR]])
diff --git 
a/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-non-byval-thunk-sret.cpp 
b/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-non-byval-thunk-sret.cpp
new file mode 100644
index 0000000000000..c7a19058466ff
--- /dev/null
+++ b/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-non-byval-thunk-sret.cpp
@@ -0,0 +1,70 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -O1 -fclangir -emit-cir %s 
-o %t.cir
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -O1 -disable-llvm-passes 
-fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --check-prefixes=LLVM,LLVM-CIR --input-file=%t-cir.ll %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -O1 -disable-llvm-passes 
-emit-llvm %s -o %t.ll
+// RUN: FileCheck --check-prefixes=LLVM,OGCG --input-file=%t.ll %s
+
+// Impl's key function is not defined here, so the vtable and its thunks are
+// emitted only when optimizing.
+
+struct Str {
+  ~Str();
+};
+
+struct BBox {
+  float a, b, c, d, e;
+};
+
+struct Base {
+  virtual void anchor();
+};
+
+struct Renderer {
+  virtual void textWithBBox(float, float, Str, unsigned);
+  virtual BBox bbox(Str, unsigned);
+};
+
+struct Impl : Base, Renderer {
+  void textWithBBox(float, float, const Str, unsigned);
+  BBox bbox(const Str, unsigned);
+};
+
+void emit() { new Impl; }
+
+// The by-value parameter sits behind two floats, so it is not argument zero.
+
+// CIR-LABEL: cir.func{{.*}} @_ZThn8_N4Impl12textWithBBoxEff3Strj
+// CIR-SAME:    %{{[^ :]+}}: !cir.ptr<!rec_Impl> {llvm.noundef}
+// CIR-SAME:    %{{[^ :]+}}: !cir.float {llvm.noundef}
+// CIR-SAME:    %{{[^ :]+}}: !cir.float {llvm.noundef}
+// CIR-SAME:    %[[STR:[^ :]+]]: !cir.ptr<!rec_Str> {llvm.align = 1 : i64, 
llvm.dereferenceable = 1 : i64, llvm.nofreeobj, llvm.noundef}
+// CIR-NOT:     cir.alloca {{.*}}!cir.ptr<!rec_Str>
+// CIR:         cir.call @_ZN4Impl12textWithBBoxEff3Strj(%{{[^,)]+}}, 
%{{[^,)]+}}, %{{[^,)]+}}, %[[STR]], %{{[^,)]+}})
+
+// An sret return prepends a block argument, so the parameter is not at the
+// index its classification sits at once the signature is rewritten.
+
+// CIR-LABEL: cir.func{{.*}} @_ZThn8_N4Impl4bboxE3Strj
+// CIR-SAME:    llvm.sret = !rec_BBox
+// CIR-SAME:    %{{[^ :]+}}: !cir.ptr<!rec_Impl> {llvm.noundef}
+// CIR-SAME:    %[[SSTR:[^ :]+]]: !cir.ptr<!rec_Str> {llvm.align = 1 : i64, 
llvm.dereferenceable = 1 : i64, llvm.nofreeobj, llvm.noundef}
+// CIR-NOT:     cir.alloca {{.*}}!cir.ptr<!rec_Str>
+// CIR:         cir.call @_ZN4Impl4bboxE3Strj(%{{[^,)]+}}, %{{[^,)]+}}, 
%[[SSTR]], %{{[^,)]+}})
+
+// LLVM-LABEL: define available_externally void 
@_ZThn8_N4Impl12textWithBBoxEff3Strj(
+// LLVM-SAME:    ptr noundef %{{[^,]+}},
+// LLVM-SAME:    float noundef %{{[^,]+}},
+// LLVM-SAME:    float noundef %{{[^,]+}},
+// LLVM-SAME:    ptr nofreeobj noundef align 1 dereferenceable(1) 
%[[LSTR:[^,]+]],
+// LLVM-SAME:    i32 noundef %{{[^,)]+}})
+// LLVM-CIR:     call void @_ZN4Impl12textWithBBoxEff3Strj(ptr noundef nonnull 
align 8 dereferenceable(16) %{{[^,)]+}}, float noundef %{{[^,)]+}}, float 
noundef %{{[^,)]+}}, ptr nofreeobj noundef align 1 dereferenceable(1) 
%[[LSTR]], i32 noundef %{{[^,)]+}})
+// OGCG:         tail call void @_ZN4Impl12textWithBBoxEff3Strj(ptr noundef 
nonnull align 8 dereferenceable(16) %{{[^,)]+}}, float noundef %{{[^,)]+}}, 
float noundef %{{[^,)]+}}, ptr nofreeobj noundef align 1 dereferenceable(1) 
%[[LSTR]], i32 noundef %{{[^,)]+}})
+
+// LLVM-LABEL: define available_externally void @_ZThn8_N4Impl4bboxE3Strj(
+// LLVM-SAME:    ptr dead_on_unwind noalias writable sret(%struct.BBox) align 
4 %{{[^,]+}},
+// LLVM-SAME:    ptr noundef %{{[^,]+}},
+// LLVM-SAME:    ptr nofreeobj noundef align 1 dereferenceable(1) 
%[[LSSTR:[^,]+]],
+// LLVM-SAME:    i32 noundef %{{[^,)]+}})
+// LLVM-CIR:     call void @_ZN4Impl4bboxE3Strj(ptr dead_on_unwind writable 
sret(%struct.BBox) align 4 %{{[^,)]+}}, ptr noundef nonnull align 8 
dereferenceable(16) %{{[^,)]+}}, ptr nofreeobj noundef align 1 
dereferenceable(1) %[[LSSTR]], i32 noundef %{{[^,)]+}})
+// OGCG:         tail call void @_ZN4Impl4bboxE3Strj(ptr dead_on_unwind 
writable sret(%struct.BBox) align 4 %{{[^,)]+}}, ptr noundef nonnull align 8 
dereferenceable(16) %{{[^,)]+}}, ptr nofreeobj noundef align 1 
dereferenceable(1) %[[LSSTR]], i32 noundef %{{[^,)]+}})
diff --git 
a/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-non-byval-thunk.cpp 
b/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-non-byval-thunk.cpp
new file mode 100644
index 0000000000000..aec0107e8a0c3
--- /dev/null
+++ b/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-non-byval-thunk.cpp
@@ -0,0 +1,50 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -O1 -fclangir -emit-cir %s 
-o %t.cir
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o 
%t-O0.cir
+// RUN: FileCheck --check-prefix=CIR --input-file=%t-O0.cir %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -O1 -disable-llvm-passes 
-fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --check-prefixes=LLVM,LLVM-CIR --input-file=%t-cir.ll %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -O1 -disable-llvm-passes 
-emit-llvm %s -o %t.ll
+// RUN: FileCheck --check-prefixes=LLVM,OGCG --input-file=%t.ll %s
+
+struct NotTrivial {
+  char buf[32];
+  NotTrivial(const NotTrivial &);
+  ~NotTrivial();
+};
+
+struct Base1 {
+  virtual ~Base1();
+  virtual int other();
+};
+
+struct Base2 {
+  virtual void take(const NotTrivial desc);
+};
+
+struct Derived : Base1, Base2 {
+  void take(const NotTrivial desc) override;
+};
+
+void Derived::take(const NotTrivial desc) {}
+
+// No copy of the parameter, at either optimization level.
+
+// CIR-LABEL: cir.func{{.*}} @_ZThn8_N7Derived4takeE10NotTrivial
+// CIR-SAME:    %{{[^ :]+}}: !cir.ptr<!rec_Derived> {llvm.noundef}
+// CIR-SAME:    %[[DESC:[^ :]+]]: !cir.ptr<!rec_NotTrivial> {llvm.align = 1 : 
i64, llvm.dereferenceable = 32 : i64, llvm.nofreeobj, llvm.noundef}
+// CIR-NOT:     cir.alloca "desc"
+// CIR:         cir.call @_ZN7Derived4takeE10NotTrivial(%{{[^,)]+}}, 
%[[DESC]]) : (!cir.ptr<!rec_Derived> {llvm.align = 8 : i64, 
llvm.dereferenceable = 16 : i64, llvm.nonnull, llvm.noundef}, 
!cir.ptr<!rec_NotTrivial> {llvm.align = 1 : i64, llvm.dereferenceable = 32 : 
i64, llvm.nofreeobj, llvm.noundef}) -> ()
+
+// The overrider takes the same pointer without byval, which is the definition
+// side of the same contract.
+
+// LLVM-LABEL: define dso_local void @_ZN7Derived4takeE10NotTrivial(
+// LLVM-SAME:    ptr noundef nonnull align 8 dereferenceable(16) %{{[^,]+}},
+// LLVM-SAME:    ptr nofreeobj noundef align 1 dereferenceable(32) %{{[^,)]+}})
+
+// LLVM-LABEL: define dso_local void @_ZThn8_N7Derived4takeE10NotTrivial(
+// LLVM-SAME:    ptr noundef %{{[^,]+}},
+// LLVM-SAME:    ptr nofreeobj noundef align 1 dereferenceable(32) 
%[[DESC:[^,)]+]])
+// LLVM-CIR:     call void @_ZN7Derived4takeE10NotTrivial(ptr noundef nonnull 
align 8 dereferenceable(16) %{{[^,)]+}}, ptr nofreeobj noundef align 1 
dereferenceable(32) %[[DESC]])
+// OGCG:         tail call void @_ZN7Derived4takeE10NotTrivial(ptr noundef 
nonnull align 8 dereferenceable(16) %{{[^,)]+}}, ptr nofreeobj noundef align 1 
dereferenceable(32) %[[DESC]])
diff --git 
a/clang/test/CIR/Transforms/abi-lowering/indirect-non-byval-forward-param.cir 
b/clang/test/CIR/Transforms/abi-lowering/indirect-non-byval-forward-param.cir
index 2fd34fd3aaedd..4019d1394bf79 100644
--- 
a/clang/test/CIR/Transforms/abi-lowering/indirect-non-byval-forward-param.cir
+++ 
b/clang/test/CIR/Transforms/abi-lowering/indirect-non-byval-forward-param.cir
@@ -393,3 +393,658 @@ module attributes {
   // CHECK:        cir.call @takes_two_expanded(%[[V0]], %[[V1]]) :
 
 }
+
+// -----
+
+!s64i = !cir.int<s, 64>
+!rec_Big = !cir.struct<"Big" {data !s64i, data !s64i, data !s64i, data !s64i}>
+
+#non_byval_arg = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8, byval = false } ]
+}
+
+module attributes {
+  dlti.dl_spec = #dlti.dl_spec<
+    #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+
+  // The parameter is handed on directly, with no load of its slot, which is
+  // what cir-simplify leaves at -O1 and above for a slot marked const.
+  cir.func @forwards_unloaded_param(%arg0: !rec_Big)
+      attributes { test_classify = #non_byval_arg } {
+    %spill = cir.alloca "arg0" align(8) init const : !cir.ptr<!rec_Big>
+    cir.store %arg0, %spill : !rec_Big, !cir.ptr<!rec_Big>
+    cir.call @takes_big_non_byval(%arg0) : (!rec_Big) -> ()
+    cir.return
+  }
+
+  // CHECK:      cir.func{{.*}} @forwards_unloaded_param(%[[PTR:.*]]: 
!cir.ptr<!rec_Big>
+  // CHECK-SAME:     {llvm.align = 8 : i64, llvm.dereferenceable = 32 : i64, 
llvm.nofreeobj, llvm.noundef})
+  // CHECK-NOT:    cir.alloca
+  // CHECK-NOT:    cir.load
+  // CHECK:        cir.call @takes_big_non_byval(%[[PTR]]) :
+  // CHECK-SAME:     (!cir.ptr<!rec_Big> {llvm.align = 8 : i64, 
llvm.dereferenceable = 32 : i64, llvm.nofreeobj, llvm.noundef}) -> ()
+
+  cir.func private @takes_big_non_byval(%arg0: !rec_Big)
+      attributes { test_classify = #non_byval_arg }
+
+}
+
+// -----
+
+!s64i = !cir.int<s, 64>
+!rec_Big = !cir.struct<"Big" {data !s64i, data !s64i, data !s64i, data !s64i}>
+
+#non_byval_arg = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8, byval = false } ]
+}
+
+module attributes {
+  dlti.dl_spec = #dlti.dl_spec<
+    #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+
+  // The same with the callee declared first, so the call is reached before
+  // the caller's own definition is rewritten, and over a slot aligned for a
+  // local copy rather than for the parameter.
+  cir.func private @takes_big_non_byval(%arg0: !rec_Big)
+      attributes { test_classify = #non_byval_arg }
+
+  cir.func @forwards_unloaded_param_early_call(%arg0: !rec_Big)
+      attributes { test_classify = #non_byval_arg } {
+    %spill = cir.alloca "arg0" align(4) init const : !cir.ptr<!rec_Big>
+    cir.store %arg0, %spill : !rec_Big, !cir.ptr<!rec_Big>
+    cir.call @takes_big_non_byval(%arg0) : (!rec_Big) -> ()
+    cir.return
+  }
+
+  // CHECK:      cir.func{{.*}} 
@forwards_unloaded_param_early_call(%[[PTR:.*]]: !cir.ptr<!rec_Big>
+  // CHECK-SAME:     {llvm.align = 8 : i64, llvm.dereferenceable = 32 : i64, 
llvm.nofreeobj, llvm.noundef})
+  // CHECK-NOT:    cir.alloca
+  // CHECK-NOT:    cir.load
+  // CHECK:        cir.call @takes_big_non_byval(%[[PTR]]) :
+  // CHECK-SAME:     (!cir.ptr<!rec_Big> {llvm.align = 8 : i64, 
llvm.dereferenceable = 32 : i64, llvm.nofreeobj, llvm.noundef}) -> ()
+
+}
+
+// -----
+
+!s64i = !cir.int<s, 64>
+!rec_Two = !cir.struct<"Two" {data !s64i, data !s64i}>
+
+#non_byval_two = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8, byval = false } ]
+}
+
+#expand_two = {
+  return = { kind = "direct" },
+  args   = [ { kind = "expand" } ]
+}
+
+module attributes {
+  dlti.dl_spec = #dlti.dl_spec<
+    #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+
+  // The unloaded parameter feeding an Expand callee.
+  cir.func private @takes_two_expand(%arg0: !rec_Two)
+      attributes { test_classify = #expand_two }
+
+  cir.func @unloaded_param_to_expand(%arg0: !rec_Two)
+      attributes { test_classify = #non_byval_two } {
+    %spill = cir.alloca "arg0" align(8) init const : !cir.ptr<!rec_Two>
+    cir.store %arg0, %spill : !rec_Two, !cir.ptr<!rec_Two>
+    cir.call @takes_two_expand(%arg0) : (!rec_Two) -> ()
+    cir.return
+  }
+
+  // CHECK:      cir.func{{.*}} @unloaded_param_to_expand(%[[PTR:.*]]: 
!cir.ptr<!rec_Two>
+  // CHECK-SAME:     {llvm.align = 8 : i64, llvm.dereferenceable = 16 : i64, 
llvm.nofreeobj, llvm.noundef})
+  // CHECK-NOT:    cir.alloca
+  // CHECK:        %[[F0:.*]] = cir.get_member %[[PTR]][0]
+  // CHECK:        %[[V0:.*]] = cir.load %[[F0]]
+  // CHECK:        %[[F1:.*]] = cir.get_member %[[PTR]][1]
+  // CHECK:        %[[V1:.*]] = cir.load %[[F1]]
+  // CHECK:        cir.call @takes_two_expand(%[[V0]], %[[V1]]) :
+
+}
+
+// -----
+
+!s64i = !cir.int<s, 64>
+!rec_Big = !cir.struct<"Big" {data !s64i, data !s64i, data !s64i, data !s64i}>
+
+#non_byval_arg = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8, byval = false } ]
+}
+
+#byval_arg = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8 } ]
+}
+
+module attributes {
+  dlti.dl_spec = #dlti.dl_spec<
+    #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+
+  // A byval callee gets its own copy, read out of the caller's storage.
+  cir.func private @takes_big_byval(%arg0: !rec_Big)
+      attributes { test_classify = #byval_arg }
+
+  cir.func @unloaded_param_to_byval(%arg0: !rec_Big)
+      attributes { test_classify = #non_byval_arg } {
+    %spill = cir.alloca "arg0" align(8) init const : !cir.ptr<!rec_Big>
+    cir.store %arg0, %spill : !rec_Big, !cir.ptr<!rec_Big>
+    cir.call @takes_big_byval(%arg0) : (!rec_Big) -> ()
+    cir.return
+  }
+
+  // CHECK:      cir.func{{.*}} @unloaded_param_to_byval(%[[PTR:.*]]: 
!cir.ptr<!rec_Big>
+  // CHECK-SAME:     {llvm.align = 8 : i64, llvm.dereferenceable = 32 : i64, 
llvm.nofreeobj, llvm.noundef})
+  // CHECK:        %[[VAL:.*]] = cir.load align(8) %[[PTR]] : 
!cir.ptr<!rec_Big>, !rec_Big
+  // CHECK-NOT:    cir.load
+  // CHECK:        %[[COPY:.*]] = cir.alloca "byval" align(8) : 
!cir.ptr<!rec_Big>
+  // CHECK:        cir.store %[[VAL]], %[[COPY]] : !rec_Big, !cir.ptr<!rec_Big>
+  // CHECK:        cir.call @takes_big_byval(%[[COPY]]) :
+  // CHECK-SAME:     (!cir.ptr<!rec_Big> {llvm.align = 8 : i64, llvm.byval = 
!rec_Big, llvm.noundef}) -> ()
+
+}
+
+// -----
+
+!s64i = !cir.int<s, 64>
+!rec_Two = !cir.struct<"Two" {data !s64i, data !s64i}>
+
+#non_byval_two = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8, byval = false } ]
+}
+
+#coerced_two = {
+  return = { kind = "direct" },
+  args   = [ { kind = "direct", coerced_type = !s64i } ]
+}
+
+module attributes {
+  dlti.dl_spec = #dlti.dl_spec<
+    #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+
+  // A Direct argument with a coerced type reads the record into a register,
+  // out of the caller's storage.
+  cir.func private @takes_two_coerced(%arg0: !rec_Two)
+      attributes { test_classify = #coerced_two }
+
+  cir.func @unloaded_param_to_coerced_direct(%arg0: !rec_Two)
+      attributes { test_classify = #non_byval_two } {
+    %spill = cir.alloca "arg0" align(8) init const : !cir.ptr<!rec_Two>
+    cir.store %arg0, %spill : !rec_Two, !cir.ptr<!rec_Two>
+    cir.call @takes_two_coerced(%arg0) : (!rec_Two) -> ()
+    cir.return
+  }
+
+  // CHECK:      cir.func{{.*}} @unloaded_param_to_coerced_direct(%[[PTR:.*]]: 
!cir.ptr<!rec_Two>
+  // CHECK-SAME:     {llvm.align = 8 : i64, llvm.dereferenceable = 16 : i64, 
llvm.nofreeobj, llvm.noundef})
+  // CHECK:        %[[SLOT:.*]] = cir.alloca "coerce" align(8) : 
!cir.ptr<!rec_Two>
+  // CHECK:        %[[VAL:.*]] = cir.load align(8) %[[PTR]] : 
!cir.ptr<!rec_Two>, !rec_Two
+  // CHECK:        cir.store %[[VAL]], %[[SLOT]] : !rec_Two, !cir.ptr<!rec_Two>
+  // CHECK:        %[[AS:.*]] = cir.cast bitcast %[[SLOT]] : 
!cir.ptr<!rec_Two> -> !cir.ptr<!s64i>
+  // CHECK:        %[[COERCED:.*]] = cir.load %[[AS]] : !cir.ptr<!s64i>, !s64i
+  // CHECK:        cir.call @takes_two_coerced(%[[COERCED]]) : (!s64i) -> ()
+
+}
+
+// -----
+
+!s64i = !cir.int<s, 64>
+!rec_Big = !cir.struct<"Big" {data !s64i, data !s64i, data !s64i, data !s64i}>
+
+#non_byval_arg = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8, byval = false } ]
+}
+
+#byval_arg = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8 } ]
+}
+
+module attributes {
+  dlti.dl_spec = #dlti.dl_spec<
+    #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+
+  // Two arguments taken from one parameter, one forwarded and one copied, so
+  // a second read of the slot would show up as a second load.
+  cir.func @unloaded_param_to_two_calls(%arg0: !rec_Big)
+      attributes { test_classify = #non_byval_arg } {
+    %spill = cir.alloca "arg0" align(8) init const : !cir.ptr<!rec_Big>
+    cir.store %arg0, %spill : !rec_Big, !cir.ptr<!rec_Big>
+    cir.call @takes_big_non_byval(%arg0) : (!rec_Big) -> ()
+    cir.call @takes_big_byval(%arg0) : (!rec_Big) -> ()
+    cir.return
+  }
+
+  // CHECK:      cir.func{{.*}} @unloaded_param_to_two_calls(%[[PTR:.*]]: 
!cir.ptr<!rec_Big>
+  // CHECK-SAME:     {llvm.align = 8 : i64, llvm.dereferenceable = 32 : i64, 
llvm.nofreeobj, llvm.noundef})
+  // CHECK:        %[[VAL:.*]] = cir.load align(8) %[[PTR]] : 
!cir.ptr<!rec_Big>, !rec_Big
+  // CHECK-NOT:    cir.load
+  // CHECK:        cir.call @takes_big_non_byval(%[[PTR]]) :
+  // CHECK:        %[[COPY:.*]] = cir.alloca "byval" align(8) : 
!cir.ptr<!rec_Big>
+  // CHECK:        cir.store %[[VAL]], %[[COPY]] : !rec_Big, !cir.ptr<!rec_Big>
+  // CHECK:        cir.call @takes_big_byval(%[[COPY]]) :
+
+  cir.func private @takes_big_non_byval(%arg0: !rec_Big)
+      attributes { test_classify = #non_byval_arg }
+
+  cir.func private @takes_big_byval(%arg0: !rec_Big)
+      attributes { test_classify = #byval_arg }
+
+}
+
+// -----
+
+!s64i = !cir.int<s, 64>
+!rec_Big = !cir.struct<"Big" {data !s64i, data !s64i, data !s64i, data !s64i}>
+
+#non_byval_arg = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8, byval = false } ]
+}
+
+module attributes {
+  dlti.dl_spec = #dlti.dl_spec<
+    #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+
+  // The argument is in a later block, which the spill still dominates.
+  cir.func @unloaded_param_read_in_successor(%arg0: !rec_Big)
+      attributes { test_classify = #non_byval_arg } {
+    %spill = cir.alloca "arg0" align(8) init const : !cir.ptr<!rec_Big>
+    cir.store %arg0, %spill : !rec_Big, !cir.ptr<!rec_Big>
+    cir.br ^bb1
+  ^bb1:
+    cir.call @takes_big_non_byval(%arg0) : (!rec_Big) -> ()
+    cir.return
+  }
+
+  // CHECK:      cir.func{{.*}} @unloaded_param_read_in_successor(%[[PTR:.*]]: 
!cir.ptr<!rec_Big>
+  // CHECK-SAME:     {llvm.align = 8 : i64, llvm.dereferenceable = 32 : i64, 
llvm.nofreeobj, llvm.noundef})
+  // CHECK-NOT:    cir.alloca
+  // CHECK-NOT:    cir.load
+  // CHECK:        cir.br ^bb1
+  // CHECK:      ^bb1:
+  // CHECK:        cir.call @takes_big_non_byval(%[[PTR]]) :
+
+  cir.func private @takes_big_non_byval(%arg0: !rec_Big)
+      attributes { test_classify = #non_byval_arg }
+
+}
+
+// -----
+
+!s64i = !cir.int<s, 64>
+!rec_Big = !cir.struct<"Big" {data !s64i, data !s64i, data !s64i, data !s64i}>
+
+#non_byval_arg = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8, byval = false } ]
+}
+
+#byval_arg = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8 } ]
+}
+
+module attributes {
+  dlti.dl_spec = #dlti.dl_spec<
+    #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+
+  // The body overwrites the slot before the argument is taken, so the copy the
+  // callee gets holds the parameter only if the read happens at the spill.
+  cir.func @unloaded_param_read_before_overwrite(%arg0: !rec_Big)
+      attributes { test_classify = #non_byval_arg } {
+    %spill = cir.alloca "arg0" align(8) init : !cir.ptr<!rec_Big>
+    cir.store %arg0, %spill : !rec_Big, !cir.ptr<!rec_Big>
+    %z = cir.const #cir.zero : !rec_Big
+    cir.store %z, %spill : !rec_Big, !cir.ptr<!rec_Big>
+    cir.call @takes_big_byval(%arg0) : (!rec_Big) -> ()
+    cir.return
+  }
+
+  // CHECK:      cir.func{{.*}} 
@unloaded_param_read_before_overwrite(%[[PTR:.*]]: !cir.ptr<!rec_Big>
+  // CHECK-SAME:     {llvm.align = 8 : i64, llvm.dereferenceable = 32 : i64, 
llvm.nofreeobj, llvm.noundef})
+  // CHECK:        %[[VAL:.*]] = cir.load align(8) %[[PTR]] : 
!cir.ptr<!rec_Big>, !rec_Big
+  // CHECK:        %[[ZERO:.*]] = cir.const #cir.zero : !rec_Big
+  // CHECK:        cir.store %[[ZERO]], %[[PTR]] : !rec_Big, !cir.ptr<!rec_Big>
+  // CHECK:        %[[COPY:.*]] = cir.alloca "byval" align(8) : 
!cir.ptr<!rec_Big>
+  // CHECK:        cir.store %[[VAL]], %[[COPY]] : !rec_Big, !cir.ptr<!rec_Big>
+  // CHECK:        cir.call @takes_big_byval(%[[COPY]]) :
+
+  cir.func private @takes_big_byval(%arg0: !rec_Big)
+      attributes { test_classify = #byval_arg }
+
+}
+
+// -----
+
+!s64i = !cir.int<s, 64>
+!rec_Big = !cir.struct<"Big" {data !s64i, data !s64i, data !s64i, data !s64i}>
+
+#sret_and_non_byval = {
+  return = { kind = "indirect", indirect_align = 8 },
+  args   = [ { kind = "indirect", indirect_align = 8, byval = false } ]
+}
+
+#non_byval_arg = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8, byval = false } ]
+}
+
+module attributes {
+  dlti.dl_spec = #dlti.dl_spec<
+    #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+
+  // An sret return prepends a block argument, so the parameter is not at the
+  // index its classification sits at once the signature is rewritten.
+  cir.func @sret_with_unloaded_param(%arg0: !rec_Big) -> !rec_Big
+      attributes { test_classify = #sret_and_non_byval } {
+    %ret = cir.alloca "__retval" align(8) : !cir.ptr<!rec_Big>
+    %spill = cir.alloca "arg0" align(8) init const : !cir.ptr<!rec_Big>
+    cir.store %arg0, %spill : !rec_Big, !cir.ptr<!rec_Big>
+    cir.call @takes_big_non_byval(%arg0) : (!rec_Big) -> ()
+    %r = cir.load %ret : !cir.ptr<!rec_Big>, !rec_Big
+    cir.return %r : !rec_Big
+  }
+
+  // CHECK:      cir.func{{.*}} @sret_with_unloaded_param
+  // CHECK-SAME:     llvm.sret = !rec_Big
+  // CHECK-SAME:     %[[PTR:[^:]*]]: !cir.ptr<!rec_Big>
+  // CHECK-SAME:     {llvm.align = 8 : i64, llvm.dereferenceable = 32 : i64, 
llvm.nofreeobj, llvm.noundef})
+  // CHECK-NOT:    cir.alloca "arg0"
+  // CHECK:        cir.call @takes_big_non_byval(%[[PTR]]) :
+  // CHECK-SAME:     (!cir.ptr<!rec_Big> {llvm.align = 8 : i64, 
llvm.dereferenceable = 32 : i64, llvm.nofreeobj, llvm.noundef}) -> ()
+
+  cir.func private @takes_big_non_byval(%arg0: !rec_Big)
+      attributes { test_classify = #non_byval_arg }
+
+}
+
+// -----
+
+!s64i = !cir.int<s, 64>
+!rec_Big = !cir.struct<"Big" {data !s64i, data !s64i, data !s64i, data !s64i}>
+!rec_Two = !cir.struct<"Two" {data !s64i, data !s64i}>
+
+#expand_then_non_byval = {
+  return = { kind = "direct" },
+  args   = [ { kind = "expand" },
+             { kind = "indirect", indirect_align = 8, byval = false } ]
+}
+
+#non_byval_arg = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8, byval = false } ]
+}
+
+module attributes {
+  dlti.dl_spec = #dlti.dl_spec<
+    #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+
+  // An Expand parameter ahead of it becomes one block argument per field, so
+  // again the parameter moves off its classification's index.
+  cir.func @expand_ahead_of_unloaded_param(%arg0: !rec_Two, %arg1: !rec_Big)
+      attributes { test_classify = #expand_then_non_byval } {
+    %s0 = cir.alloca "arg0" align(8) init : !cir.ptr<!rec_Two>
+    %s1 = cir.alloca "arg1" align(8) init const : !cir.ptr<!rec_Big>
+    cir.store %arg0, %s0 : !rec_Two, !cir.ptr<!rec_Two>
+    cir.store %arg1, %s1 : !rec_Big, !cir.ptr<!rec_Big>
+    cir.call @takes_big_non_byval(%arg1) : (!rec_Big) -> ()
+    cir.return
+  }
+
+  // CHECK:      cir.func{{.*}} @expand_ahead_of_unloaded_param(%{{[^:]*}}: 
!s64i,
+  // CHECK-SAME:     %{{[^:]*}}: !s64i,
+  // CHECK-SAME:     %[[PTR:[^:]*]]: !cir.ptr<!rec_Big>
+  // CHECK-SAME:     {llvm.align = 8 : i64, llvm.dereferenceable = 32 : i64, 
llvm.nofreeobj, llvm.noundef})
+  // CHECK-NOT:    cir.alloca "arg1"
+  // CHECK:        cir.call @takes_big_non_byval(%[[PTR]]) :
+  // CHECK-SAME:     (!cir.ptr<!rec_Big> {llvm.align = 8 : i64, 
llvm.dereferenceable = 32 : i64, llvm.nofreeobj, llvm.noundef}) -> ()
+
+  cir.func private @takes_big_non_byval(%arg0: !rec_Big)
+      attributes { test_classify = #non_byval_arg }
+
+}
+
+// -----
+
+!s64i = !cir.int<s, 64>
+!rec_Big = !cir.struct<"Big" {data !s64i, data !s64i, data !s64i, data !s64i}>
+
+#non_byval_arg = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8, byval = false } ]
+}
+
+#byval_arg = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8 } ]
+}
+
+module attributes {
+  dlti.dl_spec = #dlti.dl_spec<
+    #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+
+  // The slot is aligned above what the parameter promises, so the read of it
+  // may only claim the promise.  A byval callee keeps the read alive to check.
+  cir.func @unloaded_param_over_overaligned_slot(%arg0: !rec_Big)
+      attributes { test_classify = #non_byval_arg } {
+    %spill = cir.alloca "arg0" align(16) init const : !cir.ptr<!rec_Big>
+    cir.store %arg0, %spill : !rec_Big, !cir.ptr<!rec_Big>
+    cir.call @takes_big_byval(%arg0) : (!rec_Big) -> ()
+    cir.return
+  }
+
+  // CHECK:      cir.func{{.*}} 
@unloaded_param_over_overaligned_slot(%[[PTR:.*]]: !cir.ptr<!rec_Big>
+  // CHECK-SAME:     {llvm.align = 8 : i64, llvm.dereferenceable = 32 : i64, 
llvm.nofreeobj, llvm.noundef})
+  // CHECK:        %[[VAL:.*]] = cir.load align(8) %[[PTR]] : 
!cir.ptr<!rec_Big>, !rec_Big
+  // CHECK:        cir.store %[[VAL]], %{{.*}} : !rec_Big, !cir.ptr<!rec_Big>
+
+  cir.func private @takes_big_byval(%arg0: !rec_Big)
+      attributes { test_classify = #byval_arg }
+
+}
+
+// -----
+
+!s64i = !cir.int<s, 64>
+!rec_Big = !cir.struct<"Big" {data !s64i, data !s64i, data !s64i, data !s64i}>
+
+#non_byval_arg = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8, byval = false } ]
+}
+
+module attributes {
+  dlti.dl_spec = #dlti.dl_spec<
+    #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+
+  // A parameter with no uses at all, so there is no spill to prepare and
+  // nothing to read back.  The signature still becomes the incoming pointer.
+  cir.func @unused_non_byval_param(%arg0: !rec_Big)
+      attributes { test_classify = #non_byval_arg } {
+    cir.return
+  }
+
+  // CHECK:      cir.func{{.*}} @unused_non_byval_param(%{{.*}}: 
!cir.ptr<!rec_Big>
+  // CHECK-SAME:     {llvm.align = 8 : i64, llvm.dereferenceable = 32 : i64, 
llvm.nofreeobj, llvm.noundef})
+  // CHECK-NOT:    cir.alloca
+  // CHECK-NOT:    cir.load
+  // CHECK:        cir.return
+
+}
+
+// -----
+
+!s64i = !cir.int<s, 64>
+!rec_Big = !cir.struct<"Big" {data !s64i, data !s64i, data !s64i, data !s64i}>
+
+#non_byval_arg = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8, byval = false } ]
+}
+
+module attributes {
+  dlti.dl_spec = #dlti.dl_spec<
+    #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+
+  // The body writes the slot but nothing takes the parameter as an argument,
+  // so there is nothing to read it back for.
+  cir.func @spilled_param_no_call_arg(%arg0: !rec_Big)
+      attributes { test_classify = #non_byval_arg } {
+    %spill = cir.alloca "arg0" align(4) init : !cir.ptr<!rec_Big>
+    cir.store %arg0, %spill : !rec_Big, !cir.ptr<!rec_Big>
+    %z = cir.const #cir.zero : !rec_Big
+    cir.store %z, %spill : !rec_Big, !cir.ptr<!rec_Big>
+    cir.return
+  }
+
+  // CHECK:      cir.func{{.*}} @spilled_param_no_call_arg(%[[PTR:.*]]: 
!cir.ptr<!rec_Big>
+  // CHECK-SAME:     {llvm.align = 8 : i64, llvm.dereferenceable = 32 : i64, 
llvm.nofreeobj, llvm.noundef})
+  // CHECK-NOT:    cir.load
+  // CHECK:        %[[ZERO:.*]] = cir.const #cir.zero : !rec_Big
+  // CHECK:        cir.store %[[ZERO]], %[[PTR]] : !rec_Big, !cir.ptr<!rec_Big>
+
+}
+
+// -----
+
+!s64i = !cir.int<s, 64>
+!rec_Big = !cir.struct<"Big" {data !s64i, data !s64i, data !s64i, data !s64i}>
+
+#non_byval_arg = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8, byval = false } ]
+}
+
+module attributes {
+  dlti.dl_spec = #dlti.dl_spec<
+    #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+
+  // The parameter has no spill of its own, so it already names the caller's
+  // storage and is handed on as it stands.
+  cir.func private @takes_big_non_byval(%arg0: !rec_Big)
+      attributes { test_classify = #non_byval_arg }
+
+  cir.func @forwards_unspilled_param(%arg0: !rec_Big)
+      attributes { test_classify = #non_byval_arg } {
+    cir.call @takes_big_non_byval(%arg0) : (!rec_Big) -> ()
+    cir.return
+  }
+
+  // CHECK:      cir.func{{.*}} @forwards_unspilled_param(%[[PTR:.*]]: 
!cir.ptr<!rec_Big>
+  // CHECK-SAME:     {llvm.align = 8 : i64, llvm.dereferenceable = 32 : i64, 
llvm.nofreeobj, llvm.noundef})
+  // CHECK-NOT:    cir.alloca
+  // CHECK-NOT:    cir.load
+  // CHECK:        cir.call @takes_big_non_byval(%[[PTR]]) :
+  // CHECK-SAME:     (!cir.ptr<!rec_Big> {llvm.align = 8 : i64, 
llvm.dereferenceable = 32 : i64, llvm.nofreeobj, llvm.noundef}) -> ()
+
+}
+
+// -----
+
+!s64i = !cir.int<s, 64>
+!rec_Big = !cir.struct<"Big" {data !s64i, data !s64i, data !s64i, data !s64i}>
+
+#non_byval_arg = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8, byval = false } ]
+}
+
+#byval_arg = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8 } ]
+}
+
+module attributes {
+  dlti.dl_spec = #dlti.dl_spec<
+    #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+
+  // The same parameter handed to a callee that wants its own copy, which is
+  // read out of the caller's storage through the incoming pointer.
+  cir.func private @takes_big_byval(%arg0: !rec_Big)
+      attributes { test_classify = #byval_arg }
+
+  cir.func @unspilled_param_to_byval(%arg0: !rec_Big)
+      attributes { test_classify = #non_byval_arg } {
+    cir.call @takes_big_byval(%arg0) : (!rec_Big) -> ()
+    cir.return
+  }
+
+  // CHECK:      cir.func{{.*}} @unspilled_param_to_byval(%[[PTR:.*]]: 
!cir.ptr<!rec_Big>
+  // CHECK-SAME:     {llvm.align = 8 : i64, llvm.dereferenceable = 32 : i64, 
llvm.nofreeobj, llvm.noundef})
+  // CHECK-NOT:    cir.alloca
+  // CHECK:        %[[VAL:.*]] = cir.load align(8) %[[PTR]] : 
!cir.ptr<!rec_Big>, !rec_Big
+  // CHECK:        %[[COPY:.*]] = cir.alloca "byval" align(8) : 
!cir.ptr<!rec_Big>
+  // CHECK:        cir.store %[[VAL]], %[[COPY]] : !rec_Big, !cir.ptr<!rec_Big>
+  // CHECK:        cir.call @takes_big_byval(%[[COPY]]) :
+
+}
+
+// -----
+
+!s64i = !cir.int<s, 64>
+!rec_Big = !cir.struct<"Big" {data !s64i, data !s64i, data !s64i, data !s64i}>
+
+#non_byval_arg = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8, byval = false } ]
+}
+
+#byval_arg = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8 } ]
+}
+
+module attributes {
+  dlti.dl_spec = #dlti.dl_spec<
+    #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+
+  // The argument sits inside a nested region, which is where a full-expression
+  // scope puts it.  The read stays at the spill, outside the region.
+  cir.func @unloaded_param_read_in_scope(%arg0: !rec_Big)
+      attributes { test_classify = #non_byval_arg } {
+    %spill = cir.alloca "arg0" align(8) init const : !cir.ptr<!rec_Big>
+    cir.store %arg0, %spill : !rec_Big, !cir.ptr<!rec_Big>
+    cir.scope {
+      cir.call @takes_big_byval(%arg0) : (!rec_Big) -> ()
+    }
+    cir.return
+  }
+
+  // CHECK:      cir.func{{.*}} @unloaded_param_read_in_scope(%[[PTR:.*]]: 
!cir.ptr<!rec_Big>
+  // CHECK-SAME:     {llvm.align = 8 : i64, llvm.dereferenceable = 32 : i64, 
llvm.nofreeobj, llvm.noundef})
+  // CHECK:        %[[VAL:.*]] = cir.load align(8) %[[PTR]] : 
!cir.ptr<!rec_Big>, !rec_Big
+  // CHECK:        cir.scope {
+  // CHECK:          %[[COPY:.*]] = cir.alloca "byval" align(8) : 
!cir.ptr<!rec_Big>
+  // CHECK:          cir.store %[[VAL]], %[[COPY]] : !rec_Big, 
!cir.ptr<!rec_Big>
+  // CHECK:          cir.call @takes_big_byval(%[[COPY]]) :
+
+  cir.func private @takes_big_byval(%arg0: !rec_Big)
+      attributes { test_classify = #byval_arg }
+
+}
diff --git a/clang/test/CIR/Transforms/abi-lowering/indirect-non-byval-nyi.cir 
b/clang/test/CIR/Transforms/abi-lowering/indirect-non-byval-nyi.cir
index 864d88f490ee0..4f34978f79b20 100644
--- a/clang/test/CIR/Transforms/abi-lowering/indirect-non-byval-nyi.cir
+++ b/clang/test/CIR/Transforms/abi-lowering/indirect-non-byval-nyi.cir
@@ -545,3 +545,312 @@ module attributes {
   // CHECK-NEXT: cir.call @takes_big_non_byval(%v)
 
 }
+
+// -----
+
+!s64i = !cir.int<s, 64>
+!rec_Big = !cir.struct<"Big" {data !s64i, data !s64i, data !s64i, data !s64i}>
+
+#non_byval_arg = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8, byval = false } ]
+}
+
+module attributes {
+  dlti.dl_spec = #dlti.dl_spec<
+    #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+
+  cir.func private @takes_big_non_byval(%arg0: !rec_Big)
+      attributes { test_classify = #non_byval_arg }
+
+  // The read comes before the spill, so reading the slot there would read it
+  // before anything wrote it.
+  cir.func @param_read_before_spill(%arg0: !rec_Big)
+      attributes { test_classify = #non_byval_arg } {
+    %spill = cir.alloca "arg0" align(8) init : !cir.ptr<!rec_Big>
+    cir.call @takes_big_non_byval(%arg0) : (!rec_Big) -> ()
+    cir.store %arg0, %spill : !rec_Big, !cir.ptr<!rec_Big>
+    cir.return
+  }
+
+  // CHECK: error: 'cir.call' op non-byval parameter 0 read before its spill
+  // CHECK-SAME: is not yet implemented in CallConvLowering
+  // CHECK-NEXT: cir.call @takes_big_non_byval(%arg0)
+
+}
+
+// -----
+
+!s64i = !cir.int<s, 64>
+!rec_Big = !cir.struct<"Big" {data !s64i, data !s64i, data !s64i, data !s64i}>
+
+#non_byval_arg = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8, byval = false } ]
+}
+
+module attributes {
+  dlti.dl_spec = #dlti.dl_spec<
+    #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+
+  // Either slot could be the one the incoming pointer replaces, and there is
+  // nothing to prefer between them.
+  cir.func @param_spilled_twice(%arg0: !rec_Big)
+      attributes { test_classify = #non_byval_arg } {
+    %spill = cir.alloca "arg0" align(8) init : !cir.ptr<!rec_Big>
+    %copy = cir.alloca "copy" align(8) : !cir.ptr<!rec_Big>
+    cir.store %arg0, %spill : !rec_Big, !cir.ptr<!rec_Big>
+    cir.store %arg0, %copy : !rec_Big, !cir.ptr<!rec_Big>
+    cir.return
+  }
+
+  // CHECK: error: 'cir.store' op non-byval parameter 0 spilled more than once
+  // CHECK-SAME: is not yet implemented in CallConvLowering
+  // CHECK-NEXT: cir.store %arg0,
+
+}
+
+// -----
+
+!s64i = !cir.int<s, 64>
+!rec_Big = !cir.struct<"Big" {data !s64i, data !s64i, data !s64i, data !s64i}>
+
+#non_byval_arg = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8, byval = false } ]
+}
+
+module attributes {
+  dlti.dl_spec = #dlti.dl_spec<
+    #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+
+  cir.global "private" external @g : !rec_Big
+
+  // The spill stores to a global rather than to a slot the incoming pointer
+  // can replace.
+  cir.func @spill_to_global(%arg0: !rec_Big)
+      attributes { test_classify = #non_byval_arg } {
+    %slot = cir.get_global @g : !cir.ptr<!rec_Big>
+    cir.store %arg0, %slot : !rec_Big, !cir.ptr<!rec_Big>
+    cir.return
+  }
+
+  // CHECK: error: 'cir.store' op non-byval parameter 0 spilled to storage that
+  // CHECK-SAME: cannot take the incoming pointer is not yet implemented in
+  // CHECK-SAME: CallConvLowering
+  // CHECK-NEXT: cir.store %arg0, %slot
+
+}
+
+// -----
+
+!u8i = !cir.int<u, 8>
+!s64i = !cir.int<s, 64>
+!rec_Big = !cir.struct<"Big" {data !s64i, data !s64i, data !s64i, data !s64i}>
+
+#non_byval_arg = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8, byval = false } ]
+}
+
+module attributes {
+  dlti.dl_spec = #dlti.dl_spec<
+    #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+
+  // The spill stores through a cast rather than to the slot itself.
+  cir.func @spill_through_cast(%arg0: !rec_Big)
+      attributes { test_classify = #non_byval_arg } {
+    %raw = cir.alloca "arg0" align(8) : !cir.ptr<!cir.array<!u8i x 32>>
+    %slot = cir.cast bitcast %raw
+        : !cir.ptr<!cir.array<!u8i x 32>> -> !cir.ptr<!rec_Big>
+    cir.store %arg0, %slot : !rec_Big, !cir.ptr<!rec_Big>
+    cir.return
+  }
+
+  // CHECK: error: 'cir.store' op non-byval parameter 0 spilled to storage that
+  // CHECK-SAME: cannot take the incoming pointer is not yet implemented in
+  // CHECK-SAME: CallConvLowering
+  // CHECK-NEXT: cir.store %arg0, %slot
+
+}
+
+// -----
+
+!s64i = !cir.int<s, 64>
+!rec_Big = !cir.struct<"Big" {data !s64i, data !s64i, data !s64i, data !s64i}>
+
+#non_byval_arg = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8, byval = false } ]
+}
+
+module attributes {
+  dlti.dl_spec = #dlti.dl_spec<
+    #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+
+  // A slot in another address space is an allocation, but not one the incoming
+  // pointer can be substituted for.
+  cir.func @spill_to_other_address_space(%arg0: !rec_Big)
+      attributes { test_classify = #non_byval_arg } {
+    %slot = cir.alloca "arg0" align(8)
+        : !cir.ptr<!rec_Big, target_address_space(5)>
+    cir.store %arg0, %slot
+        : !rec_Big, !cir.ptr<!rec_Big, target_address_space(5)>
+    cir.return
+  }
+
+  // CHECK: error: 'cir.store' op non-byval parameter 0 spilled to storage that
+  // CHECK-SAME: cannot take the incoming pointer is not yet implemented in
+  // CHECK-SAME: CallConvLowering
+  // CHECK-NEXT: cir.store %arg0, %slot
+
+}
+
+// -----
+
+!s64i = !cir.int<s, 64>
+!rec_Big = !cir.struct<"Big" {data !s64i, data !s64i, data !s64i, data !s64i}>
+
+#sret_and_non_byval = {
+  return = { kind = "indirect", indirect_align = 8 },
+  args   = [ { kind = "indirect", indirect_align = 8, byval = false } ]
+}
+
+module attributes {
+  dlti.dl_spec = #dlti.dl_spec<
+    #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+
+  // The parameter is returned directly, so it is not a call argument.
+  cir.func @param_returned_directly(%arg0: !rec_Big) -> !rec_Big
+      attributes { test_classify = #sret_and_non_byval } {
+    %spill = cir.alloca "arg0" align(8) init const : !cir.ptr<!rec_Big>
+    cir.store %arg0, %spill : !rec_Big, !cir.ptr<!rec_Big>
+    cir.return %arg0 : !rec_Big
+  }
+
+  // CHECK: error: 'cir.return' op non-byval parameter 0 consumed other than as
+  // CHECK-SAME: a call argument is not yet implemented in CallConvLowering
+  // CHECK-NEXT: cir.return %arg0
+
+}
+
+// -----
+
+!s64i = !cir.int<s, 64>
+!rec_Big = !cir.struct<"Big" {data !s64i, data !s64i, data !s64i, data !s64i}>
+
+#two_non_byval = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8, byval = false },
+             { kind = "indirect", indirect_align = 8, byval = false } ]
+}
+
+module attributes {
+  dlti.dl_spec = #dlti.dl_spec<
+    #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+
+  // Each parameter is spilled exactly once, but into the same slot, and the
+  // incoming pointer can only replace that slot for one of them.
+  cir.func @params_share_a_spill_slot(%arg0: !rec_Big, %arg1: !rec_Big)
+      attributes { test_classify = #two_non_byval } {
+    %shared = cir.alloca "shared" align(8) init : !cir.ptr<!rec_Big>
+    cir.store %arg0, %shared : !rec_Big, !cir.ptr<!rec_Big>
+    cir.store %arg1, %shared : !rec_Big, !cir.ptr<!rec_Big>
+    cir.return
+  }
+
+  // CHECK: error: 'cir.store' op non-byval parameter 1 sharing its spill slot
+  // CHECK-SAME: with another parameter is not yet implemented in
+  // CHECK-SAME: CallConvLowering
+  // CHECK-NEXT: cir.store %arg1, %shared
+
+}
+
+// -----
+
+!s64i = !cir.int<s, 64>
+!rec_Big = !cir.struct<"Big" {data !s64i, data !s64i, data !s64i, data !s64i}>
+
+#non_byval_arg = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8, byval = false } ]
+}
+
+module attributes {
+  dlti.dl_spec = #dlti.dl_spec<
+    #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+
+  cir.func private @takes_big_non_byval(%arg0: !rec_Big)
+      attributes { test_classify = #non_byval_arg }
+
+  // A cir.try_call operand counts as a call argument, so the read is routed
+  // through the slot, and the call is left to the rewrite, which does not
+  // implement this op.
+  cir.func @unloaded_param_to_try_call(%arg0: !rec_Big)
+      attributes { test_classify = #non_byval_arg } {
+    %spill = cir.alloca "arg0" align(8) init const : !cir.ptr<!rec_Big>
+    cir.store %arg0, %spill : !rec_Big, !cir.ptr<!rec_Big>
+    cir.try_call @takes_big_non_byval(%arg0) ^bb1, ^bb2 : (!rec_Big) -> ()
+  ^bb1:
+    cir.return
+  ^bb2:
+    cir.return
+  }
+
+  // CHECK: error: 'cir.try_call' op TryCallOp not yet implemented in
+  // CHECK-SAME: CallConvLowering
+  // CHECK-NEXT: cir.try_call @takes_big_non_byval(%arg0)
+
+}
+
+// -----
+
+!s64i = !cir.int<s, 64>
+!rec_Big = !cir.struct<"Big" {data !s64i, data !s64i, data !s64i, data !s64i}>
+
+#non_byval_arg = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8, byval = false } ]
+}
+
+#non_byval_then_direct = {
+  return = { kind = "direct" },
+  args   = [ { kind = "indirect", indirect_align = 8, byval = false },
+             { kind = "direct" } ]
+}
+
+module attributes {
+  dlti.dl_spec = #dlti.dl_spec<
+    #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+
+  cir.func private @takes_big_non_byval(%arg0: !rec_Big)
+      attributes { test_classify = #non_byval_arg }
+
+  // The spill is on one path only, so it reaches the argument without
+  // dominating it, and a read at the spill would not either.
+  cir.func @spill_does_not_dominate_read(%arg0: !rec_Big, %c: !cir.bool)
+      attributes { test_classify = #non_byval_then_direct } {
+    cir.brcond %c ^bb1, ^bb2
+  ^bb1:
+    %spill = cir.alloca "arg0" align(8) init const : !cir.ptr<!rec_Big>
+    cir.store %arg0, %spill : !rec_Big, !cir.ptr<!rec_Big>
+    cir.br ^bb2
+  ^bb2:
+    cir.call @takes_big_non_byval(%arg0) : (!rec_Big) -> ()
+    cir.return
+  }
+
+  // CHECK: error: 'cir.call' op non-byval parameter 0 read before its spill
+  // CHECK-SAME: is not yet implemented in CallConvLowering
+  // CHECK-NEXT: cir.call @takes_big_non_byval(%arg0)
+
+}

_______________________________________________
cfe-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

Reply via email to