get_queue_checkpoint_info() reports mqd_size as the per-XCC MQD
size times NUM_XCC, and checkpoint_mqd_v9_4_3() stores one MQD per XCC.
The mqd_size check added to the CRIU restore path compares against the
single per-XCC size, so every checkpoint image of a queue on a multi-XCC
node is rejected with -EINVAL.
Worse, a single-MQD size is the only value that check accepts there, and
restore_mqd_v9_4_3() then reads one MQD per XCC from a buffer holding
only one, copying up to (NUM_XCC - 1) * sizeof(struct v9_mqd) bytes of
adjacent kernel heap into the queue's MQD.
Make mqd_size_from_queue_type() return the size checkpoint writes, so a
restore image must carry exactly one MQD per XCC. With that,
restore_mqd_v9_4_3() must also not keep the image's cp_mqd_stride_size:
checkpoint and get_wave_state() step through the per-XCC MQDs with it,
so set it to the stride of the new allocation as init_mqd_v9_4_3() does.
Restore only MQD types that have checkpoint_mqd(), since no other image
layout exists. GFX12 and GFX12.1 have restore_mqd() but no checkpoint
yet, and restore_mqd_v12_1() handles XCC0 only, so a multi-XCC GFX12.1
image would now pass the size check and leave XCC1..N with zeroed MQDs.
Fixes: c76bcb1cfce6 ("drm/amdkfd: Check bounds on CRIU restore queue type and
mqd size")
Signed-off-by: William Palacek <[email protected]>
---
.../gpu/drm/amd/amdkfd/kfd_device_queue_manager.c | 15 ++++++++++-----
.../gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c | 9 +--------
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c | 1 +
3 files changed, 12 insertions(+), 13 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index 5e57576fba21..88bbef08c39c 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -816,8 +816,9 @@ static int create_queue_nocpsch(struct device_queue_manager
*dqm,
mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
q->properties.type)];
- if (qd && !mqd_mgr->restore_mqd) {
- pr_debug("restore_mqd not implemented for queue type %d\n",
+ /* Restore takes the image layout that checkpoint_mqd() writes */
+ if (qd && (!mqd_mgr->restore_mqd || !mqd_mgr->checkpoint_mqd)) {
+ pr_debug("CRIU not implemented for queue type %d\n",
q->properties.type);
retval = -EOPNOTSUPP;
goto deallocate_vmid;
@@ -2399,8 +2400,9 @@ static int create_queue_cpsch(struct device_queue_manager
*dqm, struct queue *q,
mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
q->properties.type)];
- if (qd && !mqd_mgr->restore_mqd) {
- pr_debug("restore_mqd not implemented for queue type %d\n",
+ /* Restore takes the image layout that checkpoint_mqd() writes */
+ if (qd && (!mqd_mgr->restore_mqd || !mqd_mgr->checkpoint_mqd)) {
+ pr_debug("CRIU not implemented for queue type %d\n",
q->properties.type);
retval = -EOPNOTSUPP;
goto out_deallocate_doorbell;
@@ -4123,7 +4125,10 @@ bool kfd_dqm_is_queue_in_process(struct
device_queue_manager *dqm,
size_t mqd_size_from_queue_type(struct device_queue_manager *dqm, enum
kfd_queue_type type)
{
- return dqm->mqd_mgrs[get_mqd_type_from_queue_type(type)]->mqd_size;
+ struct mqd_manager *mqd_mgr =
dqm->mqd_mgrs[get_mqd_type_from_queue_type(type)];
+
+ /* Checkpoint stores one MQD per XCC, see get_queue_checkpoint_info() */
+ return mqd_mgr->mqd_size * NUM_XCC(mqd_mgr->dev->xcc_mask);
}
#if defined(CONFIG_DEBUG_FS)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
index 79e0d6250720..4eac85a654cb 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
@@ -670,14 +670,7 @@ static void restore_mqd_v12_1(struct mqd_manager *mm, void
**mqd,
u64 addr;
struct v12_1_compute_mqd *m;
- /*
- * GFX12.1 is multi-XCC capable but this restore handles XCC0 only.
- * Multi-XCC CRIU restore is currently unreachable because
- * kfd_criu_restore_queue() validates against unscaled mqd_size.
- */
- if (NUM_XCC(mm->dev->xcc_mask) > 1)
- pr_warn_once("GFX12.1 multi-XCC CRIU restore not fully
supported\n");
-
+ /* Handles XCC0 only, unreachable until there is a checkpoint_mqd() */
m = (struct v12_1_compute_mqd *)mqd_mem_obj->cpu_ptr;
addr = mqd_mem_obj->gpu_addr;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
index 948415908551..d95a28a80917 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
@@ -864,6 +864,7 @@ static void restore_mqd_v9_4_3(struct mqd_manager *mm, void
**mqd,
(uint8_t *)mqd_src + xcc * sizeof(*m),
(uint8_t *)ctl_stack_src + xcc *
mqd_ctl_stack_size,
mqd_ctl_stack_size);
+ m->cp_mqd_stride_size = offset;
}
if (mqd_on_vram(mm->dev->adev))
--
2.43.0
