On 2026-11-04 10:52, David Francis wrote:
CRIU queue restore doesn't acquire the bo buffers in the same way
normal queue creation does. In normal queue creation, the driver
gets an additional reference on the queue's underlying buffers
that lasts until the queue is destroyed; this prevents the
queue's buffers from being removed from underneath it.
Acquire them, but don't get the svm bo. SVM isn't yet fully
restored by this point in CRIU and so there's no svm range to get.
This will be addressed in the next patch.
v2: Don't use checkpoint ctl_stack_size
Signed-off-by: David Francis <[email protected]>
---
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 2 +-
drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 2 +-
drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c | 8 ++++++++
drivers/gpu/drm/amd/amdkfd/kfd_queue.c | 8 +++++++-
4 files changed, 17 insertions(+), 3 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index 6c710e4a5220c..d8233337b8d5e 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -433,7 +433,7 @@ static int kfd_ioctl_create_queue(struct file *filep,
struct kfd_process *p,
goto err_bind_process;
}
- err = kfd_queue_acquire_buffers(pdd, &q_properties);
+ err = kfd_queue_acquire_buffers(pdd, &q_properties, false);
if (err) {
pr_debug("failed to acquire user queue buffers\n");
goto err_acquire_queue_buf;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index f0ce5fc4a138f..eb4d35a42e917 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -1381,7 +1381,7 @@ void print_queue(struct queue *q);
int kfd_queue_buffer_get(struct amdgpu_vm *vm, void __user *addr, struct
amdgpu_bo **pbo,
u64 expected_size);
void kfd_queue_buffer_put(struct amdgpu_bo **bo);
-int kfd_queue_acquire_buffers(struct kfd_process_device *pdd, struct
queue_properties *properties);
+int kfd_queue_acquire_buffers(struct kfd_process_device *pdd, struct
queue_properties *properties, bool is_criu);
int kfd_queue_release_buffers(struct kfd_process_device *pdd, struct
queue_properties *properties);
void kfd_queue_unref_bo_va(struct amdgpu_vm *vm, struct amdgpu_bo **bo);
int kfd_queue_unref_bo_vas(struct kfd_process_device *pdd,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
index 4c4a3a8587f46..6e19be9e89f57 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
@@ -1095,9 +1095,17 @@ int kfd_criu_restore_queue(struct kfd_process *p,
print_queue_properties(&qp);
+ ret = kfd_queue_acquire_buffers(pdd, &qp, true);
+ if (ret) {
+ pr_err("Failed to acquire user queue buffers err:%d\n", ret);
+ goto exit;
+ }
+
ret = pqm_create_queue(&p->pqm, pdd->dev, &qp, &queue_id, q_data, mqd,
ctl_stack, NULL);
if (ret) {
pr_err("Failed to create new queue err:%d\n", ret);
+ kfd_queue_unref_bo_vas(pdd, &qp);
+ kfd_queue_release_buffers(pdd, &qp);
I think kfd_queue_release_buffers also needs another parameter to tell
it not to handle SVM. Otherwise it will call kfd_queue_buffer_svm_put
and try to decrement queue refcounts in pranges that weren't incremented.
Regards,
Felix
goto exit;
}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
index 6010a5d7a7eb9..18d24ddb455eb 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
@@ -239,7 +239,7 @@ static void kfd_queue_put_bos(struct queue_properties
*properties)
kfd_queue_buffer_put(&properties->cwsr_bo);
}
-int kfd_queue_acquire_buffers(struct kfd_process_device *pdd, struct queue_properties *properties)
+int kfd_queue_acquire_buffers(struct kfd_process_device *pdd, struct
queue_properties *properties, bool is_criu)
{
struct kfd_topology_device *topo_dev;
u64 expected_queue_size;
@@ -334,6 +334,12 @@ int kfd_queue_acquire_buffers(struct kfd_process_device
*pdd, struct queue_prope
amdgpu_bo_unreserve(vm->root.bo);
+ /* CRIU svm buffers are restored at a later stage than queues. They'll be in
+ * before the queue is actually used, so no risk here
+ */
+ if (is_criu)
+ return 0;
+
err = kfd_queue_buffer_svm_get(pdd,
properties->ctx_save_restore_area_address,
total_cwsr_size);
if (err)
