When a user queue is created through kfd, the driver normally acquires a reference to each buffer backing the queue, to ensure those buffers can't be freed from under the queue.
During CRIU restore, we weren't acquiring those buffers. One of them (cwsr) is svm memory. During CRIU, svm memory is restored during RESUME phase, after queues are restored. During CRIU queue restore, keep a record of the svm buffer's address and size, and then get a reference to it during RESUME phase. Signed-off-by: David Francis <[email protected]> --- drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 5 ++- drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 6 +++ drivers/gpu/drm/amd/amdkfd/kfd_process.c | 1 + .../amd/amdkfd/kfd_process_queue_manager.c | 38 +++++++++++++++++++ drivers/gpu/drm/amd/amdkfd/kfd_queue.c | 4 +- 5 files changed, 51 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index d8233337b8d5e..8157f57e9e581 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -2833,7 +2833,7 @@ static int criu_resume(struct file *filep, { struct kfd_process *target = NULL; struct pid *pid = NULL; - int ret = 0; + int i, ret = 0; pr_debug("Inside %s, target pid for criu restore: %d\n", __func__, args->pid); @@ -2861,6 +2861,9 @@ static int criu_resume(struct file *filep, goto exit; } + for (i = 0; i < target->n_pdds; i++) + kfd_criu_resume_pdd(target->pdds[i]); + ret = amdgpu_amdkfd_criu_resume(target->kgd_process_info); if (ret) pr_err("amdgpu_amdkfd_criu_resume failed for %i\n", args->pid); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index eb4d35a42e917..79f7b126ebd0d 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -895,6 +895,9 @@ struct kfd_process_device { u32 pasid; /* Indicates this process has requested PTL stay disabled */ bool ptl_disable_req; + + /* CRIU data preserved between restore and resume */ + struct list_head criu_pdd_metadata_list; }; #define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd) @@ -1371,6 +1374,8 @@ int kfd_criu_restore_event(struct file *devkfd, uint8_t __user *user_priv_data, uint64_t *priv_data_offset, uint64_t max_priv_data_size); + +int kfd_criu_resume_pdd(struct kfd_process_device *pdd); /* CRIU - End */ /* Queue Context Management */ @@ -1387,6 +1392,7 @@ void kfd_queue_unref_bo_va(struct amdgpu_vm *vm, struct amdgpu_bo **bo); int kfd_queue_unref_bo_vas(struct kfd_process_device *pdd, struct queue_properties *properties); void kfd_queue_ctx_save_restore_size(struct kfd_topology_device *dev); +int kfd_queue_buffer_svm_get(struct kfd_process_device *pdd, u64 addr, u64 size); struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type, struct kfd_node *dev); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c index ac18c2be6ecb6..ded42532e6d9c 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c @@ -1801,6 +1801,7 @@ struct kfd_process_device *kfd_create_process_device_data(struct kfd_node *dev, pdd->dev = dev; INIT_LIST_HEAD(&pdd->qpd.queues_list); INIT_LIST_HEAD(&pdd->qpd.priv_queue_list); + INIT_LIST_HEAD(&pdd->criu_pdd_metadata_list); mutex_init(&pdd->doorbell_mutex); pdd->qpd.dqm = dev->dqm; pdd->qpd.pqm = &p->pqm; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c index 509529af2dc3c..d3136f19238bb 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c @@ -1018,6 +1018,12 @@ static void set_queue_properties_from_criu(struct queue_properties *qp, qp->format = q_data->format; } +struct criu_pdd_metadata { + struct list_head list; + uint64_t ctx_save_restore_area_address; + uint32_t ctx_save_restore_area_size; +}; + int kfd_criu_restore_queue(struct kfd_process *p, uint8_t __user *user_priv_ptr, uint64_t *priv_data_offset, @@ -1026,6 +1032,7 @@ int kfd_criu_restore_queue(struct kfd_process *p, uint8_t *mqd, *ctl_stack, *q_extra_data = NULL; struct kfd_criu_queue_priv_data *q_data; struct kfd_topology_device *topo_dev; + struct criu_pdd_metadata *resume_info; struct kfd_process_device *pdd; uint64_t q_extra_data_size; struct queue_properties qp; @@ -1131,6 +1138,11 @@ int kfd_criu_restore_queue(struct kfd_process *p, goto exit; } + resume_info = kzalloc(sizeof(struct criu_pdd_metadata), GFP_KERNEL); + resume_info->ctx_save_restore_area_address = qp.ctx_save_restore_area_address; + resume_info->ctx_save_restore_area_size = qp.ctx_save_restore_area_size; + list_add_tail(&resume_info->list, &pdd->criu_pdd_metadata_list); + if (q_data->gws) ret = pqm_set_gws(&p->pqm, q_data->q_id, pdd->dev->gws); @@ -1146,6 +1158,32 @@ int kfd_criu_restore_queue(struct kfd_process *p, return ret; } +int kfd_criu_resume_pdd(struct kfd_process_device *pdd) +{ + struct kfd_topology_device *topo_dev = kfd_topology_device_by_id(pdd->dev->id); + struct criu_pdd_metadata *resume_info; + struct criu_pdd_metadata *next = NULL; + u64 total_cwsr_size; + int ret = 0; + + list_for_each_entry(resume_info, &pdd->criu_pdd_metadata_list, list) { + total_cwsr_size = resume_info->ctx_save_restore_area_size + + topo_dev->node_props.debug_memory_size; + total_cwsr_size = ALIGN(total_cwsr_size, PAGE_SIZE); + + ret = kfd_queue_buffer_svm_get(pdd, resume_info->ctx_save_restore_area_address, total_cwsr_size); + if (ret) + break; + } + + list_for_each_entry_safe(resume_info, next, &pdd->criu_pdd_metadata_list, list) { + list_del(&resume_info->list); + kfree(resume_info); + } + + return ret; +} + int pqm_get_queue_checkpoint_info(struct process_queue_manager *pqm, unsigned int qid, uint32_t *mqd_size, diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c index 18d24ddb455eb..78de556da056c 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c @@ -88,7 +88,7 @@ void uninit_queue(struct queue *q) #if IS_ENABLED(CONFIG_HSA_AMD_SVM) -static int kfd_queue_buffer_svm_get(struct kfd_process_device *pdd, u64 addr, u64 size) +int kfd_queue_buffer_svm_get(struct kfd_process_device *pdd, u64 addr, u64 size) { struct kfd_process *p = pdd->process; struct list_head update_list; @@ -182,7 +182,7 @@ static void kfd_queue_buffer_svm_put(struct kfd_process_device *pdd, u64 addr, u } #else -static int kfd_queue_buffer_svm_get(struct kfd_process_device *pdd, u64 addr, u64 size) +int kfd_queue_buffer_svm_get(struct kfd_process_device *pdd, u64 addr, u64 size) { return -EINVAL; } -- 2.34.1
