During a GPU reset, evict_process_queues_cpsch() calls remove_queue_mes() per queue, which fails with -EIO because the recovery path already holds the reset domain write lock. The loop aborts on the first queue, leaving the rest with is_active still set while the reset wipes the MES queue table.
When the process later exits, process_termination_cpsch() sends REMOVE_QUEUE for doorbells MES no longer knows about. MES rejects them, the driver reports it as a timeout and concludes MES is unrecoverable, so it requests another GPU reset. Failed to evict queue 2 GPU reset(1) succeeded! failed to remove queue from MES, doorbell=0x1002 MES might be in unrecoverable state, issue a GPU reset GPU reset begin!. Source: 3 Mark every queue as reset in kgd2kfd_post_reset(). This clears is_active, corrects the queue counts and sets is_reset so the queue is skipped later. Add a verbose argument to set_queue_as_reset() so the full reset path does not log a message for every queue. Signed-off-by: YuanShang <[email protected]> --- v2: - drop the enable_mes check and mark queues on KIQ as well (Suggested-by: Alex Deucher <[email protected]>) - add a verbose argument to set_queue_as_reset() so the full reset path does not log a message for every queue KIQ does not hit this bug. Its evict loop has no fallible call in it, so the loop always runs to completion and every queue ends up with is_active clear, leaving nothing for process teardown to send. Marking on KIQ is still worth doing: after a reset the queues are gone on both schedulers, and is_reset states that, while is_active == false only means "do not schedule it now" and is cleared again on restore. drivers/gpu/drm/amd/amdkfd/kfd_device.c | 3 ++ .../drm/amd/amdkfd/kfd_device_queue_manager.c | 36 +++++++++++++++---- .../drm/amd/amdkfd/kfd_device_queue_manager.h | 1 + 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c index 586e640f13dc..84bc04c4a55c 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c @@ -1033,6 +1033,9 @@ int kgd2kfd_post_reset(struct kfd_dev *kfd) if (!kfd->init_complete) return 0; + for (i = 0; i < kfd->num_nodes; i++) + kfd_dqm_set_queues_as_reset(kfd->nodes[i]->dqm); + for (i = 0; i < kfd->num_nodes; i++) { ret = kfd_resume(kfd->nodes[i]); if (ret) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index a23384571193..350dcb8c21cb 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -75,7 +75,7 @@ static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sd static struct queue *find_queue_by_doorbell_offset(struct device_queue_manager *dqm, u32 doorbell_offset); static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q, - struct qcm_process_device *qpd); + struct qcm_process_device *qpd, bool verbose); static int reset_queues_mes(struct device_queue_manager *dqm, struct queue *q); static inline @@ -433,7 +433,7 @@ static int reset_queue_mes(struct device_queue_manager *dqm, struct queue *q, return r; /* Proceed remove_queue with reset=true */ remove_queue_mes_on_reset_option(dqm, q, &pdd->qpd, true, true); - set_queue_as_reset(dqm, q, &pdd->qpd); + set_queue_as_reset(dqm, q, &pdd->qpd, true); return 0; } @@ -2369,12 +2369,14 @@ static int map_queues_cpsch(struct device_queue_manager *dqm) } static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q, - struct qcm_process_device *qpd) + struct qcm_process_device *qpd, bool verbose) { struct kfd_process_device *pdd = qpd_to_pdd(qpd); - dev_err(dqm->dev->adev->dev, "queue id 0x%0x at pasid %d is reset\n", - q->properties.queue_id, pdd->process->lead_thread->pid); + if (verbose) + dev_err(dqm->dev->adev->dev, + "queue id 0x%0x at pasid %d is reset\n", + q->properties.queue_id, pdd->process->lead_thread->pid); pdd->has_reset_queue = true; q->properties.is_reset = true; @@ -2384,6 +2386,26 @@ static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q } } +void kfd_dqm_set_queues_as_reset(struct device_queue_manager *dqm) +{ + struct device_process_node *cur; + struct qcm_process_device *qpd; + struct queue *q; + + dqm_lock(dqm); + if (dqm->processes_count) + dev_warn(dqm->dev->adev->dev, + "GPU reset: queues of %d process(es) are no longer usable\n", + dqm->processes_count); + + list_for_each_entry(cur, &dqm->queues, list) { + qpd = cur->qpd; + list_for_each_entry(q, &qpd->queues_list, list) + set_queue_as_reset(dqm, q, qpd, false); + } + dqm_unlock(dqm); +} + static int detect_queue_hang(struct device_queue_manager *dqm) { int i; @@ -2499,7 +2521,7 @@ static int reset_hung_queues(struct device_queue_manager *dqm) goto reset_fail; } - set_queue_as_reset(dqm, q, &pdd->qpd); + set_queue_as_reset(dqm, q, &pdd->qpd, true); reset_count++; } @@ -2546,7 +2568,7 @@ static bool set_sdma_queue_as_reset(struct device_queue_manager *dqm, if ((q->properties.type == KFD_QUEUE_TYPE_SDMA || q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) && q->properties.doorbell_off == doorbell_off) { - set_queue_as_reset(dqm, q, qpd); + set_queue_as_reset(dqm, q, qpd, true); return true; } } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h index c9f9f7a87111..8ddde9f75541 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h @@ -333,6 +333,7 @@ int debug_refresh_runlist(struct device_queue_manager *dqm); bool kfd_dqm_is_queue_in_process(struct device_queue_manager *dqm, struct qcm_process_device *qpd, int doorbell_off, u32 *queue_format); +void kfd_dqm_set_queues_as_reset(struct device_queue_manager *dqm); int kfd_reset_queue_mes(struct device_queue_manager *dqm, int queue_type, int pipe, int queue, unsigned int db); size_t mqd_size_from_queue_type(struct device_queue_manager *dqm, -- 2.25.1
