Public > -----Original Message----- > From: YuanShang Mao (River) <[email protected]> > Sent: Friday, August 28, 2026 3:42 AM > To: Kuehling, Felix <[email protected]>; Deucher, Alexander > <[email protected]> > Cc: Koenig, Christian <[email protected]>; amd- > [email protected] > Subject: [PATCH v2] drm/amdkfd: mark queues as reset after a full GPU reset > > During a GPU reset, evict_process_queues_cpsch() calls remove_queue_mes() > per queue, which fails with -EIO because the recovery path already holds the > reset domain write lock. The loop aborts on the first queue, leaving the rest > with is_active still set while the reset wipes the MES queue table. > > When the process later exits, process_termination_cpsch() sends > REMOVE_QUEUE for doorbells MES no longer knows about. MES rejects > them, the driver reports it as a timeout and concludes MES is unrecoverable, > so it requests another GPU reset. > > Failed to evict queue 2 > GPU reset(1) succeeded! > failed to remove queue from MES, doorbell=0x1002 > MES might be in unrecoverable state, issue a GPU reset > GPU reset begin!. Source: 3 > > Mark every queue as reset in kgd2kfd_post_reset(). This clears is_active, > corrects the queue counts and sets is_reset so the queue is skipped later. > > Add a verbose argument to set_queue_as_reset() so the full reset path does > not log a message for every queue. > > Signed-off-by: YuanShang <[email protected]>
Acked-by: Alex Deucher <[email protected]> > --- > v2: > - drop the enable_mes check and mark queues on KIQ as well > (Suggested-by: Alex Deucher <[email protected]>) > - add a verbose argument to set_queue_as_reset() so the full reset path > does not log a message for every queue > > KIQ does not hit this bug. Its evict loop has no fallible call in it, so the > loop > always runs to completion and every queue ends up with is_active clear, > leaving nothing for process teardown to send. Marking on KIQ is still worth > doing: after a reset the queues are gone on both schedulers, and is_reset > states that, while is_active == false only means "do not schedule it now" and > is > cleared again on restore. > > drivers/gpu/drm/amd/amdkfd/kfd_device.c | 3 ++ > .../drm/amd/amdkfd/kfd_device_queue_manager.c | 36 > +++++++++++++++---- .../drm/amd/amdkfd/kfd_device_queue_manager.h | > 1 + > 3 files changed, 33 insertions(+), 7 deletions(-) > > diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c > b/drivers/gpu/drm/amd/amdkfd/kfd_device.c > index 586e640f13dc..84bc04c4a55c 100644 > --- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c > +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c > @@ -1033,6 +1033,9 @@ int kgd2kfd_post_reset(struct kfd_dev *kfd) > if (!kfd->init_complete) > return 0; > > + for (i = 0; i < kfd->num_nodes; i++) > + kfd_dqm_set_queues_as_reset(kfd->nodes[i]->dqm); > + > for (i = 0; i < kfd->num_nodes; i++) { > ret = kfd_resume(kfd->nodes[i]); > if (ret) > diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c > b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c > index a23384571193..350dcb8c21cb 100644 > --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c > +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c > @@ -75,7 +75,7 @@ static int reset_queues_on_hws_hang(struct > device_queue_manager *dqm, bool is_sd static struct queue > *find_queue_by_doorbell_offset(struct device_queue_manager *dqm, > u32 doorbell_offset); > static void set_queue_as_reset(struct device_queue_manager *dqm, struct > queue *q, > - struct qcm_process_device *qpd); > + struct qcm_process_device *qpd, bool verbose); > static int reset_queues_mes(struct device_queue_manager *dqm, struct > queue *q); > > static inline > @@ -433,7 +433,7 @@ static int reset_queue_mes(struct > device_queue_manager *dqm, struct queue *q, > return r; > /* Proceed remove_queue with reset=true */ > remove_queue_mes_on_reset_option(dqm, q, &pdd->qpd, true, > true); > - set_queue_as_reset(dqm, q, &pdd->qpd); > + set_queue_as_reset(dqm, q, &pdd->qpd, true); > return 0; > } > > @@ -2369,12 +2369,14 @@ static int map_queues_cpsch(struct > device_queue_manager *dqm) } > > static void set_queue_as_reset(struct device_queue_manager *dqm, struct > queue *q, > - struct qcm_process_device *qpd) > + struct qcm_process_device *qpd, bool verbose) > { > struct kfd_process_device *pdd = qpd_to_pdd(qpd); > > - dev_err(dqm->dev->adev->dev, "queue id 0x%0x at pasid %d is > reset\n", > - q->properties.queue_id, pdd->process->lead_thread->pid); > + if (verbose) > + dev_err(dqm->dev->adev->dev, > + "queue id 0x%0x at pasid %d is reset\n", > + q->properties.queue_id, pdd->process->lead_thread- > >pid); > > pdd->has_reset_queue = true; > q->properties.is_reset = true; > @@ -2384,6 +2386,26 @@ static void set_queue_as_reset(struct > device_queue_manager *dqm, struct queue *q > } > } > > +void kfd_dqm_set_queues_as_reset(struct device_queue_manager *dqm) { > + struct device_process_node *cur; > + struct qcm_process_device *qpd; > + struct queue *q; > + > + dqm_lock(dqm); > + if (dqm->processes_count) > + dev_warn(dqm->dev->adev->dev, > + "GPU reset: queues of %d process(es) are no longer > usable\n", > + dqm->processes_count); > + > + list_for_each_entry(cur, &dqm->queues, list) { > + qpd = cur->qpd; > + list_for_each_entry(q, &qpd->queues_list, list) > + set_queue_as_reset(dqm, q, qpd, false); > + } > + dqm_unlock(dqm); > +} > + > static int detect_queue_hang(struct device_queue_manager *dqm) { > int i; > @@ -2499,7 +2521,7 @@ static int reset_hung_queues(struct > device_queue_manager *dqm) > goto reset_fail; > } > > - set_queue_as_reset(dqm, q, &pdd->qpd); > + set_queue_as_reset(dqm, q, &pdd->qpd, true); > reset_count++; > } > > @@ -2546,7 +2568,7 @@ static bool set_sdma_queue_as_reset(struct > device_queue_manager *dqm, > if ((q->properties.type == KFD_QUEUE_TYPE_SDMA || > q->properties.type == > KFD_QUEUE_TYPE_SDMA_XGMI) && > q->properties.doorbell_off == doorbell_off) { > - set_queue_as_reset(dqm, q, qpd); > + set_queue_as_reset(dqm, q, qpd, true); > return true; > } > } > diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h > b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h > index c9f9f7a87111..8ddde9f75541 100644 > --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h > +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h > @@ -333,6 +333,7 @@ int debug_refresh_runlist(struct > device_queue_manager *dqm); bool kfd_dqm_is_queue_in_process(struct > device_queue_manager *dqm, > struct qcm_process_device *qpd, > int doorbell_off, u32 *queue_format); > +void kfd_dqm_set_queues_as_reset(struct device_queue_manager *dqm); > int kfd_reset_queue_mes(struct device_queue_manager *dqm, int > queue_type, > int pipe, int queue, unsigned int db); size_t > mqd_size_from_queue_type(struct device_queue_manager *dqm, > -- > 2.25.1
