A process eviction fence protects every queue sharing its VM. Completing it from the per-queue fence helper releases sibling queues' BO dependencies and allows raw post-reset mapping with an already-signaled eviction fence.
Complete process eviction fences only after a successful device hardware reset, before resume can allocate BOs. Retire every old queue, then complete queue fences before acquiring manager mutexes to release the eviction dependencies. Old queues remain terminal even when VRAM survives; applications must recreate them. Safe transparent remapping would require a separate context and VM validation design, including a fresh eviction fence. Signed-off-by: Prike Liang <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 19 ++-- drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c | 6 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 98 ++++++++++++------- drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h | 3 +- .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c | 22 +++++ 5 files changed, 105 insertions(+), 43 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 83d98c9191b0..74030d588a36 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -4794,6 +4794,7 @@ static int amdgpu_device_reset_sriov(struct amdgpu_device *adev, return r; amdgpu_ras_clear_err_state(adev); + amdgpu_userq_reset_complete(adev); amdgpu_irq_gpu_reset_resume_helper(adev); /* some sw clean up VF needs to do before recover */ @@ -5024,6 +5025,8 @@ int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); int i, r; + amdgpu_userq_pre_reset(adev); + if (reset_context->reset_req_dev == adev) job = reset_context->job; @@ -5192,10 +5195,6 @@ int amdgpu_device_reinit_after_reset(struct amdgpu_reset_context *reset_context) if (r) goto out; - r = amdgpu_userq_post_reset(tmp_adev, vram_lost); - if (r) - goto out; - drm_client_dev_resume(adev_to_drm(tmp_adev)); /* @@ -5306,7 +5305,15 @@ int amdgpu_do_asic_reset(struct list_head *device_list_handle, } } - if (!r && amdgpu_ras_intr_triggered()) { + if (r) + goto out; + + if (!skip_hw_reset && need_full_reset) { + list_for_each_entry(tmp_adev, device_list_handle, reset_list) + amdgpu_userq_reset_complete(tmp_adev); + } + + if (amdgpu_ras_intr_triggered()) { list_for_each_entry(tmp_adev, device_list_handle, reset_list) { amdgpu_ras_reset_error_count(tmp_adev, AMDGPU_RAS_BLOCK__MMHUB); @@ -5536,8 +5543,6 @@ static void amdgpu_device_halt_activities(struct amdgpu_device *adev, if (!need_emergency_restart && !amdgpu_reset_in_dpc(adev)) amdgpu_ras_suspend(tmp_adev); - amdgpu_userq_pre_reset(tmp_adev); - for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { struct amdgpu_ring *ring = tmp_adev->rings[i]; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c index c8016fa55a92..fbc30e85a398 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c @@ -250,6 +250,7 @@ int amdgpu_reset_prepare_hwcontext(struct amdgpu_device *adev, int amdgpu_reset_perform_reset(struct amdgpu_device *adev, struct amdgpu_reset_context *reset_context) { + struct amdgpu_device *tmp_adev; int ret; struct amdgpu_reset_handler *reset_handler = NULL; @@ -263,6 +264,11 @@ int amdgpu_reset_perform_reset(struct amdgpu_device *adev, if (ret) return ret; + if (reset_context->reset_device_list) { + list_for_each_entry(tmp_adev, reset_context->reset_device_list, reset_list) + amdgpu_userq_reset_complete(tmp_adev); + } + return reset_handler->restore_hwcontext(adev->reset_cntl, reset_context); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c index a745c752f459..2eba234a504e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c @@ -150,6 +150,12 @@ static void amdgpu_userq_hang_detect_work(struct work_struct *work) if (!amdgpu_gpu_recovery) return; + /* A device reset retires queues and completes their fences as a group. */ + if (!down_read_trylock(&adev->reset_domain->sem)) { + schedule_delayed_work(&queue->hang_detect_work, msecs_to_jiffies(10)); + return; + } + if (queue->vm && queue->vm->pasid) { ti = amdgpu_vm_get_task_info_pasid(adev, queue->vm->pasid); if (ti) { @@ -178,6 +184,7 @@ static void amdgpu_userq_hang_detect_work(struct work_struct *work) gpu_reset = true; } amdgpu_vm_put_task_info(ti); + up_read(&adev->reset_domain->sem); /* * Don't schedule the work here! Scheduling or queue work from one reset @@ -372,6 +379,9 @@ static int amdgpu_userq_unmap_helper(struct amdgpu_usermode_queue *queue) adev->userq_funcs[queue->queue_type]; int r; + if (queue->state == AMDGPU_USERQ_STATE_RESET_PENDING) + return -EBUSY; + if ((queue->state == AMDGPU_USERQ_STATE_MAPPED) || (queue->state == AMDGPU_USERQ_STATE_PREEMPTED)) { trace_amdgpu_userq_state_start(queue); @@ -770,8 +780,11 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args) queue->doorbell_index = index; queue->doorbell_offset = (u32)args->in.doorbell_offset; trace_amdgpu_userq_create_start(queue); + /* Publish the MQD and doorbell in the same reset epoch. */ + down_read(&adev->reset_domain->sem); r = uq_funcs->mqd_create(queue, &args->in); if (r) { + up_read(&adev->reset_domain->sem); drm_file_err(uq_mgr->file, "Failed to create Queue\n"); goto clean_doorbell_bo; } @@ -781,11 +794,18 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args) r = xa_insert_irq(&adev->userq_doorbell_xa, index, queue, GFP_KERNEL); + up_read(&adev->reset_domain->sem); if (r) goto clean_mqd; map_retry: amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr); + if (queue->state == AMDGPU_USERQ_STATE_HUNG || + queue->state == AMDGPU_USERQ_STATE_RESET_PENDING) { + r = -ECANCELED; + mutex_unlock(&uq_mgr->userq_mutex); + goto erase_doorbell; + } /* don't map the queue if scheduling is halted */ if (!adev->userq_halt_for_enforce_isolation || @@ -806,7 +826,11 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args) up_read(&adev->reset_domain->sem); goto map_retry; } - r = amdgpu_userq_map_helper(queue); + if (queue->state == AMDGPU_USERQ_STATE_HUNG || + queue->state == AMDGPU_USERQ_STATE_RESET_PENDING) + r = -ECANCELED; + else + r = amdgpu_userq_map_helper(queue); up_read(&adev->reset_domain->sem); if (r) { drm_file_err(uq_mgr->file, "Failed to map Queue\n"); @@ -1582,54 +1606,58 @@ void amdgpu_userq_pre_reset(struct amdgpu_device *adev) struct amdgpu_usermode_queue *queue; unsigned long queue_id; - /* TODO: We probably need a new lock for the queue state */ + lockdep_assert_held_write(&adev->reset_domain->sem); + xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) { - if (queue->state == AMDGPU_USERQ_STATE_MAPPED) { + if (queue->state == AMDGPU_USERQ_STATE_MAPPED || + queue->state == AMDGPU_USERQ_STATE_PREEMPTED) { trace_amdgpu_userq_state_start(queue); userq_funcs = adev->userq_funcs[queue->queue_type]; userq_funcs->unmap(queue); - /* just mark all queues as hung at this point. - * if unmap succeeds, we could map again - * in amdgpu_userq_post_reset() if vram is not lost - */ - trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG); - queue->state = AMDGPU_USERQ_STATE_HUNG; } - /* Force-complete any pending fence regardless of queue state so - * that eviction/suspend and queue teardown waiters don't block - * forever on a fence that will never signal after the reset. - */ - amdgpu_userq_fence_driver_force_completion(queue); + /* A failed unmap is not proof that hardware access has stopped. */ + trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_RESET_PENDING); + queue->state = AMDGPU_USERQ_STATE_RESET_PENDING; } } -int amdgpu_userq_post_reset(struct amdgpu_device *adev, bool vram_lost) +/* Called after successful hardware reset, before allocations during resume. */ +void amdgpu_userq_reset_complete(struct amdgpu_device *adev) { - /* if any queue state is AMDGPU_USERQ_STATE_UNMAPPED - * at this point, we should be able to map it again - * and continue if vram is not lost. - */ struct amdgpu_usermode_queue *queue; - const struct amdgpu_userq_funcs *userq_funcs; unsigned long queue_id; - int ret = 0, r; + if (xa_empty(&adev->userq_doorbell_xa)) + return; + + lockdep_assert_held_write(&adev->reset_domain->sem); + + /* Release queue-fence waiters before acquiring their manager mutexes. */ xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) { - if (queue->state == AMDGPU_USERQ_STATE_HUNG && !vram_lost) { - trace_amdgpu_userq_state_start(queue); + trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG); + queue->state = AMDGPU_USERQ_STATE_HUNG; + amdgpu_userq_fence_driver_force_completion(queue); + } - userq_funcs = adev->userq_funcs[queue->queue_type]; - /* Re-map queue */ - r = userq_funcs->map(queue); - if (r) { - dev_err(adev->dev, "Failed to remap queue %ld\n", queue_id); - ret = r; - continue; - } - trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_MAPPED); - queue->state = AMDGPU_USERQ_STATE_MAPPED; + /* + * All queues are now retired. Keep them terminal even if VRAM survived: + * raw remapping would bypass VM validation and eviction-fence rearm. + */ + xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) { + struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; + struct amdgpu_fpriv *fpriv = uq_mgr->file->driver_priv; + struct dma_fence *fence; + bool cookie; + + mutex_lock(&uq_mgr->userq_mutex); + cookie = dma_fence_begin_signalling(); + fence = amdgpu_evf_mgr_get_fence(&fpriv->evf_mgr); + if (!dma_fence_is_signaled(fence)) { + dma_fence_set_error(fence, -ECANCELED); + dma_fence_signal(fence); } + dma_fence_put(fence); + dma_fence_end_signalling(cookie); + mutex_unlock(&uq_mgr->userq_mutex); } - - return ret; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h index 4dcf6151de6a..d17dbc284688 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h @@ -38,6 +38,7 @@ enum amdgpu_userq_state { AMDGPU_USERQ_STATE_PREEMPTED, AMDGPU_USERQ_STATE_HUNG, AMDGPU_USERQ_STATE_INVALID_VA, + AMDGPU_USERQ_STATE_RESET_PENDING, }; struct amdgpu_mqd_prop; @@ -180,7 +181,7 @@ int amdgpu_userq_start_sched_for_enforce_isolation(struct amdgpu_device *adev, u32 idx); void amdgpu_userq_reset_work(struct work_struct *work); void amdgpu_userq_pre_reset(struct amdgpu_device *adev); -int amdgpu_userq_post_reset(struct amdgpu_device *adev, bool vram_lost); +void amdgpu_userq_reset_complete(struct amdgpu_device *adev); void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue); void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c index c270635c907b..b9e6753ec4cc 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c @@ -556,11 +556,25 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data, /* We are here means UQ is active, make sure the eviction fence is valid */ amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr); + if (!down_read_trylock(&adev->reset_domain->sem)) { + r = -EAGAIN; + mutex_unlock(&userq_mgr->userq_mutex); + goto free_fence; + } + if (queue->state == AMDGPU_USERQ_STATE_HUNG || + queue->state == AMDGPU_USERQ_STATE_RESET_PENDING) { + r = -ECANCELED; + up_read(&adev->reset_domain->sem); + mutex_unlock(&userq_mgr->userq_mutex); + goto free_fence; + } + /* Create the new fence */ amdgpu_userq_fence_init(queue, fence, wptr); trace_amdgpu_userq_emit_fence(dev->dev, queue, fence); + up_read(&adev->reset_domain->sem); mutex_unlock(&userq_mgr->userq_mutex); /* @@ -609,6 +623,14 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data, dma_fence_put(&fence->base); drm_exec_fini(&exec); + goto put_queue; + +free_fence: + /* Allocation transferred the external fence-driver references. */ + amdgpu_userq_fence_put_fence_drv_array(fence); + amdgpu_userq_fence_driver_put(fence->fence_drv); + kvfree(fence->fence_drv_array); + kfree(fence); put_queue: amdgpu_userq_put(queue); put_gobj_write: -- 2.34.1
