Queue cleanup is asynchronous. VM work can still run while file close or VM destroy is stopping the queues.
Add a CLOSING state and set it before queue cleanup starts. Reject new exec, bind, rebind and page-fault work on closing VMs without removing their mappings yet. Cached faults bypass xe_pagefault_service(), so also check VM state at the common acknowledgment path. Hold vm->lock for read through successful reply submission to serialize it with the closing transition. Invalidate the local cache when that check fails. Keep vm->size valid so SVM invalidation can still drain existing mappings, but stop adding new SVM garbage-collector work once close starts. Also reject page faults on banned VMs. v3: - Check VM state before replying to cached and chained faults (Tejas). - Hold vm->lock through successful reply submission and invalidate the local cache when the VM lookup or state check fails. (Sashiko) - Report faults rejected by the final VM-state check. (Sashiko) - Preserve the original fault details when service replaces them with the resolved range. (Sashiko) - Account for failed prefetches and invalidate the local cache on failure. (Sashiko) Cc: Matthew Brost <[email protected]> Cc: Thomas Hellström <[email protected]> Cc: Himal Prasad Ghimiray <[email protected]> Cc: Rodrigo Vivi <[email protected]> Cc: Daniel Charles <[email protected]> Cc: Tejas Upadhyay <[email protected]> Assisted-by: LLM Tested-by: Sanjay Yadav <[email protected]> Signed-off-by: Arvind Yadav <[email protected]> --- drivers/gpu/drm/xe/xe_device.c | 4 ++ drivers/gpu/drm/xe/xe_pagefault.c | 87 ++++++++++++++++++++++++++----- drivers/gpu/drm/xe/xe_svm.c | 3 +- drivers/gpu/drm/xe/xe_vm.c | 18 ++++++- drivers/gpu/drm/xe/xe_vm.h | 9 +++- drivers/gpu/drm/xe/xe_vm_types.h | 1 + 6 files changed, 105 insertions(+), 17 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index f3c71f91c4a9..383afb2993a6 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -181,6 +181,10 @@ static void xe_file_close(struct drm_device *dev, struct drm_file *file) guard(xe_pm_runtime)(xe); + /* Block new VM work before starting asynchronous queue teardown. */ + xa_for_each(&xef->vm.xa, idx, vm) + xe_vm_close_start(vm); + /* * No need for exec_queue.lock here as there is no contention for it * when FD is closing as IOCTLs presumably can't be modifying the diff --git a/drivers/gpu/drm/xe/xe_pagefault.c b/drivers/gpu/drm/xe/xe_pagefault.c index 98cce0f12b12..f70dadcf4bd8 100644 --- a/drivers/gpu/drm/xe/xe_pagefault.c +++ b/drivers/gpu/drm/xe/xe_pagefault.c @@ -291,7 +291,7 @@ static int xe_pagefault_service(struct xe_pagefault *pf) down_read(&vm->lock); - if (xe_vm_is_closed(vm)) { + if (xe_vm_is_closed_or_banned(vm)) { err = -ENOENT; xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VM_CLOSED); goto unlock_vm; @@ -678,32 +678,37 @@ static void xe_pagefault_queue_work(struct work_struct *w) while (xe_pagefault_queue_pop(pf_queue, &pf, pf_work->id)) { const struct xe_pagefault_ops *ops = pf->producer.ops; + struct xe_vm *ack_vm = NULL; void *private = pf->producer.private; struct xe_gt *gt = pf->gt; u32 asid = FIELD_GET(XE_PAGEFAULT_ASID_MASK, pf->consumer.id); + struct xe_pagefault original = { + .gt = gt, + .consumer = { + .page_addr = pf->consumer.page_addr, + .access_type = pf->consumer.access_type, + .fault_type_level = pf->consumer.fault_type_level, + .engine_class_instance = + pf->consumer.engine_class_instance, + .id = pf->consumer.id, + }, + /* Error reporting uses the producer callback and message. */ + .producer = pf->producer, + }; + struct xe_pagefault *report_pf = pf; int err = 0; bool invalidated = false; + bool cache_hit = false; /* Last fault same address, ack immediately */ if (xe_pagefault_match(pf, cache_start, cache_end, cache_asid)) { - xe_gt_stats_incr(gt, XE_GT_STATS_ID_LAST_PAGEFAULT_COUNT, 1); + cache_hit = true; goto ack_fault; } err = xe_pagefault_service(pf); - if (err) { - if (!(pf->consumer.access_type & XE_PAGEFAULT_ACCESS_PREFETCH)) { - xe_pagefault_save_to_vm(gt_to_xe(gt), pf); - xe_pagefault_cache_start_invalidate(cache_start); - xe_pagefault_print(pf); - xe_log_err_info(pf->gt, PAGEFAULT, err, "Unsuccessful response\n"); - } else { - xe_gt_stats_incr(pf->gt, XE_GT_STATS_ID_INVALID_PREFETCH_PAGEFAULT_COUNT, 1); - xe_gt_dbg(pf->gt, "Prefetch Fault response: Unsuccessful %pe\n", - ERR_PTR(err)); - } - } else { + if (!err) { /* Cache valid fault locally */ cache_start = xe_pagefault_start_addr(pf); cache_end = xe_pagefault_end_addr(pf); @@ -711,6 +716,55 @@ static void xe_pagefault_queue_work(struct work_struct *w) } ack_fault: + /* + * Cache hits bypass fault service. Serialize successful replies + * with VM close, including replies to matching chained faults. + * Take vm->lock before ack_fault_begin() takes the CT lock and + * hold it until ack_fault_end() flushes the reply batch. + */ + if (!err) { + /* + * Successful service may replace the fault fields with + * the resolved range. Use the original fields to report + * a failure detected here. Do not restore the live entry, + * which also contains queue state and chain links. + */ + report_pf = &original; + ack_vm = xe_pagefault_asid_to_vm(report_pf, asid); + if (IS_ERR(ack_vm)) { + err = PTR_ERR(ack_vm); + ack_vm = NULL; + } else { + down_read(&ack_vm->lock); + if (xe_vm_is_closed_or_banned(ack_vm)) { + xe_pagefault_set_error(report_pf, + XE_PAGEFAULT_ERROR_VM_CLOSED); + err = -ENOENT; + up_read(&ack_vm->lock); + xe_vm_put(ack_vm); + ack_vm = NULL; + } + } + } + + if (err) { + xe_pagefault_cache_start_invalidate(cache_start); + if (!(report_pf->consumer.access_type & + XE_PAGEFAULT_ACCESS_PREFETCH)) { + xe_pagefault_save_to_vm(xe, report_pf); + xe_pagefault_print(report_pf); + xe_log_err_info(gt, PAGEFAULT, err, + "Unsuccessful response\n"); + } else { + xe_gt_stats_incr(gt, + XE_GT_STATS_ID_INVALID_PREFETCH_PAGEFAULT_COUNT, 1); + xe_gt_dbg(gt, "Prefetch Fault response: Unsuccessful %pe\n", + ERR_PTR(err)); + } + } else if (cache_hit) { + xe_gt_stats_incr(gt, XE_GT_STATS_ID_LAST_PAGEFAULT_COUNT, 1); + } + xe_assert(xe, pf->consumer.alloc_state == XE_PAGEFAULT_ALLOC_STATE_ACTIVE); xe_assert(xe, pf == pf_work->cache.pf); @@ -752,6 +806,11 @@ static void xe_pagefault_queue_work(struct work_struct *w) } ops->ack_fault_end(private); + if (ack_vm) { + up_read(&ack_vm->lock); + xe_vm_put(ack_vm); + } + if (time_after(jiffies, threshold)) { queue_work(xe->usm.pagefault_wq, w); break; diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c index bc6902b5bf57..62ccb2fe6f8f 100644 --- a/drivers/gpu/drm/xe/xe_svm.c +++ b/drivers/gpu/drm/xe/xe_svm.c @@ -219,7 +219,8 @@ xe_svm_range_notifier_event_end(struct xe_vm *vm, struct drm_gpusvm_range *r, drm_gpusvm_unmap_pages(&vm->svm.gpusvm, &(to_xe_range(r)->pages), drm_gpusvm_range_size(r) >> PAGE_SHIFT, &ctx); - if (!xe_vm_is_closed(vm) && mmu_range->event == MMU_NOTIFY_UNMAP) + if (!xe_vm_is_closed(vm) && !xe_vm_is_closing(vm) && + mmu_range->event == MMU_NOTIFY_UNMAP) xe_svm_garbage_collector_add_range(vm, to_xe_range(r), mmu_range); } diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 12afb3363e79..e6e53912448f 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -1976,6 +1976,20 @@ static void xe_vm_close(struct xe_vm *vm) drm_dev_exit(idx); } +void xe_vm_close_start(struct xe_vm *vm) +{ + down_write(&vm->lock); + if (xe_vm_in_fault_mode(vm)) + xe_svm_notifier_lock(vm); + + /* Keep size valid so SVM invalidation still performs its full drain. */ + vm->flags |= XE_VM_FLAG_CLOSING; + + if (xe_vm_in_fault_mode(vm)) + xe_svm_notifier_unlock(vm); + up_write(&vm->lock); +} + void xe_vm_close_and_put(struct xe_vm *vm) { LIST_HEAD(contested); @@ -2273,8 +2287,10 @@ int xe_vm_destroy_ioctl(struct drm_device *dev, void *data, xa_erase(&xef->vm.xa, args->vm_id); mutex_unlock(&xef->vm.lock); - if (!err) + if (!err) { + xe_vm_close_start(vm); xe_vm_close_and_put(vm); + } return err; } diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h index b1c5f940bcc5..c90c9dc1351c 100644 --- a/drivers/gpu/drm/xe/xe_vm.h +++ b/drivers/gpu/drm/xe/xe_vm.h @@ -59,6 +59,11 @@ static inline bool xe_vm_is_closed(struct xe_vm *vm) return !vm->size; } +static inline bool xe_vm_is_closing(struct xe_vm *vm) +{ + return vm->flags & XE_VM_FLAG_CLOSING; +} + static inline bool xe_vm_is_banned(struct xe_vm *vm) { return vm->flags & XE_VM_FLAG_BANNED; @@ -67,7 +72,8 @@ static inline bool xe_vm_is_banned(struct xe_vm *vm) static inline bool xe_vm_is_closed_or_banned(struct xe_vm *vm) { lockdep_assert_held(&vm->lock); - return xe_vm_is_closed(vm) || xe_vm_is_banned(vm); + return xe_vm_is_closed(vm) || xe_vm_is_banned(vm) || + xe_vm_is_closing(vm); } struct xe_vma * @@ -214,6 +220,7 @@ int xe_vm_get_property_ioctl(struct drm_device *dev, void *data, struct drm_file *file); void xe_vm_close_and_put(struct xe_vm *vm); +void xe_vm_close_start(struct xe_vm *vm); static inline bool xe_vm_in_fault_mode(struct xe_vm *vm) { diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h index fb9305124679..6ca9ef9d0c0b 100644 --- a/drivers/gpu/drm/xe/xe_vm_types.h +++ b/drivers/gpu/drm/xe/xe_vm_types.h @@ -286,6 +286,7 @@ struct xe_vm { #define XE_VM_FLAG_SET_TILE_ID(tile) FIELD_PREP(GENMASK(7, 6), (tile)->id) #define XE_VM_FLAG_GSC BIT(8) #define XE_VM_FLAG_NO_VM_OVERCOMMIT BIT(9) +#define XE_VM_FLAG_CLOSING BIT(10) unsigned long flags; /** -- 2.43.0
