Queue cleanup is asynchronous. VM work can still run while file close
or VM destroy is stopping the queues.

Add a CLOSING state and set it before queue cleanup starts. Reject
new exec, bind, rebind and page-fault work on closing VMs without
removing their mappings yet.

Cached faults bypass xe_pagefault_service(), so also check VM state
at the common acknowledgment path. Hold vm->lock for read through
successful reply submission to serialize it with the closing
transition. Invalidate the local cache when that check fails.

Keep vm->size valid so SVM invalidation can still drain existing
mappings, but stop adding new SVM garbage-collector work once close
starts. Also reject page faults on banned VMs.

v3:
- Check VM state before replying to cached and chained faults (Tejas).
- Hold vm->lock through successful reply submission and invalidate
  the local cache when the VM lookup or state check fails. (Sashiko)
- Report faults rejected by the final VM-state check. (Sashiko)
- Preserve the original fault details when service replaces them with
  the resolved range. (Sashiko)
- Account for failed prefetches and invalidate the local cache on failure.
  (Sashiko)

Cc: Matthew Brost <[email protected]>
Cc: Thomas Hellström <[email protected]>
Cc: Himal Prasad Ghimiray <[email protected]>
Cc: Rodrigo Vivi <[email protected]>
Cc: Daniel Charles <[email protected]>
Cc: Tejas Upadhyay <[email protected]>
Assisted-by: LLM
Tested-by: Sanjay Yadav <[email protected]>
Signed-off-by: Arvind Yadav <[email protected]>
---
 drivers/gpu/drm/xe/xe_device.c    |  4 ++
 drivers/gpu/drm/xe/xe_pagefault.c | 87 ++++++++++++++++++++++++++-----
 drivers/gpu/drm/xe/xe_svm.c       |  3 +-
 drivers/gpu/drm/xe/xe_vm.c        | 18 ++++++-
 drivers/gpu/drm/xe/xe_vm.h        |  9 +++-
 drivers/gpu/drm/xe/xe_vm_types.h  |  1 +
 6 files changed, 105 insertions(+), 17 deletions(-)

diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c
index f3c71f91c4a9..383afb2993a6 100644
--- a/drivers/gpu/drm/xe/xe_device.c
+++ b/drivers/gpu/drm/xe/xe_device.c
@@ -181,6 +181,10 @@ static void xe_file_close(struct drm_device *dev, struct 
drm_file *file)
 
        guard(xe_pm_runtime)(xe);
 
+       /* Block new VM work before starting asynchronous queue teardown. */
+       xa_for_each(&xef->vm.xa, idx, vm)
+               xe_vm_close_start(vm);
+
        /*
         * No need for exec_queue.lock here as there is no contention for it
         * when FD is closing as IOCTLs presumably can't be modifying the
diff --git a/drivers/gpu/drm/xe/xe_pagefault.c 
b/drivers/gpu/drm/xe/xe_pagefault.c
index 98cce0f12b12..f70dadcf4bd8 100644
--- a/drivers/gpu/drm/xe/xe_pagefault.c
+++ b/drivers/gpu/drm/xe/xe_pagefault.c
@@ -291,7 +291,7 @@ static int xe_pagefault_service(struct xe_pagefault *pf)
 
        down_read(&vm->lock);
 
-       if (xe_vm_is_closed(vm)) {
+       if (xe_vm_is_closed_or_banned(vm)) {
                err = -ENOENT;
                xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VM_CLOSED);
                goto unlock_vm;
@@ -678,32 +678,37 @@ static void xe_pagefault_queue_work(struct work_struct *w)
 
        while (xe_pagefault_queue_pop(pf_queue, &pf, pf_work->id)) {
                const struct xe_pagefault_ops *ops = pf->producer.ops;
+               struct xe_vm *ack_vm = NULL;
                void *private = pf->producer.private;
                struct xe_gt *gt = pf->gt;
                u32 asid = FIELD_GET(XE_PAGEFAULT_ASID_MASK, pf->consumer.id);
+               struct xe_pagefault original = {
+                       .gt = gt,
+                       .consumer = {
+                               .page_addr = pf->consumer.page_addr,
+                               .access_type = pf->consumer.access_type,
+                               .fault_type_level = 
pf->consumer.fault_type_level,
+                               .engine_class_instance =
+                                       pf->consumer.engine_class_instance,
+                               .id = pf->consumer.id,
+                       },
+                       /* Error reporting uses the producer callback and 
message. */
+                       .producer = pf->producer,
+               };
+               struct xe_pagefault *report_pf = pf;
                int err = 0;
                bool invalidated = false;
+               bool cache_hit = false;
 
                /* Last fault same address, ack immediately */
                if (xe_pagefault_match(pf, cache_start, cache_end, cache_asid)) 
{
-                       xe_gt_stats_incr(gt, 
XE_GT_STATS_ID_LAST_PAGEFAULT_COUNT, 1);
+                       cache_hit = true;
                        goto ack_fault;
                }
 
                err = xe_pagefault_service(pf);
 
-               if (err) {
-                       if (!(pf->consumer.access_type & 
XE_PAGEFAULT_ACCESS_PREFETCH)) {
-                               xe_pagefault_save_to_vm(gt_to_xe(gt), pf);
-                               
xe_pagefault_cache_start_invalidate(cache_start);
-                               xe_pagefault_print(pf);
-                               xe_log_err_info(pf->gt, PAGEFAULT, err, 
"Unsuccessful response\n");
-                       } else {
-                               xe_gt_stats_incr(pf->gt, 
XE_GT_STATS_ID_INVALID_PREFETCH_PAGEFAULT_COUNT, 1);
-                               xe_gt_dbg(pf->gt, "Prefetch Fault response: 
Unsuccessful %pe\n",
-                                         ERR_PTR(err));
-                       }
-               } else {
+               if (!err) {
                        /* Cache valid fault locally */
                        cache_start = xe_pagefault_start_addr(pf);
                        cache_end = xe_pagefault_end_addr(pf);
@@ -711,6 +716,55 @@ static void xe_pagefault_queue_work(struct work_struct *w)
                }
 
 ack_fault:
+               /*
+                * Cache hits bypass fault service. Serialize successful replies
+                * with VM close, including replies to matching chained faults.
+                * Take vm->lock before ack_fault_begin() takes the CT lock and
+                * hold it until ack_fault_end() flushes the reply batch.
+                */
+               if (!err) {
+                       /*
+                        * Successful service may replace the fault fields with
+                        * the resolved range. Use the original fields to report
+                        * a failure detected here. Do not restore the live 
entry,
+                        * which also contains queue state and chain links.
+                        */
+                       report_pf = &original;
+                       ack_vm = xe_pagefault_asid_to_vm(report_pf, asid);
+                       if (IS_ERR(ack_vm)) {
+                               err = PTR_ERR(ack_vm);
+                               ack_vm = NULL;
+                       } else {
+                               down_read(&ack_vm->lock);
+                               if (xe_vm_is_closed_or_banned(ack_vm)) {
+                                       xe_pagefault_set_error(report_pf,
+                                                              
XE_PAGEFAULT_ERROR_VM_CLOSED);
+                                       err = -ENOENT;
+                                       up_read(&ack_vm->lock);
+                                       xe_vm_put(ack_vm);
+                                       ack_vm = NULL;
+                               }
+                       }
+               }
+
+               if (err) {
+                       xe_pagefault_cache_start_invalidate(cache_start);
+                       if (!(report_pf->consumer.access_type &
+                             XE_PAGEFAULT_ACCESS_PREFETCH)) {
+                               xe_pagefault_save_to_vm(xe, report_pf);
+                               xe_pagefault_print(report_pf);
+                               xe_log_err_info(gt, PAGEFAULT, err,
+                                               "Unsuccessful response\n");
+                       } else {
+                               xe_gt_stats_incr(gt,
+                                                
XE_GT_STATS_ID_INVALID_PREFETCH_PAGEFAULT_COUNT, 1);
+                               xe_gt_dbg(gt, "Prefetch Fault response: 
Unsuccessful %pe\n",
+                                         ERR_PTR(err));
+                       }
+               } else if (cache_hit) {
+                       xe_gt_stats_incr(gt, 
XE_GT_STATS_ID_LAST_PAGEFAULT_COUNT, 1);
+               }
+
                xe_assert(xe, pf->consumer.alloc_state ==
                          XE_PAGEFAULT_ALLOC_STATE_ACTIVE);
                xe_assert(xe, pf == pf_work->cache.pf);
@@ -752,6 +806,11 @@ static void xe_pagefault_queue_work(struct work_struct *w)
                }
                ops->ack_fault_end(private);
 
+               if (ack_vm) {
+                       up_read(&ack_vm->lock);
+                       xe_vm_put(ack_vm);
+               }
+
                if (time_after(jiffies, threshold)) {
                        queue_work(xe->usm.pagefault_wq, w);
                        break;
diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c
index bc6902b5bf57..62ccb2fe6f8f 100644
--- a/drivers/gpu/drm/xe/xe_svm.c
+++ b/drivers/gpu/drm/xe/xe_svm.c
@@ -219,7 +219,8 @@ xe_svm_range_notifier_event_end(struct xe_vm *vm, struct 
drm_gpusvm_range *r,
 
        drm_gpusvm_unmap_pages(&vm->svm.gpusvm, &(to_xe_range(r)->pages),
                               drm_gpusvm_range_size(r) >> PAGE_SHIFT, &ctx);
-       if (!xe_vm_is_closed(vm) && mmu_range->event == MMU_NOTIFY_UNMAP)
+       if (!xe_vm_is_closed(vm) && !xe_vm_is_closing(vm) &&
+           mmu_range->event == MMU_NOTIFY_UNMAP)
                xe_svm_garbage_collector_add_range(vm, to_xe_range(r),
                                                   mmu_range);
 }
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 12afb3363e79..e6e53912448f 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -1976,6 +1976,20 @@ static void xe_vm_close(struct xe_vm *vm)
                drm_dev_exit(idx);
 }
 
+void xe_vm_close_start(struct xe_vm *vm)
+{
+       down_write(&vm->lock);
+       if (xe_vm_in_fault_mode(vm))
+               xe_svm_notifier_lock(vm);
+
+       /* Keep size valid so SVM invalidation still performs its full drain. */
+       vm->flags |= XE_VM_FLAG_CLOSING;
+
+       if (xe_vm_in_fault_mode(vm))
+               xe_svm_notifier_unlock(vm);
+       up_write(&vm->lock);
+}
+
 void xe_vm_close_and_put(struct xe_vm *vm)
 {
        LIST_HEAD(contested);
@@ -2273,8 +2287,10 @@ int xe_vm_destroy_ioctl(struct drm_device *dev, void 
*data,
                xa_erase(&xef->vm.xa, args->vm_id);
        mutex_unlock(&xef->vm.lock);
 
-       if (!err)
+       if (!err) {
+               xe_vm_close_start(vm);
                xe_vm_close_and_put(vm);
+       }
 
        return err;
 }
diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h
index b1c5f940bcc5..c90c9dc1351c 100644
--- a/drivers/gpu/drm/xe/xe_vm.h
+++ b/drivers/gpu/drm/xe/xe_vm.h
@@ -59,6 +59,11 @@ static inline bool xe_vm_is_closed(struct xe_vm *vm)
        return !vm->size;
 }
 
+static inline bool xe_vm_is_closing(struct xe_vm *vm)
+{
+       return vm->flags & XE_VM_FLAG_CLOSING;
+}
+
 static inline bool xe_vm_is_banned(struct xe_vm *vm)
 {
        return vm->flags & XE_VM_FLAG_BANNED;
@@ -67,7 +72,8 @@ static inline bool xe_vm_is_banned(struct xe_vm *vm)
 static inline bool xe_vm_is_closed_or_banned(struct xe_vm *vm)
 {
        lockdep_assert_held(&vm->lock);
-       return xe_vm_is_closed(vm) || xe_vm_is_banned(vm);
+       return xe_vm_is_closed(vm) || xe_vm_is_banned(vm) ||
+               xe_vm_is_closing(vm);
 }
 
 struct xe_vma *
@@ -214,6 +220,7 @@ int xe_vm_get_property_ioctl(struct drm_device *dev, void 
*data,
                             struct drm_file *file);
 
 void xe_vm_close_and_put(struct xe_vm *vm);
+void xe_vm_close_start(struct xe_vm *vm);
 
 static inline bool xe_vm_in_fault_mode(struct xe_vm *vm)
 {
diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h
index fb9305124679..6ca9ef9d0c0b 100644
--- a/drivers/gpu/drm/xe/xe_vm_types.h
+++ b/drivers/gpu/drm/xe/xe_vm_types.h
@@ -286,6 +286,7 @@ struct xe_vm {
 #define XE_VM_FLAG_SET_TILE_ID(tile)   FIELD_PREP(GENMASK(7, 6), (tile)->id)
 #define XE_VM_FLAG_GSC                 BIT(8)
 #define XE_VM_FLAG_NO_VM_OVERCOMMIT     BIT(9)
+#define XE_VM_FLAG_CLOSING             BIT(10)
        unsigned long flags;
 
        /**
-- 
2.43.0

Reply via email to