SVM fault handling and VM rebind work may still run when PCI error recovery starts or the device becomes permanently wedged. This work can validate BOs, update page tables or submit migration work after device I/O has been blocked.
Stop SVM fault handling, pagemap population, device-memory copies and preempt rebind work when device I/O is blocked. v2: - Protect VM and SVM hardware access with the common SRCU gate (Thomas) - Hold the gate across page-fault, migration and rebind work. - Remove the racy device state checks (Matt) v3: - Use DRM device SRCU for VM and SVM hardware access. - Protect the SVM garbage collector worker with the common I/O gate. - Keep the original TLB invalidation warning because cancellation is converted to success by the lower layer. Cc: Matthew Brost <[email protected]> Cc: Thomas Hellström <[email protected]> Cc: Himal Prasad Ghimiray <[email protected]> Cc: Rodrigo Vivi <[email protected]> Assisted-by: LLM Suggested-by: Thomas Hellström <[email protected]> Signed-off-by: Arvind Yadav <[email protected]> --- drivers/gpu/drm/xe/xe_svm.c | 26 +++++++++++++++++++++++--- drivers/gpu/drm/xe/xe_vm.c | 11 +++++++++-- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c index 909a602d091e..226dfa57cc06 100644 --- a/drivers/gpu/drm/xe/xe_svm.c +++ b/drivers/gpu/drm/xe/xe_svm.c @@ -12,6 +12,7 @@ #include <drm/ttm/ttm_bo.h> #include "xe_bo.h" +#include "xe_device.h" #include "xe_exec_queue_types.h" #include "xe_gt_stats.h" #include "xe_log.h" @@ -457,9 +458,15 @@ static void xe_svm_garbage_collector_work_func(struct work_struct *w) { struct xe_vm *vm = container_of(w, struct xe_vm, svm.garbage_collector.work); + int io_idx; guard(rwsem_read)(&vm->lock); + + if (xe_device_io_get(vm->xe, &io_idx)) + return; + xe_svm_garbage_collector(vm); + xe_device_io_put(io_idx); } #if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) @@ -600,6 +607,8 @@ static int xe_svm_copy(struct page **pages, #define XE_VRAM_ADDR_INVALID ~0x0ull u64 vram_addr = XE_VRAM_ADDR_INVALID; int err = 0, pos = 0; + bool io_held = false; + int io_idx; bool sram = dir == XE_SVM_COPY_TO_SRAM; ktime_t start = xe_gt_stats_ktime_get(); @@ -630,6 +639,12 @@ static int xe_svm_copy(struct page **pages, vr = xe_page_to_vr(spage); gt = xe_migrate_exec_queue(vr->migrate)->gt; xe = vr->xe; + + err = xe_device_io_get(xe, &io_idx); + if (err) + goto err_out; + + io_held = true; } XE_WARN_ON(spage && xe_page_to_vr(spage) != vr); @@ -750,6 +765,9 @@ static int xe_svm_copy(struct page **pages, if (pre_migrate_fence) dma_fence_wait(pre_migrate_fence, false); + if (io_held) + xe_device_io_put(io_idx); + /* * XXX: We can't derive the GT here (or anywhere in this functions, but * compute always uses the primary GT so accumulate stats on the likely @@ -1124,8 +1142,9 @@ static int xe_drm_pagemap_populate_mm(struct drm_pagemap *dpagemap, struct xe_bo *bo; int err = 0, idx; - if (!drm_dev_enter(&xe->drm, &idx)) - return -ENODEV; + err = xe_device_io_get(xe, &idx); + if (err) + return err; xe_pm_runtime_get(xe); @@ -1167,7 +1186,8 @@ static int xe_drm_pagemap_populate_mm(struct drm_pagemap *dpagemap, xe_bo_put(bo); } xe_pm_runtime_put(xe); - drm_dev_exit(idx); + + xe_device_io_put(idx); return err; } diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index d12365b11186..645c5e906224 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -541,6 +541,7 @@ static void preempt_rebind_work_func(struct work_struct *w) unsigned int fence_count = 0; LIST_HEAD(preempt_fences); int err = 0; + int io_idx; long wait; int __maybe_unused tries = 0; @@ -574,6 +575,12 @@ static void preempt_rebind_work_func(struct work_struct *w) if (err) goto out_unlock_outer; + if (xe_device_io_get(vm->xe, &io_idx)) { + xe_validation_ctx_fini(&ctx); + err = 0; + goto out_unlock_outer; + } + drm_exec_until_all_locked(&exec) { bool done = false; @@ -581,8 +588,7 @@ static void preempt_rebind_work_func(struct work_struct *w) drm_exec_retry_on_contention(&exec); xe_validation_retry_on_oom(&ctx, &err); if (err || done) { - xe_validation_ctx_fini(&ctx); - goto out_unlock_outer; + goto out_unlock; } } @@ -628,6 +634,7 @@ static void preempt_rebind_work_func(struct work_struct *w) out_unlock: xe_validation_ctx_fini(&ctx); + xe_device_io_put(io_idx); out_unlock_outer: if (err == -EAGAIN) { trace_xe_vm_rebind_worker_retry(vm); -- 2.43.0
