A device state check alone cannot stop reset or wedging from racing with a caller that has already passed the check.
Add xe_device_io_get() and xe_device_io_put() wrappers around drm_dev_enter() and drm_dev_exit(). Check the device state inside the DRM SRCU critical section so new access is rejected after PCI recovery starts or the device becomes wedged. After blocking new access, PCI recovery wedges the GTs to signal pending fences and drains existing readers before disabling the PCI device. Permanent wedge isolation uses the same DRM SRCU domain to drain active hardware users before stopping interrupts and DMA. Cc: Matthew Brost <[email protected]> Cc: Thomas Hellström <[email protected]> Cc: Himal Prasad Ghimiray <[email protected]> Cc: Rodrigo Vivi <[email protected]> Assisted-by: Claude:claude-opus-4-8 Suggested-by: Thomas Hellström <[email protected]> Signed-off-by: Arvind Yadav <[email protected]> --- drivers/gpu/drm/xe/xe_bo.c | 6 ++--- drivers/gpu/drm/xe/xe_device.c | 33 +++++++++++++++++++-------- drivers/gpu/drm/xe/xe_device.h | 29 +++++++++++++++++++++++ drivers/gpu/drm/xe/xe_guc_ct.c | 9 +++++++- drivers/gpu/drm/xe/xe_guc_tlb_inval.c | 13 +++++++---- drivers/gpu/drm/xe/xe_pci_error.c | 6 +++++ 6 files changed, 78 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index dee1a39fac13..b39b07b55f64 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -2106,9 +2106,9 @@ static vm_fault_t xe_bo_cpu_fault(struct vm_fault *vmf) struct drm_exec exec; vm_fault_t ret; int err = 0; - int idx; + int io_idx; - if (xe_device_io_blocked(xe) || !drm_dev_enter(&xe->drm, &idx)) + if (xe_device_io_get(xe, &io_idx)) return ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot); ret = xe_bo_cpu_fault_fastpath(vmf, xe, bo, needs_rpm); @@ -2195,7 +2195,7 @@ static vm_fault_t xe_bo_cpu_fault(struct vm_fault *vmf) if (retry_after_wait) xe_bo_put(bo); out: - drm_dev_exit(idx); + xe_device_io_put(io_idx); return ret; } diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index 0e3139aa96d5..9a940ba8dc1c 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -235,16 +235,20 @@ static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { struct drm_file *file_priv = file->private_data; struct xe_device *xe = to_xe_device(file_priv->minor->dev); + int io_idx; long ret; - if (xe_device_io_blocked(xe)) - return -ECANCELED; + ret = xe_device_io_get(xe, &io_idx); + if (ret) + return ret; - ACQUIRE(xe_pm_runtime_ioctl, pm)(xe); - ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm); - if (ret >= 0) + scoped_cond_guard(xe_pm_runtime_ioctl, + ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &scope), + xe) { ret = drm_ioctl(file, cmd, arg); + } + xe_device_io_put(io_idx); return ret; } @@ -253,16 +257,20 @@ static long xe_drm_compat_ioctl(struct file *file, unsigned int cmd, unsigned lo { struct drm_file *file_priv = file->private_data; struct xe_device *xe = to_xe_device(file_priv->minor->dev); + int io_idx; long ret; - if (xe_device_io_blocked(xe)) - return -ECANCELED; + ret = xe_device_io_get(xe, &io_idx); + if (ret) + return ret; - ACQUIRE(xe_pm_runtime_ioctl, pm)(xe); - ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm); - if (ret >= 0) + scoped_cond_guard(xe_pm_runtime_ioctl, + ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &scope), + xe) { ret = drm_compat_ioctl(file, cmd, arg); + } + xe_device_io_put(io_idx); return ret; } #else @@ -432,6 +440,11 @@ struct xe_device *xe_device_create(struct pci_dev *pdev) } ALLOW_ERROR_INJECTION(xe_device_create, ERRNO); /* See xe_pci_probe() */ +void xe_device_io_drain(struct xe_device *xe) +{ + drm_dev_srcu_synchronize(&xe->drm); +} + static void xe_device_parse_modparam(struct xe_device *xe) { xe->atomic_svm_timeslice_ms = 5; diff --git a/drivers/gpu/drm/xe/xe_device.h b/drivers/gpu/drm/xe/xe_device.h index ecd46e2000d2..36b8da20345a 100644 --- a/drivers/gpu/drm/xe/xe_device.h +++ b/drivers/gpu/drm/xe/xe_device.h @@ -6,6 +6,7 @@ #ifndef _XE_DEVICE_H_ #define _XE_DEVICE_H_ +#include <drm/drm_drv.h> #include <drm/drm_util.h> #include "xe_device_types.h" @@ -224,6 +225,34 @@ static inline bool xe_device_io_blocked(struct xe_device *xe) return xe_device_wedged(xe) || xe_device_is_in_reset(xe); } +/** + * xe_device_io_get - Enter a device access critical section + * @xe: Xe device + * @idx: SRCU index returned on success + * + * Return: 0 on success, -ENODEV after unplug, or -ECANCELED when + * device I/O is blocked. + */ +static inline int xe_device_io_get(struct xe_device *xe, int *idx) +{ + if (!drm_dev_enter(&xe->drm, idx)) + return -ENODEV; + + if (xe_device_io_blocked(xe)) { + drm_dev_exit(*idx); + return -ECANCELED; + } + + return 0; +} + +static inline void xe_device_io_put(int idx) +{ + drm_dev_exit(idx); +} + +void xe_device_io_drain(struct xe_device *xe); + #ifdef CONFIG_DRM_XE_DEBUG_PAGE_SIZE static inline bool xe_debug_page_size_supported(struct xe_device *xe) { diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c index f82f57e6672d..5ec95e9551cc 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct.c +++ b/drivers/gpu/drm/xe/xe_guc_ct.c @@ -1059,6 +1059,8 @@ static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, struct g2h_fence *g2h_fence, bool defer_flush) { struct xe_gt *gt = ct_to_gt(ct); + bool io_held = false; + int io_idx; u16 seqno; int ret; @@ -1069,11 +1071,13 @@ static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, xe_gt_assert(gt, g2h_len || !num_g2h); lockdep_assert_held(&ct->lock); - if (xe_device_io_blocked(ct_to_xe(ct))) { + if (xe_device_io_get(ct_to_xe(ct), &io_idx)) { ret = guc_ct_cancel_errno(ct); goto out; } + io_held = true; + if (unlikely(ct->ctbs.h2g.info.broken)) { ret = -EPIPE; goto out; @@ -1130,6 +1134,9 @@ static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, if (g2h_len) spin_unlock_irq(&ct->fast_lock); out: + if (io_held) + xe_device_io_put(io_idx); + return ret; } diff --git a/drivers/gpu/drm/xe/xe_guc_tlb_inval.c b/drivers/gpu/drm/xe/xe_guc_tlb_inval.c index 8daa9c968f4f..7a190e7cdb56 100644 --- a/drivers/gpu/drm/xe/xe_guc_tlb_inval.c +++ b/drivers/gpu/drm/xe/xe_guc_tlb_inval.c @@ -66,15 +66,13 @@ static int send_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval, u32 seqno) struct xe_guc *guc = tlb_inval->private; struct xe_gt *gt = guc_to_gt(guc); struct xe_device *xe = guc_to_xe(guc); + int io_idx; /* * Returning -ECANCELED in this function is squashed at the caller and * signals waiters. */ - if (xe_device_io_blocked(xe)) - return -ECANCELED; - if (xe_guc_ct_enabled(&guc->ct) && guc->submission_state.enabled) { u32 action[] = { XE_GUC_ACTION_TLB_INVALIDATION, @@ -86,8 +84,13 @@ static int send_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval, u32 seqno) } else if (xe_device_uc_enabled(xe)) { struct xe_mmio *mmio = >->mmio; - if (IS_SRIOV_VF(xe)) + if (xe_device_io_get(xe, &io_idx)) + return -ECANCELED; + + if (IS_SRIOV_VF(xe)) { + xe_device_io_put(io_idx); return -ECANCELED; + } CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); if (xe->info.platform == XE_PVC || GRAPHICS_VER(xe) >= 20) { @@ -99,6 +102,8 @@ static int send_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval, u32 seqno) xe_mmio_write32(mmio, GUC_TLB_INV_CR, GUC_TLB_INV_CR_INVALIDATE); } + + xe_device_io_put(io_idx); } return -ECANCELED; diff --git a/drivers/gpu/drm/xe/xe_pci_error.c b/drivers/gpu/drm/xe/xe_pci_error.c index 4b09e74af92e..3e06f504f1b3 100644 --- a/drivers/gpu/drm/xe/xe_pci_error.c +++ b/drivers/gpu/drm/xe/xe_pci_error.c @@ -25,6 +25,12 @@ static void prepare_device_for_reset(struct pci_dev *pdev) for_each_gt(gt, xe, id) xe_gt_declare_wedged(gt); + /* + * GT wedging signals pending fences. Existing hardware users can now + * finish without blocking reset. + */ + xe_device_io_drain(xe); + pci_disable_device(pdev); } -- 2.43.0
