Render-node user queues had no mechanism to install a userspace second-level trap handler equivalent to what KFD provides for compute.
On dGPUs the CWSR TMA buffer is placed in VRAM and accessed via MMIO (PCI BAR). Raw CPU pointer dereferences are unsafe on platforms that lack full load/store support for MMIO. Replace void *tma_cpu_addr with struct iosys_map tma_map and use iosys_map_wr() for all TMA writes. Writing TBA and TMA as two separate stores creates a race where a trapping wave can observe a mismatched pair. For user queues, evict all queues and flush the GPU TLB before writing, then restore queues after. For kernel queues, wait for the VM to be idle via amdgpu_vm_wait_idle() before eviction and the descriptor write. Evict user queues in amdgpu_trap_free() at file close. Without eviction a freed TMA BO can be accessed via a stale TBA, causing a GPU page fault. Export amdgpu_userq_restore_all() for immediate queue restore after SET/CLEAR without waiting for the eviction fence. Add VM_ALWAYS_VALID validation in amdgpu_trap_validate_user_addr(). SET_L2_TRAP returns -EINVAL if TBA or TMA BO does not have this flag. The old TRAP ioctl and its internal handler are removed. The new SET/CLEAR path is exposed via the VM ioctl in a later patch. SET_L2_TRAP is per-VM (per-process). Each process has its own TMA buffer allocated in amdgpu_trap_alloc(). There is no conflict between processes on user queue VMIDs. Cc: Christian König <[email protected]> Cc: Alex Deucher <[email protected]> Cc: Felix Kuehling <[email protected]> Cc: Lijo Lazar <[email protected]> Signed-off-by: Srinivasan Shanmugam <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 1 - drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c | 284 ++++++++++++++++------ drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h | 64 ++++- drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 22 +- drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h | 1 + include/uapi/drm/amdgpu_drm.h | 2 - 7 files changed, 285 insertions(+), 91 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index a93ee106ab92..6473faf18d71 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -3079,7 +3079,6 @@ const struct drm_ioctl_desc amdgpu_ioctls_kms[] = { DRM_IOCTL_DEF_DRV(AMDGPU_SCHED, amdgpu_sched_ioctl, DRM_MASTER), DRM_IOCTL_DEF_DRV(AMDGPU_BO_LIST, amdgpu_bo_list_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(AMDGPU_FENCE_TO_HANDLE, amdgpu_cs_fence_to_handle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(AMDGPU_TRAP, amdgpu_trap_ioctl, DRM_AUTH | DRM_RENDER_ALLOW), /* KMS */ DRM_IOCTL_DEF_DRV(AMDGPU_GEM_MMAP, amdgpu_gem_mmap_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(AMDGPU_GEM_WAIT_IDLE, amdgpu_gem_wait_idle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index 47e4680db797..559c9222fc48 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -1716,7 +1716,7 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev, } amdgpu_ctx_mgr_fini(&fpriv->ctx_mgr); - amdgpu_trap_free(adev, &fpriv->vm, &fpriv->trap_obj); + amdgpu_trap_free(adev, &fpriv->vm, &fpriv->userq_mgr, &fpriv->trap_obj); if (pasid) amdgpu_pasid_free_delayed(pd->tbo.base.resv, pasid); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c index 31f653ec3fb1..3313bc099cf4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c @@ -19,12 +19,23 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ +#include <linux/iosys-map.h> +#include <drm/ttm/ttm_bo.h> + #include <drm/drm_exec.h> #include "amdgpu.h" +#include "amdgpu_gmc.h" +#include "amdgpu_userq.h" +#include "amdgpu_vm.h" #include "cwsr_trap_handler.h" #include "amdgpu_trap.h" +/* First-level TMA layout shared with second-level trap handler ABI */ +#define AMDGPU_TRAP_L2_TBA_INDEX 0 +#define AMDGPU_TRAP_L2_TMA_INDEX 1 +#define AMDGPU_TRAP_DEBUG_FLAG_INDEX 2 + extern int cwsr_enable; #define AMDGPU_TRAP_TBA_MAX_SIZE (2 * AMDGPU_GPU_PAGE_SIZE) @@ -380,6 +391,7 @@ int amdgpu_trap_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_trap_obj *cwsr; struct amdgpu_bo *bo; struct drm_exec exec; + void *cpu_addr; int r; if (!amdgpu_trap_is_enabled(adev)) @@ -407,10 +419,21 @@ int amdgpu_trap_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm, r = amdgpu_bo_create_kernel(adev, AMDGPU_TRAP_TMA_MAX_SIZE, PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, &cwsr->tma_bo, NULL, - &cwsr->tma_cpu_addr); + &cpu_addr); if (r) goto err; + /* + * Build an iosys_map for the TMA CPU mapping. On dGPUs the TMA BO + * may be placed in VRAM and accessed via MMIO (PCI BAR). Check + * TTM_BO_MAP_IOMEM_MASK to select the correct write accessor. + */ + if (cwsr->tma_bo->kmap.bo_kmap_type & TTM_BO_MAP_IOMEM_MASK) + iosys_map_set_vaddr_iomem(&cwsr->tma_map, + (void __iomem *)cpu_addr); + else + iosys_map_set_vaddr(&cwsr->tma_map, cpu_addr); + r = amdgpu_trap_map_region(adev, vm, cwsr, AMDGPU_TRAP_TMA); if (r) goto err; @@ -433,6 +456,29 @@ int amdgpu_trap_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm, return r; } +void amdgpu_trap_clear_l2_handler(struct amdgpu_vm *vm, + struct amdgpu_trap_obj *trap_obj) +{ + if (!trap_obj || iosys_map_is_null(&trap_obj->tma_map)) + return; + + dma_resv_assert_held(vm->root.bo->tbo.base.resv); + + /* + * Zero TBA first to disable dispatch before clearing TMA. + */ + iosys_map_wr(&trap_obj->tma_map, + AMDGPU_TRAP_L2_TBA_INDEX * sizeof(u64), u64, 0); + dma_wmb(); + iosys_map_wr(&trap_obj->tma_map, + AMDGPU_TRAP_L2_TMA_INDEX * sizeof(u64), u64, 0); + + trap_obj->l2_tba_addr = 0; + trap_obj->l2_tba_size = 0; + trap_obj->l2_tma_addr = 0; + trap_obj->l2_tma_size = 0; +} + int amdgpu_trap_validate_params(struct amdgpu_device *adev, struct amdgpu_trap_params *trap_params, int num_xcc) @@ -464,6 +510,7 @@ int amdgpu_trap_validate_params(struct amdgpu_device *adev, } void amdgpu_trap_free(struct amdgpu_device *adev, struct amdgpu_vm *vm, + struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_trap_obj **trap_obj) { struct amdgpu_bo *tba_bo; @@ -473,32 +520,41 @@ void amdgpu_trap_free(struct amdgpu_device *adev, struct amdgpu_vm *vm, if (!trap_obj || !*trap_obj || !(*trap_obj)->tma_bo) return; + + /* + * Evict all user queues before freeing. At file-close time waves + * may still be active; without eviction a wave can trap into the + * handler after the TMA BO is freed, causing a GPU page fault. + */ + if (uq_mgr) + amdgpu_userq_evict(uq_mgr); + tba_bo = adev->trap_info->isa_bo; tma_bo = (*trap_obj)->tma_bo; if (!tba_bo || !tma_bo) return; - drm_exec_init(&exec, 0, 0); - drm_exec_until_all_locked(&exec) - { + drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0); + drm_exec_until_all_locked(&exec) { r = amdgpu_vm_lock_pd(vm, &exec, 0); - if (likely(!r)) - r = drm_exec_lock_obj(&exec, &tba_bo->tbo.base); - drm_exec_retry_on_contention(&exec); if (likely(!r)) r = drm_exec_lock_obj(&exec, &tma_bo->tbo.base); + if (likely(!r)) + r = drm_exec_lock_obj(&exec, &tba_bo->tbo.base); drm_exec_retry_on_contention(&exec); if (unlikely(r)) { dev_err(adev->dev, "failed to reserve CWSR BOs: err=%d\n", r); - goto err; + goto out; } } + amdgpu_trap_clear_l2_handler(vm, *trap_obj); + amdgpu_trap_unmap_region(adev, *trap_obj, AMDGPU_TRAP_TBA); amdgpu_trap_unmap_region(adev, *trap_obj, AMDGPU_TRAP_TMA); -err: +out: drm_exec_fini(&exec); amdgpu_bo_free_kernel(&(*trap_obj)->tma_bo, NULL, NULL); kfree(*trap_obj); @@ -509,120 +565,186 @@ static int amdgpu_trap_validate_user_addr(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_trap_usr_addr *usr_addr) { + struct amdgpu_bo *bo; struct amdgpu_bo_va_mapping *va_map; - uint64_t addr; - uint32_t size; - int r; + u64 addr; + u32 size; + + /* + * Caller holds the VM root BO reservation; assert rather than + * taking it again. + */ + dma_resv_assert_held(vm->root.bo->tbo.base.resv); addr = (usr_addr->addr & AMDGPU_GMC_HOLE_MASK) >> AMDGPU_GPU_PAGE_SHIFT; size = usr_addr->size >> AMDGPU_GPU_PAGE_SHIFT; - r = amdgpu_bo_reserve(vm->root.bo, false); - if (r) - return r; - va_map = amdgpu_vm_bo_lookup_mapping(vm, addr); - if (!va_map) { - r = -EINVAL; - goto err; - } - /* validate whether resident in the VM mapping range */ - if (addr >= va_map->start && va_map->last - addr + 1 >= size) { - amdgpu_bo_unreserve(vm->root.bo); - return 0; + if (!va_map) + return -EINVAL; + + /* + * TBA/TMA BOs must be VM_ALWAYS_VALID. The GPU jumps to TBA on + * hardware exception without any software job submission, so the + * kernel cannot add TBA/TMA to a job's used-buffer list to keep + * page tables current. Without VM_ALWAYS_VALID the BO can be + * evicted and the page table entry becomes stale, causing a GPU + * page fault or memory corruption on the next shader exception. + */ + bo = va_map->bo_va->base.bo; + if (!(bo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID)) { + dev_dbg(adev->dev, + "TBA/TMA BO at VA 0x%llx must use VM_ALWAYS_VALID\n", + usr_addr->addr); + return -EINVAL; } - r = -EINVAL; -err: - amdgpu_bo_unreserve(vm->root.bo); + if (addr >= va_map->start && va_map->last - addr + 1 >= size) + return 0; - return r; + return -EINVAL; } -static int amdgpu_trap_set_l2_trap_handler(struct amdgpu_device *adev, struct amdgpu_vm *vm, - struct amdgpu_trap_obj *cwsr_obj, - struct amdgpu_trap_usr_addr *tma, - struct amdgpu_trap_usr_addr *tba) +int amdgpu_trap_vm_set_l2_trap(struct amdgpu_device *adev, + struct amdgpu_vm *vm, + struct amdgpu_userq_mgr *uq_mgr, + struct amdgpu_trap_obj *trap_obj, + u64 tba_va, uint32_t tba_sz, + u64 tma_va, uint32_t tma_sz) { - uint64_t *l1tma; + struct amdgpu_trap_usr_addr tba = { .addr = tba_va, .size = tba_sz }; + struct amdgpu_trap_usr_addr tma = { .addr = tma_va, .size = tma_sz }; + long timeout; int r; if (!amdgpu_trap_is_enabled(adev)) return -EOPNOTSUPP; - - if (!cwsr_obj || !cwsr_obj->tma_cpu_addr || !tma || !tba) + if (!trap_obj || iosys_map_is_null(&trap_obj->tma_map)) return -EINVAL; - r = amdgpu_trap_validate_user_addr(adev, vm, tma); + + /* + * Wait for any kernel-queue work using this VM to drain before + * evicting user queues and writing the new descriptor. Without + * this wait a kernel-queue wave can observe a partially-written + * TBA/TMA pair during the update window. SET fails if the wait + * is interrupted or times out so the caller can retry. + */ + timeout = amdgpu_vm_wait_idle(vm, MAX_SCHEDULE_TIMEOUT); + if (timeout <= 0) + return timeout < 0 ? timeout : -ETIMEDOUT; + + /* + * Evict all user queues before updating TBA/TMA. Writing TBA and + * TMA as two separate stores creates a race: a wave that traps + * between them reads a mismatched pair. Evicting first ensures no + * wave is active during the update. + */ + amdgpu_userq_evict(uq_mgr); + + /* Flush GPU TLB to remove stale cached TBA/TMA values */ + amdgpu_gmc_flush_gpu_tlb_pasid(adev, vm->pasid, 2, true, 0); + + r = amdgpu_bo_reserve(vm->root.bo, false); if (r) return r; - r = amdgpu_trap_validate_user_addr(adev, vm, tba); + + r = amdgpu_trap_validate_user_addr(adev, vm, &tba); + if (r) + goto out_unreserve; + r = amdgpu_trap_validate_user_addr(adev, vm, &tma); + if (r) + goto out_unreserve; + + /* Zero TBA first to disable residual handler, then TMA, then TBA */ + iosys_map_wr(&trap_obj->tma_map, + AMDGPU_TRAP_L2_TBA_INDEX * sizeof(u64), u64, 0); + dma_wmb(); + iosys_map_wr(&trap_obj->tma_map, + AMDGPU_TRAP_L2_TMA_INDEX * sizeof(u64), u64, tma_va); + dma_wmb(); + iosys_map_wr(&trap_obj->tma_map, + AMDGPU_TRAP_L2_TBA_INDEX * sizeof(u64), u64, tba_va); + + trap_obj->l2_tba_addr = tba_va; + trap_obj->l2_tba_size = tba_sz; + trap_obj->l2_tma_addr = tma_va; + trap_obj->l2_tma_size = tma_sz; + +out_unreserve: + amdgpu_bo_unreserve(vm->root.bo); + if (!r) { + trap_obj->l2_trap_removed = false; + amdgpu_userq_restore_all(uq_mgr); + } + return r; +} + +int amdgpu_trap_vm_clear_l2_trap(struct amdgpu_device *adev, + struct amdgpu_vm *vm, + struct amdgpu_userq_mgr *uq_mgr, + struct amdgpu_trap_obj *trap_obj) +{ + int r; + + amdgpu_vm_wait_idle(vm, MAX_SCHEDULE_TIMEOUT); + amdgpu_userq_evict(uq_mgr); + amdgpu_gmc_flush_gpu_tlb_pasid(adev, vm->pasid, 2, true, 0); + + r = amdgpu_bo_reserve(vm->root.bo, false); if (r) return r; - l1tma = (uint64_t *)(cwsr_obj->tma_cpu_addr); - l1tma[0] = tma->addr; - l1tma[1] = tba->addr; + amdgpu_trap_clear_l2_handler(vm, trap_obj); + trap_obj->l2_trap_removed = false; + amdgpu_bo_unreserve(vm->root.bo); + amdgpu_userq_restore_all(uq_mgr); return 0; } -/* - * Userspace trap handler related ioctl - */ -/** - * amdgpu_trap_ioctl - Handle trap handler specific requests. - * - * @dev: drm device pointer - * @data: request object - * @filp: drm filp - * - * This function is used to perform cwsr and trap handler related operations - * Returns 0 on success, error code on failure. - */ -int amdgpu_trap_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) +void amdgpu_trap_handle_l2_unmap(struct amdgpu_device *adev, + struct amdgpu_vm *vm, + struct amdgpu_userq_mgr *uq_mgr, + struct amdgpu_trap_obj *trap_obj, + u64 addr, u64 size) { - struct amdgpu_device *adev = drm_to_adev(dev); - union drm_amdgpu_trap *cwsr = data; - struct amdgpu_fpriv *fpriv; - int r; + bool tba_overlap, tma_overlap; - fpriv = (struct amdgpu_fpriv *)filp->driver_priv; + if (!trap_obj || !size || !uq_mgr) + return; - if (!fpriv->trap_obj) - return -EOPNOTSUPP; + dma_resv_assert_held(vm->root.bo->tbo.base.resv); - switch (cwsr->in.op) { - case AMDGPU_TRAP_OP_SET_L2_TRAP: { - struct amdgpu_trap_usr_addr tba; - struct amdgpu_trap_usr_addr tma; - - tba.addr = cwsr->in.l2trap.tba_va; - tba.size = cwsr->in.l2trap.tba_sz; - tma.addr = cwsr->in.l2trap.tma_va; - tma.size = cwsr->in.l2trap.tma_sz; - r = amdgpu_trap_set_l2_trap_handler(adev, &fpriv->vm, fpriv->trap_obj, &tma, &tba); - } break; - default: - return -EINVAL; - } + tba_overlap = trap_obj->l2_tba_size && + addr < trap_obj->l2_tba_addr + trap_obj->l2_tba_size && + trap_obj->l2_tba_addr < addr + size; + tma_overlap = trap_obj->l2_tma_size && + addr < trap_obj->l2_tma_addr + trap_obj->l2_tma_size && + trap_obj->l2_tma_addr < addr + size; - return r; + if (!tba_overlap && !tma_overlap) + return; + + amdgpu_vm_wait_idle(vm, MAX_SCHEDULE_TIMEOUT); + amdgpu_userq_evict(uq_mgr); + amdgpu_gmc_flush_gpu_tlb_pasid(adev, vm->pasid, 2, true, 0); + amdgpu_trap_clear_l2_handler(vm, trap_obj); + trap_obj->l2_trap_removed = true; } int amdgpu_trap_set_trap_debug_flag(struct amdgpu_device *adev, struct amdgpu_trap_obj *cwsr_obj, bool enabled) { - uint64_t *l1tma; - if (!amdgpu_trap_is_enabled(adev)) return -EOPNOTSUPP; if (!cwsr_obj) return -EINVAL; - l1tma = (uint64_t *)(cwsr_obj->tma_cpu_addr); - l1tma[2] = enabled; + iosys_map_wr(&cwsr_obj->tma_map, + AMDGPU_TRAP_DEBUG_FLAG_INDEX * sizeof(u64), u64, + (u64)enabled); return 0; -} \ No newline at end of file +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h index 6d4664469bad..2d855433308a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h @@ -24,11 +24,13 @@ #define AMDGPU_TRAP_H #include <linux/types.h> +#include <linux/iosys-map.h> struct amdgpu_bo; struct amdgpu_bo_va; struct amdgpu_device; struct amdgpu_vm; +struct amdgpu_userq_mgr; /** * struct amdgpu_trap_obj - CWSR (Compute Wave Save Restore) buffer tracking @@ -42,7 +44,23 @@ struct amdgpu_trap_obj { struct amdgpu_bo *tma_bo; struct amdgpu_bo_va *tba_va; struct amdgpu_bo_va *tma_va; - void *tma_cpu_addr; + struct iosys_map tma_map; /* GTT or VRAM MMIO, set at alloc time */ + + /* + * Active second-level trap-handler VA ranges. + * Protected by the VM root BO reservation object. + */ + u64 l2_tba_addr; + u64 l2_tba_size; + u64 l2_tma_addr; + u64 l2_tma_size; + + /* + * Set when TBA or TMA was removed via GEM_VA UNMAP without calling + * CLEAR_L2_TRAP first. Prevents queues restarting with a missing + * trap handler. Cleared when SET_L2_TRAP or CLEAR_L2_TRAP is called. + */ + bool l2_trap_removed; }; struct amdgpu_trap_info { @@ -75,16 +93,31 @@ void amdgpu_trap_fini(struct amdgpu_device *adev); int amdgpu_trap_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_trap_obj **trap_obj); void amdgpu_trap_free(struct amdgpu_device *adev, struct amdgpu_vm *vm, + struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_trap_obj **trap_obj); + static inline bool amdgpu_trap_is_enabled(struct amdgpu_device *adev) { return adev->trap_info; } +/** + * amdgpu_trap_l2_was_removed - check if TBA/TMA was forcibly removed. + * + * Returns true if the second-level trap handler TBA or TMA was removed + * via GEM_VA UNMAP without calling CLEAR_L2_TRAP first. Queues must + * not restart with a missing trap handler. + */ +static inline bool amdgpu_trap_l2_was_removed(struct amdgpu_trap_obj *obj) +{ + return obj && obj->l2_trap_removed; +} + uint32_t amdgpu_trap_size_needed(struct amdgpu_device *adev, int num_xcc); int amdgpu_trap_validate_params(struct amdgpu_device *adev, struct amdgpu_trap_params *trap_params, int num_xcc); + static inline bool amdgpu_trap_has_dbg_wa(struct amdgpu_device *adev) { uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0); @@ -92,8 +125,33 @@ static inline bool amdgpu_trap_has_dbg_wa(struct amdgpu_device *adev) return gc_ver >= IP_VERSION(11, 0, 0) && gc_ver <= IP_VERSION(11, 0, 3); } -int amdgpu_trap_ioctl(struct drm_device *dev, void *data, - struct drm_file *filp); +void amdgpu_trap_clear_l2_handler(struct amdgpu_vm *vm, + struct amdgpu_trap_obj *trap_obj); + +/** + * amdgpu_trap_vm_set_l2_trap - install the second-level trap handler. + * + * Exclusivity: SET_L2_TRAP is per-VM (per-process). Each process has its + * own per-process TMA buffer created in amdgpu_trap_alloc(). App A's + * handler address lives in App A's TMA; App B's in App B's TMA. There + * is no conflict between processes on user queue VMIDs. + */ +int amdgpu_trap_vm_set_l2_trap(struct amdgpu_device *adev, + struct amdgpu_vm *vm, + struct amdgpu_userq_mgr *uq_mgr, + struct amdgpu_trap_obj *trap_obj, + u64 tba_va, uint32_t tba_sz, + u64 tma_va, uint32_t tma_sz); +int amdgpu_trap_vm_clear_l2_trap(struct amdgpu_device *adev, + struct amdgpu_vm *vm, + struct amdgpu_userq_mgr *uq_mgr, + struct amdgpu_trap_obj *trap_obj); +void amdgpu_trap_handle_l2_unmap(struct amdgpu_device *adev, + struct amdgpu_vm *vm, + struct amdgpu_userq_mgr *uq_mgr, + struct amdgpu_trap_obj *trap_obj, + u64 addr, u64 size); + int amdgpu_trap_set_trap_debug_flag(struct amdgpu_device *adev, struct amdgpu_trap_obj *cwsr_obj, bool enabled); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c index db09e879b85e..aef11786e119 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c @@ -997,9 +997,10 @@ int amdgpu_userq_ioctl(struct drm_device *dev, void *data, return r; } -static int +int amdgpu_userq_restore_all(struct amdgpu_userq_mgr *uq_mgr) { + struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr); struct amdgpu_usermode_queue *queue; unsigned long queue_id; int ret = 0, r; @@ -1011,7 +1012,21 @@ amdgpu_userq_restore_all(struct amdgpu_userq_mgr *uq_mgr) if (!amdgpu_userq_buffer_vas_mapped(queue)) { drm_file_err(uq_mgr->file, "trying restore queue without va mapping\n"); - trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_INVALID_VA); + trace_amdgpu_userq_state_changed(queue, + AMDGPU_USERQ_STATE_INVALID_VA); + queue->state = AMDGPU_USERQ_STATE_INVALID_VA; + continue; + } + + /* + * If the second-level trap handler TBA or TMA was forcibly + * removed by GEM_VA UNMAP without CLEAR_L2_TRAP, do not + * restart the queue. A queue must not run without a valid + * resident trap handler if one was previously configured. + */ + if (amdgpu_trap_l2_was_removed(fpriv->trap_obj)) { + trace_amdgpu_userq_state_changed(queue, + AMDGPU_USERQ_STATE_INVALID_VA); queue->state = AMDGPU_USERQ_STATE_INVALID_VA; continue; } @@ -1024,7 +1039,8 @@ amdgpu_userq_restore_all(struct amdgpu_userq_mgr *uq_mgr) if (ret) drm_file_err(uq_mgr->file, - "Failed to map all the queues, restore failed ret=%d\n", ret); + "Failed to map all the queues, restore failed ret=%d\n", + ret); return ret; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h index 3e19bc6103a0..d1569346bfae 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h @@ -164,6 +164,7 @@ void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr); void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr); void amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr); +int amdgpu_userq_restore_all(struct amdgpu_userq_mgr *uq_mgr); void amdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *userq_mgr, struct amdgpu_eviction_fence_mgr *evf_mgr); diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h index 552cbf483543..4785cf48da17 100644 --- a/include/uapi/drm/amdgpu_drm.h +++ b/include/uapi/drm/amdgpu_drm.h @@ -59,7 +59,6 @@ extern "C" { #define DRM_AMDGPU_USERQ_WAIT 0x18 #define DRM_AMDGPU_GEM_LIST_HANDLES 0x19 #define DRM_AMDGPU_PROC_OPTIONS 0x1A -#define DRM_AMDGPU_TRAP 0x1B #define DRM_IOCTL_AMDGPU_GEM_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_CREATE, union drm_amdgpu_gem_create) #define DRM_IOCTL_AMDGPU_GEM_MMAP DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_MMAP, union drm_amdgpu_gem_mmap) @@ -82,7 +81,6 @@ extern "C" { #define DRM_IOCTL_AMDGPU_USERQ_WAIT DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_USERQ_WAIT, struct drm_amdgpu_userq_wait) #define DRM_IOCTL_AMDGPU_GEM_LIST_HANDLES DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_LIST_HANDLES, struct drm_amdgpu_gem_list_handles) #define DRM_IOCTL_AMDGPU_PROC_OPTIONS DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_PROC_OPTIONS, struct drm_amdgpu_proc_options) -#define DRM_IOCTL_AMDGPU_TRAP DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_TRAP, union drm_amdgpu_trap) /** * DOC: memory domains -- 2.34.1
