From: Christian König <[email protected]> Taking the eviction lock is actually just one step which we need to do in the critical section handling.
Rename the functions to reflect that, use the update parameters instead of the vm to save the GFP flags. v2: rebased and reordered Signed-off-by: Christian König <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 36 ++++++---------- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 1 - .../gpu/drm/amd/amdgpu/amdgpu_vm_internal.h | 41 ++++++++++++++----- drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 28 ++++++------- 4 files changed, 58 insertions(+), 48 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 0930b3fdade7..a994141e9df5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -1167,11 +1167,9 @@ int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm, params.override_pte = allow_override && adev->gmc.override_pte; INIT_LIST_HEAD(¶ms.tlb_flush_waitlist); - amdgpu_vm_eviction_lock(vm); - if (vm->evicting) { - r = -EBUSY; + r = amdgpu_vm_begin_critical(¶ms); + if (r) goto error_free; - } if (!dma_fence_is_signaled(vm->last_unlocked)) { struct dma_fence *tmp = dma_fence_get_stub(); @@ -1253,7 +1251,7 @@ int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm, error_free: kfree(tlb_cb); - amdgpu_vm_eviction_unlock(vm); + amdgpu_vm_end_critical(¶ms); drm_dev_exit(idx); return r; } @@ -1301,11 +1299,9 @@ int amdgpu_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_vm *vm, params.unlocked = true; INIT_LIST_HEAD(¶ms.tlb_flush_waitlist); - amdgpu_vm_eviction_lock(vm); - if (vm->evicting) { - r = -EBUSY; + r = amdgpu_vm_begin_critical(¶ms); + if (r) goto error_free; - } r = vm->update_funcs->prepare(¶ms, sync, AMDGPU_KERNEL_JOB_ID_VM_UNMAP_RANGE); @@ -1324,7 +1320,7 @@ int amdgpu_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_vm *vm, error_free: kfree(tlb_cb); - amdgpu_vm_eviction_unlock(vm); + amdgpu_vm_end_critical(¶ms); drm_dev_exit(idx); return r; } @@ -3156,32 +3152,26 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid, goto error_unlock; } - amdgpu_vm_eviction_lock(vm); - if (vm->evicting) { - r = -EBUSY; - goto error_dev_exit; - } - memset(¶ms, 0, sizeof(params)); params.adev = adev; params.vm = vm; params.immediate = true; params.pages_addr = NULL; + r = amdgpu_vm_begin_critical(¶ms); + if (r) + goto error_end_critical; + r = vm->update_funcs->prepare(¶ms, NULL, AMDGPU_KERNEL_JOB_ID_VM_UPDATE_PDES); if (r) - goto error_dev_exit; + goto error_end_critical; amdgpu_vm_pt_leaves(¶ms, addr, addr, values, flags); - r = vm->update_funcs->commit(¶ms, &vm->last_update); - if (r) - goto error_dev_exit; - - amdgpu_vm_eviction_unlock(vm); -error_dev_exit: +error_end_critical: + amdgpu_vm_end_critical(¶ms); drm_dev_exit(idx); error_unlock: diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index e871a9e3e1a9..fbbd9e725799 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h @@ -287,7 +287,6 @@ struct amdgpu_vm { */ struct mutex eviction_lock; bool evicting; - unsigned int saved_flags; /* Memory statistics for this vm, protected by stats_lock */ spinlock_t stats_lock; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h index 7f6442d0b647..a3328a48fecd 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h @@ -93,6 +93,11 @@ struct amdgpu_vm_update_params { */ bool override_pte; + /** + * @saved_flags: Saved flags for GFP reduction. + */ + unsigned int saved_flags; + /** * @tlb_flush_waitlist: temporary storage for BOs until tlb_flush */ @@ -130,21 +135,37 @@ int amdgpu_vm_pt_map_tables(struct amdgpu_device *adev, struct amdgpu_vm *vm); int amdgpu_vm_pt_alloc_dummies(struct amdgpu_device *adev); void amdgpu_vm_pt_free_dummies(struct amdgpu_device *adev); -/* - * vm eviction_lock can be taken in MMU notifiers. Make sure no reclaim-FS - * happens while holding this lock anywhere to prevent deadlocks when - * an MMU notifier runs in reclaim-FS context. +/** + * amdgpu_vm_begin_critical - start the critical section of the update + * @p: The update parameters + * + * Serialize all updates, check parameters and make sure that memory allocations + * don't enter the reclaim path so that we don't deadlock with MMU notifiers. + * + * Returns: + * + * 0 on success or a negative error code on failure. + * Even on error amdgpu_vm_end_critical() must still be called to clean up! */ -static inline void amdgpu_vm_eviction_lock(struct amdgpu_vm *vm) +static inline int amdgpu_vm_begin_critical(struct amdgpu_vm_update_params *p) { - mutex_lock(&vm->eviction_lock); - vm->saved_flags = memalloc_noreclaim_save(); + mutex_lock(&p->vm->eviction_lock); + p->saved_flags = memalloc_noreclaim_save(); + if (p->vm->evicting) + return -EBUSY; + return 0; } -static inline void amdgpu_vm_eviction_unlock(struct amdgpu_vm *vm) +/** + * amdgpu_vm_end_critical - end the critical section of the update + * @p: The update parameters + * + * Restore the GFP flags and drop the lock. + */ +static inline void amdgpu_vm_end_critical(struct amdgpu_vm_update_params *p) { - memalloc_noreclaim_restore(vm->saved_flags); - mutex_unlock(&vm->eviction_lock); + memalloc_noreclaim_restore(p->saved_flags); + mutex_unlock(&p->vm->eviction_lock); } #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c index 5bb1a3300aa4..5385f662d0ac 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c @@ -478,42 +478,42 @@ int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm, /** * amdgpu_vm_pt_alloc - Allocate a specific page table * - * @adev: amdgpu_device pointer - * @vm: VM to allocate page tables for + * @p: see amdgpu_vm_update_params definition * @cursor: Which page table to allocate * * Make sure a specific page table or directory is allocated. * * Returns: - * 1 if page table needed to be allocated, 0 if page table was already - * allocated, negative errno if an error occurred. + * + * 0 on success or a negative error code on failure. */ -static int amdgpu_vm_pt_alloc(struct amdgpu_device *adev, - struct amdgpu_vm *vm, +static int amdgpu_vm_pt_alloc(struct amdgpu_vm_update_params *p, struct amdgpu_vm_pt_cursor *cursor) { struct amdgpu_vm_bo_base *entry = cursor->entry; struct amdgpu_bo *pt_bo; struct amdgpu_bo_vm *pt; - int r; + int r, r2; if (entry->bo) return 0; - amdgpu_vm_eviction_unlock(vm); - r = amdgpu_vm_pt_create(adev, vm, cursor->level, &pt, - vm->root.bo->xcp_id); - amdgpu_vm_eviction_lock(vm); + amdgpu_vm_end_critical(p); + r = amdgpu_vm_pt_create(p->adev, p->vm, cursor->level, &pt, + p->vm->root.bo->xcp_id); + r2 = amdgpu_vm_begin_critical(p); if (r) return r; + if (r2) + return r2; /* Keep a reference to the root directory to avoid * freeing them up in the wrong order. */ pt_bo = &pt->bo; pt_bo->parent = amdgpu_bo_ref(cursor->parent->bo); - amdgpu_vm_bo_base_init(entry, vm, pt_bo); - r = amdgpu_vm_pt_clear(adev, vm, pt); + amdgpu_vm_bo_base_init(entry, p->vm, pt_bo); + r = amdgpu_vm_pt_clear(p->adev, p->vm, pt); if (r) goto error_free_pt; @@ -802,7 +802,7 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params, /* make sure that the page tables covering the * address range are actually allocated */ - r = amdgpu_vm_pt_alloc(params->adev, params->vm, &cursor); + r = amdgpu_vm_pt_alloc(params, &cursor); if (r) return r; -- 2.53.0
