Split amdgpu_vm_update_range into two functions. amdgpu_vm_map_range() is for mapping PTEs into a range and updates which can be done while holding the VM lock.
amdgpu_vm_unmap_range() is for unmapping PTEs without holding the VM lock in MMU notifiers. Signed-off-by: Christian König <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_job.h | 3 +- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 112 ++++++++++++++++++---- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 14 ++- drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 35 ++----- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 16 ++-- 5 files changed, 120 insertions(+), 60 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h index 44fe40f9e8df..653ffa9ca0f3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h @@ -47,7 +47,7 @@ enum amdgpu_ib_pool_type; /* Internal kernel job ids. (decreasing values, starting from U64_MAX). */ #define AMDGPU_KERNEL_JOB_ID_VM_UPDATE (18446744073709551615ULL) #define AMDGPU_KERNEL_JOB_ID_VM_UPDATE_PDES (18446744073709551614ULL) -#define AMDGPU_KERNEL_JOB_ID_VM_UPDATE_RANGE (18446744073709551613ULL) +#define AMDGPU_KERNEL_JOB_ID_VM_MAP_RANGE (18446744073709551613ULL) #define AMDGPU_KERNEL_JOB_ID_VM_PT_CLEAR (18446744073709551612ULL) #define AMDGPU_KERNEL_JOB_ID_TTM_MAP_BUFFER (18446744073709551611ULL) #define AMDGPU_KERNEL_JOB_ID_TTM_ACCESS_MEMORY_SDMA (18446744073709551610ULL) @@ -63,6 +63,7 @@ enum amdgpu_ib_pool_type; #define AMDGPU_KERNEL_JOB_ID_SDMA_RING_TEST (18446744073709551600ULL) #define AMDGPU_KERNEL_JOB_ID_VPE_RING_TEST (18446744073709551599ULL) #define AMDGPU_KERNEL_JOB_ID_RUN_SHADER (18446744073709551598ULL) +#define AMDGPU_KERNEL_JOB_ID_VM_UNMAP_RANGE (18446744073709551597ULL) struct amdgpu_job { struct drm_sched_job base; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index edc8b1ca2d3e..b5adfcacc55a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -1080,11 +1080,10 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, } /** - * amdgpu_vm_update_range - update a range in the vm page table + * amdgpu_vm_map_range - map something to a range in the vm page tables * * @adev: amdgpu_device pointer to use for commands * @vm: the VM to update the range - * @unlocked: unlocked invalidation during MM callback * @flush_tlb: trigger tlb invalidation after update completed * @allow_override: change MTYPE for local NUMA nodes * @sync: fences we need to sync to @@ -1097,23 +1096,26 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, * @pages_addr: DMA addresses to use for mapping * @fence: optional resulting fence * - * Fill in the page table entries between @start and @last. + * Fill in the page table entries between @start and @last. Allocate and free + * new page tables as needed. Can only be called while holding the VM lock. * * Returns: * 0 for success, negative erro code for failure. */ -int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm, - bool unlocked, bool flush_tlb, bool allow_override, - struct amdgpu_sync *sync, uint64_t start, - uint64_t last, uint64_t flags, uint64_t offset, - uint64_t vram_base, struct ttm_resource *res, - dma_addr_t *pages_addr, struct dma_fence **fence) +int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm, + bool flush_tlb, bool allow_override, + struct amdgpu_sync *sync, uint64_t start, + uint64_t last, uint64_t flags, uint64_t offset, + uint64_t vram_base, struct ttm_resource *res, + dma_addr_t *pages_addr, struct dma_fence **fence) { struct amdgpu_vm_tlb_seq_struct *tlb_cb; struct amdgpu_vm_update_params params; struct amdgpu_res_cursor cursor; int r, idx; + amdgpu_vm_assert_locked(vm); + if (!drm_dev_enter(adev_to_drm(adev), &idx)) return -ENODEV; @@ -1138,7 +1140,6 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm, params.adev = adev; params.vm = vm; params.pages_addr = pages_addr; - params.unlocked = unlocked; params.needs_flush = flush_tlb; params.override_pte = allow_override && adev->gmc.override_pte; INIT_LIST_HEAD(¶ms.tlb_flush_waitlist); @@ -1149,7 +1150,7 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm, goto error_free; } - if (!unlocked && !dma_fence_is_signaled(vm->last_unlocked)) { + if (!dma_fence_is_signaled(vm->last_unlocked)) { struct dma_fence *tmp = dma_fence_get_stub(); amdgpu_bo_fence(vm->root.bo, vm->last_unlocked, true); @@ -1158,7 +1159,7 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm, } r = vm->update_funcs->prepare(¶ms, sync, - AMDGPU_KERNEL_JOB_ID_VM_UPDATE_RANGE); + AMDGPU_KERNEL_JOB_ID_VM_MAP_RANGE); if (r) goto error_free; @@ -1234,6 +1235,77 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm, return r; } +/** + * amdgpu_vm_unmap_range - clear leave PTEs to unmap something + * + * @adev: amdgpu_device pointer to use for commands + * @vm: the VM to update the range + * @sync: fences we need to sync to + * @start: start of unmapped range + * @last: last unmapped entry + * @flags: flags for the entries + * @fence: optional resulting fence + * + * Fill in the page table entries between @start and @last with a fixed flags + * value without allocating or freeing page tables. Can be used without locking + * the VM. + * + * Returns: + * 0 for success, negative erro code for failure. + */ +int amdgpu_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_vm *vm, + struct amdgpu_sync *sync, uint64_t start, + uint64_t last, uint64_t flags, + struct dma_fence **fence) +{ + struct amdgpu_vm_tlb_seq_struct *tlb_cb; + struct amdgpu_vm_update_params params; + int r, idx; + + if (!drm_dev_enter(adev_to_drm(adev), &idx)) + return -ENODEV; + + tlb_cb = kmalloc(sizeof(*tlb_cb), GFP_KERNEL); + if (!tlb_cb) { + drm_dev_exit(idx); + return -ENOMEM; + } + + memset(¶ms, 0, sizeof(params)); + params.adev = adev; + params.vm = vm; + params.needs_flush = true; + params.unlocked = true; + INIT_LIST_HEAD(¶ms.tlb_flush_waitlist); + + amdgpu_vm_eviction_lock(vm); + if (vm->evicting) { + r = -EBUSY; + goto error_free; + } + + r = vm->update_funcs->prepare(¶ms, sync, + AMDGPU_KERNEL_JOB_ID_VM_UNMAP_RANGE); + if (r) + goto error_free; + + amdgpu_vm_update_leaves(¶ms, start, last, 0, flags); + + r = vm->update_funcs->commit(¶ms, fence); + if (r) + goto error_free; + + amdgpu_vm_tlb_flush(¶ms, fence, tlb_cb); + amdgpu_vm_pt_free_list(adev, ¶ms); + tlb_cb = NULL; + +error_free: + kfree(tlb_cb); + amdgpu_vm_eviction_unlock(vm); + drm_dev_exit(idx); + return r; +} + void amdgpu_vm_get_memory(struct amdgpu_vm *vm, struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM]) { @@ -1362,11 +1434,11 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va, trace_amdgpu_vm_bo_update(mapping); - r = amdgpu_vm_update_range(adev, vm, false, flush_tlb, - !uncached, &sync, mapping->start, - mapping->last, update_flags, - mapping->offset, vram_base, mem, - pages_addr, last_update); + r = amdgpu_vm_map_range(adev, vm, flush_tlb, !uncached, &sync, + mapping->start, mapping->last, + update_flags, mapping->offset, + vram_base, mem, pages_addr, + last_update); if (r) goto error_free; } @@ -1565,9 +1637,9 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev, struct amdgpu_bo_va_mapping, list); list_del(&mapping->list); - r = amdgpu_vm_update_range(adev, vm, false, true, false, - &sync, mapping->start, mapping->last, - 0, 0, 0, NULL, NULL, &f); + r = amdgpu_vm_map_range(adev, vm, true, false, + &sync, mapping->start, mapping->last, + 0, 0, 0, NULL, NULL, &f); amdgpu_vm_free_mapping(adev, vm, mapping, f); if (r) { dma_fence_put(f); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index 3e86a2a470f0..561f2873d2ec 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h @@ -529,12 +529,16 @@ int amdgpu_vm_flush_compute_tlb(struct amdgpu_device *adev, uint32_t xcc_mask); void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base, struct amdgpu_vm *vm, struct amdgpu_bo *bo); -int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm, - bool unlocked, bool flush_tlb, bool allow_override, +int amdgpu_vm_map_range(struct amdgpu_device *adev, struct amdgpu_vm *vm, + bool flush_tlb, bool allow_override, + struct amdgpu_sync *sync, uint64_t start, + uint64_t last, uint64_t flags, uint64_t offset, + uint64_t vram_base, struct ttm_resource *res, + dma_addr_t *pages_addr, struct dma_fence **fence); +int amdgpu_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_sync *sync, uint64_t start, - uint64_t last, uint64_t flags, uint64_t offset, - uint64_t vram_base, struct ttm_resource *res, - dma_addr_t *pages_addr, struct dma_fence **fence); + uint64_t last, uint64_t flags, + struct dma_fence **fence); int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va, bool clear); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c index 6f5415d5a1bc..ac3f3e31e2e2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c @@ -553,7 +553,6 @@ void amdgpu_vm_pt_free_list(struct amdgpu_device *adev, struct amdgpu_vm_update_params *params) { struct amdgpu_vm_bo_base *entry, *next; - bool unlocked = params->unlocked; if (list_empty(¶ms->tlb_flush_waitlist)) return; @@ -561,7 +560,7 @@ void amdgpu_vm_pt_free_list(struct amdgpu_device *adev, /* * unlocked unmap clear page table leaves, warning to free the page entry. */ - WARN_ON(unlocked); + WARN_ON(params->unlocked); list_for_each_entry_safe(entry, next, ¶ms->tlb_flush_waitlist, vm_status) amdgpu_vm_pt_free(entry); @@ -801,24 +800,17 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params, uint64_t incr, entry_end, pe_start; struct amdgpu_bo *pt; - if (!params->unlocked) { - /* make sure that the page tables covering the - * address range are actually allocated - */ - r = amdgpu_vm_pt_alloc(params->adev, params->vm, - &cursor); - if (r) - return r; - } + /* make sure that the page tables covering the + * address range are actually allocated + */ + r = amdgpu_vm_pt_alloc(params->adev, params->vm, &cursor); + if (r) + return r; shift = amdgpu_vm_pt_level_shift(adev, cursor.level); parent_shift = amdgpu_vm_pt_level_shift(adev, cursor.level - 1); - if (params->unlocked) { - /* Unlocked updates are only allowed on the leaves */ - if (amdgpu_vm_pt_descendant(adev, &cursor)) - continue; - } else if (adev->asic_type < CHIP_VEGA10 && - (flags & AMDGPU_PTE_VALID)) { + if (adev->asic_type < CHIP_VEGA10 && + (flags & AMDGPU_PTE_VALID)) { /* No huge page support before GMC v9 */ if (cursor.level != AMDGPU_VM_PTB) { if (!amdgpu_vm_pt_descendant(adev, &cursor)) @@ -864,14 +856,7 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params, mask = amdgpu_vm_pt_entries_mask(adev, cursor.level); pe_start = ((cursor.pfn >> shift) & mask) * 8; - if (cursor.level < AMDGPU_VM_PTB && params->unlocked) - /* - * MMU notifier callback unlocked unmap huge page, leave is PDE entry, - * only clear one entry. Next entry search again for PDE or PTE leave. - */ - entry_end = 1ULL << shift; - else - entry_end = ((uint64_t)mask + 1) << shift; + entry_end = ((uint64_t)mask + 1) << shift; entry_end += cursor.pfn & ~(entry_end - 1); entry_end = min(entry_end, end); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index 37b5166e9a14..d0ea20dea3e1 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -1372,9 +1372,8 @@ svm_range_unmap_from_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm, return -EINVAL; } - return amdgpu_vm_update_range(adev, vm, true, true, false, NULL, gpu_start, - gpu_end, init_pte_value, 0, 0, NULL, NULL, - fence); + return amdgpu_vm_unmap_range(adev, vm, NULL, gpu_start, gpu_end, + init_pte_value, fence); } static int @@ -1489,12 +1488,11 @@ svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange, (last_domain == SVM_RANGE_VRAM_DOMAIN) ? 1 : 0, pte_flags); - r = amdgpu_vm_update_range(adev, vm, false, flush_tlb, true, - NULL, gpu_start, gpu_end, - pte_flags, - (last_start - prange->start) << PAGE_SHIFT, - bo_adev ? bo_adev->vm_manager.vram_base_offset : 0, - NULL, dma_addr, &vm->last_update); + r = amdgpu_vm_map_range(adev, vm, flush_tlb, true, NULL, + gpu_start, gpu_end, pte_flags, + (last_start - prange->start) << PAGE_SHIFT, + bo_adev ? bo_adev->vm_manager.vram_base_offset : 0, + NULL, dma_addr, &vm->last_update); for (j = last_start - prange->start; j <= i; j++) dma_addr[j] |= last_domain; -- 2.43.0
