From: Honglei Huang <[email protected]> Add MMU notifier event handling and garbage collection infrastructure: - amdgpu_svm_range_remove: unmap pages and remove from gpusvm - amdgpu_svm_range_notifier_event_begin: compute the [start_page, last_page] window from the notifier range and the gpusvm range, then zap PTEs via amdgpu_svm_range_zap_ptes and invalidate the GPU mapping - amdgpu_svm_gc_enqueue: queue range for GC worker and set queue state - amdgpu_svm_gc_add_range: mark range unmapped and enqueue for GC - amdgpu_svm_range_notifier_event_end: DMA unmap and GC on MMU_UNMAP - amdgpu_svm_range_invalidate_interval: cursor based PTE clearing across notifiers/ranges; clears PTEs through amdgpu_svm_range_zap_ptes - amdgpu_svm_range_dequeue_locked: dequeue work item with atomic pending state transfer - amdgpu_svm_range_put_if_dequeued: release range ref after dequeue, re-dispatch if new work was enqueued during processing - amdgpu_svm_range_unqueue: locked context helper that pulls a range off the GC list
Signed-off-by: Honglei Huang <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c | 255 ++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h | 2 + 2 files changed, 257 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c index e1b392b2cf56a..57bcbaf0fdd2b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c @@ -486,3 +486,258 @@ amdgpu_svm_range_map_attrs(struct amdgpu_svm *svm, return 0; } +void amdgpu_svm_range_remove(struct amdgpu_svm *svm, + struct amdgpu_svm_range *range, + struct drm_gpusvm_ctx *ctx) +{ + struct drm_gpusvm_range *base = &range->base; + + amdgpu_svm_assert_locked(svm); + + if (!base->pages.flags.unmapped && !base->pages.flags.partial_unmap) + drm_gpusvm_range_unmap_pages(&svm->gpusvm, base, ctx); + + amdgpu_svm_range_invalidate_gpu_mapping(range); + drm_gpusvm_range_remove(&svm->gpusvm, base); +} + +bool +amdgpu_svm_range_notifier_event_begin(struct amdgpu_svm *svm, + struct drm_gpusvm_range *range, + const struct mmu_notifier_range *mmu_range) +{ + struct amdgpu_svm_range *svm_range = to_amdgpu_svm_range(range); + unsigned long start_page, last_page; + + amdgpu_svm_assert_in_notifier(svm); + + AMDGPU_SVM_RANGE_DEBUG(svm_range, "NOTIFIER"); + + if (range->pages.flags.unmapped || !svm_range->gpu_mapped) + return false; + + AMDGPU_SVM_RANGE_DEBUG(svm_range, "NOTIFIER - EXECUTE"); + + start_page = max(drm_gpusvm_range_start(range), + mmu_range->start) >> PAGE_SHIFT; + last_page = (min(drm_gpusvm_range_end(range), + mmu_range->end) >> PAGE_SHIFT) - 1; + + amdgpu_svm_range_zap_ptes(svm, svm_range, start_page, last_page); + amdgpu_svm_range_invalidate_gpu_mapping(svm_range); + + return true; +} + +static void +amdgpu_svm_gc_enqueue(struct amdgpu_svm *svm, + struct amdgpu_svm_range *range, + unsigned long start_page, unsigned long last_page) +{ + if (atomic_read(&svm->exiting)) + return; + + spin_lock(&svm->work_lock); + if (range->queue_state == AMDGPU_SVM_RANGE_NOT_QUEUED) { + drm_gpusvm_range_get(&range->base); + range->queue_state = AMDGPU_SVM_RANGE_IN_GC; + } + + range->pending_start_page = min(range->pending_start_page, start_page); + range->pending_last_page = max(range->pending_last_page, last_page); + if (range->pending_ops == AMDGPU_SVM_RANGE_OP_NONE) + list_add_tail(&range->work_node, &svm->gc.list); + range->pending_ops |= AMDGPU_SVM_RANGE_OP_UNMAP; + spin_unlock(&svm->work_lock); + + queue_work(svm->gc.wq, &svm->gc.work); +} + +static void +amdgpu_svm_gc_add_range(struct amdgpu_svm *svm, + struct amdgpu_svm_range *svm_range, + const struct mmu_notifier_range *mmu_range) +{ + unsigned long start_page = max(drm_gpusvm_range_start(&svm_range->base), + mmu_range->start) >> PAGE_SHIFT; + unsigned long last_page = (min(drm_gpusvm_range_end(&svm_range->base), + mmu_range->end) >> PAGE_SHIFT) - 1; + + AMDGPU_SVM_RANGE_DEBUG(svm_range, "GARBAGE COLLECTOR ADD"); + + drm_gpusvm_range_set_unmapped(&svm_range->base, mmu_range); + amdgpu_svm_gc_enqueue(svm, svm_range, start_page, last_page); +} + +static void +amdgpu_svm_range_notifier_event_end(struct amdgpu_svm *svm, + struct drm_gpusvm_range *range, + const struct mmu_notifier_range *mmu_range) +{ + struct drm_gpusvm_ctx ctx = { .in_notifier = true, }; + + amdgpu_svm_assert_in_notifier(svm); + + drm_gpusvm_range_unmap_pages(&svm->gpusvm, range, &ctx); + if (mmu_range->event == MMU_NOTIFY_UNMAP) + amdgpu_svm_gc_add_range(svm, to_amdgpu_svm_range(range), + mmu_range); +} + +int +amdgpu_svm_range_invalidate_interval(struct amdgpu_svm *svm, + unsigned long start_page, + unsigned long last_page) +{ + unsigned long start = start_page << PAGE_SHIFT; + unsigned long end = (last_page + 1) << PAGE_SHIFT; + struct drm_gpusvm_notifier *notifier, *next_notifier; + struct drm_gpusvm_ctx ctx = { .in_notifier = false }; + struct drm_exec exec; + bool needs_flush = false; + int ret; + + amdgpu_svm_assert_locked(svm); + + ret = amdgpu_svm_range_lock_vm_pd(svm, &exec, true); + if (ret) + return ret; + + drm_gpusvm_for_each_notifier_safe(notifier, next_notifier, &svm->gpusvm, + start, end) { + struct drm_gpusvm_range *range, *next_range; + + drm_gpusvm_for_each_range_safe(range, next_range, notifier, + start, end) { + struct amdgpu_svm_range *svm_range = to_amdgpu_svm_range(range); + bool crosses_boundary = start > drm_gpusvm_range_start(range) || + end < drm_gpusvm_range_end(range); + + if (svm_range->gpu_mapped) { + AMDGPU_SVM_RANGE_DEBUG(svm_range, + crosses_boundary ? "ATTR DESTROY" : + "ATTR ZAP PTE"); + + ret = amdgpu_svm_range_zap_ptes(svm, svm_range, + drm_gpusvm_range_start(range) >> PAGE_SHIFT, + (drm_gpusvm_range_end(range) >> PAGE_SHIFT) - 1); + if (ret < 0) { + drm_exec_fini(&exec); + return ret; + } + + amdgpu_svm_range_invalidate_gpu_mapping(svm_range); + needs_flush = true; + } + + if (crosses_boundary) { + /* remove ranges crossing the boundary so GPU fault + * creates new ranges bounded by the updated + * attr_range boundaries. + * Evict devmem-backed pages back to sysmem first + * so VRAM-resident data is not lost when the range + * is destroyed. No-op for sysmem-only ranges. + */ + drm_gpusvm_range_get(range); + + amdgpu_svm_range_evict(svm, range); + amdgpu_svm_range_remove(svm, svm_range, &ctx); + + amdgpu_svm_range_unqueue(svm, svm_range); + + drm_gpusvm_range_put(range); + } + } + } + + drm_exec_fini(&exec); + + if (needs_flush) + svm->flush_tlb(svm); + + AMDGPU_SVM_TRACE("attr invalidate done [0x%lx-0x%lx]-0x%lx needs_flush=%d\n", + start_page, last_page, last_page - start_page + 1, + needs_flush ? 1 : 0); + + return 0; +} + +bool +amdgpu_svm_range_dequeue_locked(struct amdgpu_svm *svm, + struct list_head *work_list, + struct amdgpu_svm_range_op_ctx *op_ctx) +{ + struct amdgpu_svm_range *range; + + lockdep_assert_held(&svm->work_lock); + + range = list_first_entry_or_null(work_list, struct amdgpu_svm_range, + work_node); + if (!range) + return false; + + list_del_init(&range->work_node); + range->queue_state = AMDGPU_SVM_RANGE_PROCESSING; + + op_ctx->range = range; + op_ctx->start_page = range->pending_start_page; + op_ctx->last_page = range->pending_last_page; + op_ctx->pending_ops = range->pending_ops; + + range->pending_start_page = ULONG_MAX; + range->pending_last_page = 0; + range->pending_ops = AMDGPU_SVM_RANGE_OP_NONE; + + return true; +} + +void +amdgpu_svm_range_put_if_dequeued(struct amdgpu_svm *svm, + struct amdgpu_svm_range *range) +{ + bool release_kref = false; + bool queue_gc = false; + + spin_lock(&svm->work_lock); + + if (range->queue_state != AMDGPU_SVM_RANGE_PROCESSING) { + spin_unlock(&svm->work_lock); + return; + } + + if (UNMAP_WORK(range->pending_ops)) { + list_add_tail(&range->work_node, &svm->gc.list); + range->queue_state = AMDGPU_SVM_RANGE_IN_GC; + queue_gc = true; + } else { + range->queue_state = AMDGPU_SVM_RANGE_NOT_QUEUED; + release_kref = true; + } + + spin_unlock(&svm->work_lock); + + if (queue_gc) + queue_work(svm->gc.wq, &svm->gc.work); + if (release_kref) + drm_gpusvm_range_put(&range->base); +} + +void amdgpu_svm_range_unqueue(struct amdgpu_svm *svm, + struct amdgpu_svm_range *range) +{ + bool put = false; + + amdgpu_svm_assert_locked(svm); + + spin_lock(&svm->work_lock); + if (range->queue_state == AMDGPU_SVM_RANGE_IN_GC) { + list_del_init(&range->work_node); + range->queue_state = AMDGPU_SVM_RANGE_NOT_QUEUED; + range->pending_ops = AMDGPU_SVM_RANGE_OP_NONE; + put = true; + } + spin_unlock(&svm->work_lock); + + if (put) + drm_gpusvm_range_put(&range->base); +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h index a079eff86020e..1a5eff10f6cb3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h @@ -131,6 +131,8 @@ bool amdgpu_svm_range_dequeue_locked(struct amdgpu_svm *svm, struct amdgpu_svm_range_op_ctx *op_ctx); void amdgpu_svm_range_put_if_dequeued(struct amdgpu_svm *svm, struct amdgpu_svm_range *range); +void amdgpu_svm_range_unqueue(struct amdgpu_svm *svm, + struct amdgpu_svm_range *range); void amdgpu_svm_range_remove(struct amdgpu_svm *svm, struct amdgpu_svm_range *range, struct drm_gpusvm_ctx *ctx); -- 2.53.0
