From: Honglei Huang <[email protected]> Add MMU notifier event handling and garbage collection infrastructure: - amdgpu_svm_range_remove: unmap pages and remove from gpusvm - amdgpu_svm_range_notifier_event_begin: compute the [start_page, last_page] window from the notifier range and the gpusvm range, then zap PTEs via amdgpu_svm_range_zap_ptes and invalidate the GPU mapping - amdgpu_svm_gc_enqueue: queue range for GC worker and set queue state - amdgpu_svm_gc_add_range: mark range unmapped and enqueue for GC - amdgpu_svm_range_notifier_event_end: DMA unmap and GC on MMU_UNMAP - amdgpu_svm_range_invalidate_interval: cursor based PTE clearing across notifiers/ranges; clears PTEs through amdgpu_svm_range_zap_ptes - amdgpu_svm_range_dequeue_locked: dequeue work item with atomic pending state transfer - amdgpu_svm_range_put_if_dequeued: release range ref after dequeue, re-dispatch if new work was enqueued during processing - amdgpu_svm_range_unqueue: locked context helper that pulls a range off the GC list
Signed-off-by: Honglei Huang <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c | 311 ++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h | 16 + 2 files changed, 327 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c index 335fe5a551f0f..9e348390795a6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c @@ -417,3 +417,314 @@ amdgpu_svm_range_map_attrs(struct amdgpu_svm *svm, return 0; } +void amdgpu_svm_range_remove(struct amdgpu_svm *svm, + struct amdgpu_svm_range *range, + struct drm_gpusvm_ctx *ctx) +{ + struct drm_gpusvm_range *base = &range->base; + + amdgpu_svm_assert_locked(svm); + + if (!base->pages.flags.unmapped && !base->pages.flags.partial_unmap) + drm_gpusvm_range_unmap_pages(&svm->gpusvm, base, ctx); + + amdgpu_svm_range_invalidate_gpu_mapping(range); + drm_gpusvm_range_remove(&svm->gpusvm, base); +} + +/** + * amdgpu_svm_range_notifier_event_begin() - First MMU notifier phase + * @svm: The SVM context. + * @range: The GPU SVM range covered by the notifier event. + * @mmu_range: The MMU notifier range describing the invalidation window. + * + * Invalidate the GPU view of @range over the notifier window: zap the + * affected PTEs and clear the gpu_mapped flag. The range is not removed + * here; structural removal if needed happens in the event-end phase. + * + * Return: true if PTEs were zapped, false otherwise. + */ +bool +amdgpu_svm_range_notifier_event_begin(struct amdgpu_svm *svm, + struct drm_gpusvm_range *range, + const struct mmu_notifier_range *mmu_range) +{ + struct amdgpu_svm_range *svm_range = to_amdgpu_svm_range(range); + unsigned long start_page, last_page; + + amdgpu_svm_assert_in_notifier(svm); + + AMDGPU_SVM_RANGE_TRACE(svm_range, "NOTIFIER"); + + if (range->pages.flags.unmapped || !svm_range->gpu_mapped) + return false; + + AMDGPU_SVM_RANGE_TRACE(svm_range, "NOTIFIER - EXECUTE"); + + start_page = max(drm_gpusvm_range_start(range), + mmu_range->start) >> PAGE_SHIFT; + last_page = (min(drm_gpusvm_range_end(range), + mmu_range->end) >> PAGE_SHIFT) - 1; + + amdgpu_svm_range_zap_ptes(svm, svm_range, start_page, last_page); + amdgpu_svm_range_invalidate_gpu_mapping(svm_range); + + return true; +} + +/** + * amdgpu_svm_gc_enqueue() - Queue a range for work queue processing + * @svm: The SVM context. + * @range: The SVM range to queue. + * @start_page: First page of the pending invalidation interval. + * @last_page: Last page of the pending invalidation interval. + * + * Queue @range on the GC list and kick the GC worker. A range reference + * is taken when it first enters IN_GC and dropped only when it returns + * to NOT_QUEUED, so the range stays alive while on the GC list or being + * processed. All transitions are serialized by svm->work_lock. + */ +static void +amdgpu_svm_gc_enqueue(struct amdgpu_svm *svm, + struct amdgpu_svm_range *range, + unsigned long start_page, unsigned long last_page) +{ + if (atomic_read(&svm->exiting)) + return; + + spin_lock(&svm->work_lock); + if (range->queue_state == AMDGPU_SVM_RANGE_NOT_QUEUED) { + drm_gpusvm_range_get(&range->base); + range->queue_state = AMDGPU_SVM_RANGE_IN_GC; + } + + range->pending_start_page = min(range->pending_start_page, start_page); + range->pending_last_page = max(range->pending_last_page, last_page); + if (range->pending_ops == AMDGPU_SVM_RANGE_OP_NONE) + list_add_tail(&range->work_node, &svm->gc.list); + range->pending_ops |= AMDGPU_SVM_RANGE_OP_UNMAP; + spin_unlock(&svm->work_lock); + + queue_work(svm->gc.wq, &svm->gc.work); +} + +static void +amdgpu_svm_gc_add_range(struct amdgpu_svm *svm, + struct amdgpu_svm_range *svm_range, + const struct mmu_notifier_range *mmu_range) +{ + unsigned long start_page = max(drm_gpusvm_range_start(&svm_range->base), + mmu_range->start) >> PAGE_SHIFT; + unsigned long last_page = (min(drm_gpusvm_range_end(&svm_range->base), + mmu_range->end) >> PAGE_SHIFT) - 1; + + AMDGPU_SVM_RANGE_TRACE(svm_range, "GARBAGE COLLECTOR ADD"); + + drm_gpusvm_range_set_unmapped(&svm_range->base, mmu_range); + amdgpu_svm_gc_enqueue(svm, svm_range, start_page, last_page); +} + +/** + * amdgpu_svm_range_notifier_event_end() - Second MMU notifier phase + * @svm: The SVM context. + * @range: The GPU SVM range covered by the notifier event. + * @mmu_range: The MMU notifier range describing the event. + * + * DMA-unmap the pages for @range. On an actual unmap event, also queue the + * range to the GC so it gets structurally removed from the gpusvm trees. + */ +static void +amdgpu_svm_range_notifier_event_end(struct amdgpu_svm *svm, + struct drm_gpusvm_range *range, + const struct mmu_notifier_range *mmu_range) +{ + struct drm_gpusvm_ctx ctx = { .in_notifier = true, }; + + amdgpu_svm_assert_in_notifier(svm); + + drm_gpusvm_range_unmap_pages(&svm->gpusvm, range, &ctx); + if (mmu_range->event == MMU_NOTIFY_UNMAP) + amdgpu_svm_gc_add_range(svm, to_amdgpu_svm_range(range), + mmu_range); +} + +int +amdgpu_svm_range_invalidate_interval(struct amdgpu_svm *svm, + unsigned long start_page, + unsigned long last_page) +{ + unsigned long start = start_page << PAGE_SHIFT; + unsigned long end = (last_page + 1) << PAGE_SHIFT; + struct drm_gpusvm_notifier *notifier, *next_notifier; + struct drm_gpusvm_ctx ctx = { .in_notifier = false }; + struct drm_exec exec; + bool needs_flush = false; + int ret; + + amdgpu_svm_assert_locked(svm); + + ret = amdgpu_svm_range_lock_vm_pd(svm, &exec, true); + if (ret) + return ret; + + drm_gpusvm_for_each_notifier_safe(notifier, next_notifier, &svm->gpusvm, + start, end) { + struct drm_gpusvm_range *range, *next_range; + + drm_gpusvm_for_each_range_safe(range, next_range, notifier, + start, end) { + struct amdgpu_svm_range *svm_range = to_amdgpu_svm_range(range); + bool crosses_boundary = start > drm_gpusvm_range_start(range) || + end < drm_gpusvm_range_end(range); + + if (svm_range->gpu_mapped) { + AMDGPU_SVM_RANGE_TRACE(svm_range, + crosses_boundary ? "ATTR DESTROY" : + "ATTR ZAP PTE"); + + ret = amdgpu_svm_range_zap_ptes(svm, svm_range, + drm_gpusvm_range_start(range) >> PAGE_SHIFT, + (drm_gpusvm_range_end(range) >> PAGE_SHIFT) - 1); + if (ret < 0) { + drm_exec_fini(&exec); + return ret; + } + + amdgpu_svm_range_invalidate_gpu_mapping(svm_range); + needs_flush = true; + } + + if (crosses_boundary) { + /* remove ranges crossing the boundary so GPU fault + * creates new ranges bounded by the updated + * attr_range boundaries. + * Evict devmem-backed pages back to sysmem first + * so VRAM-resident data is not lost when the range + * is destroyed. No-op for sysmem-only ranges. + */ + drm_gpusvm_range_get(range); + + amdgpu_svm_range_evict(svm, range); + amdgpu_svm_range_remove(svm, svm_range, &ctx); + + amdgpu_svm_range_unqueue(svm, svm_range); + + drm_gpusvm_range_put(range); + } + } + } + + drm_exec_fini(&exec); + + if (needs_flush) + amdgpu_svm_flush_tlb(svm); + + AMDGPU_SVM_DBG("attr invalidate done [0x%lx-0x%lx]-0x%lx needs_flush=%d\n", + start_page, last_page, last_page - start_page + 1, + needs_flush ? 1 : 0); + + return 0; +} + +/** + * amdgpu_svm_range_dequeue_locked() - Pull the next range off a work list + * @svm: The SVM context. + * @work_list: The work list to dequeue from. + * @op_ctx: Output context filled with the dequeued range's pending state. + * + * Move the first range on @work_list to PROCESSING and atomically snapshot + * its pending op/interval into @op_ctx while resetting them on the range, so + * invalidations that arrive during processing accumulate afresh. Caller must + * hold svm->work_lock. + * + * Return: true if a range was dequeued, false if @work_list was empty. + */ +bool +amdgpu_svm_range_dequeue_locked(struct amdgpu_svm *svm, + struct list_head *work_list, + struct amdgpu_svm_range_op_ctx *op_ctx) +{ + struct amdgpu_svm_range *range; + + lockdep_assert_held(&svm->work_lock); + + range = list_first_entry_or_null(work_list, struct amdgpu_svm_range, + work_node); + if (!range) + return false; + + list_del_init(&range->work_node); + range->queue_state = AMDGPU_SVM_RANGE_PROCESSING; + + op_ctx->range = range; + op_ctx->start_page = range->pending_start_page; + op_ctx->last_page = range->pending_last_page; + op_ctx->pending_ops = range->pending_ops; + + range->pending_start_page = ULONG_MAX; + range->pending_last_page = 0; + range->pending_ops = AMDGPU_SVM_RANGE_OP_NONE; + + return true; +} + +/** + * amdgpu_svm_range_put_if_dequeued() - Finish processing a dequeued range + * @svm: The SVM context. + * @range: The range previously moved to PROCESSING. + * + * Called after a PROCESSING range has been handled. If new UNMAP work was + * enqueued while it was being processed, re-queue it on the GC list and kick + * the worker again; otherwise return it to NOT_QUEUED and drop the reference + * taken at enqueue time. + */ +void +amdgpu_svm_range_put_if_dequeued(struct amdgpu_svm *svm, + struct amdgpu_svm_range *range) +{ + bool release_kref = false; + bool queue_gc = false; + + spin_lock(&svm->work_lock); + + if (range->queue_state != AMDGPU_SVM_RANGE_PROCESSING) { + spin_unlock(&svm->work_lock); + return; + } + + if (range->pending_ops & AMDGPU_SVM_RANGE_OP_UNMAP) { + list_add_tail(&range->work_node, &svm->gc.list); + range->queue_state = AMDGPU_SVM_RANGE_IN_GC; + queue_gc = true; + } else { + range->queue_state = AMDGPU_SVM_RANGE_NOT_QUEUED; + release_kref = true; + } + + spin_unlock(&svm->work_lock); + + if (queue_gc) + queue_work(svm->gc.wq, &svm->gc.work); + if (release_kref) + drm_gpusvm_range_put(&range->base); +} + +void amdgpu_svm_range_unqueue(struct amdgpu_svm *svm, + struct amdgpu_svm_range *range) +{ + bool put = false; + + amdgpu_svm_assert_locked(svm); + + spin_lock(&svm->work_lock); + if (range->queue_state == AMDGPU_SVM_RANGE_IN_GC) { + list_del_init(&range->work_node); + range->queue_state = AMDGPU_SVM_RANGE_NOT_QUEUED; + range->pending_ops = AMDGPU_SVM_RANGE_OP_NONE; + put = true; + } + spin_unlock(&svm->work_lock); + + if (put) + drm_gpusvm_range_put(&range->base); +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h index e5a5f52166576..f423be277a85a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h @@ -149,6 +149,16 @@ int amdgpu_svm_range_update_mapping(struct amdgpu_svm *svm, bool read_only, bool intr, bool wait, bool flush_tlb); +bool amdgpu_svm_range_dequeue_locked(struct amdgpu_svm *svm, + struct list_head *work_list, + struct amdgpu_svm_range_op_ctx *op_ctx); +void amdgpu_svm_range_put_if_dequeued(struct amdgpu_svm *svm, + struct amdgpu_svm_range *range); +void amdgpu_svm_range_unqueue(struct amdgpu_svm *svm, + struct amdgpu_svm_range *range); +void amdgpu_svm_range_remove(struct amdgpu_svm *svm, + struct amdgpu_svm_range *range, + struct drm_gpusvm_ctx *ctx); int amdgpu_svm_range_map_attrs(struct amdgpu_svm *svm, const struct amdgpu_svm_attrs *attrs, unsigned long start, unsigned long end); @@ -165,5 +175,11 @@ amdgpu_svm_range_find_or_insert(struct amdgpu_svm *svm, unsigned long addr, int amdgpu_svm_range_get_pages(struct amdgpu_svm *svm, struct drm_gpusvm_range *range, struct drm_gpusvm_ctx *ctx); +bool amdgpu_svm_range_notifier_event_begin(struct amdgpu_svm *svm, + struct drm_gpusvm_range *range, + const struct mmu_notifier_range *mmu_range); +int amdgpu_svm_range_invalidate_interval(struct amdgpu_svm *svm, + unsigned long start_page, + unsigned long last_page); #endif /* __AMDGPU_SVM_RANGE_H__ */ -- 2.53.0
