From: Honglei Huang <[email protected]> Add delayed restore worker for xnack-off range restoration: - svm_restore_range: restore single range by looking up attributes and updating GPU mappings. Uses the per-segment PTE-flag pipeline: amdgpu_svm_range_is_valid() and amdgpu_svm_range_update_mapping() consume the attribute set plus the read_only flag directly, so no pte_flags is pre-computed here. - amdgpu_svm_restore_worker: main delayed work loop that dequeues ranges from restore list, attempts restoration, handles retries with re-enqueue on failure, checks for completion to invoke end callback when all evictions are resolved
Signed-off-by: Honglei Huang <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_userptr.c | 147 ++++++++++++++++++++ 1 file changed, 147 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userptr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userptr.c index b231c7d44..89e8b687b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userptr.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userptr.c @@ -222,6 +222,153 @@ svm_restore_enqueue_work(struct amdgpu_svm *svm, msecs_to_jiffies(AMDGPU_SVM_RANGE_RESTORE_DELAY_MS)); } +static int +svm_restore_range(struct amdgpu_svm *svm, struct amdgpu_svm_range *range) +{ + struct amdgpu_svm_attr_tree *attr_tree = svm->attr_tree; + struct amdgpu_svm_attr_range *attr_range; + struct amdgpu_svm_attrs attrs; + unsigned long range_start_page; + int ret; + struct drm_gpusvm_ctx map_ctx; + bool devmem_possible, need_vram_migration; + + amdgpu_svm_assert_locked(svm); + + range_start_page = drm_gpusvm_range_start(&range->base) >> PAGE_SHIFT; + + mutex_lock(&attr_tree->lock); + attr_range = amdgpu_svm_attr_find_locked(attr_tree, range_start_page); + if (attr_range) + attrs = attr_range->attrs; + mutex_unlock(&attr_tree->lock); + + if (!attr_range || !amdgpu_svm_attr_has_access(attrs.access)) + return 0; + + devmem_possible = amdgpu_svm_attr_devmem_possible(svm, &attrs); + need_vram_migration = amdgpu_svm_attr_prefer_vram(svm, &attrs); + devmem_possible = false; /* TODO: add migration */ + map_ctx = (struct drm_gpusvm_ctx){ + .read_only = !!(attrs.flags & AMDGPU_SVM_ATTR_BIT_GPU_RO), + .devmem_possible = devmem_possible, + .devmem_only = need_vram_migration, + .check_pages_threshold = devmem_possible ? SZ_64K : 0, + }; + + if (amdgpu_svm_range_is_valid(svm, range, &attrs)) + return 0; + + AMDGPU_SVM_RANGE_DEBUG(range, "RESTORE - GET PAGES"); + + ret = amdgpu_svm_range_get_pages(svm, &range->base, &map_ctx); + if (ret) + return ret; + + AMDGPU_SVM_RANGE_DEBUG(range, "RESTORE - UPDATE MAPPING"); + + ret = amdgpu_svm_range_update_mapping(svm, range, &attrs, + map_ctx.read_only, + true, true, true); + return ret; +} + +static void amdgpu_svm_restore_worker(struct work_struct *w) +{ + struct delayed_work *dwork = to_delayed_work(w); + struct amdgpu_svm_restore *rst = container_of(dwork, struct amdgpu_svm_restore, work); + struct amdgpu_svm *svm = container_of(rst, struct amdgpu_svm, restore); + unsigned long resched_delay = + max_t(unsigned long, 1, + msecs_to_jiffies(AMDGPU_SVM_RANGE_RESTORE_DELAY_MS)); + struct amdgpu_svm_range_op_ctx op_ctx; + int evicted_record; + bool need_resched = false; + bool has_pending; + int ret; + + if (atomic_read(&svm->exiting)) + return; + + evicted_record = atomic_read(&svm->restore.evicted_ranges); + if (!evicted_record) + return; + + if (!svm->gpusvm.mm) { + atomic_set(&svm->restore.evicted_ranges, 0); + svm->restore.end(svm); + return; + } + + spin_lock(&svm->work_lock); + while (amdgpu_svm_range_dequeue_locked(svm, &svm->restore.list, + &op_ctx)) { + spin_unlock(&svm->work_lock); + + down_write(&svm->svm_lock); + if (UNMAP_WORK(op_ctx.pending_ops)) + ret = 0; + else + ret = svm_restore_range(svm, op_ctx.range); + up_write(&svm->svm_lock); + + if (ret) { + AMDGPU_SVM_TRACE("restore work retry ret=%d start=0x%lx last=0x%lx\n", + ret, op_ctx.start_page, op_ctx.last_page); + spin_lock(&svm->work_lock); + if (!UNMAP_WORK(op_ctx.range->pending_ops)) + op_ctx.range->pending_ops |= + AMDGPU_SVM_RANGE_OP_RESTORE; + op_ctx.range->pending_start_page = + min(op_ctx.range->pending_start_page, + op_ctx.start_page); + op_ctx.range->pending_last_page = + max(op_ctx.range->pending_last_page, + op_ctx.last_page); + spin_unlock(&svm->work_lock); + need_resched = true; + } + + amdgpu_svm_range_put_if_dequeued(svm, op_ctx.range); + spin_lock(&svm->work_lock); + } + spin_unlock(&svm->work_lock); + + spin_lock(&svm->work_lock); + has_pending = !list_empty(&svm->restore.list) || + !list_empty(&svm->gc.list); + spin_unlock(&svm->work_lock); + + if (!need_resched && !has_pending) { + flush_work(&svm->gc.work); + + spin_lock(&svm->work_lock); + has_pending = !list_empty(&svm->restore.list) || + !list_empty(&svm->gc.list); + spin_unlock(&svm->work_lock); + } + + if (!need_resched && !has_pending) { + drm_gpusvm_notifier_lock(&svm->gpusvm); + spin_lock(&svm->work_lock); + + has_pending = !list_empty(&svm->restore.list) || + !list_empty(&svm->gc.list); + + spin_unlock(&svm->work_lock); + + if (!has_pending && + atomic_cmpxchg(&svm->restore.evicted_ranges, evicted_record, 0) == + evicted_record) { + drm_gpusvm_notifier_unlock(&svm->gpusvm); + svm->restore.end(svm); + return; + } + drm_gpusvm_notifier_unlock(&svm->gpusvm); + } + + queue_delayed_work(svm->restore.wq, &svm->restore.work, resched_delay); +} int amdgpu_svm_restore_init(struct amdgpu_svm *svm, void (*begin)(struct amdgpu_svm *), void (*end)(struct amdgpu_svm *)) -- 2.34.1
