From: Honglei Huang <[email protected]> amdgpu_svm.c: - amdgpu_svm_work_init/fini: call amdgpu_svm_restore_init/fini for XNACK_OFF; unwind gc on restore_init failure. - amdgpu_svm_init_with_ops: accept begin_restore/end_restore callbacks and forward them to work_init. - amdgpu_svm_init_compute: xnack-off path now calls init_with_ops with amdgpu_svm_restore_invalidate, amdgpu_svm_restore_gc_work_func, and amdgpu_svm_quiesce_compute/resume_compute as the begin/end-restore hooks (replacing the previous -EOPNOTSUPP stub). - amdgpu_svm_sync_work: XNACK_OFF path flushes restore -> gc -> restore.wq -> gc.wq so any work re-queued during draining is fully drained. - amdgpu_svm_apply_attr_change: in XNACK_OFF mode, dispatch to amdgpu_svm_map_attrs_with_restore(); XNACK_ON path unchanged.
amdgpu_svm_range.c: - amdgpu_svm_range_put_if_dequeued: add RESTORE_WORK branch to re-queue ranges into the restore list when restore ops are pending, mirroring the GC re-queue logic. Makefile: - Add amdgpu_userptr.o to obj-y/build/clean targets so the restore worker translation unit gets linked into amdgpu.ko. Signed-off-by: Honglei Huang <[email protected]> --- drivers/gpu/drm/amd/amdgpu/Makefile | 6 +-- drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c | 44 ++++++++++++++++--- drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c | 8 ++++ 3 files changed, 50 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile index a40a42995..0485128c4 100644 --- a/drivers/gpu/drm/amd/amdgpu/Makefile +++ b/drivers/gpu/drm/amd/amdgpu/Makefile @@ -323,12 +323,12 @@ amdgpu-$(CONFIG_HMM_MIRROR) += amdgpu_hmm.o # svm support amdgpu-$(CONFIG_DRM_AMDGPU_SVM) += amdgpu_svm.o amdgpu_svm_attr.o \ - amdgpu_svm_fault.o amdgpu_svm_range.o + amdgpu_svm_fault.o amdgpu_svm_range.o amdgpu_userptr.o .PHONY: clean-svm clean-svm: - rm -f $(obj)/amdgpu_svm.o $(obj)/amdgpu_svm_attr.o $(obj)/amdgpu_svm_fault.o $(obj)/amdgpu_svm_range.o \ - $(obj)/.amdgpu_svm.o.cmd $(obj)/.amdgpu_svm_attr.o.cmd $(obj)/.amdgpu_svm_fault.o.cmd $(obj)/.amdgpu_svm_range.o.cmd + rm -f $(obj)/amdgpu_svm.o $(obj)/amdgpu_svm_attr.o $(obj)/amdgpu_svm_fault.o $(obj)/amdgpu_svm_range.o $(obj)/amdgpu_userptr.o \ + $(obj)/.amdgpu_svm.o.cmd $(obj)/.amdgpu_svm_attr.o.cmd $(obj)/.amdgpu_svm_fault.o.cmd $(obj)/.amdgpu_svm_range.o.cmd $(obj)/.amdgpu_userptr.o.cmd include $(FULL_AMD_PATH)/pm/Makefile diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c index b3cca1ff5..67a94086a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c @@ -315,6 +315,11 @@ int amdgpu_svm_apply_attr_change(struct amdgpu_svm *svm, if (!update_mapping) return 0; + if (XNACK_OFF(svm)) + return amdgpu_svm_map_attrs_with_restore(svm, new_attrs, + start_page, + last_page); + return amdgpu_svm_range_map_attrs(svm, new_attrs, start_page << PAGE_SHIFT, (last_page + 1) << PAGE_SHIFT); @@ -350,7 +355,9 @@ static void amdgpu_svm_flush_tlb_compute(struct amdgpu_svm *svm) } static int amdgpu_svm_work_init(struct amdgpu_svm *svm, - void (*gc_work_func)(struct work_struct *)); + void (*gc_work_func)(struct work_struct *), + void (*begin_restore)(struct amdgpu_svm *), + void (*end_restore)(struct amdgpu_svm *)); static void amdgpu_svm_work_fini(struct amdgpu_svm *svm); static int amdgpu_svm_init_xnack_mode(struct amdgpu_device *adev, @@ -389,6 +396,8 @@ static int amdgpu_svm_init_with_ops(struct amdgpu_svm *svm, struct drm_gpusvm_range *, uint64_t, uint64_t), void (*gc_work_func)(struct work_struct *), + void (*begin_restore)(struct amdgpu_svm *), + void (*end_restore)(struct amdgpu_svm *), void (*flush_tlb)(struct amdgpu_svm *)) { struct amdgpu_device *adev = svm->adev; @@ -397,7 +406,8 @@ static int amdgpu_svm_init_with_ops(struct amdgpu_svm *svm, svm->flush_tlb = flush_tlb; svm->invalidate_ranges = invalidate_ranges; - ret = amdgpu_svm_work_init(svm, gc_work_func); + ret = amdgpu_svm_work_init(svm, gc_work_func, begin_restore, + end_restore); if (ret) return ret; @@ -465,10 +475,15 @@ static int amdgpu_svm_init_compute(struct amdgpu_device *adev, ret = amdgpu_svm_init_with_ops(svm, amdgpu_svm_range_invalidate, amdgpu_svm_gc_work_func, + NULL, NULL, amdgpu_svm_flush_tlb_compute); } else { - AMDGPU_SVM_ERR("xnack off is not supported yet\n"); - ret = -EOPNOTSUPP; + ret = amdgpu_svm_init_with_ops(svm, + amdgpu_svm_restore_invalidate, + amdgpu_svm_restore_gc_work_func, + amdgpu_svm_quiesce_compute, + amdgpu_svm_resume_compute, + amdgpu_svm_flush_tlb_compute); } if (ret) @@ -617,7 +632,9 @@ static void amdgpu_svm_gc_flush(struct amdgpu_svm *svm) } static int amdgpu_svm_work_init(struct amdgpu_svm *svm, - void (*gc_work_func)(struct work_struct *)) + void (*gc_work_func)(struct work_struct *), + void (*begin_restore)(struct amdgpu_svm *), + void (*end_restore)(struct amdgpu_svm *)) { int ret; @@ -628,17 +645,34 @@ static int amdgpu_svm_work_init(struct amdgpu_svm *svm, if (ret) return ret; + if (XNACK_OFF(svm)) { + ret = amdgpu_svm_restore_init(svm, begin_restore, + end_restore); + if (ret) { + amdgpu_svm_gc_fini(svm); + return ret; + } + } + return 0; } static void amdgpu_svm_work_fini(struct amdgpu_svm *svm) { + if (XNACK_OFF(svm)) + amdgpu_svm_restore_fini(svm); amdgpu_svm_gc_fini(svm); } void amdgpu_svm_sync_work(struct amdgpu_svm *svm) { amdgpu_svm_gc_flush(svm); + if (XNACK_OFF(svm)) { + amdgpu_svm_restore_flush(svm); + amdgpu_svm_gc_flush(svm); + flush_workqueue(svm->restore.wq); + } + flush_workqueue(svm->gc.wq); } int amdgpu_gem_svm_ioctl(struct drm_device *dev, void *data, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c index 98e66a301..5cd0d1540 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c @@ -674,6 +674,7 @@ amdgpu_svm_range_put_if_dequeued(struct amdgpu_svm *svm, { bool release_kref = false; bool queue_gc = false; + bool queue_restore = false; spin_lock(&svm->work_lock); @@ -686,6 +687,10 @@ amdgpu_svm_range_put_if_dequeued(struct amdgpu_svm *svm, list_add_tail(&range->work_node, &svm->gc.list); range->queue_state = AMDGPU_SVM_RANGE_IN_GC; queue_gc = true; + } else if (RESTORE_WORK(range->pending_ops)) { + list_add_tail(&range->work_node, &svm->restore.list); + range->queue_state = AMDGPU_SVM_RANGE_IN_RESTORE; + queue_restore = true; } else { range->queue_state = AMDGPU_SVM_RANGE_NOT_QUEUED; release_kref = true; @@ -695,6 +700,9 @@ amdgpu_svm_range_put_if_dequeued(struct amdgpu_svm *svm, if (queue_gc) queue_work(svm->gc.wq, &svm->gc.work); + if (queue_restore) + queue_delayed_work(svm->restore.wq, &svm->restore.work, + msecs_to_jiffies(AMDGPU_SVM_RANGE_RESTORE_DELAY_MS)); if (release_kref) drm_gpusvm_range_put(&range->base); } -- 2.34.1
