From: Honglei Huang <[email protected]> amdgpu_svm_attr.h: - Add AMDGPU_SVM_ATTR_TRIGGER_NEED_REMAP macro as an alias for AMDGPU_SVM_ATTR_TRIGGER_NEED_INVALIDATE. Both cover the same trigger set today but carry different semantics: NEED_INVALIDATE is used by xnack-on (invalidate GPU PTEs, rely on fault to rebuild), NEED_REMAP is used by xnack-off (eagerly rebuild GPU mapping since no fault mechanism exists). Keeping them as separate macros allows future divergence without code churn.
amdgpu_svm.c: - amdgpu_svm_apply_attr_change: xnack-off remap decision now uses AMDGPU_SVM_ATTR_TRIGGER_NEED_REMAP instead of open-coding the four trigger flags. - amdgpu_svm_work_init/fini: call amdgpu_svm_restore_init/fini for XNACK_OFF; unwind gc on restore_init failure. - amdgpu_svm_init_with_ops: accept begin_restore/end_restore callbacks and forward them to work_init. - amdgpu_svm_init_compute: xnack-off path now calls init_with_ops with amdgpu_svm_restore_invalidate, amdgpu_svm_restore_gc_work_func, and amdgpu_svm_quiesce_compute/resume_compute as the begin/end-restore hooks (replacing the previous -EOPNOTSUPP stub). - amdgpu_svm_sync_work: XNACK_OFF path flushes restore -> gc -> restore.wq -> gc.wq so any work re-queued during draining is fully drained. - amdgpu_svm_apply_attr_change: in XNACK_OFF mode, dispatch to amdgpu_svm_map_attrs_with_restore(); XNACK_ON path unchanged. amdgpu_svm_range.c: - amdgpu_svm_range_put_if_dequeued: add RESTORE_WORK branch to re-queue ranges into the restore list when restore ops are pending, mirroring the GC re-queue logic. Makefile: - Add amdgpu_userptr.o to obj-y/build/clean targets so the restore worker translation unit gets linked into amdgpu.ko. Signed-off-by: Honglei Huang <[email protected]> --- drivers/gpu/drm/amd/amdgpu/Makefile | 6 +- drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c | 56 +++++++++++++++++-- drivers/gpu/drm/amd/amdgpu/amdgpu_svm_attr.h | 3 + drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c | 8 +++ 4 files changed, 65 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile index 608d30d9a..b15a8e23e 100644 --- a/drivers/gpu/drm/amd/amdgpu/Makefile +++ b/drivers/gpu/drm/amd/amdgpu/Makefile @@ -328,12 +328,12 @@ amdgpu-$(CONFIG_HMM_MIRROR) += amdgpu_hmm.o # svm support amdgpu-$(CONFIG_DRM_AMDGPU_SVM) += amdgpu_svm.o amdgpu_svm_attr.o \ - amdgpu_svm_fault.o amdgpu_svm_range.o + amdgpu_svm_fault.o amdgpu_svm_range.o amdgpu_userptr.o .PHONY: clean-svm clean-svm: - rm -f $(obj)/amdgpu_svm.o $(obj)/amdgpu_svm_attr.o $(obj)/amdgpu_svm_fault.o $(obj)/amdgpu_svm_range.o \ - $(obj)/.amdgpu_svm.o.cmd $(obj)/.amdgpu_svm_attr.o.cmd $(obj)/.amdgpu_svm_fault.o.cmd $(obj)/.amdgpu_svm_range.o.cmd + rm -f $(obj)/amdgpu_svm.o $(obj)/amdgpu_svm_attr.o $(obj)/amdgpu_svm_fault.o $(obj)/amdgpu_svm_range.o $(obj)/amdgpu_userptr.o \ + $(obj)/.amdgpu_svm.o.cmd $(obj)/.amdgpu_svm_attr.o.cmd $(obj)/.amdgpu_svm_fault.o.cmd $(obj)/.amdgpu_svm_range.o.cmd $(obj)/.amdgpu_userptr.o.cmd include $(FULL_AMD_PATH)/pm/Makefile diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c index 96b8f9454..626c5790e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c @@ -270,6 +270,18 @@ int amdgpu_svm_apply_attr_change(struct amdgpu_svm *svm, if (trigger & AMDGPU_SVM_ATTR_TRIGGER_PREFETCH) needs_mapping = true; + /* + * With xnack off there is no page fault mechanism to lazily establish + * GPU mappings, so they must be in place before the GPU runs. The attr + * layer has no visibility into actual GPU mapping state, so force a + * mapping pass whenever the range is accessible, even when attrs are + * unchanged, to guarantee the GPU mapping is current. + */ + if (XNACK_OFF(svm) && new_access && !trigger) + needs_mapping = true; + if (XNACK_OFF(svm) && new_access && + (trigger & AMDGPU_SVM_ATTR_TRIGGER_NEED_REMAP)) + needs_mapping = true; if (!trigger && !needs_mapping) return 0; @@ -296,6 +308,11 @@ int amdgpu_svm_apply_attr_change(struct amdgpu_svm *svm, if (!needs_mapping) return 0; + if (XNACK_OFF(svm)) + return amdgpu_svm_map_attrs_with_restore(svm, new_attrs, + start_page, + last_page); + return amdgpu_svm_range_map_attrs(svm, new_attrs, start_page << PAGE_SHIFT, (last_page + 1) << PAGE_SHIFT); @@ -331,7 +348,9 @@ static void amdgpu_svm_flush_tlb_compute(struct amdgpu_svm *svm) } static int amdgpu_svm_work_init(struct amdgpu_svm *svm, - void (*gc_work_func)(struct work_struct *)); + void (*gc_work_func)(struct work_struct *), + void (*begin_restore)(struct amdgpu_svm *), + void (*end_restore)(struct amdgpu_svm *)); static void amdgpu_svm_work_fini(struct amdgpu_svm *svm); static int amdgpu_svm_init_xnack_mode(struct amdgpu_device *adev, @@ -370,6 +389,8 @@ static int amdgpu_svm_init_with_ops(struct amdgpu_svm *svm, struct drm_gpusvm_range *, uint64_t, uint64_t), void (*gc_work_func)(struct work_struct *), + void (*begin_restore)(struct amdgpu_svm *), + void (*end_restore)(struct amdgpu_svm *), void (*flush_tlb)(struct amdgpu_svm *)) { struct amdgpu_device *adev = svm->adev; @@ -378,7 +399,8 @@ static int amdgpu_svm_init_with_ops(struct amdgpu_svm *svm, svm->flush_tlb = flush_tlb; svm->invalidate_ranges = invalidate_ranges; - ret = amdgpu_svm_work_init(svm, gc_work_func); + ret = amdgpu_svm_work_init(svm, gc_work_func, begin_restore, + end_restore); if (ret) return ret; @@ -445,10 +467,15 @@ static int amdgpu_svm_init_compute(struct amdgpu_device *adev, ret = amdgpu_svm_init_with_ops(svm, amdgpu_svm_range_invalidate, amdgpu_svm_gc_work_func, + NULL, NULL, amdgpu_svm_flush_tlb_compute); } else { - AMDGPU_SVM_ERR("xnack off is not supported yet\n"); - ret = -EOPNOTSUPP; + ret = amdgpu_svm_init_with_ops(svm, + amdgpu_svm_restore_invalidate, + amdgpu_svm_restore_gc_work_func, + amdgpu_svm_quiesce_compute, + amdgpu_svm_resume_compute, + amdgpu_svm_flush_tlb_compute); } if (ret) @@ -597,7 +624,9 @@ static void amdgpu_svm_gc_flush(struct amdgpu_svm *svm) } static int amdgpu_svm_work_init(struct amdgpu_svm *svm, - void (*gc_work_func)(struct work_struct *)) + void (*gc_work_func)(struct work_struct *), + void (*begin_restore)(struct amdgpu_svm *), + void (*end_restore)(struct amdgpu_svm *)) { int ret; @@ -608,17 +637,34 @@ static int amdgpu_svm_work_init(struct amdgpu_svm *svm, if (ret) return ret; + if (XNACK_OFF(svm)) { + ret = amdgpu_svm_restore_init(svm, begin_restore, + end_restore); + if (ret) { + amdgpu_svm_gc_fini(svm); + return ret; + } + } + return 0; } static void amdgpu_svm_work_fini(struct amdgpu_svm *svm) { + if (XNACK_OFF(svm)) + amdgpu_svm_restore_fini(svm); amdgpu_svm_gc_fini(svm); } void amdgpu_svm_sync_work(struct amdgpu_svm *svm) { amdgpu_svm_gc_flush(svm); + if (XNACK_OFF(svm)) { + amdgpu_svm_restore_flush(svm); + amdgpu_svm_gc_flush(svm); + flush_workqueue(svm->restore.wq); + } + flush_workqueue(svm->gc.wq); } int amdgpu_gem_svm_ioctl(struct drm_device *dev, void *data, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_attr.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_attr.h index d8eedf578..ce7be1bc4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_attr.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_attr.h @@ -121,6 +121,9 @@ enum amdgpu_svm_attr_change_trigger { AMDGPU_SVM_ATTR_TRIGGER_MAPPING_FLAG_CHANGE | \ AMDGPU_SVM_ATTR_TRIGGER_LOCATION_CHANGE) +#define AMDGPU_SVM_ATTR_TRIGGER_NEED_REMAP \ + AMDGPU_SVM_ATTR_TRIGGER_NEED_INVALIDATE + struct amdgpu_svm_attr_tree * amdgpu_svm_attr_tree_create(struct amdgpu_svm *svm); void amdgpu_svm_attr_tree_destroy(struct amdgpu_svm_attr_tree *attr_tree); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c index f1be2f2d5..ef913a236 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c @@ -699,6 +699,7 @@ amdgpu_svm_range_put_if_dequeued(struct amdgpu_svm *svm, { bool release_kref = false; bool queue_gc = false; + bool queue_restore = false; spin_lock(&svm->work_lock); @@ -711,6 +712,10 @@ amdgpu_svm_range_put_if_dequeued(struct amdgpu_svm *svm, list_add_tail(&range->work_node, &svm->gc.list); range->queue_state = AMDGPU_SVM_RANGE_IN_GC; queue_gc = true; + } else if (RESTORE_WORK(range->pending_ops)) { + list_add_tail(&range->work_node, &svm->restore.list); + range->queue_state = AMDGPU_SVM_RANGE_IN_RESTORE; + queue_restore = true; } else { range->queue_state = AMDGPU_SVM_RANGE_NOT_QUEUED; release_kref = true; @@ -720,6 +725,9 @@ amdgpu_svm_range_put_if_dequeued(struct amdgpu_svm *svm, if (queue_gc) queue_work(svm->gc.wq, &svm->gc.work); + if (queue_restore) + queue_delayed_work(svm->restore.wq, &svm->restore.work, + msecs_to_jiffies(AMDGPU_SVM_RANGE_RESTORE_DELAY_MS)); if (release_kref) drm_gpusvm_range_put(&range->base); } -- 2.34.1
