From: Junhua Shen <[email protected]> - Call amdgpu_svm_range_migrate_to_vram() before GPU mapping in fault and prefetch paths when VRAM migration is needed - Set device_private_page_owner in drm_gpusvm_ctx for devmem awareness - Allow AMDGPU_INTERCONNECT_VRAM entries in update_gpu_range proto check - Skip valid-mapping fast path in prefetch when VRAM migration is requested, so prefetch can actively move pages to VRAM - Enable amdgpu_pagemap_capable() check in amdgpu_svm_devmem_possible() - Move AMDGPU_INTERCONNECT_VRAM/P2P defines to migration header
Signed-off-by: Junhua Shen <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c | 4 +++- drivers/gpu/drm/amd/amdgpu/amdgpu_svm.h | 3 --- drivers/gpu/drm/amd/amdgpu/amdgpu_svm_fault.c | 11 ++++++++- drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c | 23 ++++++++++++++++--- 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c index 9d25357de5d3c..c785d22fe5bea 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c @@ -35,6 +35,7 @@ #include "amdgpu_svm_attr.h" #include "amdgpu_svm_fault.h" #include "amdgpu_svm_range.h" +#include "amdgpu_svm_range_migrate.h" #include "amdgpu_vm.h" #if IS_ENABLED(CONFIG_DRM_AMDGPU_SVM) @@ -308,7 +309,8 @@ bool amdgpu_svm_devmem_possible(struct amdgpu_svm *svm) if (svm->adev->apu_prefer_gtt) return false; - /* TODO: add amdgpu_pagemap_capable() */ + if (amdgpu_pagemap_capable(svm)) + return true; return false; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.h index ac84b80f750f8..0cab194263588 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.h @@ -45,9 +45,6 @@ struct amdgpu_svm_attrs; struct drm_device; struct drm_file; -#define AMDGPU_INTERCONNECT_VRAM DRM_INTERCONNECT_DRIVER -#define AMDGPU_INTERCONNECT_P2P (AMDGPU_INTERCONNECT_VRAM + 1) - enum amdgpu_svm_xnack_mode { AMDGPU_SVM_XNACK_OFF, AMDGPU_SVM_XNACK_ON, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_fault.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_fault.c index a92a1a4f2436f..4bb72d32794cb 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_fault.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_fault.c @@ -26,6 +26,8 @@ #include "amdgpu_svm_attr.h" #include "amdgpu_svm_fault.h" #include "amdgpu_svm_range.h" +#include "amdgpu_svm_range_migrate.h" +#include "amdgpu_migrate.h" #include "amdgpu.h" #include "amdgpu_vm.h" #include "amdgpu_gmc.h" @@ -160,6 +162,8 @@ static int fault_map_range(struct amdgpu_svm *svm, .check_pages_threshold = devmem_possible ? SZ_64K : 0, .devmem_only = need_vram_migration, .timeslice_ms = need_vram_migration ? 5 : 0, + .device_private_page_owner = devmem_possible ? + AMDGPU_PGMAP_OWNER(svm->adev) : NULL, }; struct amdgpu_svm_range *range; ktime_t timestamp = ktime_get_boottime(); @@ -229,7 +233,12 @@ static int fault_map_range(struct amdgpu_svm *svm, } AMDGPU_SVM_RANGE_DEBUG(range, "PAGE FAULT"); - /* TODO: add migration*/ + if (need_vram_migration) { + AMDGPU_SVM_RANGE_DEBUG(range, "MIGRATE PAGES"); + ret = amdgpu_svm_range_migrate_to_vram(svm, &range->base); + if (ret) + return ret; + } AMDGPU_SVM_RANGE_DEBUG(range, "GET PAGES"); ret = amdgpu_svm_range_get_pages(svm, &range->base, &map_ctx); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c index 6ac785a3b5cba..62b4e0c306682 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c @@ -26,6 +26,8 @@ #include "amdgpu_svm_attr.h" #include "amdgpu_svm_range.h" #include "amdgpu_svm_fault.h" +#include "amdgpu_svm_range_migrate.h" +#include "amdgpu_migrate.h" #include "amdgpu.h" #include "amdgpu_vm.h" @@ -275,7 +277,8 @@ amdgpu_svm_range_update_gpu_range(struct amdgpu_svm *svm, unsigned long start_page, last_page; bool is_last_seg; - if (entry->proto != DRM_INTERCONNECT_SYSTEM) + if (entry->proto != DRM_INTERCONNECT_SYSTEM && + entry->proto != AMDGPU_INTERCONNECT_VRAM) return -EOPNOTSUPP; pte_flags = amdgpu_svm_range_attr_pte_flags(svm, attrs, @@ -427,6 +430,8 @@ amdgpu_svm_range_map_attrs(struct amdgpu_svm *svm, .devmem_possible = devmem_possible, .devmem_only = need_vram_migration, .check_pages_threshold = devmem_possible ? SZ_64K : 0, + .device_private_page_owner = devmem_possible ? + AMDGPU_PGMAP_OWNER(svm->adev) : NULL, }; while (addr < end) { @@ -449,12 +454,24 @@ amdgpu_svm_range_map_attrs(struct amdgpu_svm *svm, if (next_addr <= addr) return -EINVAL; - if (amdgpu_svm_range_is_valid(svm, range, attrs)) { + /* + * Prefetch to device memory should still run migration logic even if + * the current GPU mapping is already valid. This allows prefetch + * requests to actively move backing pages to VRAM instead of being + * skipped by the valid-mapping fast path. + */ + if (amdgpu_svm_range_is_valid(svm, range, attrs) && + !need_vram_migration) { addr = next_addr; continue; } - /* TODO: add migration */ + if (need_vram_migration) { + AMDGPU_SVM_RANGE_DEBUG(range, "MIGRATE PAGES"); + ret = amdgpu_svm_range_migrate_to_vram(svm, &range->base); + if (ret) + return ret; + } AMDGPU_SVM_RANGE_DEBUG(range, "GET PAGES"); -- 2.53.0
