Wire the migration layer into the SVM range map and GPU fault call chains: - Add amdgpu_pagemap_capable() guard in amdgpu_svm_attr_devmem_possible() to disable devmem when pagemap is not supported - Add device_private_page_owner to drm_gpusvm_ctx in both prefetch and fault map contexts - Call amdgpu_svm_range_migrate_range() before GPU mapping in both fault and prefetch paths to perform VRAM migration when preferred - Support AMDGPU_INTERCONNECT_VRAM in amdgpu_svm_range_update_gpu_range() by clearing AMDGPU_PTE_SYSTEM and AMDGPU_PTE_SNOOPED flags for VRAM pages
Signed-off-by: Junhua Shen <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_svm_attr.c | 4 ++++ drivers/gpu/drm/amd/amdgpu/amdgpu_svm_fault.c | 9 ++++++-- drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c | 21 +++++++++++++++---- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_attr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_attr.c index e50b67540c99..115bda12e625 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_attr.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_attr.c @@ -25,6 +25,7 @@ #include "amdgpu_svm.h" #include "amdgpu_svm_attr.h" #include "amdgpu.h" +#include "amdgpu_svm_range_migrate.h" #include <linux/err.h> #include <linux/errno.h> @@ -55,6 +56,9 @@ struct attr_get_ctx { bool amdgpu_svm_attr_devmem_possible(struct amdgpu_svm *svm, const struct amdgpu_svm_attrs *attrs) { + if (!amdgpu_pagemap_capable(svm)) + return false; + if (svm->adev->apu_prefer_gtt) return false; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_fault.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_fault.c index 7763eb029eaa..b5d21a66a228 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_fault.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_fault.c @@ -26,6 +26,8 @@ #include "amdgpu_svm_attr.h" #include "amdgpu_svm_fault.h" #include "amdgpu_svm_range.h" +#include "amdgpu_svm_range_migrate.h" +#include "amdgpu_migrate.h" #include "amdgpu.h" #include "amdgpu_vm.h" #include "amdgpu_gmc.h" @@ -153,13 +155,14 @@ static int amdgpu_svm_range_map_fault(struct amdgpu_svm *svm, const struct amdgpu_svm_attrs *attrs = &attr_range->attrs; bool devmem_possible = amdgpu_svm_attr_devmem_possible(svm, attrs); bool need_vram_migration = amdgpu_svm_attr_prefer_vram(svm, attrs); - devmem_possible = false; /* TODO: add migration */ struct drm_gpusvm_ctx map_ctx = { .read_only = !!(attrs->flags & AMDGPU_SVM_ATTR_BIT_GPU_RO), .devmem_possible = devmem_possible, .check_pages_threshold = devmem_possible ? SZ_64K : 0, .devmem_only = need_vram_migration && devmem_possible, .timeslice_ms = need_vram_migration && devmem_possible ? 5 : 0, + .device_private_page_owner = devmem_possible ? + AMDGPU_PGMAP_OWNER(svm->adev) : NULL, }; struct amdgpu_svm_range *range; ktime_t timestamp = ktime_get_boottime(); @@ -228,7 +231,9 @@ static int amdgpu_svm_range_map_fault(struct amdgpu_svm *svm, } AMDGPU_SVM_RANGE_DEBUG(range, "PAGE FAULT"); - /* TODO: add migration*/ + if (need_vram_migration) + amdgpu_svm_range_migrate_range(svm, &range->base, + AMDGPU_SVM_MIGRATE_TO_VRAM); AMDGPU_SVM_RANGE_DEBUG(range, "GET PAGES"); ret = amdgpu_svm_range_get_pages(svm, &range->base, &map_ctx); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c index fe543a16b399..b77f3a52f3ae 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c @@ -26,6 +26,8 @@ #include "amdgpu_svm_attr.h" #include "amdgpu_svm_range.h" #include "amdgpu_svm_fault.h" +#include "amdgpu_svm_range_migrate.h" +#include "amdgpu_migrate.h" #include "amdgpu.h" #include "amdgpu_vm.h" @@ -244,9 +246,11 @@ amdgpu_svm_range_update_gpu_range(struct amdgpu_svm *svm, unsigned long seg_pages = min_t(unsigned long, 1UL << entry->order, npages - mapped_pages); unsigned long start_page, last_page; + uint64_t seg_pte_flags = pte_flags; bool is_last_seg; - if (entry->proto != DRM_INTERCONNECT_SYSTEM) + if (entry->proto != DRM_INTERCONNECT_SYSTEM && + entry->proto != AMDGPU_INTERCONNECT_VRAM) return -EOPNOTSUPP; start_page = range_start_page + mapped_pages; @@ -254,9 +258,13 @@ amdgpu_svm_range_update_gpu_range(struct amdgpu_svm *svm, mapped_pages += seg_pages; is_last_seg = mapped_pages == npages; + /* For VRAM pages, clear the SYSTEM and SNOOPED bits */ + if (entry->proto == AMDGPU_INTERCONNECT_VRAM) + seg_pte_flags &= ~(AMDGPU_PTE_SYSTEM | AMDGPU_PTE_SNOOPED); + ret = amdgpu_vm_update_range(svm->adev, svm->vm, false, false, flush_tlb && is_last_seg, true, NULL, - start_page, last_page, pte_flags, + start_page, last_page, seg_pte_flags, 0, entry->addr, NULL, NULL, wait_fence && is_last_seg ? fence : NULL); if (ret) @@ -365,12 +373,13 @@ amdgpu_svm_range_map_attrs(struct amdgpu_svm *svm, int ret; bool devmem_possible = amdgpu_svm_attr_devmem_possible(svm, attrs); bool need_vram_migration = amdgpu_svm_attr_prefer_vram(svm, attrs); - devmem_possible = false; /* TODO: add migration */ struct drm_gpusvm_ctx map_ctx = { .read_only = !!(attrs->flags & AMDGPU_SVM_ATTR_BIT_GPU_RO), .devmem_possible = devmem_possible, .devmem_only = need_vram_migration && devmem_possible, .check_pages_threshold = devmem_possible ? SZ_64K : 0, + .device_private_page_owner = devmem_possible ? + AMDGPU_PGMAP_OWNER(svm->adev) : NULL, }; while (addr < end) { @@ -399,7 +408,11 @@ amdgpu_svm_range_map_attrs(struct amdgpu_svm *svm, continue; } - /* TODO: add migration */ + if (need_vram_migration) { + AMDGPU_SVM_RANGE_DEBUG(range, "PREFETCH - MIGRATION PAGES"); + amdgpu_svm_range_migrate_range(svm, &range->base, + AMDGPU_SVM_MIGRATE_TO_VRAM); + } AMDGPU_SVM_RANGE_DEBUG(range, "PREFETCH - GET PAGES"); -- 2.34.1
