From: Mika Penttilä <[email protected]> For migration to happen after hmm_range_fault(), the vma has to retrieved. Luckily, pagewalk already resolves that for us, so just have to save it for further use. For migration, mmap_lock has to be hold for the whole operation, so vma stays stable.
Take care also of firing the mmu_notifier when migrating. Also, prepare for differencies of how mm_struct * is retrieved in fault and migrate_vma paths. Cc: David Hildenbrand <[email protected]> Cc: Jason Gunthorpe <[email protected]> Cc: Leon Romanovsky <[email protected]> Cc: Alistair Popple <[email protected]> Cc: Balbir Singh <[email protected]> Cc: Zi Yan <[email protected]> Cc: Matthew Brost <[email protected]> Suggested-by: Alistair Popple <[email protected]> Signed-off-by: Mika Penttilä <[email protected]> --- include/linux/migrate.h | 45 ++++++++++++++++++++- lib/test_hmm.c | 2 +- mm/hmm.c | 86 ++++++++++++++++++++++++++++++++++++++--- 3 files changed, 125 insertions(+), 8 deletions(-) diff --git a/include/linux/migrate.h b/include/linux/migrate.h index 425ab5242da0..7380a89756ee 100644 --- a/include/linux/migrate.h +++ b/include/linux/migrate.h @@ -106,6 +106,28 @@ static inline void softleaf_entry_wait_on_locked(softleaf_t entry, spinlock_t *p spin_unlock(ptl); } +enum migrate_vma_info { + MIGRATE_VMA_SELECT_NONE = 0, + MIGRATE_VMA_SELECT_COMPOUND = MIGRATE_VMA_SELECT_NONE, +}; + +static inline enum migrate_vma_info hmm_select_migrate(struct hmm_range *range) +{ + return MIGRATE_VMA_SELECT_NONE; +} + +static inline void hmm_fill_migrate_vma(struct hmm_range *range, + struct vm_area_struct *vma, + unsigned long start, + unsigned long end) +{ +} + +static inline struct mm_struct *hmm_range_fault_mm(struct hmm_range *range) +{ + return range->notifier->mm; +} + #endif /* CONFIG_MIGRATION */ #ifdef CONFIG_NUMA_BALANCING @@ -149,7 +171,7 @@ static inline unsigned long migrate_pfn(unsigned long pfn) return (pfn << MIGRATE_PFN_SHIFT) | MIGRATE_PFN_VALID; } -enum migrate_vma_direction { +enum migrate_vma_info { MIGRATE_VMA_SELECT_SYSTEM = 1 << 0, MIGRATE_VMA_SELECT_DEVICE_PRIVATE = 1 << 1, MIGRATE_VMA_SELECT_DEVICE_COHERENT = 1 << 2, @@ -191,6 +213,27 @@ struct migrate_vma { struct page *fault_page; }; +// TODO: enable migration +static inline enum migrate_vma_info hmm_select_migrate(struct hmm_range *range) +{ + return 0; +} + +static inline struct mm_struct *hmm_range_fault_mm(struct hmm_range *range) +{ + return range->notifier ? range->notifier->mm : range->migrate->vma->vm_mm; +} + +static inline void hmm_fill_migrate_vma(struct hmm_range *range, + struct vm_area_struct *vma, + unsigned long start, + unsigned long end) +{ + range->migrate->vma = vma; + range->migrate->start = start; + range->migrate->end = end; +} + int migrate_vma_setup(struct migrate_vma *args); void migrate_vma_pages(struct migrate_vma *migrate); void migrate_vma_finalize(struct migrate_vma *migrate); diff --git a/lib/test_hmm.c b/lib/test_hmm.c index c4adbf98fac7..79cb7487d211 100644 --- a/lib/test_hmm.c +++ b/lib/test_hmm.c @@ -145,7 +145,7 @@ static bool dmirror_is_private_zone(struct dmirror_device *mdevice) HMM_DMIRROR_MEMORY_DEVICE_PRIVATE); } -static enum migrate_vma_direction +static enum migrate_vma_info dmirror_select_device(struct dmirror *dmirror) { return (dmirror->mdevice->zone_device_type == diff --git a/mm/hmm.c b/mm/hmm.c index c72c9ddfdb2f..637c00188b3e 100644 --- a/mm/hmm.c +++ b/mm/hmm.c @@ -20,6 +20,7 @@ #include <linux/pagemap.h> #include <linux/leafops.h> #include <linux/hugetlb.h> +#include <linux/migrate.h> #include <linux/memremap.h> #include <linux/sched/mm.h> #include <linux/jump_label.h> @@ -27,12 +28,17 @@ #include <linux/pci-p2pdma.h> #include <linux/mmu_notifier.h> #include <linux/memory_hotplug.h> +#include <asm/tlbflush.h> #include "internal.h" struct hmm_vma_walk { - struct hmm_range *range; - unsigned long last; + struct mmu_notifier_range mmu_range; + struct vm_area_struct *vma; + struct hmm_range *range; + unsigned long start; + unsigned long end; + unsigned long last; }; enum { @@ -393,6 +399,57 @@ static int hmm_vma_handle_absent_pmd(struct mm_walk *walk, unsigned long start, } #endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +static int hmm_vma_capture_migrate_range(unsigned long start, + unsigned long end, + struct mm_walk *walk) +{ + struct hmm_vma_walk *hmm_vma_walk = walk->private; + struct hmm_range *range = hmm_vma_walk->range; + + if (!hmm_select_migrate(range)) + return 0; + + if (hmm_vma_walk->vma && (hmm_vma_walk->vma != walk->vma)) + return -ERANGE; + + hmm_vma_walk->vma = walk->vma; + hmm_vma_walk->start = start; + hmm_vma_walk->end = end; + + if (end - start > range->end - range->start) + return -ERANGE; + + if (!hmm_vma_walk->mmu_range.owner) { + mmu_notifier_range_init_owner(&hmm_vma_walk->mmu_range, MMU_NOTIFY_MIGRATE, 0, + walk->vma->vm_mm, start, end, + range->dev_private_owner); + mmu_notifier_invalidate_range_start(&hmm_vma_walk->mmu_range); + } + + return 0; +} + +static void hmm_vma_post_range_fault(struct hmm_vma_walk *hmm_vma_walk) +{ + + struct hmm_range *range = hmm_vma_walk->range; + + if (hmm_select_migrate(range) && + hmm_vma_walk->mmu_range.owner) { + /* + * The migrate_vma path has the following initialized, + * so take care of fault path below. + */ + if (range->notifier) { + hmm_fill_migrate_vma(range, + hmm_vma_walk->vma, + hmm_vma_walk->start, + hmm_vma_walk->end); + } + mmu_notifier_invalidate_range_end(&hmm_vma_walk->mmu_range); + } +} + static int hmm_vma_walk_pmd(pmd_t *pmdp, unsigned long start, unsigned long end, @@ -600,6 +657,11 @@ static int hmm_vma_walk_test(unsigned long start, unsigned long end, struct hmm_vma_walk *hmm_vma_walk = walk->private; struct hmm_range *range = hmm_vma_walk->range; struct vm_area_struct *vma = walk->vma; + int r; + + r = hmm_vma_capture_migrate_range(start, end, walk); + if (r) + return r; if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)) && vma->vm_flags & VM_READ) @@ -662,16 +724,25 @@ int hmm_range_fault(struct hmm_range *range) .range = range, .last = range->start, }; - struct mm_struct *mm = range->notifier->mm; + /* + * Could be serving a device fault or come from migrate + * entry point. For the former we have not resolved the vma + * yet, and the latter we don't have a notifier (but have a vma). + * + */ + struct mm_struct *mm = hmm_range_fault_mm(range); int ret; mmap_assert_locked(mm); do { /* If range is no longer valid force retry. */ - if (mmu_interval_check_retry(range->notifier, - range->notifier_seq)) - return -EBUSY; + if (range->notifier && mmu_interval_check_retry(range->notifier, + range->notifier_seq)) { + ret = -EBUSY; + break; + } + ret = walk_page_range(mm, hmm_vma_walk.last, range->end, &hmm_walk_ops, &hmm_vma_walk); /* @@ -681,6 +752,9 @@ int hmm_range_fault(struct hmm_range *range) * output, and all >= are still at their input values. */ } while (ret == -EBUSY); + + hmm_vma_post_range_fault(&hmm_vma_walk); + return ret; } EXPORT_SYMBOL(hmm_range_fault); -- 2.55.0
