From: Mika Penttilä <[email protected]>
Implement the needed hmm_vma_handle_migrate_prepare_pmd() function
which is mostly carried over from migrate_device.c's
migrate_vma_collect_huge_pmd() function.
With HMM pagewalk based migration, the idea is that
hmm_vma_handle_*() are responsible for faulting,
and the pfn collecting part. hmm_vma_handle_migrate_prepare*()
do the migration decisions (with HMM_PFN_MIGRATE), possibly split
folios, and insert migration ptes/pmds.
HMM pagewalk based migration is enabled in later commit, for now
now hmm_select_migrate() returns 0.
Also avoid the problems present in the migrate_vma_collect_huge_pmd()
split fallback path this HMM path supersedes:
- extra refcount for fault_folio makes split_folio() fail
- wrong folio unlocked after split_folio()
Cc: David Hildenbrand <[email protected]>
Cc: Jason Gunthorpe <[email protected]>
Cc: Leon Romanovsky <[email protected]>
Cc: Alistair Popple <[email protected]>
Cc: Balbir Singh <[email protected]>
Cc: Zi Yan <[email protected]>
Cc: Matthew Brost <[email protected]>
Suggested-by: Alistair Popple <[email protected]>
Signed-off-by: Mika Penttilä <[email protected]>
---
mm/hmm.c | 173 ++++++++++++++++++++++++++++++++++++++++++++++++-------
1 file changed, 151 insertions(+), 22 deletions(-)
diff --git a/mm/hmm.c b/mm/hmm.c
index a60b66963049..ea77429c5178 100644
--- a/mm/hmm.c
+++ b/mm/hmm.c
@@ -492,31 +492,19 @@ static int hmm_vma_handle_absent_pmd(struct mm_walk
*walk, unsigned long start,
#ifdef CONFIG_DEVICE_MIGRATION
/**
- * migrate_vma_split_folio() - Helper function to split a THP folio
+ * __migrate_vma_split_folio() - split a folio and move the lock/ref to the
+ * order-0 folio backing @fault_page after the split
* @folio: the folio to split
- * @fault_page: struct page associated with the fault if any
- * @hmm_vma_walk: walk in progress
- * @ptep: pte_t * for unmap and unlock ptl
+ * @fault_page: fault page if any
*
- * Returns 0 on success
+ * Returns 0 on success.
*/
-static int migrate_vma_split_folio(struct folio *folio,
- struct page *fault_page,
- struct hmm_vma_walk *hmm_vma_walk,
- pte_t *ptep)
+static int __migrate_vma_split_folio(struct folio *folio,
+ struct page *fault_page)
{
- int ret;
struct folio *fault_folio = fault_page ? page_folio(fault_page) : NULL;
struct folio *new_fault_folio = NULL;
-
- if (folio != fault_folio)
- folio_get(folio);
-
- pte_unmap_unlock(ptep, hmm_vma_walk->ptl);
- hmm_vma_walk->ptelocked = false;
-
- if (folio != fault_folio)
- folio_lock(folio);
+ int ret;
ret = split_folio(folio);
if (ret) {
@@ -548,14 +536,145 @@ static int migrate_vma_split_folio(struct folio *folio,
return 0;
}
+/**
+ * migrate_vma_split_folio() - drop the pte lock and split a THP folio
+ * @folio: the folio to split
+ * @fault_page: struct page associated with the fault if any
+ * @hmm_vma_walk: walk in progress
+ * @ptep: pte_t * for unmap and unlock ptl
+ *
+ * Returns 0 on success
+ */
+static int migrate_vma_split_folio(struct folio *folio,
+ struct page *fault_page,
+ struct hmm_vma_walk *hmm_vma_walk,
+ pte_t *ptep)
+{
+ struct folio *fault_folio = fault_page ? page_folio(fault_page) : NULL;
+
+ if (folio != fault_folio)
+ folio_get(folio);
+
+ pte_unmap_unlock(ptep, hmm_vma_walk->ptl);
+ hmm_vma_walk->ptelocked = false;
+
+ if (folio != fault_folio)
+ folio_lock(folio);
+
+ return __migrate_vma_split_folio(folio, fault_page);
+}
+
static int hmm_vma_handle_migrate_prepare_pmd(const struct mm_walk *walk,
pmd_t *pmdp,
unsigned long start,
unsigned long end,
unsigned long *hmm_pfn)
{
- // TODO: implement migration entry insertion
- return 0;
+ struct hmm_vma_walk *hmm_vma_walk = walk->private;
+ struct hmm_range *range = hmm_vma_walk->range;
+ struct migrate_vma *migrate = range->migrate;
+ struct folio *fault_folio = NULL;
+ enum migrate_vma_info minfo;
+ struct folio *folio;
+ unsigned long i;
+ int r = 0;
+
+ // Do we want to migrate at all?
+ minfo = hmm_select_migrate(range);
+ if (!minfo)
+ return r;
+
+ WARN_ON_ONCE(!migrate);
+ HMM_ASSERT_PMD_LOCKED(hmm_vma_walk, true);
+
+ fault_folio = migrate->fault_page ?
+ page_folio(migrate->fault_page) : NULL;
+
+ if (pmd_none(*pmdp))
+ return hmm_pfns_fill(start, end, hmm_vma_walk, 0);
+
+ if (!(hmm_pfn[0] & HMM_PFN_VALID))
+ goto out;
+
+ if (pmd_trans_huge(*pmdp)) {
+ if (!(minfo & MIGRATE_VMA_SELECT_SYSTEM))
+ goto out;
+
+ folio = pmd_folio(*pmdp);
+ if (is_huge_zero_folio(folio))
+ return hmm_pfns_fill(start, end, hmm_vma_walk, 0);
+
+ } else if (!pmd_present(*pmdp)) {
+ const softleaf_t entry = softleaf_from_pmd(*pmdp);
+
+ if (!softleaf_is_device_private(entry))
+ goto out;
+
+ if (!(minfo & MIGRATE_VMA_SELECT_DEVICE_PRIVATE))
+ goto out;
+
+ folio = softleaf_to_folio(entry);
+ if (folio->pgmap->owner != migrate->pgmap_owner)
+ goto out;
+ } else {
+ hmm_vma_walk->last = start;
+ return -EBUSY;
+ }
+
+ folio_get(folio);
+
+ if (folio != fault_folio && unlikely(!folio_trylock(folio))) {
+ folio_put(folio);
+ hmm_pfns_fill(start, end, hmm_vma_walk, HMM_PFN_ERROR);
+ return 0;
+ }
+
+ if (thp_migration_supported() &&
+ (migrate->flags & MIGRATE_VMA_SELECT_COMPOUND) &&
+ (IS_ALIGNED(start, HPAGE_PMD_SIZE) &&
+ IS_ALIGNED(end, HPAGE_PMD_SIZE))) {
+ struct page_vma_mapped_walk pvmw = {
+ .ptl = hmm_vma_walk->ptl,
+ .address = start,
+ .pmd = pmdp,
+ .vma = walk->vma,
+ };
+
+ hmm_pfn[0] |= HMM_PFN_MIGRATE | HMM_PFN_COMPOUND;
+
+ r = set_pmd_migration_entry(&pvmw, folio_page(folio, 0));
+ if (r) {
+ hmm_pfn[0] &= ~(HMM_PFN_MIGRATE | HMM_PFN_COMPOUND);
+ goto split; /* fall back to splitting the pmd */
+ }
+ for (i = 1, start += PAGE_SIZE; start < end; start +=
PAGE_SIZE, i++)
+ hmm_pfn[i] &= HMM_PFN_INOUT_FLAGS;
+
+ } else {
+ goto split; /* fall back to splitting the pmd */
+ }
+
+out:
+ return r;
+
+split:
+ spin_unlock(hmm_vma_walk->ptl);
+ hmm_vma_walk->pmdlocked = false;
+
+ /*
+ * folio_get() above took an extra reference. For the fault folio the
+ * caller still holds a reference and the lock, which is the
+ * precondition of __migrate_vma_split_folio(), so drop the extra one.
+ */
+ if (folio == fault_folio)
+ folio_put(folio);
+
+ r = __migrate_vma_split_folio(folio, migrate->fault_page);
+ if (r)
+ return r;
+
+ hmm_vma_walk->last = start;
+ return -EBUSY;
}
/*
@@ -958,8 +1077,18 @@ static int hmm_vma_walk_pmd(pmd_t *pmdp,
hmm_vma_walk->pmdlocked = false;
}
+ /*
+ * hmm_vma_handle_migrate_prepare_pmd() splits the huge pmd in
+ * place when needed and returns -EBUSY to re-walk the range as
+ * PTEs; any other error means the split failed.
+ */
+ if (r == -EBUSY)
+ return -EBUSY;
+ if (r) {
+ /* Split not successful, skip */
+ return hmm_pfns_fill(start, end, hmm_vma_walk,
HMM_PFN_ERROR);
+ }
return r;
-
}
if (hmm_vma_walk->pmdlocked) {
--
2.55.0
