Author: jhibbits Date: Sat Apr 11 00:16:50 2020 New Revision: 359792 URL: https://svnweb.freebsd.org/changeset/base/359792
Log: powerpc/booke: Add pte_find_next() to find the next in-use PTE Summary: Iterating over VM_MIN_ADDRESS->VM_MAXUSER_ADDRESS can take a very long time iterating one page at a time (2**(log_2(SIZE)-12) operations), yielding possibly several days or even weeks on 64-bit Book-E, even for a largely empty, which can happen when swapping out a process by vmdaemon. Speed this up by instead finding the next PTE at or equal to the given VA. Reviewed by: bdragon Differential Revision: https://reviews.freebsd.org/D24238 Modified: head/sys/powerpc/booke/pmap.c head/sys/powerpc/booke/pmap_32.c head/sys/powerpc/booke/pmap_64.c Modified: head/sys/powerpc/booke/pmap.c ============================================================================== --- head/sys/powerpc/booke/pmap.c Sat Apr 11 00:12:34 2020 (r359791) +++ head/sys/powerpc/booke/pmap.c Sat Apr 11 00:16:50 2020 (r359792) @@ -1532,9 +1532,12 @@ mmu_booke_remove(mmu_t mmu, pmap_t pmap, vm_offset_t v rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); for (; va < endva; va += PAGE_SIZE) { - pte = pte_find(mmu, pmap, va); - if ((pte != NULL) && PTE_ISVALID(pte)) - pte_remove(mmu, pmap, va, hold_flag); + pte = pte_find_next(mmu, pmap, &va); + if ((pte == NULL) || !PTE_ISVALID(pte)) + break; + if (va >= endva) + break; + pte_remove(mmu, pmap, va, hold_flag); } PMAP_UNLOCK(pmap); rw_wunlock(&pvh_global_lock); Modified: head/sys/powerpc/booke/pmap_32.c ============================================================================== --- head/sys/powerpc/booke/pmap_32.c Sat Apr 11 00:12:34 2020 (r359791) +++ head/sys/powerpc/booke/pmap_32.c Sat Apr 11 00:16:50 2020 (r359792) @@ -598,6 +598,35 @@ pte_find(mmu_t mmu, pmap_t pmap, vm_offset_t va) return (NULL); } +/* Get a pointer to a PTE in a page table, or the next closest (greater) one. */ +static __inline pte_t * +pte_find_next(mmu_t mmu, pmap_t pmap, vm_offset_t *pva) +{ + vm_offset_t va; + pte_t **pdir; + pte_t *pte; + unsigned long i, j; + + KASSERT((pmap != NULL), ("pte_find: invalid pmap")); + + va = *pva; + i = PDIR_IDX(va); + j = PTBL_IDX(va); + pdir = pmap->pm_pdir; + for (; i < PDIR_NENTRIES; i++, j = 0) { + if (pdir[i] == NULL) + continue; + for (; j < PTBL_NENTRIES; j++) { + pte = &pdir[i][j]; + if (!PTE_ISVALID(pte)) + continue; + *pva = PDIR_SIZE * i + PAGE_SIZE * j; + return (pte); + } + } + return (NULL); +} + /* Set up kernel page tables. */ static void kernel_pte_alloc(vm_offset_t data_end, vm_offset_t addr) Modified: head/sys/powerpc/booke/pmap_64.c ============================================================================== --- head/sys/powerpc/booke/pmap_64.c Sat Apr 11 00:12:34 2020 (r359791) +++ head/sys/powerpc/booke/pmap_64.c Sat Apr 11 00:16:50 2020 (r359792) @@ -145,6 +145,7 @@ static vm_paddr_t pte_vatopa(mmu_t, pmap_t, vm_offset_ static int pte_enter(mmu_t, pmap_t, vm_page_t, vm_offset_t, uint32_t, boolean_t); static int pte_remove(mmu_t, pmap_t, vm_offset_t, uint8_t); static pte_t *pte_find(mmu_t, pmap_t, vm_offset_t); +static pte_t *pte_find_next(mmu_t, pmap_t, vm_offset_t *); static void kernel_pte_alloc(vm_offset_t, vm_offset_t); /**************************************************************************/ @@ -202,6 +203,50 @@ pte_find(mmu_t mmu, pmap_t pmap, vm_offset_t va) ptbl = pdir[PDIR_IDX(va)]; return ((ptbl != NULL) ? &ptbl[PTBL_IDX(va)] : NULL); +} + +/* Get a pointer to a PTE in a page table, or the next closest (greater) one. */ +static __inline pte_t * +pte_find_next(mmu_t mmu, pmap_t pmap, vm_offset_t *pva) +{ + vm_offset_t va; + pte_t ****pm_root; + pte_t *pte; + unsigned long i, j, k, l; + + KASSERT((pmap != NULL), ("pte_find: invalid pmap")); + + va = *pva; + i = PG_ROOT_IDX(va); + j = PDIR_L1_IDX(va); + k = PDIR_IDX(va); + l = PTBL_IDX(va); + pm_root = pmap->pm_root; + /* truncate the VA for later. */ + va &= ~((1UL << (PG_ROOT_H + 1)) - 1); + for (; i < PG_ROOT_NENTRIES; i++, j = 0) { + if (pm_root[i] == 0) + continue; + for (; j < PDIR_L1_NENTRIES; j++, k = 0) { + if (pm_root[i][j] == 0) + continue; + for (; k < PDIR_NENTRIES; k++, l = 0) { + if (pm_root[i][j][k] == NULL) + continue; + for (; l < PTBL_NENTRIES; l++) { + pte = &pm_root[i][j][k][l]; + if (!PTE_ISVALID(pte)) + continue; + *pva = va + PG_ROOT_SIZE * i + + PDIR_L1_SIZE * j + + PDIR_SIZE * k + + PAGE_SIZE * l; + return (pte); + } + } + } + } + return (NULL); } static bool _______________________________________________ svn-src-all@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-all To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
