Author: scottph Date: Mon Sep 21 22:23:57 2020 New Revision: 365981 URL: https://svnweb.freebsd.org/changeset/base/365981
Log: arm64/pmap: Sparsify pv_table Reviewed by: markj, kib Approved by: scottl (implicit) MFC after: 1 week Sponsored by: Ampere Computing, Inc. Differential Revision: https://reviews.freebsd.org/D26132 Modified: head/sys/arm64/arm64/pmap.c head/sys/vm/vm_phys.h Modified: head/sys/arm64/arm64/pmap.c ============================================================================== --- head/sys/arm64/arm64/pmap.c Mon Sep 21 22:22:53 2020 (r365980) +++ head/sys/arm64/arm64/pmap.c Mon Sep 21 22:23:57 2020 (r365981) @@ -178,8 +178,32 @@ __FBSDID("$FreeBSD$"); #endif #define pmap_l2_pindex(v) ((v) >> L2_SHIFT) -#define pa_to_pvh(pa) (&pv_table[pmap_l2_pindex(pa)]) +static struct md_page * +pa_to_pvh(vm_paddr_t pa) +{ + struct vm_phys_seg *seg; + int segind; + + for (segind = 0; segind < vm_phys_nsegs; segind++) { + seg = &vm_phys_segs[segind]; + if (pa >= seg->start && pa < seg->end) + return ((struct md_page *)seg->md_first + + pmap_l2_pindex(pa) - pmap_l2_pindex(seg->start)); + } + panic("pa 0x%jx not within vm_phys_segs", (uintmax_t)pa); +} + +static struct md_page * +page_to_pvh(vm_page_t m) +{ + struct vm_phys_seg *seg; + + seg = &vm_phys_segs[m->segind]; + return ((struct md_page *)seg->md_first + + pmap_l2_pindex(VM_PAGE_TO_PHYS(m)) - pmap_l2_pindex(seg->start)); +} + #define NPV_LIST_LOCKS MAXCPU #define PHYS_TO_PV_LIST_LOCK(pa) \ @@ -1049,6 +1073,8 @@ pmap_init_asids(struct asid_set *set, int bits) void pmap_init(void) { + struct vm_phys_seg *seg, *next_seg; + struct md_page *pvh; vm_size_t s; uint64_t mmfr1; int i, pv_npg, vmid_bits; @@ -1093,7 +1119,12 @@ pmap_init(void) /* * Calculate the size of the pv head table for superpages. */ - pv_npg = howmany(vm_phys_segs[vm_phys_nsegs - 1].end, L2_SIZE); + pv_npg = 0; + for (i = 0; i < vm_phys_nsegs; i++) { + seg = &vm_phys_segs[i]; + pv_npg += pmap_l2_pindex(roundup2(seg->end, L2_SIZE)) - + pmap_l2_pindex(seg->start); + } /* * Allocate memory for the pv head table for superpages. @@ -1105,6 +1136,31 @@ pmap_init(void) TAILQ_INIT(&pv_table[i].pv_list); TAILQ_INIT(&pv_dummy.pv_list); + /* + * Set pointers from vm_phys_segs to pv_table. + */ + for (i = 0, pvh = pv_table; i < vm_phys_nsegs; i++) { + seg = &vm_phys_segs[i]; + seg->md_first = pvh; + pvh += pmap_l2_pindex(roundup2(seg->end, L2_SIZE)) - + pmap_l2_pindex(seg->start); + + /* + * If there is a following segment, and the final + * superpage of this segment and the initial superpage + * of the next segment are the same then adjust the + * pv_table entry for that next segment down by one so + * that the pv_table entries will be shared. + */ + if (i + 1 < vm_phys_nsegs) { + next_seg = &vm_phys_segs[i + 1]; + if (pmap_l2_pindex(roundup2(seg->end, L2_SIZE)) - 1 == + pmap_l2_pindex(next_seg->start)) { + pvh--; + } + } + } + vm_initialized = 1; } @@ -2247,7 +2303,7 @@ reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **l m->md.pv_gen++; if (TAILQ_EMPTY(&m->md.pv_list) && (m->flags & PG_FICTITIOUS) == 0) { - pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); + pvh = page_to_pvh(m); if (TAILQ_EMPTY(&pvh->pv_list)) { vm_page_aflag_clear(m, PGA_WRITEABLE); @@ -2788,7 +2844,7 @@ pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_ pmap_pvh_free(&m->md, pmap, va); if (TAILQ_EMPTY(&m->md.pv_list) && (m->flags & PG_FICTITIOUS) == 0) { - pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); + pvh = page_to_pvh(m); if (TAILQ_EMPTY(&pvh->pv_list)) vm_page_aflag_clear(m, PGA_WRITEABLE); } @@ -2858,7 +2914,7 @@ pmap_remove_l3_range(pmap_t pmap, pd_entry_t l2e, vm_o pmap_pvh_free(&m->md, pmap, sva); if (TAILQ_EMPTY(&m->md.pv_list) && (m->flags & PG_FICTITIOUS) == 0) { - pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); + pvh = page_to_pvh(m); if (TAILQ_EMPTY(&pvh->pv_list)) vm_page_aflag_clear(m, PGA_WRITEABLE); } @@ -2997,8 +3053,7 @@ pmap_remove_all(vm_page_t m) ("pmap_remove_all: page %p is not managed", m)); SLIST_INIT(&free); lock = VM_PAGE_TO_PV_LIST_LOCK(m); - pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : - pa_to_pvh(VM_PAGE_TO_PHYS(m)); + pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : page_to_pvh(m); retry: rw_wlock(lock); while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) { @@ -4480,7 +4535,7 @@ pmap_page_exists_quick(pmap_t pmap, vm_page_t m) break; } if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) { - pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); + pvh = page_to_pvh(m); TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { if (PV_PMAP(pv) == pmap) { rv = TRUE; @@ -4535,7 +4590,7 @@ restart: PMAP_UNLOCK(pmap); } if ((m->flags & PG_FICTITIOUS) == 0) { - pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); + pvh = page_to_pvh(m); TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { pmap = PV_PMAP(pv); if (!PMAP_TRYLOCK(pmap)) { @@ -4577,7 +4632,7 @@ pmap_page_is_mapped(vm_page_t m) rw_rlock(lock); rv = !TAILQ_EMPTY(&m->md.pv_list) || ((m->flags & PG_FICTITIOUS) == 0 && - !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list)); + !TAILQ_EMPTY(&page_to_pvh(m)->pv_list)); rw_runlock(lock); return (rv); } @@ -4740,8 +4795,7 @@ pmap_remove_pages(pmap_t pmap) if ((m->a.flags & PGA_WRITEABLE) != 0 && TAILQ_EMPTY(&m->md.pv_list) && (m->flags & PG_FICTITIOUS) == 0) { - pvh = pa_to_pvh( - VM_PAGE_TO_PHYS(m)); + pvh = page_to_pvh(m); if (TAILQ_EMPTY(&pvh->pv_list)) vm_page_aflag_clear(m, PGA_WRITEABLE); @@ -4818,7 +4872,7 @@ restart: goto out; } if ((m->flags & PG_FICTITIOUS) == 0) { - pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); + pvh = page_to_pvh(m); TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { pmap = PV_PMAP(pv); PMAP_ASSERT_STAGE1(pmap); @@ -4938,8 +4992,7 @@ pmap_remove_write(vm_page_t m) if (!pmap_page_is_write_mapped(m)) return; lock = VM_PAGE_TO_PV_LIST_LOCK(m); - pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : - pa_to_pvh(VM_PAGE_TO_PHYS(m)); + pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : page_to_pvh(m); retry_pv_loop: rw_wlock(lock); TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) { @@ -5035,7 +5088,7 @@ pmap_ts_referenced(vm_page_t m) cleared = 0; pa = VM_PAGE_TO_PHYS(m); lock = PHYS_TO_PV_LIST_LOCK(pa); - pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : pa_to_pvh(pa); + pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : page_to_pvh(m); rw_wlock(lock); retry: not_cleared = 0; @@ -5312,8 +5365,7 @@ pmap_clear_modify(vm_page_t m) if (!pmap_page_is_write_mapped(m)) return; - pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : - pa_to_pvh(VM_PAGE_TO_PHYS(m)); + pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : page_to_pvh(m); lock = VM_PAGE_TO_PV_LIST_LOCK(m); rw_wlock(lock); restart: Modified: head/sys/vm/vm_phys.h ============================================================================== --- head/sys/vm/vm_phys.h Mon Sep 21 22:22:53 2020 (r365980) +++ head/sys/vm/vm_phys.h Mon Sep 21 22:23:57 2020 (r365981) @@ -72,6 +72,7 @@ struct vm_phys_seg { #if VM_NRESERVLEVEL > 0 vm_reserv_t first_reserv; #endif + void *md_first; int domain; struct vm_freelist (*free_queues)[VM_NFREEPOOL][VM_NFREEORDER_MAX]; }; _______________________________________________ svn-src-all@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-all To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
