Author: thompsa
Date: Thu Jun 18 20:42:37 2009
New Revision: 194459
URL: http://svn.freebsd.org/changeset/base/194459
Log:
Track the kernel mapping of a physical page by a new entry in vm_page
structure. When the page is shared, the kernel mapping becomes a special
type of managed page to force the cache off the page mappings. This is
needed to avoid stale entries on all ARM VIVT caches, and VIPT caches
with cache color issue.
Submitted by: Mark Tinguely
Reviewed by: alc
Tested by: Grzegorz Bernacki, thompsa
Modified:
head/sys/arm/arm/pmap.c
head/sys/arm/include/pmap.h
head/sys/vm/vm_phys.c
Modified: head/sys/arm/arm/pmap.c
==============================================================================
--- head/sys/arm/arm/pmap.c Thu Jun 18 19:58:05 2009 (r194458)
+++ head/sys/arm/arm/pmap.c Thu Jun 18 20:42:37 2009 (r194459)
@@ -200,8 +200,7 @@ static pv_entry_t pmap_get_pv_entry(void
static void pmap_enter_locked(pmap_t, vm_offset_t, vm_page_t,
vm_prot_t, boolean_t, int);
-static __inline void pmap_fix_cache(struct vm_page *, pmap_t,
- vm_offset_t);
+static void pmap_fix_cache(struct vm_page *, pmap_t, vm_offset_t);
static void pmap_alloc_l1(pmap_t);
static void pmap_free_l1(pmap_t);
static void pmap_use_l1(pmap_t);
@@ -406,7 +405,7 @@ int pmap_needs_pte_sync;
#define pmap_is_current(pm) ((pm) == pmap_kernel() || \
curproc->p_vmspace->vm_map.pmap == (pm))
-static uma_zone_t pvzone;
+static uma_zone_t pvzone = NULL;
uma_zone_t l2zone;
static uma_zone_t l2table_zone;
static vm_offset_t pmap_kernel_l2dtable_kva;
@@ -1451,6 +1450,7 @@ pmap_fix_cache(struct vm_page *pg, pmap_
* kernel writable or kernel readable with writable user entry
*/
if ((kwritable && entries) ||
+ (kwritable > 1) ||
((kwritable != writable) && kentries &&
(pv->pv_pmap == pmap_kernel() ||
(pv->pv_flags & PVF_WRITE) ||
@@ -1472,7 +1472,8 @@ pmap_fix_cache(struct vm_page *pg, pmap_
continue;
}
/* user is no longer sharable and writable */
- if (pm != pmap_kernel() && (pv->pv_pmap == pm) &&
+ if (pm != pmap_kernel() &&
+ (pv->pv_pmap == pm || pv->pv_pmap == pmap_kernel()) &&
!pmwc && (pv->pv_flags & PVF_NC)) {
pv->pv_flags &= ~(PVF_NC | PVF_MWC);
@@ -1674,7 +1675,29 @@ pmap_enter_pv(struct vm_page *pg, struct
vm_offset_t va, u_int flags)
{
+ int km;
+
mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+
+ if (pg->md.pv_kva) {
+ /* PMAP_ASSERT_LOCKED(pmap_kernel()); */
+ pve->pv_pmap = pmap_kernel();
+ pve->pv_va = pg->md.pv_kva;
+ pve->pv_flags = PVF_WRITE | PVF_UNMAN;
+ pg->md.pv_kva = 0;
+
+ TAILQ_INSERT_HEAD(&pg->md.pv_list, pve, pv_list);
+ TAILQ_INSERT_HEAD(&pm->pm_pvlist, pve, pv_plist);
+ if ((km = PMAP_OWNED(pmap_kernel())))
+ PMAP_UNLOCK(pmap_kernel());
+ vm_page_unlock_queues();
+ if ((pve = pmap_get_pv_entry()) == NULL)
+ panic("pmap_kenter_internal: no pv entries");
+ vm_page_lock_queues();
+ if (km)
+ PMAP_LOCK(pmap_kernel());
+ }
+
PMAP_ASSERT_LOCKED(pm);
pve->pv_pmap = pm;
pve->pv_va = va;
@@ -1742,6 +1765,7 @@ static void
pmap_nuke_pv(struct vm_page *pg, pmap_t pm, struct pv_entry *pve)
{
+ struct pv_entry *pv;
mtx_assert(&vm_page_queue_mtx, MA_OWNED);
PMAP_ASSERT_LOCKED(pm);
TAILQ_REMOVE(&pg->md.pv_list, pve, pv_list);
@@ -1766,6 +1790,20 @@ pmap_nuke_pv(struct vm_page *pg, pmap_t
vm_page_flag_clear(pg, PG_WRITEABLE);
}
}
+ pv = TAILQ_FIRST(&pg->md.pv_list);
+ if (pv != NULL && (pv->pv_flags & PVF_UNMAN) &&
+ TAILQ_NEXT(pv, pv_list) == NULL) {
+ pg->md.pv_kva = pv->pv_va;
+ /* a recursive pmap_nuke_pv */
+ TAILQ_REMOVE(&pg->md.pv_list, pv, pv_list);
+ TAILQ_REMOVE(&pm->pm_pvlist, pv, pv_plist);
+ if (pv->pv_flags & PVF_WIRED)
+ --pm->pm_stats.wired_count;
+ pg->md.pvh_attrs &= ~PVF_REF;
+ pg->md.pvh_attrs &= ~PVF_MOD;
+ vm_page_flag_clear(pg, PG_WRITEABLE);
+ pmap_free_pv_entry(pv);
+ }
}
static struct pv_entry *
@@ -1784,6 +1822,9 @@ pmap_remove_pv(struct vm_page *pg, pmap_
pve = TAILQ_NEXT(pve, pv_list);
}
+ if (pve == NULL && pg->md.pv_kva == va)
+ pg->md.pv_kva = 0;
+
return(pve); /* return removed pve */
}
/*
@@ -2711,8 +2752,8 @@ pmap_remove_pages(pmap_t pmap)
cpu_idcache_wbinv_all();
cpu_l2cache_wbinv_all();
for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) {
- if (pv->pv_flags & PVF_WIRED) {
- /* The page is wired, cannot remove it now. */
+ if (pv->pv_flags & PVF_WIRED || pv->pv_flags & PVF_UNMAN) {
+ /* Cannot remove wired or unmanaged pages now. */
npv = TAILQ_NEXT(pv, pv_plist);
continue;
}
@@ -2822,6 +2863,9 @@ pmap_kenter_internal(vm_offset_t va, vm_
struct l2_bucket *l2b;
pt_entry_t *pte;
pt_entry_t opte;
+ struct pv_entry *pve;
+ vm_page_t m;
+
PDEBUG(1, printf("pmap_kenter: va = %08x, pa = %08x\n",
(uint32_t) va, (uint32_t) pa));
@@ -2835,10 +2879,7 @@ pmap_kenter_internal(vm_offset_t va, vm_
PDEBUG(1, printf("pmap_kenter: pte = %08x, opte = %08x, npte = %08x\n",
(uint32_t) pte, opte, *pte));
if (l2pte_valid(opte)) {
- cpu_dcache_wbinv_range(va, PAGE_SIZE);
- cpu_l2cache_wbinv_range(va, PAGE_SIZE);
- cpu_tlb_flushD_SE(va);
- cpu_cpwait();
+ pmap_kremove(va);
} else {
if (opte == 0)
l2b->l2b_occupancy++;
@@ -2850,6 +2891,33 @@ pmap_kenter_internal(vm_offset_t va, vm_
if (flags & KENTER_USER)
*pte |= L2_S_PROT_U;
PTE_SYNC(pte);
+
+ /* kernel direct mappings can be shared, so use a pv_entry
+ * to ensure proper caching.
+ *
+ * The pvzone is used to delay the recording of kernel
+ * mappings until the VM is running.
+ *
+ * This expects the physical memory to have vm_page_array entry.
+ */
+ if (pvzone != NULL && (m = vm_phys_paddr_to_vm_page(pa))) {
+ vm_page_lock_queues();
+ if (!TAILQ_EMPTY(&m->md.pv_list) || m->md.pv_kva) {
+ /* release vm_page lock for pv_entry UMA */
+ vm_page_unlock_queues();
+ if ((pve = pmap_get_pv_entry()) == NULL)
+ panic("pmap_kenter_internal: no pv entries");
+ vm_page_lock_queues();
+ PMAP_LOCK(pmap_kernel());
+ pmap_enter_pv(m, pve, pmap_kernel(), va,
+ PVF_WRITE | PVF_UNMAN);
+ pmap_fix_cache(m, pmap_kernel(), va);
+ PMAP_UNLOCK(pmap_kernel());
+ } else {
+ m->md.pv_kva = va;
+ }
+ vm_page_unlock_queues();
+ }
}
void
@@ -2886,6 +2954,9 @@ pmap_kremove(vm_offset_t va)
{
struct l2_bucket *l2b;
pt_entry_t *pte, opte;
+ struct pv_entry *pve;
+ vm_page_t m;
+ vm_offset_t pa;
l2b = pmap_get_l2_bucket(pmap_kernel(), va);
if (!l2b)
@@ -2894,6 +2965,25 @@ pmap_kremove(vm_offset_t va)
pte = &l2b->l2b_kva[l2pte_index(va)];
opte = *pte;
if (l2pte_valid(opte)) {
+ /* pa = vtophs(va) taken from pmap_extract() */
+ switch (opte & L2_TYPE_MASK) {
+ case L2_TYPE_L:
+ pa = (opte & L2_L_FRAME) | (va & L2_L_OFFSET);
+ break;
+ default:
+ pa = (opte & L2_S_FRAME) | (va & L2_S_OFFSET);
+ break;
+ }
+ /* note: should never have to remove an allocation
+ * before the pvzone is initialized.
+ */
+ vm_page_lock_queues();
+ PMAP_LOCK(pmap_kernel());
+ if (pvzone != NULL && (m = vm_phys_paddr_to_vm_page(pa)) &&
+ (pve = pmap_remove_pv(m, pmap_kernel(), va)))
+ pmap_free_pv_entry(pve);
+ PMAP_UNLOCK(pmap_kernel());
+ vm_page_unlock_queues();
cpu_dcache_wbinv_range(va, PAGE_SIZE);
cpu_l2cache_wbinv_range(va, PAGE_SIZE);
cpu_tlb_flushD_SE(va);
@@ -3137,16 +3227,24 @@ pmap_remove_all(vm_page_t m)
cpu_l2cache_inv_range(pv->pv_va, PAGE_SIZE);
}
- l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va);
- KASSERT(l2b != NULL, ("No l2 bucket"));
- ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
- *ptep = 0;
- PTE_SYNC_CURRENT(pv->pv_pmap, ptep);
- pmap_free_l2_bucket(pv->pv_pmap, l2b, 1);
- if (pv->pv_flags & PVF_WIRED)
- pv->pv_pmap->pm_stats.wired_count--;
- pv->pv_pmap->pm_stats.resident_count--;
- flags |= pv->pv_flags;
+ if (pv->pv_flags & PVF_UNMAN) {
+ /* remove the pv entry, but do not remove the mapping
+ * and remember this is a kernel mapped page
+ */
+ m->md.pv_kva = pv->pv_va;
+ } else {
+ /* remove the mapping and pv entry */
+ l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va);
+ KASSERT(l2b != NULL, ("No l2 bucket"));
+ ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
+ *ptep = 0;
+ PTE_SYNC_CURRENT(pv->pv_pmap, ptep);
+ pmap_free_l2_bucket(pv->pv_pmap, l2b, 1);
+ if (pv->pv_flags & PVF_WIRED)
+ pv->pv_pmap->pm_stats.wired_count--;
+ pv->pv_pmap->pm_stats.resident_count--;
+ flags |= pv->pv_flags;
+ }
pmap_nuke_pv(m, pv->pv_pmap, pv);
PMAP_UNLOCK(pv->pv_pmap);
pmap_free_pv_entry(pv);
@@ -3428,25 +3526,19 @@ do_l2b_alloc:
* It is part of our managed memory so we
* must remove it from the PV list
*/
- pve = pmap_remove_pv(opg, pmap, va);
- if (m && (m->flags & (PG_UNMANAGED | PG_FICTITIOUS)) &&
- pve)
- pmap_free_pv_entry(pve);
- else if (!pve &&
- !(m->flags & (PG_UNMANAGED | PG_FICTITIOUS)))
- pve = pmap_get_pv_entry();
- KASSERT(pve != NULL || m->flags & (PG_UNMANAGED |
- PG_FICTITIOUS), ("No pv"));
- oflags = pve->pv_flags;
-
+ if ((pve = pmap_remove_pv(opg, pmap, va))) {
+
+ /* note for patch: the oflags/invalidation was moved
+ * because PG_FICTITIOUS pages could free the pve
+ */
+ oflags = pve->pv_flags;
/*
* If the old mapping was valid (ref/mod
* emulation creates 'invalid' mappings
* initially) then make sure to frob
* the cache.
*/
- if ((oflags & PVF_NC) == 0 &&
- l2pte_valid(opte)) {
+ if ((oflags & PVF_NC) == 0 && l2pte_valid(opte)) {
if (PV_BEEN_EXECD(oflags)) {
pmap_idcache_wbinv_range(pmap, va,
PAGE_SIZE);
@@ -3456,15 +3548,43 @@ do_l2b_alloc:
PAGE_SIZE, TRUE,
(oflags & PVF_WRITE) == 0);
}
- }
- } else if (m && !(m->flags & (PG_UNMANAGED | PG_FICTITIOUS)))
- if ((pve = pmap_get_pv_entry()) == NULL) {
- panic("pmap_enter: no pv entries");
- }
- if (m && !(m->flags & (PG_UNMANAGED | PG_FICTITIOUS))) {
+ }
+
+ /* free/allocate a pv_entry for UNMANAGED pages if
+ * this physical page is not/is already mapped.
+ */
+
+ if (m && ((m->flags & PG_FICTITIOUS) ||
+ ((m->flags & PG_UNMANAGED) &&
+ !m->md.pv_kva &&
+ TAILQ_EMPTY(&m->md.pv_list)))) {
+ pmap_free_pv_entry(pve);
+ pve = NULL;
+ }
+ } else if (m && !(m->flags & PG_FICTITIOUS) &&
+ (!(m->flags & PG_UNMANAGED) || m->md.pv_kva ||
+ !TAILQ_EMPTY(&m->md.pv_list)))
+ pve = pmap_get_pv_entry();
+ } else if (m && !(m->flags & PG_FICTITIOUS) &&
+ (!(m->flags & PG_UNMANAGED) || m->md.pv_kva ||
+ !TAILQ_EMPTY(&m->md.pv_list)))
+ pve = pmap_get_pv_entry();
+
+ if (m && !(m->flags & PG_FICTITIOUS)) {
KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva,
- ("pmap_enter: managed mapping within the clean
submap"));
- pmap_enter_pv(m, pve, pmap, va, nflags);
+ ("pmap_enter: managed mapping within the clean
submap"));
+ if (m->flags & PG_UNMANAGED) {
+ if (!TAILQ_EMPTY(&m->md.pv_list) ||
+ m->md.pv_kva) {
+ KASSERT(pve != NULL, ("No pv"));
+ nflags |= PVF_UNMAN;
+ pmap_enter_pv(m, pve, pmap, va, nflags);
+ } else
+ m->md.pv_kva = va;
+ } else {
+ KASSERT(pve != NULL, ("No pv"));
+ pmap_enter_pv(m, pve, pmap, va, nflags);
+ }
}
}
/*
Modified: head/sys/arm/include/pmap.h
==============================================================================
--- head/sys/arm/include/pmap.h Thu Jun 18 19:58:05 2009 (r194458)
+++ head/sys/arm/include/pmap.h Thu Jun 18 20:42:37 2009 (r194459)
@@ -94,6 +94,7 @@ struct pv_entry;
struct md_page {
int pvh_attrs;
+ vm_offset_t pv_kva; /* first kernel VA mapping */
TAILQ_HEAD(,pv_entry) pv_list;
};
@@ -494,6 +495,7 @@ void pmap_use_minicache(vm_offset_t, vm_
#define PVF_EXEC 0x10 /* mapping is executable */
#define PVF_NC 0x20 /* mapping is non-cacheable */
#define PVF_MWC 0x40 /* mapping is used multiple
times in userland */
+#define PVF_UNMAN 0x80 /* mapping is unmanaged */
void vector_page_setprot(int);
Modified: head/sys/vm/vm_phys.c
==============================================================================
--- head/sys/vm/vm_phys.c Thu Jun 18 19:58:05 2009 (r194458)
+++ head/sys/vm/vm_phys.c Thu Jun 18 20:42:37 2009 (r194459)
@@ -382,8 +382,7 @@ vm_phys_paddr_to_vm_page(vm_paddr_t pa)
if (pa >= seg->start && pa < seg->end)
return (&seg->first_page[atop(pa - seg->start)]);
}
- panic("vm_phys_paddr_to_vm_page: paddr %#jx is not in any segment",
- (uintmax_t)pa);
+ return (NULL);
}
/*
_______________________________________________
[email protected] mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "[email protected]"
