Author: nwhitehorn
Date: Fri Apr 20 00:45:23 2012
New Revision: 234476
URL: http://svn.freebsd.org/changeset/base/234476
Log:
MFC r233949,233957,233964,234149,234155:
More PMAP concurrency and performance improvements by only flushing icaches
when needed and executing ptesync only when architecturally required.
Modified:
stable/9/sys/powerpc/aim/machdep.c
stable/9/sys/powerpc/aim/mmu_oea.c
stable/9/sys/powerpc/aim/mmu_oea64.c
stable/9/sys/powerpc/aim/moea64_native.c
stable/9/sys/vm/vm_page.h
Directory Properties:
stable/9/sys/ (props changed)
Modified: stable/9/sys/powerpc/aim/machdep.c
==============================================================================
--- stable/9/sys/powerpc/aim/machdep.c Fri Apr 20 00:34:15 2012
(r234475)
+++ stable/9/sys/powerpc/aim/machdep.c Fri Apr 20 00:45:23 2012
(r234476)
@@ -400,6 +400,9 @@ powerpc_init(vm_offset_t startkernel, vm
cacheline_size = 32;
}
+ /* Make sure the kernel icache is valid before we go too much further */
+ __syncicache((caddr_t)startkernel, endkernel - startkernel);
+
#ifndef __powerpc64__
/*
* Figure out whether we need to use the 64 bit PMAP. This works by
Modified: stable/9/sys/powerpc/aim/mmu_oea.c
==============================================================================
--- stable/9/sys/powerpc/aim/mmu_oea.c Fri Apr 20 00:34:15 2012
(r234475)
+++ stable/9/sys/powerpc/aim/mmu_oea.c Fri Apr 20 00:45:23 2012
(r234476)
@@ -1055,7 +1055,7 @@ moea_enter_locked(pmap_t pmap, vm_offset
struct pvo_head *pvo_head;
uma_zone_t zone;
vm_page_t pg;
- u_int pte_lo, pvo_flags, was_exec;
+ u_int pte_lo, pvo_flags;
int error;
if (!moea_initialized) {
@@ -1063,13 +1063,11 @@ moea_enter_locked(pmap_t pmap, vm_offset
zone = moea_upvo_zone;
pvo_flags = 0;
pg = NULL;
- was_exec = PTE_EXEC;
} else {
pvo_head = vm_page_to_pvoh(m);
pg = m;
zone = moea_mpvo_zone;
pvo_flags = PVO_MANAGED;
- was_exec = 0;
}
if (pmap_bootstrapped)
mtx_assert(&vm_page_queue_mtx, MA_OWNED);
@@ -1085,18 +1083,6 @@ moea_enter_locked(pmap_t pmap, vm_offset
zone = moea_upvo_zone;
}
- /*
- * If this is a managed page, and it's the first reference to the page,
- * clear the execness of the page. Otherwise fetch the execness.
- */
- if ((pg != NULL) && ((m->oflags & VPO_UNMANAGED) == 0)) {
- if (LIST_EMPTY(pvo_head)) {
- moea_attr_clear(pg, PTE_EXEC);
- } else {
- was_exec = moea_attr_fetch(pg) & PTE_EXEC;
- }
- }
-
pte_lo = moea_calc_wimg(VM_PAGE_TO_PHYS(m), pmap_page_get_memattr(m));
if (prot & VM_PROT_WRITE) {
@@ -1117,22 +1103,14 @@ moea_enter_locked(pmap_t pmap, vm_offset
pte_lo, pvo_flags);
/*
- * Flush the real page from the instruction cache if this page is
- * mapped executable and cacheable and was not previously mapped (or
- * was not mapped executable).
+ * Flush the real page from the instruction cache. This has be done
+ * for all user mappings to prevent information leakage via the
+ * instruction cache. moea_pvo_enter() returns ENOENT for the first
+ * mapping for a page.
*/
- if (error == 0 && (pvo_flags & PVO_EXECUTABLE) &&
- (pte_lo & PTE_I) == 0 && was_exec == 0) {
- /*
- * Flush the real memory from the cache.
- */
+ if (pmap != kernel_pmap && error == ENOENT &&
+ (pte_lo & (PTE_I | PTE_G)) == 0)
moea_syncicache(VM_PAGE_TO_PHYS(m), PAGE_SIZE);
- if (pg != NULL)
- moea_attr_save(pg, PTE_EXEC);
- }
-
- /* XXX syncicache always until problems are sorted */
- moea_syncicache(VM_PAGE_TO_PHYS(m), PAGE_SIZE);
}
/*
@@ -1454,12 +1432,6 @@ moea_kenter_attr(mmu_t mmu, vm_offset_t
panic("moea_kenter: failed to enter va %#x pa %#x: %d", va,
pa, error);
- /*
- * Flush the real memory from the instruction cache.
- */
- if ((pte_lo & (PTE_I | PTE_G)) == 0) {
- moea_syncicache(pa, PAGE_SIZE);
- }
PMAP_UNLOCK(kernel_pmap);
}
Modified: stable/9/sys/powerpc/aim/mmu_oea64.c
==============================================================================
--- stable/9/sys/powerpc/aim/mmu_oea64.c Fri Apr 20 00:34:15 2012
(r234475)
+++ stable/9/sys/powerpc/aim/mmu_oea64.c Fri Apr 20 00:45:23 2012
(r234476)
@@ -283,8 +283,6 @@ static struct pvo_entry *moea64_pvo_find
/*
* Utility routines.
*/
-static void moea64_enter_locked(mmu_t, pmap_t, vm_offset_t,
- vm_page_t, vm_prot_t, boolean_t);
static boolean_t moea64_query_bit(mmu_t, vm_page_t, u_int64_t);
static u_int moea64_clear_bit(mmu_t, vm_page_t, u_int64_t);
static void moea64_kremove(mmu_t, vm_offset_t);
@@ -1167,30 +1165,11 @@ moea64_zero_page_idle(mmu_t mmu, vm_page
* target pmap with the protection requested. If specified the page
* will be wired down.
*/
+
void
moea64_enter(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m,
vm_prot_t prot, boolean_t wired)
{
-
- LOCK_TABLE_WR();
- PMAP_LOCK(pmap);
- moea64_enter_locked(mmu, pmap, va, m, prot, wired);
- UNLOCK_TABLE_WR();
- PMAP_UNLOCK(pmap);
-}
-
-/*
- * Map the given physical page at the specified virtual address in the
- * target pmap with the protection requested. If specified the page
- * will be wired down.
- *
- * The table (write) and pmap must be locked.
- */
-
-static void
-moea64_enter_locked(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m,
- vm_prot_t prot, boolean_t wired)
-{
struct pvo_head *pvo_head;
uma_zone_t zone;
vm_page_t pg;
@@ -1210,10 +1189,9 @@ moea64_enter_locked(mmu_t mmu, pmap_t pm
pvo_flags = PVO_MANAGED;
}
- PMAP_LOCK_ASSERT(pmap, MA_OWNED);
KASSERT((m->oflags & (VPO_UNMANAGED | VPO_BUSY)) != 0 ||
VM_OBJECT_LOCKED(m->object),
- ("moea64_enter_locked: page %p is not busy", m));
+ ("moea64_enter: page %p is not busy", m));
/* XXX change the pvo head for fake pages */
if ((m->oflags & VPO_UNMANAGED) != 0) {
@@ -1238,15 +1216,22 @@ moea64_enter_locked(mmu_t mmu, pmap_t pm
if (wired)
pvo_flags |= PVO_WIRED;
+ LOCK_TABLE_WR();
+ PMAP_LOCK(pmap);
error = moea64_pvo_enter(mmu, pmap, zone, pvo_head, va,
VM_PAGE_TO_PHYS(m), pte_lo, pvo_flags);
+ PMAP_UNLOCK(pmap);
+ UNLOCK_TABLE_WR();
/*
* Flush the page from the instruction cache if this page is
* mapped executable and cacheable.
*/
- if ((pte_lo & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0)
+ if (pmap != kernel_pmap && !(m->aflags & PGA_EXECUTABLE) &&
+ (pte_lo & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
+ vm_page_aflag_set(m, PGA_EXECUTABLE);
moea64_syncicache(mmu, pmap, va, VM_PAGE_TO_PHYS(m), PAGE_SIZE);
+ }
}
static void
@@ -1307,15 +1292,11 @@ moea64_enter_object(mmu_t mmu, pmap_t pm
psize = atop(end - start);
m = m_start;
- LOCK_TABLE_WR();
- PMAP_LOCK(pm);
while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
- moea64_enter_locked(mmu, pm, start + ptoa(diff), m, prot &
+ moea64_enter(mmu, pm, start + ptoa(diff), m, prot &
(VM_PROT_READ | VM_PROT_EXECUTE), FALSE);
m = TAILQ_NEXT(m, listq);
}
- UNLOCK_TABLE_WR();
- PMAP_UNLOCK(pm);
}
void
@@ -1323,12 +1304,8 @@ moea64_enter_quick(mmu_t mmu, pmap_t pm,
vm_prot_t prot)
{
- LOCK_TABLE_WR();
- PMAP_LOCK(pm);
- moea64_enter_locked(mmu, pm, va, m,
+ moea64_enter(mmu, pm, va, m,
prot & (VM_PROT_READ | VM_PROT_EXECUTE), FALSE);
- UNLOCK_TABLE_WR();
- PMAP_UNLOCK(pm);
}
vm_paddr_t
@@ -1664,12 +1641,6 @@ moea64_kenter_attr(mmu_t mmu, vm_offset_
if (error != 0 && error != ENOENT)
panic("moea64_kenter: failed to enter va %#zx pa %#zx: %d", va,
pa, error);
-
- /*
- * Flush the memory from the instruction cache.
- */
- if ((pte_lo & (LPTE_I | LPTE_G)) == 0)
- __syncicache((void *)va, PAGE_SIZE);
}
void
@@ -1900,6 +1871,7 @@ static void
moea64_pvo_protect(mmu_t mmu, pmap_t pm, struct pvo_entry *pvo, vm_prot_t
prot)
{
uintptr_t pt;
+ struct vm_page *pg;
uint64_t oldlo;
PMAP_LOCK_ASSERT(pm, MA_OWNED);
@@ -1923,18 +1895,20 @@ moea64_pvo_protect(mmu_t mmu, pmap_t pm
else
pvo->pvo_pte.lpte.pte_lo |= LPTE_BR;
+ pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN);
+
/*
* If the PVO is in the page table, update that pte as well.
*/
- if (pt != -1) {
+ if (pt != -1)
MOEA64_PTE_CHANGE(mmu, pt, &pvo->pvo_pte.lpte,
pvo->pvo_vpn);
- if ((pvo->pvo_pte.lpte.pte_lo &
- (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
- moea64_syncicache(mmu, pm, PVO_VADDR(pvo),
- pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN,
- PAGE_SIZE);
- }
+ if (pm != kernel_pmap && pg != NULL && !(pg->aflags & PGA_EXECUTABLE) &&
+ (pvo->pvo_pte.lpte.pte_lo & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
+ if ((pg->oflags & VPO_UNMANAGED) == 0)
+ vm_page_aflag_set(pg, PGA_EXECUTABLE);
+ moea64_syncicache(mmu, pm, PVO_VADDR(pvo),
+ pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN, PAGE_SIZE);
}
/*
@@ -1943,9 +1917,6 @@ moea64_pvo_protect(mmu_t mmu, pmap_t pm
*/
if ((pvo->pvo_vaddr & PVO_MANAGED) == PVO_MANAGED &&
(oldlo & LPTE_PP) != LPTE_BR && !(prot && VM_PROT_WRITE)) {
- struct vm_page *pg;
-
- pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN);
if (pg != NULL) {
if (pvo->pvo_pte.lpte.pte_lo & LPTE_CHG)
vm_page_dirty(pg);
@@ -2128,15 +2099,11 @@ moea64_remove(mmu_t mmu, pmap_t pm, vm_o
void
moea64_remove_all(mmu_t mmu, vm_page_t m)
{
- struct pvo_head *pvo_head;
struct pvo_entry *pvo, *next_pvo;
pmap_t pmap;
- pvo_head = vm_page_to_pvoh(m);
LOCK_TABLE_WR();
- for (pvo = LIST_FIRST(pvo_head); pvo != NULL; pvo = next_pvo) {
- next_pvo = LIST_NEXT(pvo, pvo_vlink);
-
+ LIST_FOREACH_SAFE(pvo, vm_page_to_pvoh(m), pvo_vlink, next_pvo) {
pmap = pvo->pvo_pmap;
PMAP_LOCK(pmap);
moea64_pvo_remove(mmu, pvo);
@@ -2146,6 +2113,7 @@ moea64_remove_all(mmu_t mmu, vm_page_t m
if ((m->aflags & PGA_WRITEABLE) && moea64_is_modified(mmu, m))
vm_page_dirty(m);
vm_page_aflag_clear(m, PGA_WRITEABLE);
+ vm_page_aflag_clear(m, PGA_EXECUTABLE);
}
/*
@@ -2350,6 +2318,7 @@ moea64_pvo_enter(mmu_t mmu, pmap_t pm, u
static void
moea64_pvo_remove(mmu_t mmu, struct pvo_entry *pvo)
{
+ struct vm_page *pg;
uintptr_t pt;
PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);
@@ -2389,12 +2358,10 @@ moea64_pvo_remove(mmu_t mmu, struct pvo_
/*
* Update vm about the REF/CHG bits if the page is managed.
*/
- if ((pvo->pvo_vaddr & PVO_MANAGED) == PVO_MANAGED &&
- (pvo->pvo_pte.lpte.pte_lo & LPTE_PP) != LPTE_BR) {
- struct vm_page *pg;
+ pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN);
- pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN);
- if (pg != NULL) {
+ if ((pvo->pvo_vaddr & PVO_MANAGED) == PVO_MANAGED && pg != NULL) {
+ if ((pvo->pvo_pte.lpte.pte_lo & LPTE_PP) != LPTE_BR) {
if (pvo->pvo_pte.lpte.pte_lo & LPTE_CHG)
vm_page_dirty(pg);
if (pvo->pvo_pte.lpte.pte_lo & LPTE_REF)
@@ -2402,6 +2369,8 @@ moea64_pvo_remove(mmu_t mmu, struct pvo_
if (LIST_EMPTY(vm_page_to_pvoh(pg)))
vm_page_aflag_clear(pg, PGA_WRITEABLE);
}
+ if (LIST_EMPTY(vm_page_to_pvoh(pg)))
+ vm_page_aflag_clear(pg, PGA_EXECUTABLE);
}
moea64_pvo_entries--;
Modified: stable/9/sys/powerpc/aim/moea64_native.c
==============================================================================
--- stable/9/sys/powerpc/aim/moea64_native.c Fri Apr 20 00:34:15 2012
(r234475)
+++ stable/9/sys/powerpc/aim/moea64_native.c Fri Apr 20 00:45:23 2012
(r234476)
@@ -153,13 +153,10 @@ TLBIE(uint64_t vpn) {
vpn &= ~(0xffffULL << 48);
#ifdef __powerpc64__
- sched_pin();
- __asm __volatile("ptesync");
mtx_lock(&tlbie_mutex);
__asm __volatile("tlbie %0" :: "r"(vpn) : "memory");
mtx_unlock(&tlbie_mutex);
__asm __volatile("eieio; tlbsync; ptesync");
- sched_unpin();
#else
vpn_hi = (uint32_t)(vpn >> 32);
vpn_lo = (uint32_t)vpn;
@@ -171,7 +168,6 @@ TLBIE(uint64_t vpn) {
mr %1, %0; \
insrdi %1,%5,1,0; \
mtmsrd %1; isync; \
- ptesync; \
\
sld %1,%2,%4; \
or %1,%1,%3; \
@@ -265,7 +261,9 @@ moea64_pte_clear_native(mmu_t mmu, uintp
* As shown in Section 7.6.3.2.3
*/
pt->pte_lo &= ~ptebit;
+ sched_pin();
TLBIE(vpn);
+ sched_unpin();
}
static void
@@ -295,21 +293,16 @@ moea64_pte_unset_native(mmu_t mmu, uintp
{
struct lpte *pt = (struct lpte *)pt_cookie;
- pvo_pt->pte_hi &= ~LPTE_VALID;
-
- /* Finish all pending operations */
- isync();
-
- /*
- * Force the reg & chg bits back into the PTEs.
- */
- SYNC();
-
/*
* Invalidate the pte.
*/
+ isync();
+ sched_pin();
+ pvo_pt->pte_hi &= ~LPTE_VALID;
pt->pte_hi &= ~LPTE_VALID;
+ PTESYNC();
TLBIE(vpn);
+ sched_unpin();
/*
* Save the reg & chg bits.
Modified: stable/9/sys/vm/vm_page.h
==============================================================================
--- stable/9/sys/vm/vm_page.h Fri Apr 20 00:34:15 2012 (r234475)
+++ stable/9/sys/vm/vm_page.h Fri Apr 20 00:45:23 2012 (r234476)
@@ -248,9 +248,13 @@ extern struct vpglocks pa_lock[];
*
* PGA_WRITEABLE is set exclusively on managed pages by pmap_enter(). When it
* does so, the page must be VPO_BUSY.
+ *
+ * PGA_EXECUTABLE may be set by pmap routines, and indicates that a page has
+ * at least one executable mapping. It is not consumed by the VM layer.
*/
#define PGA_WRITEABLE 0x01 /* page may be mapped writeable
*/
#define PGA_REFERENCED 0x02 /* page has been referenced */
+#define PGA_EXECUTABLE 0x04 /* page may be mapped
executable */
/*
* Page flags. If changed at any other time than page allocation or
_______________________________________________
[email protected] mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-stable-9
To unsubscribe, send any mail to "[email protected]"
