Module Name: src
Committed By: matt
Date: Mon Jul 9 17:23:37 UTC 2012
Modified Files:
src/sys/arch/mips/include [matt-nb5-mips64]: cpu.h pmap.h
src/sys/arch/mips/mips [matt-nb5-mips64]: pmap.c pmap_segtab.c
pmap_tlb.c
Log Message:
Use a spinlock to protect the segtab queues. Use union pmap_segmap and
pmap_segmap_t to track -HEAD. Use KERNEL_PID for the same reason.
To generate a diff of this commit:
cvs rdiff -u -r1.90.16.44 -r1.90.16.45 src/sys/arch/mips/include/cpu.h
cvs rdiff -u -r1.54.26.25 -r1.54.26.26 src/sys/arch/mips/include/pmap.h
cvs rdiff -u -r1.179.16.44 -r1.179.16.45 src/sys/arch/mips/mips/pmap.c
cvs rdiff -u -r1.1.2.16 -r1.1.2.17 src/sys/arch/mips/mips/pmap_segtab.c
cvs rdiff -u -r1.1.2.23 -r1.1.2.24 src/sys/arch/mips/mips/pmap_tlb.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/mips/include/cpu.h
diff -u src/sys/arch/mips/include/cpu.h:1.90.16.44 src/sys/arch/mips/include/cpu.h:1.90.16.45
--- src/sys/arch/mips/include/cpu.h:1.90.16.44 Mon Feb 27 17:04:39 2012
+++ src/sys/arch/mips/include/cpu.h Mon Jul 9 17:23:37 2012
@@ -120,9 +120,9 @@ struct cpu_info {
int ci_tlb_slot; /* reserved tlb entry for cpu_info */
u_int ci_pmap_asid_cur; /* current ASID */
struct pmap_tlb_info *ci_tlb_info; /* tlb information for this cpu */
- union segtab *ci_pmap_seg0tab;
+ union pmap_segtab *ci_pmap_seg0tab;
#ifdef _LP64
- union segtab *ci_pmap_segtab;
+ union pmap_segtab *ci_pmap_segtab;
#else
vaddr_t ci_pmap_srcbase; /* starting VA of ephemeral src space */
vaddr_t ci_pmap_dstbase; /* starting VA of ephemeral dst space */
Index: src/sys/arch/mips/include/pmap.h
diff -u src/sys/arch/mips/include/pmap.h:1.54.26.25 src/sys/arch/mips/include/pmap.h:1.54.26.26
--- src/sys/arch/mips/include/pmap.h:1.54.26.25 Mon Feb 27 16:57:05 2012
+++ src/sys/arch/mips/include/pmap.h Mon Jul 9 17:23:37 2012
@@ -104,6 +104,8 @@
* dynamically allocated at boot time.
*/
+#define KERNEL_PID 0
+
#define mips_trunc_seg(x) ((vaddr_t)(x) & ~SEGOFSET)
#define mips_round_seg(x) (((vaddr_t)(x) + SEGOFSET) & ~SEGOFSET)
@@ -115,17 +117,15 @@
union pt_entry;
-union segtab {
-#ifdef _LP64
- union segtab *seg_seg[PMAP_SEGTABSIZE];
-#endif
- union pt_entry *seg_tab[PMAP_SEGTABSIZE];
-};
+typedef union pmap_segtab {
+ union pmap_segtab * seg_seg[PMAP_SEGTABSIZE];
+ union pt_entry * seg_tab[PMAP_SEGTABSIZE];
+} pmap_segtab_t;
#else
/*
* Modules don't need to know this.
*/
-union segtab;
+typedef union pmap_segtab pmap_segtab_t;
#endif
/*
@@ -155,6 +155,7 @@ void pmap_pte_process(struct pmap *, vad
void pmap_segtab_activate(struct pmap *, struct lwp *);
void pmap_segtab_init(struct pmap *);
void pmap_segtab_destroy(struct pmap *, pte_callback_t, uintptr_t);
+extern kmutex_t pmap_segtab_lock;
#endif /* _KERNEL */
/*
@@ -182,7 +183,7 @@ typedef struct pmap {
volatile uint32_t pm_onproc; /* pmap is active on ... */
volatile uint32_t pm_shootdown_pending;
#endif
- union segtab *pm_segtab; /* pointers to pages of PTEs */
+ pmap_segtab_t * pm_segtab; /* pointers to pages of PTEs */
u_int pm_count; /* pmap reference count */
u_int pm_flags;
#define PMAP_DEFERRED_ACTIVATE 0x0001
@@ -304,8 +305,10 @@ struct vm_page *mips_pmap_alloc_poolpage
#define POOL_VTOPHYS(va) (MIPS_KSEG0_P(va) \
? MIPS_KSEG0_TO_PHYS(va) \
: MIPS_XKPHYS_TO_PHYS(va))
+#define POOL_PHYSTOV(pa) MIPS_PHYS_TO_XKPHYS_CACHED((paddr_t)(pa))
#else
#define POOL_VTOPHYS(va) MIPS_KSEG0_TO_PHYS((vaddr_t)(va))
+#define POOL_PHYSTOV(pa) MIPS_PHYS_TO_KSEG0((paddr_t)(pa))
#endif
/*
Index: src/sys/arch/mips/mips/pmap.c
diff -u src/sys/arch/mips/mips/pmap.c:1.179.16.44 src/sys/arch/mips/mips/pmap.c:1.179.16.45
--- src/sys/arch/mips/mips/pmap.c:1.179.16.44 Mon Feb 27 16:57:06 2012
+++ src/sys/arch/mips/mips/pmap.c Mon Jul 9 17:23:37 2012
@@ -760,6 +760,11 @@ pmap_init(void)
#endif
/*
+ * Initialize the segtab lock.
+ */
+ mutex_init(&pmap_segtab_lock, MUTEX_DEFAULT, IPL_HIGH);
+
+ /*
* Set a low water mark on the pv_entry pool, so that we are
* more likely to have these around even in extreme memory
* starvation.
Index: src/sys/arch/mips/mips/pmap_segtab.c
diff -u src/sys/arch/mips/mips/pmap_segtab.c:1.1.2.16 src/sys/arch/mips/mips/pmap_segtab.c:1.1.2.17
--- src/sys/arch/mips/mips/pmap_segtab.c:1.1.2.16 Thu Jul 5 17:28:39 2012
+++ src/sys/arch/mips/mips/pmap_segtab.c Mon Jul 9 17:23:37 2012
@@ -119,13 +119,8 @@ __KERNEL_RCSID(0, "pmap_segtab.c,v 1.1.2
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/proc.h>
-#include <sys/pool.h>
#include <sys/mutex.h>
#include <sys/atomic.h>
-#ifdef SYSVSHM
-#include <sys/shm.h>
-#endif
-#include <sys/socketvar.h> /* XXX: for sock_loan_thresh */
#include <uvm/uvm.h>
@@ -134,25 +129,29 @@ __KERNEL_RCSID(0, "pmap_segtab.c,v 1.1.2
#include <mips/locore.h>
#include <mips/pte.h>
-CTASSERT(NBPG >= sizeof(union segtab));
+CTASSERT(NBPG >= sizeof(pmap_segtab_t));
+#define PMAP_PTP_CACHE
-union segtab * volatile free_segtab; /* free list kept locally */
+struct pmap_segtab_info {
+ pmap_segtab_t *free_segtab; /* free list kept locally */
#ifdef DEBUG
-uint32_t nget_segtab;
-uint32_t nput_segtab;
-uint32_t npage_segtab;
-#define SEGTAB_ADD(n, v) (n ## _segtab += (v))
+ uint32_t nget_segtab;
+ uint32_t nput_segtab;
+ uint32_t npage_segtab;
+#define SEGTAB_ADD(n, v) (pmap_segtab_info.n ## _segtab += (v))
#else
#define SEGTAB_ADD(n, v) ((void) 0)
#endif
-
-/*
- * Keep a list of idle page tables.
- */
-#define PMAP_PTP_CACHE
#ifdef PMAP_PTP_CACHE
-static struct pgflist ptp_pgflist = LIST_HEAD_INITIALIZER(ptp_pgflist);
+ struct pgflist ptp_pgflist; /* Keep a list of idle page tables. */
#endif
+} pmap_segtab_info = {
+#ifdef PMAP_PTP_CACHE
+ .ptp_pgflist = LIST_HEAD_INITIALIZER(pmap_segtab_info.ptp_pgflist),
+#endif
+};
+
+kmutex_t pmap_segtab_lock;
static inline struct vm_page *
pmap_pte_pagealloc(void)
@@ -171,17 +170,17 @@ pmap_pte_pagealloc(void)
return pg;
}
-static inline pt_entry_t *
+static inline pt_entry_t *
pmap_segmap(struct pmap *pmap, vaddr_t va)
{
- union segtab *stp = pmap->pm_segtab;
+ pmap_segtab_t *stp = pmap->pm_segtab;
KASSERT(!mm_md_direct_mapped_virt(va, NULL, NULL));
#ifdef _LP64
stp = stp->seg_seg[(va >> XSEGSHIFT) & (NSEGPG - 1)];
if (stp == NULL)
return NULL;
#endif
-
+
return stp->seg_tab[(va >> SEGSHIFT) & (PMAP_SEGTABSIZE - 1)];
}
@@ -196,28 +195,26 @@ pmap_pte_lookup(pmap_t pmap, vaddr_t va)
}
static void
-pmap_segtab_free(union segtab *stp)
+pmap_segtab_free(pmap_segtab_t *stp)
{
/*
* Insert the the segtab into the segtab freelist.
*/
- for (;;) {
- void *tmp = free_segtab;
- stp->seg_tab[0] = tmp;
- if (tmp == atomic_cas_ptr(&free_segtab, tmp, stp)) {
- SEGTAB_ADD(nput, 1);
- break;
- }
- }
+ mutex_spin_enter(&pmap_segtab_lock);
+ stp->seg_seg[0] = pmap_segtab_info.free_segtab;
+ pmap_segtab_info.free_segtab = stp;
+ SEGTAB_ADD(nput, 1);
+ mutex_spin_exit(&pmap_segtab_lock);
}
static void
-pmap_segtab_release(pmap_t pmap, union segtab **stp_p, bool free_stp,
+pmap_segtab_release(pmap_t pmap, pmap_segtab_t **stp_p, bool free_stp,
pte_callback_t callback, uintptr_t flags,
vaddr_t va, vsize_t vinc)
{
- union segtab *stp = *stp_p;
+ pmap_segtab_t *stp = *stp_p;
+ KASSERT(((va / vinc) & (PMAP_SEGTABSIZE - 1)) == 0);
for (size_t i = 0; i < PMAP_SEGTABSIZE; i++, va += vinc) {
#ifdef _LP64
if (vinc > NBSEG) {
@@ -241,12 +238,12 @@ pmap_segtab_release(pmap_t pmap, union s
*/
if (callback != NULL) {
(*callback)(pmap, va, va + vinc, pte, flags);
- }
+ }
#ifdef DEBUG
for (size_t j = 0; j < NPTEPG; j++) {
if (pte[j].pt_entry)
panic("%s: pte entry %p not 0 (%#x)",
- __func__, &pte[j], pte[j].pt_entry);
+ __func__, &pte[j], pte[j].pt_entry);
}
#endif
@@ -262,12 +259,12 @@ pmap_segtab_release(pmap_t pmap, union s
if (MIPS_CACHE_VIRTUAL_ALIAS)
mips_dcache_inv_range((vaddr_t)pte, PAGE_SIZE);
#endif /* MIPS3_PLUS */
- paddr_t pa = mips_pmap_unmap_poolpage((vaddr_t)pte);
+ paddr_t pa = PMAP_UNMAP_POOLPAGE((vaddr_t)pte);
struct vm_page *pg = PHYS_TO_VM_PAGE(pa);
#ifdef PMAP_PTP_CACHE
- mutex_spin_enter(&uvm_fpageqlock);
- LIST_INSERT_HEAD(&ptp_pgflist, pg, listq.list);
- mutex_spin_exit(&uvm_fpageqlock);
+ mutex_spin_enter(&pmap_segtab_lock);
+ LIST_INSERT_HEAD(&pmap_segtab_info.ptp_pgflist, pg, listq.list);
+ mutex_spin_exit(&pmap_segtab_lock);
#else
uvm_km_pagefree(pg);
#endif
@@ -293,25 +290,21 @@ pmap_segtab_release(pmap_t pmap, union s
* the map will be used in software only, and
* is bounded by that size.
*/
-static union segtab *
+static pmap_segtab_t *
pmap_segtab_alloc(void)
{
- union segtab *stp;
+ pmap_segtab_t *stp;
+
again:
- stp = NULL;
- while (__predict_true(free_segtab != NULL)) {
- union segtab *next_stp;
- stp = free_segtab;
- next_stp = (union segtab *)stp->seg_tab[0];
- if (stp == atomic_cas_ptr(&free_segtab, stp, next_stp)) {
- SEGTAB_ADD(nget, 1);
- break;
- }
+ mutex_spin_enter(&pmap_segtab_lock);
+ if (__predict_true((stp = pmap_segtab_info.free_segtab) != NULL)) {
+ pmap_segtab_info.free_segtab = stp->seg_seg[0];
+ stp->seg_seg[0] = NULL;
+ SEGTAB_ADD(nget, 1);
}
-
- if (__predict_true(stp != NULL)) {
- stp->seg_tab[0] = NULL;
- } else {
+ mutex_spin_exit(&pmap_segtab_lock);
+
+ if (__predict_false(stp == NULL)) {
struct vm_page * const stp_pg = pmap_pte_pagealloc();
if (__predict_false(stp_pg == NULL)) {
@@ -327,25 +320,23 @@ pmap_segtab_alloc(void)
#ifdef _LP64
KASSERT(mips_options.mips3_xkphys_cached);
#endif
- stp = (union segtab *)mips_pmap_map_poolpage(stp_pa);
- const size_t n = NBPG / sizeof(union segtab);
+ stp = (pmap_segtab_t *)PMAP_MAP_POOLPAGE(stp_pa);
+ const size_t n = NBPG / sizeof(*stp);
if (n > 1) {
/*
* link all the segtabs in this page together
*/
for (size_t i = 1; i < n - 1; i++) {
- stp[i].seg_tab[0] = (void *)&stp[i+1];
+ stp[i].seg_seg[0] = &stp[i+1];
}
/*
* Now link the new segtabs into the free segtab list.
*/
- for (;;) {
- void *tmp = free_segtab;
- stp[n-1].seg_tab[0] = tmp;
- if (tmp == atomic_cas_ptr(&free_segtab, tmp, stp+1))
- break;
- }
+ mutex_spin_enter(&pmap_segtab_lock);
+ stp[n-1].seg_seg[0] = pmap_segtab_info.free_segtab;
+ pmap_segtab_info.free_segtab = stp + 1;
SEGTAB_ADD(nput, n - 1);
+ mutex_spin_exit(&pmap_segtab_lock);
}
}
@@ -380,12 +371,12 @@ pmap_segtab_destroy(pmap_t pmap, pte_cal
return;
#ifdef _LP64
- pmap_segtab_release(pmap, &pmap->pm_segtab,
- func == NULL, func, flags, 0, NBXSEG);
+ const vsize_t vinc = NBXSEG;
#else
- pmap_segtab_release(pmap, &pmap->pm_segtab,
- func == NULL, func, flags, 0, NBSEG);
+ const vsize_t vinc = NBSEG;
#endif
+ pmap_segtab_release(pmap, &pmap->pm_segtab,
+ func == NULL, func, flags, 0, vinc);
}
/*
@@ -455,18 +446,18 @@ pmap_pte_process(pmap_t pmap, vaddr_t sv
pt_entry_t *
pmap_pte_reserve(pmap_t pmap, vaddr_t va, int flags)
{
- union segtab *stp = pmap->pm_segtab;
+ pmap_segtab_t *stp = pmap->pm_segtab;
pt_entry_t *pte;
pte = pmap_pte_lookup(pmap, va);
if (__predict_false(pte == NULL)) {
#ifdef _LP64
- union segtab ** const stp_p =
+ pmap_segtab_t ** const stp_p =
&stp->seg_seg[(va >> XSEGSHIFT) & (NSEGPG - 1)];
if (__predict_false((stp = *stp_p) == NULL)) {
- union segtab *nstp = pmap_segtab_alloc();
+ pmap_segtab_t *nstp = pmap_segtab_alloc();
#ifdef MULTIPROCESSOR
- union segtab *ostp = atomic_cas_ptr(stp_p, NULL, nstp);
+ pmap_segtab_t *ostp = atomic_cas_ptr(stp_p, NULL, nstp);
if (__predict_false(ostp != NULL)) {
pmap_segtab_free(nstp);
nstp = ostp;
@@ -479,13 +470,13 @@ pmap_pte_reserve(pmap_t pmap, vaddr_t va
KASSERT(stp == pmap->pm_segtab->seg_seg[(va >> XSEGSHIFT) & (NSEGPG - 1)]);
#endif /* _LP64 */
struct vm_page *pg = NULL;
-#if defined(PMAP_PTP_CACHE)
- mutex_spin_enter(&uvm_fpageqlock);
- if ((pg = LIST_FIRST(&ptp_pgflist)) != NULL) {
+#ifdef PMAP_PTP_CACHE
+ mutex_spin_enter(&pmap_segtab_lock);
+ if ((pg = LIST_FIRST(&pmap_segtab_info.ptp_pgflist)) != NULL) {
LIST_REMOVE(pg, listq.list);
- KASSERT(LIST_FIRST(&ptp_pgflist) != pg);
+ KASSERT(LIST_FIRST(&pmap_segtab_info.ptp_pgflist) != pg);
}
- mutex_spin_exit(&uvm_fpageqlock);
+ mutex_spin_exit(&pmap_segtab_lock);
#endif
if (pg == NULL)
pg = pmap_pte_pagealloc();
@@ -511,9 +502,10 @@ pmap_pte_reserve(pmap_t pmap, vaddr_t va
*/
if (__predict_false(opte != NULL)) {
#ifdef PMAP_PTP_CACHE
- mutex_spin_enter(&uvm_fpageqlock);
- LIST_INSERT_HEAD(&ptp_pgflist, pg, listq.list);
- mutex_spin_exit(&uvm_fpageqlock);
+ mutex_spin_enter(&pmap_segtab_lock);
+ LIST_INSERT_HEAD(&pmap_segtab_info.ptp_pgflist,
+ pg, listq.list);
+ mutex_spin_exit(&pmap_segtab_lock);
#else
uvm_km_pagefree(pg);
#endif
Index: src/sys/arch/mips/mips/pmap_tlb.c
diff -u src/sys/arch/mips/mips/pmap_tlb.c:1.1.2.23 src/sys/arch/mips/mips/pmap_tlb.c:1.1.2.24
--- src/sys/arch/mips/mips/pmap_tlb.c:1.1.2.23 Thu Feb 16 23:02:22 2012
+++ src/sys/arch/mips/mips/pmap_tlb.c Mon Jul 9 17:23:37 2012
@@ -191,7 +191,7 @@ pmap_pai_reset(struct pmap_tlb_info *ti,
/*
* We must have an ASID but it must not be onproc (on a processor).
*/
- KASSERT(pai->pai_asid);
+ KASSERT(pai->pai_asid > KERNEL_PID);
#ifdef MULTIPROCESSOR
KASSERT((pm->pm_onproc & ti->ti_cpu_mask) == 0);
#endif
@@ -412,7 +412,7 @@ pmap_tlb_asid_reinitialize(struct pmap_t
for (pai = LIST_FIRST(&ti->ti_pais); pai != NULL; pai = next) {
struct pmap * const pm = PAI_PMAP(pai, ti);
next = LIST_NEXT(pai, pai_link);
- KASSERT(pai->pai_asid != 0);
+ KASSERT(pai->pai_asid > KERNEL_PID);
#ifdef MULTIPROCESSOR
if (pm->pm_onproc & ti->ti_cpu_mask) {
if (!TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) {
@@ -470,7 +470,7 @@ pmap_tlb_shootdown_process(void)
* The victim is an active pmap so we will just
* invalidate its TLB entries.
*/
- KASSERT(pai->pai_asid != 0);
+ KASSERT(pai->pai_asid > KERNEL_PID);
pmap_tlb_asid_check();
tlb_invalidate_asids(pai->pai_asid, pai->pai_asid + 1);
pmap_tlb_asid_check();
@@ -568,7 +568,7 @@ pmap_tlb_target_bystander(struct pmap_tl
ti->ti_tlbinvop = TLBINV_KERNEL_MAP(ti->ti_tlbinvop);
ti->ti_victim = NULL;
} else {
- KASSERT(pai->pai_asid);
+ KASSERT(pai->pai_asid > KERNEL_PID);
if (__predict_false(ti->ti_victim == pm)) {
KASSERT(ti->ti_tlbinvop == TLBINV_ONE);
/*
