CVS commit: [rmind-uvmplock] src/sys/arch/x86/x86
Module Name:src
Committed By: rmind
Date: Tue Mar 8 23:41:09 UTC 2011
Modified Files:
src/sys/arch/x86/x86 [rmind-uvmplock]: pmap_tlb.c
Log Message:
- pmap_tlb_shootdown: fix a bug when state for full TLB flush can be
reverted to a single page invalidation(s).
- pmap_tlb_init: clear pmap_tlb_packet and pmap_tlb_mailbox structs.
To generate a diff of this commit:
cvs rdiff -u -r1.1.2.3 -r1.1.2.4 src/sys/arch/x86/x86/pmap_tlb.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/x86/x86/pmap_tlb.c
diff -u src/sys/arch/x86/x86/pmap_tlb.c:1.1.2.3 src/sys/arch/x86/x86/pmap_tlb.c:1.1.2.4
--- src/sys/arch/x86/x86/pmap_tlb.c:1.1.2.3 Fri Jul 2 02:11:21 2010
+++ src/sys/arch/x86/x86/pmap_tlb.c Tue Mar 8 23:41:09 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap_tlb.c,v 1.1.2.3 2010/07/02 02:11:21 rmind Exp $ */
+/* $NetBSD: pmap_tlb.c,v 1.1.2.4 2011/03/08 23:41:09 rmind Exp $ */
/*-
* Copyright (c) 2008, 2010 The NetBSD Foundation, Inc.
@@ -40,7 +40,7 @@
*/
#include
-__KERNEL_RCSID(0, "$NetBSD: pmap_tlb.c,v 1.1.2.3 2010/07/02 02:11:21 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap_tlb.c,v 1.1.2.4 2011/03/08 23:41:09 rmind Exp $");
#include
#include
@@ -59,9 +59,9 @@
/*
* TLB shootdown state.
*/
-static struct evcnt pmap_tlb_evcnt __aligned(64);
-struct pmap_tlb_packet pmap_tlb_packet __aligned(64);
-struct pmap_tlb_mailbox pmap_tlb_mailbox __aligned(64);
+static struct evcnt pmap_tlb_evcnt __cacheline_aligned;
+struct pmap_tlb_packet pmap_tlb_packet __cacheline_aligned;
+struct pmap_tlb_mailbox pmap_tlb_mailbox __cacheline_aligned;
/*
* TLB shootdown statistics.
@@ -93,6 +93,10 @@
void
pmap_tlb_init(void)
{
+ int i = 0;
+
+ memset(&pmap_tlb_packet, 0, sizeof(struct pmap_tlb_packet));
+ memset(&pmap_tlb_mailbox, 0, sizeof(struct pmap_tlb_mailbox));
evcnt_attach_dynamic(&pmap_tlb_evcnt, EVCNT_TYPE_INTR,
NULL, "TLB", "shootdown");
@@ -177,9 +181,14 @@
CTASSERT(PG_G == (uint16_t)PG_G);
tp->tp_pte |= (uint16_t)pte;
- if (tp->tp_count < TP_MAXVA && va != (vaddr_t)-1LL) {
+ if (tp->tp_count == (uint16_t)-1) {
+ /*
+ * Already flushing everything.
+ */
+ } else if (tp->tp_count < TP_MAXVA && va != (vaddr_t)-1LL) {
/* Flush a single page. */
tp->tp_va[tp->tp_count++] = va;
+ KASSERT(tp->tp_count > 0);
} else {
/* Flush everything. */
tp->tp_count = (uint16_t)-1;
CVS commit: [rmind-uvmplock] src/sys/arch/x86/x86
Module Name:src
Committed By: rmind
Date: Tue Mar 8 23:26:36 UTC 2011
Modified Files:
src/sys/arch/x86/x86 [rmind-uvmplock]: pmap.c
Log Message:
pmap_deactivate: improve comment.
To generate a diff of this commit:
cvs rdiff -u -r1.105.2.11 -r1.105.2.12 src/sys/arch/x86/x86/pmap.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/x86/x86/pmap.c
diff -u src/sys/arch/x86/x86/pmap.c:1.105.2.11 src/sys/arch/x86/x86/pmap.c:1.105.2.12
--- src/sys/arch/x86/x86/pmap.c:1.105.2.11 Tue Mar 8 20:07:30 2011
+++ src/sys/arch/x86/x86/pmap.c Tue Mar 8 23:26:35 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.105.2.11 2011/03/08 20:07:30 rmind Exp $ */
+/* $NetBSD: pmap.c,v 1.105.2.12 2011/03/08 23:26:35 rmind Exp $ */
/*-
* Copyright (c) 2008, 2010 The NetBSD Foundation, Inc.
@@ -171,7 +171,7 @@
*/
#include
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.105.2.11 2011/03/08 20:07:30 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.105.2.12 2011/03/08 23:26:35 rmind Exp $");
#include "opt_user_ldt.h"
#include "opt_lockdebug.h"
@@ -2712,11 +2712,10 @@
}
/*
- * pmap_deactivate: deactivate a process' pmap
+ * pmap_deactivate: deactivate a process' pmap.
*
- * => must be called with kernel preemption disabled (high SPL is enough)
+ * => Must be called with kernel preemption disabled (high IPL is enough).
*/
-
void
pmap_deactivate(struct lwp *l)
{
@@ -2730,10 +2729,10 @@
}
/*
- * wait for pending TLB shootdowns to complete. necessary
- * because TLB shootdown state is per-CPU, and the LWP may
- * be coming off the CPU before it has a chance to call
- * pmap_update().
+ * Wait for pending TLB shootdowns to complete. Necessary because
+ * TLB shootdown state is per-CPU, and the LWP may be coming off
+ * the CPU before it has a chance to call pmap_update(), e.g. due
+ * to kernel preemption or blocking routine in between.
*/
pmap_tlb_shootnow();
CVS commit: [rmind-uvmplock] src/sys/arch/x86/x86
Module Name:src
Committed By: rmind
Date: Tue Mar 8 20:07:31 UTC 2011
Modified Files:
src/sys/arch/x86/x86 [rmind-uvmplock]: pmap.c
Log Message:
- pmap_remove_ptes: simplify by removing the duplicate code and re-using
equivalent functionality in pmap_remove_pte().
- pmap_remove_pte: fix assert to allow page owner lock to be unacquired
if pmap is pmap_kernel(); relevant for UVM_KMF_PAGEABLE memory case.
To generate a diff of this commit:
cvs rdiff -u -r1.105.2.10 -r1.105.2.11 src/sys/arch/x86/x86/pmap.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/x86/x86/pmap.c
diff -u src/sys/arch/x86/x86/pmap.c:1.105.2.10 src/sys/arch/x86/x86/pmap.c:1.105.2.11
--- src/sys/arch/x86/x86/pmap.c:1.105.2.10 Sat Mar 5 20:52:31 2011
+++ src/sys/arch/x86/x86/pmap.c Tue Mar 8 20:07:30 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.105.2.10 2011/03/05 20:52:31 rmind Exp $ */
+/* $NetBSD: pmap.c,v 1.105.2.11 2011/03/08 20:07:30 rmind Exp $ */
/*-
* Copyright (c) 2008, 2010 The NetBSD Foundation, Inc.
@@ -171,7 +171,7 @@
*/
#include
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.105.2.10 2011/03/05 20:52:31 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.105.2.11 2011/03/08 20:07:30 rmind Exp $");
#include "opt_user_ldt.h"
#include "opt_lockdebug.h"
@@ -3121,9 +3121,7 @@
pmap_remove_ptes(struct pmap *pmap, struct vm_page *ptp, vaddr_t ptpva,
vaddr_t startva, vaddr_t endva, struct pv_entry **pv_tofree)
{
- struct pv_entry *pve;
- pt_entry_t *pte = (pt_entry_t *) ptpva;
- pt_entry_t opte;
+ pt_entry_t *pte = (pt_entry_t *)ptpva;
KASSERT(pmap == pmap_kernel() || mutex_owned(pmap->pm_lock));
KASSERT(kpreempt_disabled());
@@ -3136,74 +3134,16 @@
* and the wire_count is greater than 1 (because we use the wire_count
* to keep track of the number of real PTEs in the PTP).
*/
-
- for (/*null*/; startva < endva && (ptp == NULL || ptp->wire_count > 1)
- ; pte++, startva += PAGE_SIZE) {
- struct vm_page *pg;
- struct pmap_page *pp;
-
- if (!pmap_valid_entry(*pte))
- continue; /* VA not mapped */
-
- /* atomically save the old PTE and zap! it */
- opte = pmap_pte_testset(pte, 0);
- if (!pmap_valid_entry(opte)) {
- continue;
- }
-
- pmap_exec_account(pmap, startva, opte, 0);
- pmap_stats_update_bypte(pmap, 0, opte);
-
- if (ptp) {
- ptp->wire_count--; /* dropping a PTE */
- /* Make sure that the PDE is flushed */
- if (ptp->wire_count <= 1)
-opte |= PG_U;
- }
-
- if ((opte & PG_U) != 0) {
- pmap_tlb_shootdown(pmap, startva, opte,
- TLBSHOOT_REMOVE_PTES);
- }
-
- /*
- * if we are not on a pv_head list we are done.
- */
-
- if ((opte & PG_PVLIST) == 0) {
-#if defined(DIAGNOSTIC) && !defined(DOM0OPS)
- if (PHYS_TO_VM_PAGE(pmap_pte2pa(opte)) != NULL)
-panic("pmap_remove_ptes: managed page without "
- "PG_PVLIST for %#" PRIxVADDR, startva);
-#endif
- continue;
- }
-
- pg = PHYS_TO_VM_PAGE(pmap_pte2pa(opte));
-
- KASSERTMSG(pg != NULL, ("pmap_remove_ptes: unmanaged page "
- "marked PG_PVLIST, va = %#" PRIxVADDR ", pa = %#" PRIxPADDR,
- startva, (paddr_t)pmap_pte2pa(opte)));
-
- KASSERT(uvm_page_locked_p(pg));
-
- /* sync R/M bits */
- pp = VM_PAGE_TO_PP(pg);
- pp->pp_attrs |= opte;
- pve = pmap_remove_pv(pp, ptp, startva);
-
- if (pve != NULL) {
- pve->pve_next = *pv_tofree;
- *pv_tofree = pve;
- }
-
- /* end of "for" loop: time for next pte */
+ while (startva < endva && (ptp == NULL || ptp->wire_count > 1)) {
+ (void)pmap_remove_pte(pmap, ptp, pte, startva, pv_tofree);
+ startva += PAGE_SIZE;
+ pte++;
}
}
/*
- * pmap_remove_pte: remove a single PTE from a PTP
+ * pmap_remove_pte: remove a single PTE from a PTP.
*
* => caller must hold pmap's lock
* => PTP must be mapped into KVA
@@ -3211,23 +3151,24 @@
* => returns true if we removed a mapping
* => must be called with kernel preemption disabled
*/
-
static bool
pmap_remove_pte(struct pmap *pmap, struct vm_page *ptp, pt_entry_t *pte,
vaddr_t va, struct pv_entry **pv_tofree)
{
- pt_entry_t opte;
struct pv_entry *pve;
struct vm_page *pg;
struct pmap_page *pp;
+ pt_entry_t opte;
KASSERT(pmap == pmap_kernel() || mutex_owned(pmap->pm_lock));
- KASSERT(pmap == pmap_kernel() || kpreempt_disabled());
+ KASSERT(kpreempt_disabled());
- if (!pmap_valid_entry(*pte))
- return(false); /* VA not mapped */
+ if (!pmap_valid_entry(*pte)) {
+ /* VA not mapped. */
+ return false;
+ }
- /* atomically save the old PTE and zap! it */
+ /* Atomically save the old PTE and zap it. */
opte = pmap_pte_testset(pte, 0);
if (!pmap_valid_entry(opte)) {
return false;
@@ -3237,10 +3178,13 @@
pmap_stats_update_bypte(pmap, 0, opte);
if (ptp) {
- ptp->wire_count--; /* dropping a PTE */
- /* Make sure that the PDE is flushed */
- if (ptp->wire_count <= 1)
+ /*
+ * Dropping a PTE. Make sure that the PDE is flu
CVS commit: [rmind-uvmplock] src/sys/arch/x86/x86
Module Name:src
Committed By: rmind
Date: Fri Jul 2 02:11:21 UTC 2010
Modified Files:
src/sys/arch/x86/x86 [rmind-uvmplock]: pmap_tlb.c
Log Message:
pmap_tlb_shootdown: add assert demonstrating assumption.
To generate a diff of this commit:
cvs rdiff -u -r1.1.2.2 -r1.1.2.3 src/sys/arch/x86/x86/pmap_tlb.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/x86/x86/pmap_tlb.c
diff -u src/sys/arch/x86/x86/pmap_tlb.c:1.1.2.2 src/sys/arch/x86/x86/pmap_tlb.c:1.1.2.3
--- src/sys/arch/x86/x86/pmap_tlb.c:1.1.2.2 Mon May 31 01:12:14 2010
+++ src/sys/arch/x86/x86/pmap_tlb.c Fri Jul 2 02:11:21 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap_tlb.c,v 1.1.2.2 2010/05/31 01:12:14 rmind Exp $ */
+/* $NetBSD: pmap_tlb.c,v 1.1.2.3 2010/07/02 02:11:21 rmind Exp $ */
/*-
* Copyright (c) 2008, 2010 The NetBSD Foundation, Inc.
@@ -40,7 +40,7 @@
*/
#include
-__KERNEL_RCSID(0, "$NetBSD: pmap_tlb.c,v 1.1.2.2 2010/05/31 01:12:14 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap_tlb.c,v 1.1.2.3 2010/07/02 02:11:21 rmind Exp $");
#include
#include
@@ -172,7 +172,11 @@
*/
s = splvm();
tp = (struct pmap_tlb_packet *)curcpu()->ci_pmap_data;
+
+ /* Whole address flush will be needed if PG_G is set. */
+ CTASSERT(PG_G == (uint16_t)PG_G);
tp->tp_pte |= (uint16_t)pte;
+
if (tp->tp_count < TP_MAXVA && va != (vaddr_t)-1LL) {
/* Flush a single page. */
tp->tp_va[tp->tp_count++] = va;
CVS commit: [rmind-uvmplock] src/sys/arch/x86/x86
Module Name:src
Committed By: rmind
Date: Wed May 26 04:57:22 UTC 2010
Modified Files:
src/sys/arch/x86/x86 [rmind-uvmplock]: pmap.c
Log Message:
pmap_map_ptes: handle emap on TLB flush.
To generate a diff of this commit:
cvs rdiff -u -r1.105.2.6 -r1.105.2.7 src/sys/arch/x86/x86/pmap.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/x86/x86/pmap.c
diff -u src/sys/arch/x86/x86/pmap.c:1.105.2.6 src/sys/arch/x86/x86/pmap.c:1.105.2.7
--- src/sys/arch/x86/x86/pmap.c:1.105.2.6 Wed May 26 04:55:23 2010
+++ src/sys/arch/x86/x86/pmap.c Wed May 26 04:57:21 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.105.2.6 2010/05/26 04:55:23 rmind Exp $ */
+/* $NetBSD: pmap.c,v 1.105.2.7 2010/05/26 04:57:21 rmind Exp $ */
/*-
* Copyright (c) 2008, 2010 The NetBSD Foundation, Inc.
@@ -177,7 +177,7 @@
*/
#include
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.105.2.6 2010/05/26 04:55:23 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.105.2.7 2010/05/26 04:57:21 rmind Exp $");
#include "opt_user_ldt.h"
#include "opt_lockdebug.h"
@@ -900,7 +900,7 @@
curpmap = ci->ci_pmap;
if (vm_map_pmap(&l->l_proc->p_vmspace->vm_map) == pmap) {
/* Our own pmap so just load it: easy. */
- if (ci->ci_want_pmapload) {
+ if (__predict_false(ci->ci_want_pmapload)) {
mutex_exit(pmap->pm_lock);
pmap_load();
goto retry;
@@ -913,7 +913,9 @@
* to the kernel pmap in order to destroy a user pmap.
*/
if (!pmap_reactivate(pmap)) {
+ u_int gen = uvm_emap_gen_return();
tlbflush();
+ uvm_emap_update(gen);
}
} else {
/*
@@ -980,7 +982,7 @@
/*
* We cannot tolerate context switches while mapped in.
- * if it's our own pmap all we have to do is unlock.
+ * If it is our own pmap all we have to do is unlock.
*/
KASSERT(pmap->pm_ncsw == curlwp->l_ncsw);
mypmap = vm_map_pmap(&curproc->p_vmspace->vm_map);
@@ -990,7 +992,7 @@
}
/*
- * Mark whatever's on the cpu now as lazy and unlock.
+ * Mark whatever's on the CPU now as lazy and unlock.
* If the pmap was already installed, we are done.
*/
ci = curcpu();
