Author: alc
Date: Thu Jul 8 03:35:00 2010
New Revision: 209789
URL: http://svn.freebsd.org/changeset/base/209789
Log:
Correctly maintain the per-cpu field "curpmap" on amd64 just like we
do on i386. The consequences of not doing so on amd64 became apparent
with the introduction of the COUNT_IPIS and COUNT_XINVLTLB_HITS
options. Specifically, single-threaded applications were generating
unnecessary IPIs to shoot-down the TLB on other processors. However,
this is clearly nonsensical because a single-threaded application is
only running on the current processor. The reason that this happens
is that pmap_activate() is unable to properly update the old pmap's
field "pm_active" without the correct "curpmap". So, in effect, stale
bits in "pm_active" were leading pmap_protect(), pmap_remove(),
pmap_remove_pages(), etc. to flush the TLB contents on some arbitrary
processor that wasn't even running the same application.
Reviewed by: kib
MFC after: 3 weeks
Modified:
head/sys/amd64/amd64/cpu_switch.S
head/sys/amd64/amd64/pmap.c
Modified: head/sys/amd64/amd64/cpu_switch.S
==============================================================================
--- head/sys/amd64/amd64/cpu_switch.S Thu Jul 8 03:28:25 2010
(r209788)
+++ head/sys/amd64/amd64/cpu_switch.S Thu Jul 8 03:35:00 2010
(r209789)
@@ -69,16 +69,13 @@
* %rsi = newtd
*/
ENTRY(cpu_throw)
+ movl PCPU(CPUID),%eax
testq %rdi,%rdi
- jnz 1f
- movq PCPU(IDLETHREAD),%rdi
-1:
- movq TD_PCB(%rdi),%r8 /* Old pcb */
- movl PCPU(CPUID), %eax
+ jz 1f
/* release bit from old pm_active */
- movq TD_PROC(%rdi), %rdx /* oldtd->td_proc */
- movq P_VMSPACE(%rdx), %rdx /* proc->p_vmspace */
- LK btrl %eax, VM_PMAP+PM_ACTIVE(%rdx) /* clear old */
+ movq PCPU(CURPMAP),%rdx
+ LK btrl %eax,PM_ACTIVE(%rdx) /* clear old */
+1:
movq TD_PCB(%rsi),%r8 /* newtd->td_proc */
movq PCB_CR3(%r8),%rdx
movq %rdx,%cr3 /* new address space */
@@ -140,15 +137,16 @@ swinact:
movq %rcx,%cr3 /* new address space */
movl PCPU(CPUID), %eax
/* Release bit from old pmap->pm_active */
- movq TD_PROC(%rdi), %rcx /* oldproc */
- movq P_VMSPACE(%rcx), %rcx
- LK btrl %eax, VM_PMAP+PM_ACTIVE(%rcx) /* clear old */
+ movq PCPU(CURPMAP),%rcx
+ LK btrl %eax,PM_ACTIVE(%rcx) /* clear old */
SETLK %rdx, TD_LOCK(%rdi) /* Release the old thread */
swact:
/* Set bit in new pmap->pm_active */
movq TD_PROC(%rsi),%rdx /* newproc */
movq P_VMSPACE(%rdx), %rdx
- LK btsl %eax, VM_PMAP+PM_ACTIVE(%rdx) /* set new */
+ addq $VM_PMAP,%rdx
+ LK btsl %eax,PM_ACTIVE(%rdx) /* set new */
+ movq %rdx,PCPU(CURPMAP)
sw1:
#if defined(SCHED_ULE) && defined(SMP)
Modified: head/sys/amd64/amd64/pmap.c
==============================================================================
--- head/sys/amd64/amd64/pmap.c Thu Jul 8 03:28:25 2010 (r209788)
+++ head/sys/amd64/amd64/pmap.c Thu Jul 8 03:35:00 2010 (r209789)
@@ -1574,6 +1574,7 @@ pmap_pinit0(pmap_t pmap)
pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(KPML4phys);
pmap->pm_root = NULL;
pmap->pm_active = 0;
+ PCPU_SET(curpmap, pmap);
TAILQ_INIT(&pmap->pm_pvchunk);
bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
}
@@ -5008,6 +5009,7 @@ if (oldpmap) /* XXX FIXME */
cr3 = DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml4);
td->td_pcb->pcb_cr3 = cr3;
load_cr3(cr3);
+ PCPU_SET(curpmap, pmap);
critical_exit();
}
_______________________________________________
svn-src-all@freebsd.org mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
