Author: attilio
Date: Mon Jul 4 12:04:52 2011
New Revision: 223758
URL: http://svn.freebsd.org/changeset/base/223758
Log:
With retirement of cpumask_t and usage of cpuset_t for representing a
mask of CPUs, pc_other_cpus and pc_cpumask become highly inefficient.
Remove them and replace their usage with custom pc_cpuid magic (as,
atm, pc_cpumask can be easilly represented by (1 << pc_cpuid) and
pc_other_cpus by (all_cpus & ~(1 << pc_cpuid))).
This change is not targeted for MFC because of struct pcpu members
removal and dependency by cpumask_t retirement.
MD review by: marcel, marius, alc
Tested by: pluknet
MD testing by: marcel, marius, gonzo, andreast
Modified:
head/lib/libkvm/kvm_pcpu.c
head/sys/amd64/acpica/acpi_wakeup.c
head/sys/amd64/amd64/mp_machdep.c
head/sys/amd64/amd64/pmap.c
head/sys/amd64/amd64/vm_machdep.c
head/sys/cddl/dev/dtrace/amd64/dtrace_subr.c
head/sys/cddl/dev/dtrace/i386/dtrace_subr.c
head/sys/dev/xen/control/control.c
head/sys/i386/i386/mp_machdep.c
head/sys/i386/i386/pmap.c
head/sys/i386/i386/vm_machdep.c
head/sys/i386/xen/mp_machdep.c
head/sys/i386/xen/pmap.c
head/sys/ia64/ia64/mp_machdep.c
head/sys/kern/kern_rmlock.c
head/sys/kern/sched_4bsd.c
head/sys/kern/subr_kdb.c
head/sys/kern/subr_pcpu.c
head/sys/kern/subr_smp.c
head/sys/mips/mips/mp_machdep.c
head/sys/mips/mips/pmap.c
head/sys/powerpc/aim/mmu_oea.c
head/sys/powerpc/aim/mmu_oea64.c
head/sys/powerpc/booke/pmap.c
head/sys/powerpc/powerpc/mp_machdep.c
head/sys/sparc64/sparc64/mp_machdep.c
head/sys/sys/pcpu.h
Directory Properties:
head/ (props changed)
head/cddl/contrib/opensolaris/ (props changed)
head/contrib/bind9/ (props changed)
head/contrib/binutils/ (props changed)
head/contrib/bzip2/ (props changed)
head/contrib/compiler-rt/ (props changed)
head/contrib/dialog/ (props changed)
head/contrib/ee/ (props changed)
head/contrib/expat/ (props changed)
head/contrib/file/ (props changed)
head/contrib/gcc/ (props changed)
head/contrib/gdb/ (props changed)
head/contrib/gdtoa/ (props changed)
head/contrib/gnu-sort/ (props changed)
head/contrib/groff/ (props changed)
head/contrib/less/ (props changed)
head/contrib/libpcap/ (props changed)
head/contrib/libstdc++/ (props changed)
head/contrib/llvm/ (props changed)
head/contrib/llvm/tools/clang/ (props changed)
head/contrib/ncurses/ (props changed)
head/contrib/netcat/ (props changed)
head/contrib/ntp/ (props changed)
head/contrib/one-true-awk/ (props changed)
head/contrib/openbsm/ (props changed)
head/contrib/openpam/ (props changed)
head/contrib/pf/ (props changed)
head/contrib/sendmail/ (props changed)
head/contrib/tcpdump/ (props changed)
head/contrib/tcsh/ (props changed)
head/contrib/tnftp/ (props changed)
head/contrib/top/ (props changed)
head/contrib/top/install-sh (props changed)
head/contrib/tzcode/stdtime/ (props changed)
head/contrib/tzcode/zic/ (props changed)
head/contrib/tzdata/ (props changed)
head/contrib/wpa/ (props changed)
head/contrib/xz/ (props changed)
head/crypto/openssh/ (props changed)
head/crypto/openssl/ (props changed)
head/gnu/lib/ (props changed)
head/gnu/usr.bin/binutils/ (props changed)
head/gnu/usr.bin/cc/cc_tools/ (props changed)
head/gnu/usr.bin/gdb/ (props changed)
head/lib/libc/ (props changed)
head/lib/libc/stdtime/ (props changed)
head/lib/libutil/ (props changed)
head/lib/libz/ (props changed)
head/sbin/ (props changed)
head/sbin/ipfw/ (props changed)
head/share/mk/bsd.arch.inc.mk (props changed)
head/share/zoneinfo/ (props changed)
head/sys/ (props changed)
head/sys/amd64/include/xen/ (props changed)
head/sys/boot/ (props changed)
head/sys/boot/i386/efi/ (props changed)
head/sys/boot/ia64/efi/ (props changed)
head/sys/boot/ia64/ski/ (props changed)
head/sys/boot/powerpc/boot1.chrp/ (props changed)
head/sys/boot/powerpc/ofw/ (props changed)
head/sys/cddl/contrib/opensolaris/ (props changed)
head/sys/conf/ (props changed)
head/sys/contrib/dev/acpica/ (props changed)
head/sys/contrib/octeon-sdk/ (props changed)
head/sys/contrib/pf/ (props changed)
head/sys/contrib/x86emu/ (props changed)
head/usr.bin/calendar/ (props changed)
head/usr.bin/csup/ (props changed)
head/usr.bin/procstat/ (props changed)
head/usr.sbin/ndiscvt/ (props changed)
head/usr.sbin/zic/ (props changed)
Modified: head/lib/libkvm/kvm_pcpu.c
==============================================================================
--- head/lib/libkvm/kvm_pcpu.c Mon Jul 4 11:13:00 2011 (r223757)
+++ head/lib/libkvm/kvm_pcpu.c Mon Jul 4 12:04:52 2011 (r223758)
@@ -39,13 +39,11 @@
__FBSDID("$FreeBSD$");
#include <sys/param.h>
-#include <sys/cpuset.h>
#include <sys/pcpu.h>
#include <sys/sysctl.h>
#include <kvm.h>
#include <limits.h>
#include <stdlib.h>
-#include <unistd.h>
#include "kvm_private.h"
@@ -120,9 +118,6 @@ _kvm_pcpu_clear(void)
void *
kvm_getpcpu(kvm_t *kd, int cpu)
{
- long kcpusetsize;
- ssize_t nbytes;
- uintptr_t readptr;
char *buf;
if (kd == NULL) {
@@ -130,10 +125,6 @@ kvm_getpcpu(kvm_t *kd, int cpu)
return (NULL);
}
- kcpusetsize = sysconf(_SC_CPUSET_SIZE);
- if (kcpusetsize == -1 || (u_long)kcpusetsize > sizeof(cpuset_t))
- return ((void *)-1);
-
if (maxcpu == 0)
if (_kvm_pcpu_init(kd) < 0)
return ((void *)-1);
@@ -146,26 +137,8 @@ kvm_getpcpu(kvm_t *kd, int cpu)
_kvm_err(kd, kd->program, "out of memory");
return ((void *)-1);
}
- nbytes = sizeof(struct pcpu) - 2 * kcpusetsize;
- readptr = (uintptr_t)pcpu_data[cpu];
- if (kvm_read(kd, readptr, buf, nbytes) != nbytes) {
- _kvm_err(kd, kd->program, "unable to read per-CPU data");
- free(buf);
- return ((void *)-1);
- }
-
- /* Fetch the valid cpuset_t objects. */
- CPU_ZERO((cpuset_t *)(buf + nbytes));
- CPU_ZERO((cpuset_t *)(buf + nbytes + sizeof(cpuset_t)));
- readptr += nbytes;
- if (kvm_read(kd, readptr, buf + nbytes, kcpusetsize) != kcpusetsize) {
- _kvm_err(kd, kd->program, "unable to read per-CPU data");
- free(buf);
- return ((void *)-1);
- }
- readptr += kcpusetsize;
- if (kvm_read(kd, readptr, buf + nbytes + sizeof(cpuset_t),
- kcpusetsize) != kcpusetsize) {
+ if (kvm_read(kd, (uintptr_t)pcpu_data[cpu], buf,
+ sizeof(struct pcpu)) != sizeof(struct pcpu)) {
_kvm_err(kd, kd->program, "unable to read per-CPU data");
free(buf);
return ((void *)-1);
Modified: head/sys/amd64/acpica/acpi_wakeup.c
==============================================================================
--- head/sys/amd64/acpica/acpi_wakeup.c Mon Jul 4 11:13:00 2011
(r223757)
+++ head/sys/amd64/acpica/acpi_wakeup.c Mon Jul 4 12:04:52 2011
(r223758)
@@ -226,7 +226,8 @@ acpi_sleep_machdep(struct acpi_softc *sc
return (ret);
#ifdef SMP
- wakeup_cpus = PCPU_GET(other_cpus);
+ wakeup_cpus = all_cpus;
+ CPU_CLR(PCPU_GET(cpuid), &wakeup_cpus);
#endif
AcpiSetFirmwareWakingVector(WAKECODE_PADDR(sc));
Modified: head/sys/amd64/amd64/mp_machdep.c
==============================================================================
--- head/sys/amd64/amd64/mp_machdep.c Mon Jul 4 11:13:00 2011
(r223757)
+++ head/sys/amd64/amd64/mp_machdep.c Mon Jul 4 12:04:52 2011
(r223758)
@@ -604,10 +604,10 @@ cpu_mp_announce(void)
void
init_secondary(void)
{
- cpuset_t tcpuset, tallcpus;
struct pcpu *pc;
struct nmi_pcpu *np;
u_int64_t msr, cr0;
+ u_int cpuid;
int cpu, gsel_tss, x;
struct region_descriptor ap_gdt;
@@ -711,8 +711,9 @@ init_secondary(void)
fpuinit();
/* A quick check from sanity claus */
+ cpuid = PCPU_GET(cpuid);
if (PCPU_GET(apic_id) != lapic_id()) {
- printf("SMP: cpuid = %d\n", PCPU_GET(cpuid));
+ printf("SMP: cpuid = %d\n", cpuid);
printf("SMP: actual apic_id = %d\n", lapic_id());
printf("SMP: correct apic_id = %d\n", PCPU_GET(apic_id));
panic("cpuid mismatch! boom!!");
@@ -734,19 +735,13 @@ init_secondary(void)
smp_cpus++;
- CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", PCPU_GET(cpuid));
- printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid));
- tcpuset = PCPU_GET(cpumask);
+ CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", cpuid);
+ printf("SMP: AP CPU #%d Launched!\n", cpuid);
/* Determine if we are a logical CPU. */
/* XXX Calculation depends on cpu_logical being a power of 2, e.g. 2 */
if (cpu_logical > 1 && PCPU_GET(apic_id) % cpu_logical != 0)
- CPU_OR(&logical_cpus_mask, &tcpuset);
-
- /* Build our map of 'other' CPUs. */
- tallcpus = all_cpus;
- CPU_NAND(&tallcpus, &tcpuset);
- PCPU_SET(other_cpus, tallcpus);
+ CPU_SET(cpuid, &logical_cpus_mask);
if (bootverbose)
lapic_dump("AP");
@@ -893,7 +888,6 @@ assign_cpu_ids(void)
static int
start_all_aps(void)
{
- cpuset_t tallcpus, tcpuset;
vm_offset_t va = boot_address + KERNBASE;
u_int64_t *pt4, *pt3, *pt2;
u_int32_t mpbioswarmvec;
@@ -961,12 +955,6 @@ start_all_aps(void)
CPU_SET(cpu, &all_cpus); /* record AP in CPU map */
}
- /* build our map of 'other' CPUs */
- tallcpus = all_cpus;
- tcpuset = PCPU_GET(cpumask);
- CPU_NAND(&tallcpus, &tcpuset);
- PCPU_SET(other_cpus, tallcpus);
-
/* restore the warmstart vector */
*(u_int32_t *) WARMBOOT_OFF = mpbioswarmvec;
@@ -1150,9 +1138,7 @@ smp_targeted_tlb_shootdown(cpuset_t mask
if (othercpus < 1)
return;
} else {
- sched_pin();
- CPU_NAND(&mask, PCPU_PTR(cpumask));
- sched_unpin();
+ CPU_CLR(PCPU_GET(cpuid), &mask);
if (CPU_EMPTY(&mask))
return;
}
@@ -1349,11 +1335,13 @@ ipi_cpu(int cpu, u_int ipi)
void
ipi_all_but_self(u_int ipi)
{
+ cpuset_t other_cpus;
+
+ other_cpus = all_cpus;
+ CPU_CLR(PCPU_GET(cpuid), &other_cpus);
- sched_pin();
if (IPI_IS_BITMAPED(ipi)) {
- ipi_selected(PCPU_GET(other_cpus), ipi);
- sched_unpin();
+ ipi_selected(other_cpus, ipi);
return;
}
@@ -1363,8 +1351,7 @@ ipi_all_but_self(u_int ipi)
* Set the mask of receiving CPUs for this purpose.
*/
if (ipi == IPI_STOP_HARD)
- CPU_OR_ATOMIC(&ipi_nmi_pending, PCPU_PTR(other_cpus));
- sched_unpin();
+ CPU_OR_ATOMIC(&ipi_nmi_pending, &other_cpus);
CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS);
@@ -1373,7 +1360,7 @@ ipi_all_but_self(u_int ipi)
int
ipi_nmi_handler()
{
- cpuset_t cpumask;
+ u_int cpuid;
/*
* As long as there is not a simple way to know about a NMI's
@@ -1381,13 +1368,11 @@ ipi_nmi_handler()
* the global pending bitword an IPI_STOP_HARD has been issued
* and should be handled.
*/
- sched_pin();
- cpumask = PCPU_GET(cpumask);
- sched_unpin();
- if (!CPU_OVERLAP(&ipi_nmi_pending, &cpumask))
+ cpuid = PCPU_GET(cpuid);
+ if (!CPU_ISSET(cpuid, &ipi_nmi_pending))
return (1);
- CPU_NAND_ATOMIC(&ipi_nmi_pending, &cpumask);
+ CPU_CLR_ATOMIC(cpuid, &ipi_nmi_pending);
cpustop_handler();
return (0);
}
@@ -1399,25 +1384,21 @@ ipi_nmi_handler()
void
cpustop_handler(void)
{
- cpuset_t cpumask;
u_int cpu;
- sched_pin();
cpu = PCPU_GET(cpuid);
- cpumask = PCPU_GET(cpumask);
- sched_unpin();
savectx(&stoppcbs[cpu]);
/* Indicate that we are stopped */
- CPU_OR_ATOMIC(&stopped_cpus, &cpumask);
+ CPU_SET_ATOMIC(cpu, &stopped_cpus);
/* Wait for restart */
- while (!CPU_OVERLAP(&started_cpus, &cpumask))
+ while (!CPU_ISSET(cpu, &started_cpus))
ia32_pause();
- CPU_NAND_ATOMIC(&started_cpus, &cpumask);
- CPU_NAND_ATOMIC(&stopped_cpus, &cpumask);
+ CPU_CLR_ATOMIC(cpu, &started_cpus);
+ CPU_CLR_ATOMIC(cpu, &stopped_cpus);
if (cpu == 0 && cpustop_restartfunc != NULL) {
cpustop_restartfunc();
@@ -1432,19 +1413,17 @@ cpustop_handler(void)
void
cpususpend_handler(void)
{
- cpuset_t cpumask;
register_t cr3, rf;
u_int cpu;
cpu = PCPU_GET(cpuid);
- cpumask = PCPU_GET(cpumask);
rf = intr_disable();
cr3 = rcr3();
if (savectx(susppcbs[cpu])) {
wbinvd();
- CPU_OR_ATOMIC(&stopped_cpus, &cpumask);
+ CPU_SET_ATOMIC(cpu, &stopped_cpus);
} else {
pmap_init_pat();
PCPU_SET(switchtime, 0);
@@ -1452,11 +1431,11 @@ cpususpend_handler(void)
}
/* Wait for resume */
- while (!CPU_OVERLAP(&started_cpus, &cpumask))
+ while (!CPU_ISSET(cpu, &started_cpus))
ia32_pause();
- CPU_NAND_ATOMIC(&started_cpus, &cpumask);
- CPU_NAND_ATOMIC(&stopped_cpus, &cpumask);
+ CPU_CLR_ATOMIC(cpu, &started_cpus);
+ CPU_CLR_ATOMIC(cpu, &stopped_cpus);
/* Restore CR3 and enable interrupts */
load_cr3(cr3);
Modified: head/sys/amd64/amd64/pmap.c
==============================================================================
--- head/sys/amd64/amd64/pmap.c Mon Jul 4 11:13:00 2011 (r223757)
+++ head/sys/amd64/amd64/pmap.c Mon Jul 4 12:04:52 2011 (r223758)
@@ -925,16 +925,18 @@ pmap_update_pde_invalidate(vm_offset_t v
void
pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
{
- cpuset_t cpumask, other_cpus;
+ cpuset_t other_cpus;
+ u_int cpuid;
sched_pin();
if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) {
invlpg(va);
smp_invlpg(va);
} else {
- cpumask = PCPU_GET(cpumask);
- other_cpus = PCPU_GET(other_cpus);
- if (CPU_OVERLAP(&pmap->pm_active, &cpumask))
+ cpuid = PCPU_GET(cpuid);
+ other_cpus = all_cpus;
+ CPU_CLR(cpuid, &other_cpus);
+ if (CPU_ISSET(cpuid, &pmap->pm_active))
invlpg(va);
CPU_AND(&other_cpus, &pmap->pm_active);
if (!CPU_EMPTY(&other_cpus))
@@ -946,8 +948,9 @@ pmap_invalidate_page(pmap_t pmap, vm_off
void
pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
{
- cpuset_t cpumask, other_cpus;
+ cpuset_t other_cpus;
vm_offset_t addr;
+ u_int cpuid;
sched_pin();
if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) {
@@ -955,9 +958,10 @@ pmap_invalidate_range(pmap_t pmap, vm_of
invlpg(addr);
smp_invlpg_range(sva, eva);
} else {
- cpumask = PCPU_GET(cpumask);
- other_cpus = PCPU_GET(other_cpus);
- if (CPU_OVERLAP(&pmap->pm_active, &cpumask))
+ cpuid = PCPU_GET(cpuid);
+ other_cpus = all_cpus;
+ CPU_CLR(cpuid, &other_cpus);
+ if (CPU_ISSET(cpuid, &pmap->pm_active))
for (addr = sva; addr < eva; addr += PAGE_SIZE)
invlpg(addr);
CPU_AND(&other_cpus, &pmap->pm_active);
@@ -970,16 +974,18 @@ pmap_invalidate_range(pmap_t pmap, vm_of
void
pmap_invalidate_all(pmap_t pmap)
{
- cpuset_t cpumask, other_cpus;
+ cpuset_t other_cpus;
+ u_int cpuid;
sched_pin();
if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) {
invltlb();
smp_invltlb();
} else {
- cpumask = PCPU_GET(cpumask);
- other_cpus = PCPU_GET(other_cpus);
- if (CPU_OVERLAP(&pmap->pm_active, &cpumask))
+ cpuid = PCPU_GET(cpuid);
+ other_cpus = all_cpus;
+ CPU_CLR(cpuid, &other_cpus);
+ if (CPU_ISSET(cpuid, &pmap->pm_active))
invltlb();
CPU_AND(&other_cpus, &pmap->pm_active);
if (!CPU_EMPTY(&other_cpus))
@@ -999,11 +1005,11 @@ pmap_invalidate_cache(void)
}
struct pde_action {
- cpuset_t store; /* processor that updates the PDE */
cpuset_t invalidate; /* processors that invalidate their TLB */
vm_offset_t va;
pd_entry_t *pde;
pd_entry_t newpde;
+ u_int store; /* processor that updates the PDE */
};
static void
@@ -1011,12 +1017,8 @@ pmap_update_pde_action(void *arg)
{
struct pde_action *act = arg;
- sched_pin();
- if (!CPU_CMP(&act->store, PCPU_PTR(cpumask))) {
- sched_unpin();
+ if (act->store == PCPU_GET(cpuid))
pde_store(act->pde, act->newpde);
- } else
- sched_unpin();
}
static void
@@ -1024,12 +1026,8 @@ pmap_update_pde_teardown(void *arg)
{
struct pde_action *act = arg;
- sched_pin();
- if (CPU_OVERLAP(&act->invalidate, PCPU_PTR(cpumask))) {
- sched_unpin();
+ if (CPU_ISSET(PCPU_GET(cpuid), &act->invalidate))
pmap_update_pde_invalidate(act->va, act->newpde);
- } else
- sched_unpin();
}
/*
@@ -1044,28 +1042,30 @@ static void
pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t
newpde)
{
struct pde_action act;
- cpuset_t active, cpumask, other_cpus;
+ cpuset_t active, other_cpus;
+ u_int cpuid;
sched_pin();
- cpumask = PCPU_GET(cpumask);
- other_cpus = PCPU_GET(other_cpus);
+ cpuid = PCPU_GET(cpuid);
+ other_cpus = all_cpus;
+ CPU_CLR(cpuid, &other_cpus);
if (pmap == kernel_pmap)
active = all_cpus;
else
active = pmap->pm_active;
if (CPU_OVERLAP(&active, &other_cpus)) {
- act.store = cpumask;
+ act.store = cpuid;
act.invalidate = active;
act.va = va;
act.pde = pde;
act.newpde = newpde;
- CPU_OR(&cpumask, &active);
- smp_rendezvous_cpus(cpumask,
+ CPU_SET(cpuid, &active);
+ smp_rendezvous_cpus(active,
smp_no_rendevous_barrier, pmap_update_pde_action,
pmap_update_pde_teardown, &act);
} else {
pde_store(pde, newpde);
- if (CPU_OVERLAP(&active, &cpumask))
+ if (CPU_ISSET(cpuid, &active))
pmap_update_pde_invalidate(va, newpde);
}
sched_unpin();
@@ -5095,17 +5095,19 @@ void
pmap_activate(struct thread *td)
{
pmap_t pmap, oldpmap;
+ u_int cpuid;
u_int64_t cr3;
critical_enter();
pmap = vmspace_pmap(td->td_proc->p_vmspace);
oldpmap = PCPU_GET(curpmap);
+ cpuid = PCPU_GET(cpuid);
#ifdef SMP
- CPU_NAND_ATOMIC(&oldpmap->pm_active, PCPU_PTR(cpumask));
- CPU_OR_ATOMIC(&pmap->pm_active, PCPU_PTR(cpumask));
+ CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active);
+ CPU_SET_ATOMIC(cpuid, &pmap->pm_active);
#else
- CPU_NAND(&oldpmap->pm_active, PCPU_PTR(cpumask));
- CPU_OR(&pmap->pm_active, PCPU_PTR(cpumask));
+ CPU_CLR(cpuid, &oldpmap->pm_active);
+ CPU_SET(cpuid, &pmap->pm_active);
#endif
cr3 = DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml4);
td->td_pcb->pcb_cr3 = cr3;
Modified: head/sys/amd64/amd64/vm_machdep.c
==============================================================================
--- head/sys/amd64/amd64/vm_machdep.c Mon Jul 4 11:13:00 2011
(r223757)
+++ head/sys/amd64/amd64/vm_machdep.c Mon Jul 4 12:04:52 2011
(r223758)
@@ -59,7 +59,6 @@ __FBSDID("$FreeBSD$");
#include <sys/mutex.h>
#include <sys/pioctl.h>
#include <sys/proc.h>
-#include <sys/sched.h>
#include <sys/sf_buf.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
@@ -535,8 +534,8 @@ cpu_reset()
u_int cnt;
if (smp_active) {
- sched_pin();
- map = PCPU_GET(other_cpus);
+ map = all_cpus;
+ CPU_CLR(PCPU_GET(cpuid), &map);
CPU_NAND(&map, &stopped_cpus);
if (!CPU_EMPTY(&map)) {
printf("cpu_reset: Stopping other CPUs\n");
@@ -545,7 +544,6 @@ cpu_reset()
if (PCPU_GET(cpuid) != 0) {
cpu_reset_proxyid = PCPU_GET(cpuid);
- sched_unpin();
cpustop_restartfunc = cpu_reset_proxy;
cpu_reset_proxy_active = 0;
printf("cpu_reset: Restarting BSP\n");
@@ -564,8 +562,7 @@ cpu_reset()
while (1);
/* NOTREACHED */
- } else
- sched_unpin();
+ }
DELAY(1000000);
}
Modified: head/sys/cddl/dev/dtrace/amd64/dtrace_subr.c
==============================================================================
--- head/sys/cddl/dev/dtrace/amd64/dtrace_subr.c Mon Jul 4 11:13:00
2011 (r223757)
+++ head/sys/cddl/dev/dtrace/amd64/dtrace_subr.c Mon Jul 4 12:04:52
2011 (r223758)
@@ -412,8 +412,8 @@ dtrace_gethrtime_init(void *arg)
continue;
pc = pcpu_find(i);
- map = PCPU_GET(cpumask);
- CPU_OR(&map, &pc->pc_cpumask);
+ CPU_SETOF(PCPU_GET(cpuid), &map);
+ CPU_SET(pc->pc_cpuid, &map);
smp_rendezvous_cpus(map, NULL,
dtrace_gethrtime_init_cpu,
Modified: head/sys/cddl/dev/dtrace/i386/dtrace_subr.c
==============================================================================
--- head/sys/cddl/dev/dtrace/i386/dtrace_subr.c Mon Jul 4 11:13:00 2011
(r223757)
+++ head/sys/cddl/dev/dtrace/i386/dtrace_subr.c Mon Jul 4 12:04:52 2011
(r223758)
@@ -413,8 +413,8 @@ dtrace_gethrtime_init(void *arg)
continue;
pc = pcpu_find(i);
- map = PCPU_GET(cpumask);
- CPU_OR(&map, &pc->pc_cpumask);
+ CPU_SETOF(PCPU_GET(cpuid), &map);
+ CPU_SET(pc->pc_cpuid, &map);
smp_rendezvous_cpus(map, NULL,
dtrace_gethrtime_init_cpu,
Modified: head/sys/dev/xen/control/control.c
==============================================================================
--- head/sys/dev/xen/control/control.c Mon Jul 4 11:13:00 2011
(r223757)
+++ head/sys/dev/xen/control/control.c Mon Jul 4 12:04:52 2011
(r223758)
@@ -197,6 +197,7 @@ extern void xencons_resume(void);
static void
xctrl_suspend()
{
+ u_int cpuid;
int i, j, k, fpp;
unsigned long max_pfn, start_info_mfn;
@@ -210,11 +211,11 @@ xctrl_suspend()
thread_lock(td);
sched_bind(td, 0);
thread_unlock(td);
- KASSERT(PCPU_GET(cpuid) == 0, ("xen_suspend: not running on cpu 0"));
+ cpuid = PCPU_GET(cpuid);
+ KASSERT(cpuid == 0, ("xen_suspend: not running on cpu 0"));
- sched_pin();
- map = PCPU_GET(other_cpus);
- sched_unpin();
+ map = all_cpus;
+ CPU_CLR(cpuid, &map);
CPU_NAND(&map, &stopped_cpus);
if (!CPU_EMPTY(&map))
stop_cpus(map);
Modified: head/sys/i386/i386/mp_machdep.c
==============================================================================
--- head/sys/i386/i386/mp_machdep.c Mon Jul 4 11:13:00 2011
(r223757)
+++ head/sys/i386/i386/mp_machdep.c Mon Jul 4 12:04:52 2011
(r223758)
@@ -658,12 +658,11 @@ cpu_mp_announce(void)
void
init_secondary(void)
{
- cpuset_t tcpuset, tallcpus;
struct pcpu *pc;
vm_offset_t addr;
int gsel_tss;
int x, myid;
- u_int cr0;
+ u_int cpuid, cr0;
/* bootAP is set in start_ap() to our ID. */
myid = bootAP;
@@ -758,8 +757,9 @@ init_secondary(void)
#endif
/* A quick check from sanity claus */
+ cpuid = PCPU_GET(cpuid);
if (PCPU_GET(apic_id) != lapic_id()) {
- printf("SMP: cpuid = %d\n", PCPU_GET(cpuid));
+ printf("SMP: cpuid = %d\n", cpuid);
printf("SMP: actual apic_id = %d\n", lapic_id());
printf("SMP: correct apic_id = %d\n", PCPU_GET(apic_id));
panic("cpuid mismatch! boom!!");
@@ -781,19 +781,13 @@ init_secondary(void)
smp_cpus++;
- CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", PCPU_GET(cpuid));
- printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid));
- tcpuset = PCPU_GET(cpumask);
+ CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", cpuid);
+ printf("SMP: AP CPU #%d Launched!\n", cpuid);
/* Determine if we are a logical CPU. */
/* XXX Calculation depends on cpu_logical being a power of 2, e.g. 2 */
if (cpu_logical > 1 && PCPU_GET(apic_id) % cpu_logical != 0)
- CPU_OR(&logical_cpus_mask, &tcpuset);
-
- /* Build our map of 'other' CPUs. */
- tallcpus = all_cpus;
- CPU_NAND(&tallcpus, &tcpuset);
- PCPU_SET(other_cpus, tallcpus);
+ CPU_SET(cpuid, &logical_cpus_mask);
if (bootverbose)
lapic_dump("AP");
@@ -934,7 +928,6 @@ assign_cpu_ids(void)
static int
start_all_aps(void)
{
- cpuset_t tallcpus;
#ifndef PC98
u_char mpbiosreason;
#endif
@@ -997,11 +990,6 @@ start_all_aps(void)
CPU_SET(cpu, &all_cpus); /* record AP in CPU map */
}
- /* build our map of 'other' CPUs */
- tallcpus = all_cpus;
- CPU_NAND(&tallcpus, PCPU_PTR(cpumask));
- PCPU_SET(other_cpus, tallcpus);
-
/* restore the warmstart vector */
*(u_int32_t *) WARMBOOT_OFF = mpbioswarmvec;
@@ -1253,9 +1241,7 @@ smp_targeted_tlb_shootdown(cpuset_t mask
if (othercpus < 1)
return;
} else {
- sched_pin();
- CPU_NAND(&mask, PCPU_PTR(cpumask));
- sched_unpin();
+ CPU_CLR(PCPU_GET(cpuid), &mask);
if (CPU_EMPTY(&mask))
return;
}
@@ -1452,11 +1438,12 @@ ipi_cpu(int cpu, u_int ipi)
void
ipi_all_but_self(u_int ipi)
{
+ cpuset_t other_cpus;
- sched_pin();
+ other_cpus = all_cpus;
+ CPU_CLR(PCPU_GET(cpuid), &other_cpus);
if (IPI_IS_BITMAPED(ipi)) {
- ipi_selected(PCPU_GET(other_cpus), ipi);
- sched_unpin();
+ ipi_selected(other_cpus, ipi);
return;
}
@@ -1466,8 +1453,7 @@ ipi_all_but_self(u_int ipi)
* Set the mask of receiving CPUs for this purpose.
*/
if (ipi == IPI_STOP_HARD)
- CPU_OR_ATOMIC(&ipi_nmi_pending, PCPU_PTR(other_cpus));
- sched_unpin();
+ CPU_OR_ATOMIC(&ipi_nmi_pending, &other_cpus);
CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS);
@@ -1476,7 +1462,7 @@ ipi_all_but_self(u_int ipi)
int
ipi_nmi_handler()
{
- cpuset_t cpumask;
+ u_int cpuid;
/*
* As long as there is not a simple way to know about a NMI's
@@ -1484,13 +1470,11 @@ ipi_nmi_handler()
* the global pending bitword an IPI_STOP_HARD has been issued
* and should be handled.
*/
- sched_pin();
- cpumask = PCPU_GET(cpumask);
- sched_unpin();
- if (!CPU_OVERLAP(&ipi_nmi_pending, &cpumask))
+ cpuid = PCPU_GET(cpuid);
+ if (!CPU_ISSET(cpuid, &ipi_nmi_pending))
return (1);
- CPU_NAND_ATOMIC(&ipi_nmi_pending, &cpumask);
+ CPU_CLR_ATOMIC(cpuid, &ipi_nmi_pending);
cpustop_handler();
return (0);
}
@@ -1502,25 +1486,21 @@ ipi_nmi_handler()
void
cpustop_handler(void)
{
- cpuset_t cpumask;
u_int cpu;
- sched_pin();
cpu = PCPU_GET(cpuid);
- cpumask = PCPU_GET(cpumask);
- sched_unpin();
savectx(&stoppcbs[cpu]);
/* Indicate that we are stopped */
- CPU_OR_ATOMIC(&stopped_cpus, &cpumask);
+ CPU_SET_ATOMIC(cpu, &stopped_cpus);
/* Wait for restart */
- while (!CPU_OVERLAP(&started_cpus, &cpumask))
+ while (!CPU_ISSET(cpu, &started_cpus))
ia32_pause();
- CPU_NAND_ATOMIC(&started_cpus, &cpumask);
- CPU_NAND_ATOMIC(&stopped_cpus, &cpumask);
+ CPU_CLR_ATOMIC(cpu, &started_cpus);
+ CPU_CLR_ATOMIC(cpu, &stopped_cpus);
if (cpu == 0 && cpustop_restartfunc != NULL) {
cpustop_restartfunc();
Modified: head/sys/i386/i386/pmap.c
==============================================================================
--- head/sys/i386/i386/pmap.c Mon Jul 4 11:13:00 2011 (r223757)
+++ head/sys/i386/i386/pmap.c Mon Jul 4 12:04:52 2011 (r223758)
@@ -932,16 +932,18 @@ pmap_update_pde_invalidate(vm_offset_t v
void
pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
{
- cpuset_t cpumask, other_cpus;
+ cpuset_t other_cpus;
+ u_int cpuid;
sched_pin();
if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) {
invlpg(va);
smp_invlpg(va);
} else {
- cpumask = PCPU_GET(cpumask);
- other_cpus = PCPU_GET(other_cpus);
- if (CPU_OVERLAP(&pmap->pm_active, &cpumask))
+ cpuid = PCPU_GET(cpuid);
+ other_cpus = all_cpus;
+ CPU_CLR(cpuid, &other_cpus);
+ if (CPU_ISSET(cpuid, &pmap->pm_active))
invlpg(va);
CPU_AND(&other_cpus, &pmap->pm_active);
if (!CPU_EMPTY(&other_cpus))
@@ -953,8 +955,9 @@ pmap_invalidate_page(pmap_t pmap, vm_off
void
pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
{
- cpuset_t cpumask, other_cpus;
+ cpuset_t other_cpus;
vm_offset_t addr;
+ u_int cpuid;
sched_pin();
if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) {
@@ -962,9 +965,10 @@ pmap_invalidate_range(pmap_t pmap, vm_of
invlpg(addr);
smp_invlpg_range(sva, eva);
} else {
- cpumask = PCPU_GET(cpumask);
- other_cpus = PCPU_GET(other_cpus);
- if (CPU_OVERLAP(&pmap->pm_active, &cpumask))
+ cpuid = PCPU_GET(cpuid);
+ other_cpus = all_cpus;
+ CPU_CLR(cpuid, &other_cpus);
+ if (CPU_ISSET(cpuid, &pmap->pm_active))
for (addr = sva; addr < eva; addr += PAGE_SIZE)
invlpg(addr);
CPU_AND(&other_cpus, &pmap->pm_active);
@@ -977,16 +981,18 @@ pmap_invalidate_range(pmap_t pmap, vm_of
void
pmap_invalidate_all(pmap_t pmap)
{
- cpuset_t cpumask, other_cpus;
+ cpuset_t other_cpus;
+ u_int cpuid;
sched_pin();
if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) {
invltlb();
smp_invltlb();
} else {
- cpumask = PCPU_GET(cpumask);
- other_cpus = PCPU_GET(other_cpus);
- if (CPU_OVERLAP(&pmap->pm_active, &cpumask))
+ cpuid = PCPU_GET(cpuid);
+ other_cpus = all_cpus;
+ CPU_CLR(cpuid, &other_cpus);
+ if (CPU_ISSET(cpuid, &pmap->pm_active))
invltlb();
CPU_AND(&other_cpus, &pmap->pm_active);
if (!CPU_EMPTY(&other_cpus))
@@ -1006,11 +1012,11 @@ pmap_invalidate_cache(void)
}
struct pde_action {
- cpuset_t store; /* processor that updates the PDE */
cpuset_t invalidate; /* processors that invalidate their TLB */
vm_offset_t va;
pd_entry_t *pde;
pd_entry_t newpde;
+ u_int store; /* processor that updates the PDE */
};
static void
@@ -1020,9 +1026,7 @@ pmap_update_pde_kernel(void *arg)
pd_entry_t *pde;
pmap_t pmap;
- sched_pin();
- if (!CPU_CMP(&act->store, PCPU_PTR(cpumask))) {
- sched_unpin();
+ if (act->store == PCPU_GET(cpuid)) {
/*
* Elsewhere, this operation requires allpmaps_lock for
@@ -1033,8 +1037,7 @@ pmap_update_pde_kernel(void *arg)
pde = pmap_pde(pmap, act->va);
pde_store(pde, act->newpde);
}
- } else
- sched_unpin();
+ }
}
static void
@@ -1042,12 +1045,8 @@ pmap_update_pde_user(void *arg)
{
struct pde_action *act = arg;
- sched_pin();
- if (!CPU_CMP(&act->store, PCPU_PTR(cpumask))) {
- sched_unpin();
+ if (act->store == PCPU_GET(cpuid))
pde_store(act->pde, act->newpde);
- } else
- sched_unpin();
}
static void
@@ -1055,12 +1054,8 @@ pmap_update_pde_teardown(void *arg)
{
struct pde_action *act = arg;
- sched_pin();
- if (CPU_OVERLAP(&act->invalidate, PCPU_PTR(cpumask))) {
- sched_unpin();
+ if (CPU_ISSET(PCPU_GET(cpuid), &act->invalidate))
pmap_update_pde_invalidate(act->va, act->newpde);
- } else
- sched_unpin();
}
/*
@@ -1075,23 +1070,25 @@ static void
pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t
newpde)
{
struct pde_action act;
- cpuset_t active, cpumask, other_cpus;
+ cpuset_t active, other_cpus;
+ u_int cpuid;
sched_pin();
- cpumask = PCPU_GET(cpumask);
- other_cpus = PCPU_GET(other_cpus);
+ cpuid = PCPU_GET(cpuid);
+ other_cpus = all_cpus;
+ CPU_CLR(cpuid, &other_cpus);
if (pmap == kernel_pmap)
active = all_cpus;
else
active = pmap->pm_active;
if (CPU_OVERLAP(&active, &other_cpus)) {
- act.store = cpumask;
+ act.store = cpuid;
act.invalidate = active;
act.va = va;
act.pde = pde;
act.newpde = newpde;
- CPU_OR(&cpumask, &active);
- smp_rendezvous_cpus(cpumask,
+ CPU_SET(cpuid, &active);
+ smp_rendezvous_cpus(active,
smp_no_rendevous_barrier, pmap == kernel_pmap ?
pmap_update_pde_kernel : pmap_update_pde_user,
pmap_update_pde_teardown, &act);
@@ -1100,7 +1097,7 @@ pmap_update_pde(pmap_t pmap, vm_offset_t
pmap_kenter_pde(va, newpde);
else
pde_store(pde, newpde);
- if (CPU_OVERLAP(&active, &cpumask))
+ if (CPU_ISSET(cpuid, &active))
pmap_update_pde_invalidate(va, newpde);
}
sched_unpin();
@@ -1925,12 +1922,12 @@ pmap_lazyfix_action(void)
}
static void
-pmap_lazyfix_self(cpuset_t mymask)
+pmap_lazyfix_self(u_int cpuid)
{
if (rcr3() == lazyptd)
load_cr3(PCPU_GET(curpcb)->pcb_cr3);
- CPU_NAND_ATOMIC(lazymask, &mymask);
+ CPU_CLR_ATOMIC(cpuid, lazymask);
}
@@ -1938,7 +1935,7 @@ static void
pmap_lazyfix(pmap_t pmap)
{
cpuset_t mymask, mask;
- u_int spins;
+ u_int cpuid, spins;
int lsb;
mask = pmap->pm_active;
@@ -1956,10 +1953,13 @@ pmap_lazyfix(pmap_t pmap)
#else
lazyptd = vtophys(pmap->pm_pdir);
#endif
- mymask = PCPU_GET(cpumask);
+ cpuid = PCPU_GET(cpuid);
+
+ /* Use a cpuset just for having an easy check. */
+ CPU_SETOF(cpuid, &mymask);
if (!CPU_CMP(&mask, &mymask)) {
lazymask = &pmap->pm_active;
- pmap_lazyfix_self(mymask);
+ pmap_lazyfix_self(cpuid);
} else {
atomic_store_rel_int((u_int *)&lazymask,
(u_int)&pmap->pm_active);
@@ -5098,17 +5098,19 @@ void
pmap_activate(struct thread *td)
{
pmap_t pmap, oldpmap;
+ u_int cpuid;
u_int32_t cr3;
critical_enter();
pmap = vmspace_pmap(td->td_proc->p_vmspace);
oldpmap = PCPU_GET(curpmap);
+ cpuid = PCPU_GET(cpuid);
#if defined(SMP)
- CPU_NAND_ATOMIC(&oldpmap->pm_active, PCPU_PTR(cpumask));
- CPU_OR_ATOMIC(&pmap->pm_active, PCPU_PTR(cpumask));
+ CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active);
+ CPU_SET_ATOMIC(cpuid, &pmap->pm_active);
#else
- CPU_NAND(&oldpmap->pm_active, PCPU_PTR(cpumask));
- CPU_OR(&pmap->pm_active, PCPU_PTR(cpumask));
+ CPU_CLR(cpuid, &oldpmap->pm_active);
+ CPU_SET(cpuid, &pmap->pm_active);
#endif
#ifdef PAE
cr3 = vtophys(pmap->pm_pdpt);
Modified: head/sys/i386/i386/vm_machdep.c
==============================================================================
--- head/sys/i386/i386/vm_machdep.c Mon Jul 4 11:13:00 2011
(r223757)
+++ head/sys/i386/i386/vm_machdep.c Mon Jul 4 12:04:52 2011
(r223758)
@@ -602,8 +602,8 @@ cpu_reset()
u_int cnt;
if (smp_active) {
- sched_pin();
- map = PCPU_GET(other_cpus);
+ map = all_cpus;
+ CPU_CLR(PCPU_GET(cpuid), &map);
CPU_NAND(&map, &stopped_cpus);
if (!CPU_EMPTY(&map)) {
printf("cpu_reset: Stopping other CPUs\n");
@@ -612,7 +612,6 @@ cpu_reset()
if (PCPU_GET(cpuid) != 0) {
cpu_reset_proxyid = PCPU_GET(cpuid);
- sched_unpin();
cpustop_restartfunc = cpu_reset_proxy;
cpu_reset_proxy_active = 0;
printf("cpu_reset: Restarting BSP\n");
@@ -632,8 +631,7 @@ cpu_reset()
while (1);
/* NOTREACHED */
- } else
- sched_unpin();
+ }
DELAY(1000000);
}
@@ -802,7 +800,8 @@ sf_buf_alloc(struct vm_page *m, int flag
struct sf_head *hash_list;
struct sf_buf *sf;
#ifdef SMP
- cpuset_t cpumask, other_cpus;
+ cpuset_t other_cpus;
+ u_int cpuid;
#endif
int error;
@@ -877,13 +876,14 @@ sf_buf_alloc(struct vm_page *m, int flag
CPU_ZERO(&sf->cpumask);
shootdown:
sched_pin();
- cpumask = PCPU_GET(cpumask);
- if (!CPU_OVERLAP(&cpumask, &sf->cpumask)) {
- CPU_OR(&sf->cpumask, &cpumask);
+ cpuid = PCPU_GET(cpuid);
+ if (!CPU_ISSET(cpuid, &sf->cpumask)) {
+ CPU_SET(cpuid, &sf->cpumask);
invlpg(sf->kva);
}
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
_______________________________________________
svn-src-all@freebsd.org mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
