Module Name: src
Committed By: cherry
Date: Thu Aug 11 17:59:00 UTC 2011
Modified Files:
src/sys/arch/xen/include: evtchn.h intr.h
src/sys/arch/xen/x86: intr.c
src/sys/arch/xen/xen: evtchn.c xenevt.c
Log Message:
Make event/interrupt handling MP aware
To generate a diff of this commit:
cvs rdiff -u -r1.18 -r1.19 src/sys/arch/xen/include/evtchn.h
cvs rdiff -u -r1.32 -r1.33 src/sys/arch/xen/include/intr.h
cvs rdiff -u -r1.27 -r1.28 src/sys/arch/xen/x86/intr.c
cvs rdiff -u -r1.49 -r1.50 src/sys/arch/xen/xen/evtchn.c
cvs rdiff -u -r1.37 -r1.38 src/sys/arch/xen/xen/xenevt.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/xen/include/evtchn.h
diff -u src/sys/arch/xen/include/evtchn.h:1.18 src/sys/arch/xen/include/evtchn.h:1.19
--- src/sys/arch/xen/include/evtchn.h:1.18 Fri Oct 23 02:32:33 2009
+++ src/sys/arch/xen/include/evtchn.h Thu Aug 11 17:58:59 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: evtchn.h,v 1.18 2009/10/23 02:32:33 snj Exp $ */
+/* $NetBSD: evtchn.h,v 1.19 2011/08/11 17:58:59 cherry Exp $ */
/*
*
@@ -41,8 +41,9 @@
int event_set_handler(int, int (*func)(void *), void *, int, const char *);
int event_remove_handler(int, int (*func)(void *), void *);
+struct cpu_info;
struct intrhand;
-void event_set_iplhandler(struct intrhand *, int);
+void event_set_iplhandler(struct cpu_info *, struct intrhand *, int);
extern int debug_port;
extern int xen_debug_handler(void *);
@@ -52,6 +53,8 @@
int unbind_pirq_from_evtch(int);
int unbind_virq_from_evtch(int);
+evtchn_port_t bind_vcpu_to_evtch(cpuid_t);
+
struct pintrhand {
int pirq;
int evtch;
Index: src/sys/arch/xen/include/intr.h
diff -u src/sys/arch/xen/include/intr.h:1.32 src/sys/arch/xen/include/intr.h:1.33
--- src/sys/arch/xen/include/intr.h:1.32 Wed Aug 10 11:39:45 2011
+++ src/sys/arch/xen/include/intr.h Thu Aug 11 17:58:59 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: intr.h,v 1.32 2011/08/10 11:39:45 cherry Exp $ */
+/* $NetBSD: intr.h,v 1.33 2011/08/11 17:58:59 cherry Exp $ */
/* NetBSD intr.h,v 1.15 2004/10/31 10:39:34 yamt Exp */
/*-
@@ -54,6 +54,7 @@
uint32_t ev_imask; /* interrupt mask */
struct intrhand *ev_handlers; /* handler chain */
struct evcnt ev_evcnt; /* interrupt counter */
+ struct cpu_info *ev_cpu; /* cpu on which this event is bound */
char ev_evname[32]; /* event counter name */
};
@@ -161,7 +162,6 @@
void intr_default_setup(void);
int x86_nmi(void);
-void intr_calculatemasks(struct evtsource *);
void *intr_establish(int, struct pic *, int, int, int, int (*)(void *), void *, bool);
void intr_disestablish(struct intrhand *);
@@ -180,9 +180,9 @@
int xen_send_ipi(struct cpu_info *, uint32_t);
void xen_broadcast_ipi(uint32_t);
#else
-#define xen_ipi_init(_1) do {} while(0) /* nothing */
-#define xen_send_ipi(_i1, _i2) do {} while(0) /* nothing */
-#define xen_broadcast_ipi(_i1) do {} while(0) /* nothing */
+#define xen_ipi_init(_1) ((void) 0) /* nothing */
+#define xen_send_ipi(_i1, _i2) (0) /* nothing */
+#define xen_broadcast_ipi(_i1) ((void) 0) /* nothing */
#endif /* MULTIPROCESSOR */
#endif /* !_LOCORE */
Index: src/sys/arch/xen/x86/intr.c
diff -u src/sys/arch/xen/x86/intr.c:1.27 src/sys/arch/xen/x86/intr.c:1.28
--- src/sys/arch/xen/x86/intr.c:1.27 Fri Mar 19 23:27:12 2010
+++ src/sys/arch/xen/x86/intr.c Thu Aug 11 17:59:00 2011
@@ -103,7 +103,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: intr.c,v 1.27 2010/03/19 23:27:12 dyoung Exp $");
+__KERNEL_RCSID(0, "$NetBSD: intr.c,v 1.28 2011/08/11 17:59:00 cherry Exp $");
#include "opt_multiprocessor.h"
#include "opt_xen.h"
@@ -119,6 +119,7 @@
#include <sys/proc.h>
#include <sys/errno.h>
#include <sys/cpu.h>
+#include <sys/simplelock.h>
#include <uvm/uvm_extern.h>
@@ -153,24 +154,6 @@
#endif
/*
- * Recalculate the interrupt from scratch for an event source.
- */
-void
-intr_calculatemasks(struct evtsource *evts)
-{
- struct intrhand *ih;
-
- evts->ev_maxlevel = IPL_NONE;
- evts->ev_imask = 0;
- for (ih = evts->ev_handlers; ih != NULL; ih = ih->ih_evt_next) {
- if (ih->ih_level > evts->ev_maxlevel)
- evts->ev_maxlevel = ih->ih_level;
- evts->ev_imask |= (1 << ih->ih_level);
- }
-
-}
-
-/*
* Fake interrupt handler structures for the benefit of symmetry with
* other interrupt sources.
*/
Index: src/sys/arch/xen/xen/evtchn.c
diff -u src/sys/arch/xen/xen/evtchn.c:1.49 src/sys/arch/xen/xen/evtchn.c:1.50
--- src/sys/arch/xen/xen/evtchn.c:1.49 Wed Aug 10 21:46:02 2011
+++ src/sys/arch/xen/xen/evtchn.c Thu Aug 11 17:59:00 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: evtchn.c,v 1.49 2011/08/10 21:46:02 cherry Exp $ */
+/* $NetBSD: evtchn.c,v 1.50 2011/08/11 17:59:00 cherry Exp $ */
/*
* Copyright (c) 2006 Manuel Bouyer.
@@ -54,20 +54,21 @@
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: evtchn.c,v 1.49 2011/08/10 21:46:02 cherry Exp $");
+__KERNEL_RCSID(0, "$NetBSD: evtchn.c,v 1.50 2011/08/11 17:59:00 cherry Exp $");
#include "opt_xen.h"
#include "isa.h"
#include "pci.h"
#include <sys/param.h>
+#include <sys/cpu.h>
#include <sys/kernel.h>
#include <sys/systm.h>
#include <sys/device.h>
#include <sys/proc.h>
#include <sys/malloc.h>
#include <sys/reboot.h>
-#include <sys/simplelock.h>
+#include <sys/mutex.h>
#include <uvm/uvm.h>
@@ -82,14 +83,23 @@
* This lock protects updates to the following mapping and reference-count
* arrays. The lock does not need to be acquired to read the mapping tables.
*/
-static struct simplelock irq_mapping_update_lock = SIMPLELOCK_INITIALIZER;
+static kmutex_t evtchn_lock;
/* event handlers */
struct evtsource *evtsource[NR_EVENT_CHANNELS];
-/* Reference counts for bindings to event channels */
+/* channel locks */
+static kmutex_t evtlock[NR_EVENT_CHANNELS];
+
+/* Reference counts for bindings to event channels XXX: redo for SMP */
static uint8_t evtch_bindcount[NR_EVENT_CHANNELS];
+/* event-channel <-> VCPU mapping for IPIs. XXX: redo for SMP. */
+static evtchn_port_t vcpu_ipi_to_evtch[MAX_VIRT_CPUS];
+
+/* event-channel <-> VCPU mapping for VIRQ_TIMER. XXX: redo for SMP. */
+static int virq_timer_to_evtch[MAX_VIRT_CPUS];
+
/* event-channel <-> VIRQ mapping. */
static int virq_to_evtch[NR_VIRQS];
@@ -137,6 +147,14 @@
{
int i;
+ /* No VCPU -> event mappings. */
+ for (i = 0; i < MAX_VIRT_CPUS; i++)
+ vcpu_ipi_to_evtch[i] = -1;
+
+ /* No VIRQ_TIMER -> event mappings. */
+ for (i = 0; i < MAX_VIRT_CPUS; i++)
+ virq_timer_to_evtch[i] = -1;
+
/* No VIRQ -> event mappings. */
for (i = 0; i < NR_VIRQS; i++)
virq_to_evtch[i] = -1;
@@ -162,6 +180,8 @@
events_init(void)
{
debug_port = bind_virq_to_evtch(VIRQ_DEBUG);
+ KASSERT(debug_port != -1);
+
aprint_verbose("debug virtual interrupt using event channel %d\n",
debug_port);
/*
@@ -197,7 +217,7 @@
if (evtch == IRQ_DEBUG)
printf("evtchn_do_event: evtch %d\n", evtch);
#endif
- ci = &cpu_info_primary;
+ ci = curcpu();
/*
* Shortcut for the debug handler, we want it to always run,
@@ -217,31 +237,46 @@
ci->ci_data.cpu_nintr++;
evtsource[evtch]->ev_evcnt.ev_count++;
ilevel = ci->ci_ilevel;
- if (evtsource[evtch]->ev_maxlevel <= ilevel) {
+ if (evtsource[evtch]->ev_maxlevel <= ilevel ||
+ evtsource[evtch]->ev_cpu != ci /* XXX: get stats */) {
#ifdef IRQ_DEBUG
if (evtch == IRQ_DEBUG)
printf("evtsource[%d]->ev_maxlevel %d <= ilevel %d\n",
evtch, evtsource[evtch]->ev_maxlevel, ilevel);
#endif
- hypervisor_set_ipending(ci, evtsource[evtch]->ev_imask,
- evtch >> LONG_SHIFT, evtch & LONG_MASK);
- /* leave masked */
+ hypervisor_set_ipending(evtsource[evtch]->ev_cpu,
+ evtsource[evtch]->ev_imask,
+ evtch >> LONG_SHIFT,
+ evtch & LONG_MASK);
+
+ if (evtsource[evtch]->ev_cpu != ci) {
+ /* facilitate spllower() on remote cpu */
+ struct cpu_info *rci = evtsource[evtch]->ev_cpu;
+ if (xen_send_ipi(rci, XEN_IPI_KICK) != 0) {
+ panic("xen_send_ipi(%s, XEN_IPI_KICK) failed\n", cpu_name(rci));
+ }
+ }
+
+ /* leave masked */
return 0;
}
ci->ci_ilevel = evtsource[evtch]->ev_maxlevel;
iplmask = evtsource[evtch]->ev_imask;
sti();
+ mutex_spin_enter(&evtlock[evtch]);
ih = evtsource[evtch]->ev_handlers;
while (ih != NULL) {
- if (ih->ih_level <= ilevel) {
+ if (ih->ih_level <= ilevel ||
+ ih->ih_cpu != ci) {
#ifdef IRQ_DEBUG
if (evtch == IRQ_DEBUG)
printf("ih->ih_level %d <= ilevel %d\n", ih->ih_level, ilevel);
#endif
cli();
- hypervisor_set_ipending(ci, iplmask,
+ hypervisor_set_ipending(ih->ih_cpu, iplmask,
evtch >> LONG_SHIFT, evtch & LONG_MASK);
/* leave masked */
+ mutex_spin_exit(&evtlock[evtch]);
goto splx;
}
iplmask &= ~IUNMASK(ci, ih->ih_level);
@@ -250,6 +285,7 @@
ih_fun(ih->ih_arg, regs);
ih = ih->ih_evt_next;
}
+ mutex_spin_exit(&evtlock[evtch]);
cli();
hypervisor_enable_event(evtch);
splx:
@@ -267,6 +303,7 @@
ci->ci_ilevel = i;
for (ih = ci->ci_isources[i]->ipl_handlers;
ih != NULL; ih = ih->ih_ipl_next) {
+ KASSERT(ih->ih_cpu == ci);
sti();
ih_fun = (void *)ih->ih_fun;
ih_fun(ih->ih_arg, regs);
@@ -292,6 +329,37 @@
return 0;
}
+#define PRIuCPUID "lu" /* XXX: move this somewhere more appropriate */
+
+evtchn_port_t
+bind_vcpu_to_evtch(cpuid_t vcpu)
+{
+ evtchn_op_t op;
+ evtchn_port_t evtchn;
+ int s;
+
+ s = splhigh();
+ mutex_spin_enter(&evtchn_lock);
+
+ evtchn = vcpu_ipi_to_evtch[vcpu];
+ if (evtchn == -1) {
+ op.cmd = EVTCHNOP_bind_ipi;
+ op.u.bind_ipi.vcpu = (uint32_t) vcpu;
+ if (HYPERVISOR_event_channel_op(&op) != 0)
+ panic("Failed to bind ipi to VCPU %"PRIuCPUID"\n", vcpu);
+ evtchn = op.u.bind_ipi.port;
+
+ vcpu_ipi_to_evtch[vcpu] = evtchn;
+ }
+
+ evtch_bindcount[evtchn]++;
+
+ mutex_spin_exit(&evtchn_lock);
+ splx(s);
+
+ return evtchn;
+}
+
int
bind_virq_to_evtch(int virq)
{
@@ -299,13 +367,31 @@
int evtchn, s;
s = splhigh();
- simple_lock(&irq_mapping_update_lock);
+ mutex_spin_enter(&evtchn_lock);
+
+ /*
+ * XXX: The only per-cpu VIRQ we currently use is VIRQ_TIMER.
+ * Please re-visit this implementation when others are used.
+ * Note: VIRQ_DEBUG is special-cased, and not used or bound on APs.
+ * XXX: event->virq/ipi can be unified in a linked-list
+ * implementation.
+ */
+ struct cpu_info *ci = curcpu();
+
+ if (virq == VIRQ_DEBUG && ci != &cpu_info_primary) {
+ return -1;
+ }
- evtchn = virq_to_evtch[virq];
+ if (virq == VIRQ_TIMER) {
+ evtchn = virq_timer_to_evtch[ci->ci_cpuid];
+ }
+ else {
+ evtchn = virq_to_evtch[virq];
+ }
if (evtchn == -1) {
op.cmd = EVTCHNOP_bind_virq;
op.u.bind_virq.virq = virq;
- op.u.bind_virq.vcpu = 0;
+ op.u.bind_virq.vcpu = ci->ci_cpuid;
if (HYPERVISOR_event_channel_op(&op) != 0)
panic("Failed to bind virtual IRQ %d\n", virq);
evtchn = op.u.bind_virq.port;
@@ -315,7 +401,7 @@
evtch_bindcount[evtchn]++;
- simple_unlock(&irq_mapping_update_lock);
+ mutex_spin_exit(&evtchn_lock);
splx(s);
return evtchn;
@@ -325,10 +411,24 @@
unbind_virq_from_evtch(int virq)
{
evtchn_op_t op;
- int evtchn = virq_to_evtch[virq];
- int s = splhigh();
+ int evtchn;
+ int s;
+
+ struct cpu_info *ci = curcpu();
+
+ if (virq == VIRQ_TIMER) {
+ evtchn = virq_timer_to_evtch[ci->ci_cpuid];
+ }
+ else {
+ evtchn = virq_to_evtch[virq];
+ }
+
+ if (evtchn == -1) {
+ return -1;
+ }
- simple_lock(&irq_mapping_update_lock);
+ s = splhigh();
+ mutex_spin_enter(&evtchn_lock);
evtch_bindcount[evtchn]--;
if (evtch_bindcount[evtchn] == 0) {
@@ -340,7 +440,7 @@
virq_to_evtch[virq] = -1;
}
- simple_unlock(&irq_mapping_update_lock);
+ mutex_spin_exit(&evtchn_lock);
splx(s);
return evtchn;
@@ -358,7 +458,7 @@
}
s = splhigh();
- simple_lock(&irq_mapping_update_lock);
+ mutex_spin_enter(&evtchn_lock);
evtchn = pirq_to_evtch[pirq];
if (evtchn == -1) {
@@ -377,7 +477,7 @@
evtch_bindcount[evtchn]++;
- simple_unlock(&irq_mapping_update_lock);
+ mutex_spin_exit(&evtchn_lock);
splx(s);
return evtchn;
@@ -390,7 +490,7 @@
int evtchn = pirq_to_evtch[pirq];
int s = splhigh();
- simple_lock(&irq_mapping_update_lock);
+ mutex_spin_enter(&evtchn_lock);
evtch_bindcount[evtchn]--;
if (evtch_bindcount[evtchn] == 0) {
@@ -402,7 +502,7 @@
pirq_to_evtch[pirq] = -1;
}
- simple_unlock(&irq_mapping_update_lock);
+ mutex_spin_exit(&evtchn_lock);
splx(s);
return evtchn;
@@ -420,8 +520,10 @@
printf("pirq_establish: can't malloc handler info\n");
return NULL;
}
-
- event_set_handler(evtch, pirq_interrupt, ih, level, evname);
+ if (event_set_handler(evtch, pirq_interrupt, ih, level, evname) != 0) {
+ free(ih, M_DEVBUF);
+ return NULL;
+ }
ih->pirq = pirq;
ih->evtch = evtch;
ih->func = func;
@@ -459,11 +561,34 @@
#endif /* NPCI > 0 || NISA > 0 */
+
+/*
+ * Recalculate the interrupt from scratch for an event source.
+ */
+static void
+intr_calculatemasks(struct evtsource *evts, int evtch)
+{
+ struct intrhand *ih;
+
+#ifdef MULTIPROCESSOR
+ KASSERT(!mutex_owned(&evtlock[evtch]));
+#endif
+ mutex_spin_enter(&evtlock[evtch]);
+ evts->ev_maxlevel = IPL_NONE;
+ evts->ev_imask = 0;
+ for (ih = evts->ev_handlers; ih != NULL; ih = ih->ih_evt_next) {
+ if (ih->ih_level > evts->ev_maxlevel)
+ evts->ev_maxlevel = ih->ih_level;
+ evts->ev_imask |= (1 << ih->ih_level);
+ }
+ mutex_spin_exit(&evtlock[evtch]);
+}
+
int
event_set_handler(int evtch, int (*func)(void *), void *arg, int level,
const char *evname)
{
- struct cpu_info *ci = &cpu_info_primary;
+ struct cpu_info *ci = curcpu(); /* XXX: pass in ci ? */
struct evtsource *evts;
struct intrhand *ih, **ihp;
int s;
@@ -497,6 +622,7 @@
ih->ih_arg = ih->ih_realarg = arg;
ih->ih_evt_next = NULL;
ih->ih_ipl_next = NULL;
+ ih->ih_cpu = ci;
#ifdef MULTIPROCESSOR
if (!mpsafe) {
ih->ih_fun = intr_biglock_wrapper;
@@ -506,8 +632,8 @@
s = splhigh();
- /* register handler for spllower() */
- event_set_iplhandler(ih, level);
+ /* register per-cpu handler for spllower() */
+ event_set_iplhandler(ci, ih, level);
/* register handler for event channel */
if (evtsource[evtch] == NULL) {
@@ -515,7 +641,16 @@
M_DEVBUF, M_WAITOK|M_ZERO);
if (evts == NULL)
panic("can't allocate fixed interrupt source");
+
evts->ev_handlers = ih;
+ /*
+ * XXX: We're assuming here that ci is the same cpu as
+ * the one on which this event/port is bound on. The
+ * api needs to be reshuffled so that this assumption
+ * is more explicitly implemented.
+ */
+ evts->ev_cpu = ci;
+ mutex_init(&evtlock[evtch], MUTEX_DEFAULT, IPL_VM);
evtsource[evtch] = evts;
if (evname)
strncpy(evts->ev_evname, evname,
@@ -528,6 +663,7 @@
} else {
evts = evtsource[evtch];
/* sort by IPL order, higher first */
+ mutex_spin_enter(&evtlock[evtch]);
for (ihp = &evts->ev_handlers; ; ihp = &((*ihp)->ih_evt_next)) {
if ((*ihp)->ih_level < ih->ih_level) {
/* insert before *ihp */
@@ -540,20 +676,23 @@
break;
}
}
+ mutex_spin_exit(&evtlock[evtch]);
}
- intr_calculatemasks(evts);
+ intr_calculatemasks(evts, evtch);
splx(s);
return 0;
}
void
-event_set_iplhandler(struct intrhand *ih, int level)
+event_set_iplhandler(struct cpu_info *ci,
+ struct intrhand *ih,
+ int level)
{
- struct cpu_info *ci = &cpu_info_primary;
struct iplsource *ipls;
+ KASSERT(ci == ih->ih_cpu);
if (ci->ci_isources[level] == NULL) {
ipls = malloc(sizeof (struct iplsource),
M_DEVBUF, M_WAITOK|M_ZERO);
@@ -577,21 +716,25 @@
struct evtsource *evts;
struct intrhand *ih;
struct intrhand **ihp;
- struct cpu_info *ci = &cpu_info_primary;
+ struct cpu_info *ci = curcpu();
evts = evtsource[evtch];
if (evts == NULL)
return ENOENT;
+ mutex_spin_enter(&evtlock[evtch]);
for (ihp = &evts->ev_handlers, ih = evts->ev_handlers;
ih != NULL;
ihp = &ih->ih_evt_next, ih = ih->ih_evt_next) {
if (ih->ih_fun == func && ih->ih_arg == arg)
break;
}
- if (ih == NULL)
+ if (ih == NULL) {
+ mutex_spin_exit(&evtlock[evtch]);
return ENOENT;
+ }
*ihp = ih->ih_evt_next;
+ mutex_spin_exit(&evtlock[evtch]);
ipls = ci->ci_isources[ih->ih_level];
for (ihp = &ipls->ipl_handlers, ih = ipls->ipl_handlers;
@@ -609,7 +752,7 @@
free(evts, M_DEVBUF);
evtsource[evtch] = NULL;
} else {
- intr_calculatemasks(evts);
+ intr_calculatemasks(evts, evtch);
}
return 0;
}
Index: src/sys/arch/xen/xen/xenevt.c
diff -u src/sys/arch/xen/xen/xenevt.c:1.37 src/sys/arch/xen/xen/xenevt.c:1.38
--- src/sys/arch/xen/xen/xenevt.c:1.37 Sun May 22 04:27:15 2011
+++ src/sys/arch/xen/xen/xenevt.c Thu Aug 11 17:59:00 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: xenevt.c,v 1.37 2011/05/22 04:27:15 rmind Exp $ */
+/* $NetBSD: xenevt.c,v 1.38 2011/08/11 17:59:00 cherry Exp $ */
/*
* Copyright (c) 2005 Manuel Bouyer.
@@ -26,7 +26,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: xenevt.c,v 1.37 2011/05/22 04:27:15 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: xenevt.c,v 1.38 2011/08/11 17:59:00 cherry Exp $");
#include "opt_xen.h"
#include <sys/param.h>
@@ -160,6 +160,7 @@
ih->ih_fun = ih->ih_realfun = xenevt_processevt;
ih->ih_arg = ih->ih_realarg = NULL;
ih->ih_ipl_next = NULL;
+ ih->ih_cpu = curcpu();
#ifdef MULTIPROCESSOR
if (!mpsafe) {
ih->ih_fun = intr_biglock_wrapper;
@@ -168,7 +169,7 @@
#endif /* MULTIPROCESSOR */
s = splhigh();
- event_set_iplhandler(ih, level);
+ event_set_iplhandler(ih->ih_cpu, ih, level);
splx(s);
}
@@ -178,7 +179,7 @@
{
xenevt_ev1 |= 1UL << l1;
xenevt_ev2[l1] |= 1UL << l2;
- curcpu()->ci_ipending |= 1 << IPL_HIGH;
+ curcpu()/*XXX*/->ci_ipending |= 1 << IPL_HIGH;
}
/* process pending events */
