Since interrupt handlers are running asynchronously it is difficult to
make sure that one isn't running on another cpu while you're frobbing
data structures that you share with it. This is an issue for example
in our network drivers, where we want to clean up the rings when we
bring the interface down, but where the interrupt handler might still
be processing incoming packets. Currently we rely on the kernel lock
to prevent this from happening.
The diff below adds an interface (called intr_sync()) to help solving
this problem. The idea is that after you've called intr_sync() you
can be sure that an interrupt handler that was running before the
intr_sync() call will have finished when intr_sync() returns. This
means that if your interrupt handler checks the IFF_RUNNING flag
before touhcing the rings, the function that brings down the interface
can:
1. clear the IFF_RUNNING flag
2. call intr_sync()
3. free the rings
The intr_sync() function does its job by scheduling a task and make
sure that it runs on the same CPU as the interrupt handler. dlg@
tells me this is what Linux does. The current implementation "knows"
that all interrupt handlers run on the primary CPU. But this is
something that may change in the near future (I have a working diff
for sparc64 to do this). That's why intr_sync() has a void *
argument. The idea is that you pass the "cookie" you got back when
establishing the interrupt handler. The MD code should be able to
figure out the CPU the handler istied to from that.
Unfortunately this means we'll need to implement this function for all
our architectures before we can use it. But I can probably split of
the guts of it into a common function.
Does this make sense? I'd like to get all the bikeshedding out of the
way before I go on and implement this for all our architectures.
Index: sparc64/intr.c
===================================================================
RCS file: /cvs/src/sys/arch/sparc64/sparc64/intr.c,v
retrieving revision 1.52
diff -u -p -r1.52 intr.c
--- sparc64/intr.c 5 Oct 2014 11:40:37 -0000 1.52
+++ sparc64/intr.c 12 Sep 2015 12:36:44 -0000
@@ -45,6 +45,8 @@
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
+#include <sys/proc.h>
+#include <sys/task.h>
#include <dev/cons.h>
@@ -310,6 +312,41 @@ intr_establish(int level, struct intrhan
#endif
splx(s);
+}
+
+void
+intr_sync_task(void *arg)
+{
+ CPU_INFO_ITERATOR cii;
+ struct cpu_info *ci;
+ volatile int *done = arg;
+
+ CPU_INFO_FOREACH(cii, ci) {
+ if (CPU_IS_PRIMARY(ci)) {
+ sched_peg_curproc(ci);
+ break;
+ }
+ }
+
+ *done = 1;
+ wakeup(arg);
+
+ atomic_clearbits_int(&curproc->p_flag, P_CPUPEG);
+}
+
+void
+intr_sync(void *ih)
+{
+ struct sleep_state sls;
+ struct task task;
+ volatile int done = 0;
+
+ task_set(&task, intr_sync_task, (void *)&done);
+ task_add(systq, &task);
+ while (!done) {
+ sleep_setup(&sls, &done, PWAIT, "isync");
+ sleep_finish(&sls, !done);
+ }
}
void *
Index: include/intr.h
===================================================================
RCS file: /cvs/src/sys/arch/sparc64/include/intr.h,v
retrieving revision 1.16
diff -u -p -r1.16 intr.h
--- include/intr.h 17 May 2013 18:26:37 -0000 1.16
+++ include/intr.h 12 Sep 2015 12:36:44 -0000
@@ -84,6 +84,8 @@ void intr_establish(int, struct intrh
#define IPL_MPSAFE 0x100
+void intr_sync(void *);
+
void *softintr_establish(int, void (*)(void *), void *);
void softintr_disestablish(void *);
void softintr_schedule(void *);
Index: dev/vnet.c
===================================================================
RCS file: /cvs/src/sys/arch/sparc64/dev/vnet.c,v
retrieving revision 1.46
diff -u -p -r1.46 vnet.c
--- dev/vnet.c 24 Jun 2015 09:40:53 -0000 1.46
+++ dev/vnet.c 12 Sep 2015 12:36:45 -0000
@@ -1436,6 +1436,9 @@ vnet_stop(struct ifnet *ifp)
cbus_intr_setenabled(sc->sc_bustag, sc->sc_tx_ino, INTR_DISABLED);
cbus_intr_setenabled(sc->sc_bustag, sc->sc_rx_ino, INTR_DISABLED);
+ intr_sync(sc->sc_tx_ih);
+ intr_sync(sc->sc_rx_ih);
+
hv_ldc_tx_qconf(lc->lc_id, 0, 0);
hv_ldc_rx_qconf(lc->lc_id, 0, 0);
lc->lc_tx_seqid = 0;
